diff --git a/.ci/update_windows/update.py b/.ci/update_windows/update.py
index ef9374c44..127247b2f 100755
--- a/.ci/update_windows/update.py
+++ b/.ci/update_windows/update.py
@@ -1,6 +1,9 @@
import pygit2
from datetime import datetime
import sys
+import os
+import shutil
+import filecmp
def pull(repo, remote_name='origin', branch='master'):
for remote in repo.remotes:
@@ -42,7 +45,8 @@ def pull(repo, remote_name='origin', branch='master'):
raise AssertionError('Unknown merge analysis result')
pygit2.option(pygit2.GIT_OPT_SET_OWNER_VALIDATION, 0)
-repo = pygit2.Repository(str(sys.argv[1]))
+repo_path = str(sys.argv[1])
+repo = pygit2.Repository(repo_path)
ident = pygit2.Signature('comfyui', 'comfy@ui')
try:
print("stashing current changes")
@@ -51,7 +55,10 @@ except KeyError:
print("nothing to stash")
backup_branch_name = 'backup_branch_{}'.format(datetime.today().strftime('%Y-%m-%d_%H_%M_%S'))
print("creating backup branch: {}".format(backup_branch_name))
-repo.branches.local.create(backup_branch_name, repo.head.peel())
+try:
+ repo.branches.local.create(backup_branch_name, repo.head.peel())
+except:
+ pass
print("checking out master branch")
branch = repo.lookup_branch('master')
@@ -63,3 +70,41 @@ pull(repo)
print("Done!")
+self_update = True
+if len(sys.argv) > 2:
+ self_update = '--skip_self_update' not in sys.argv
+
+update_py_path = os.path.realpath(__file__)
+repo_update_py_path = os.path.join(repo_path, ".ci/update_windows/update.py")
+
+cur_path = os.path.dirname(update_py_path)
+
+
+req_path = os.path.join(cur_path, "current_requirements.txt")
+repo_req_path = os.path.join(repo_path, "requirements.txt")
+
+
+def files_equal(file1, file2):
+ try:
+ return filecmp.cmp(file1, file2, shallow=False)
+ except:
+ return False
+
+def file_size(f):
+ try:
+ return os.path.getsize(f)
+ except:
+ return 0
+
+
+if self_update and not files_equal(update_py_path, repo_update_py_path) and file_size(repo_update_py_path) > 10:
+ shutil.copy(repo_update_py_path, os.path.join(cur_path, "update_new.py"))
+ exit()
+
+if not os.path.exists(req_path) or not files_equal(repo_req_path, req_path):
+ import subprocess
+ try:
+ subprocess.check_call([sys.executable, '-s', '-m', 'pip', 'install', '-r', repo_req_path])
+ shutil.copy(repo_req_path, req_path)
+ except:
+ pass
diff --git a/.ci/update_windows/update_comfyui.bat b/.ci/update_windows/update_comfyui.bat
index 60d1e694f..bb08c0de0 100755
--- a/.ci/update_windows/update_comfyui.bat
+++ b/.ci/update_windows/update_comfyui.bat
@@ -1,2 +1,8 @@
+@echo off
..\python_embeded\python.exe .\update.py ..\ComfyUI\
-pause
+if exist update_new.py (
+ move /y update_new.py update.py
+ echo Running updater again since it got updated.
+ ..\python_embeded\python.exe .\update.py ..\ComfyUI\ --skip_self_update
+)
+if "%~1"=="" pause
diff --git a/.ci/update_windows/update_comfyui_and_python_dependencies.bat b/.ci/update_windows/update_comfyui_and_python_dependencies.bat
deleted file mode 100755
index b7308550d..000000000
--- a/.ci/update_windows/update_comfyui_and_python_dependencies.bat
+++ /dev/null
@@ -1,3 +0,0 @@
-..\python_embeded\python.exe .\update.py ..\ComfyUI\
-..\python_embeded\python.exe -s -m pip install --upgrade torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cu117 xformers -r ../ComfyUI/requirements.txt pygit2
-pause
diff --git a/.ci/update_windows_cu118/update_comfyui_and_python_dependencies.bat b/.ci/update_windows_cu118/update_comfyui_and_python_dependencies.bat
deleted file mode 100755
index c33adc0a7..000000000
--- a/.ci/update_windows_cu118/update_comfyui_and_python_dependencies.bat
+++ /dev/null
@@ -1,11 +0,0 @@
-@echo off
-..\python_embeded\python.exe .\update.py ..\ComfyUI\
-echo
-echo This will try to update pytorch and all python dependencies, if you get an error wait for pytorch/xformers to fix their stuff
-echo You should not be running this anyways unless you really have to
-echo
-echo If you just want to update normally, close this and run update_comfyui.bat instead.
-echo
-pause
-..\python_embeded\python.exe -s -m pip install --upgrade torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cu118 xformers -r ../ComfyUI/requirements.txt pygit2
-pause
diff --git a/.github/workflows/windows_release_cu118_dependencies.yml b/.github/workflows/windows_release_cu118_dependencies.yml
deleted file mode 100644
index 75c42b624..000000000
--- a/.github/workflows/windows_release_cu118_dependencies.yml
+++ /dev/null
@@ -1,71 +0,0 @@
-name: "Windows Release cu118 dependencies"
-
-on:
- workflow_dispatch:
-# push:
-# branches:
-# - master
-
-jobs:
- build_dependencies:
- env:
- # you need at least cuda 5.0 for some of the stuff compiled here.
- TORCH_CUDA_ARCH_LIST: "5.0+PTX 6.0 6.1 7.0 7.5 8.0 8.6 8.9"
- FORCE_CUDA: 1
- MAX_JOBS: 1 # will crash otherwise
- DISTUTILS_USE_SDK: 1 # otherwise distutils will complain on windows about multiple versions of msvc
- XFORMERS_BUILD_TYPE: "Release"
- runs-on: windows-latest
- steps:
- - name: Cache Built Dependencies
- uses: actions/cache@v3
- id: cache-cu118_python_stuff
- with:
- path: cu118_python_deps.tar
- key: ${{ runner.os }}-build-cu118
-
- - if: steps.cache-cu118_python_stuff.outputs.cache-hit != 'true'
- uses: actions/checkout@v3
-
- - if: steps.cache-cu118_python_stuff.outputs.cache-hit != 'true'
- uses: actions/setup-python@v4
- with:
- python-version: '3.10.9'
-
- - if: steps.cache-cu118_python_stuff.outputs.cache-hit != 'true'
- uses: comfyanonymous/cuda-toolkit@test
- id: cuda-toolkit
- with:
- cuda: '11.8.0'
- # copied from xformers github
- - name: Setup MSVC
- uses: ilammy/msvc-dev-cmd@v1
- - name: Configure Pagefile
- # windows runners will OOM with many CUDA architectures
- # we cheat here with a page file
- uses: al-cheb/configure-pagefile-action@v1.3
- with:
- minimum-size: 2GB
- # really unfortunate: https://github.com/ilammy/msvc-dev-cmd#name-conflicts-with-shell-bash
- - name: Remove link.exe
- shell: bash
- run: rm /usr/bin/link
-
- - if: steps.cache-cu118_python_stuff.outputs.cache-hit != 'true'
- shell: bash
- run: |
- python -m pip wheel --no-cache-dir torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cu118 -r requirements.txt pygit2 -w ./temp_wheel_dir
- python -m pip install --no-cache-dir ./temp_wheel_dir/*
- echo installed basic
- git clone --recurse-submodules https://github.com/facebookresearch/xformers.git
- cd xformers
- python -m pip install --no-cache-dir wheel setuptools twine
- echo building xformers
- python setup.py bdist_wheel -d ../temp_wheel_dir/
- cd ..
- rm -rf xformers
- ls -lah temp_wheel_dir
- mv temp_wheel_dir cu118_python_deps
- tar cf cu118_python_deps.tar cu118_python_deps
-
-
diff --git a/.github/workflows/windows_release_cu118_dependencies_2.yml b/.github/workflows/windows_release_cu118_dependencies_2.yml
deleted file mode 100644
index a7760b21e..000000000
--- a/.github/workflows/windows_release_cu118_dependencies_2.yml
+++ /dev/null
@@ -1,37 +0,0 @@
-name: "Windows Release cu118 dependencies 2"
-
-on:
- workflow_dispatch:
- inputs:
- xformers:
- description: 'xformers version'
- required: true
- type: string
- default: "xformers"
-
-# push:
-# branches:
-# - master
-
-jobs:
- build_dependencies:
- runs-on: windows-latest
- steps:
- - uses: actions/checkout@v3
- - uses: actions/setup-python@v4
- with:
- python-version: '3.10.9'
-
- - shell: bash
- run: |
- python -m pip wheel --no-cache-dir torch torchvision torchaudio ${{ inputs.xformers }} --extra-index-url https://download.pytorch.org/whl/cu118 -r requirements.txt pygit2 -w ./temp_wheel_dir
- python -m pip install --no-cache-dir ./temp_wheel_dir/*
- echo installed basic
- ls -lah temp_wheel_dir
- mv temp_wheel_dir cu118_python_deps
- tar cf cu118_python_deps.tar cu118_python_deps
-
- - uses: actions/cache/save@v3
- with:
- path: cu118_python_deps.tar
- key: ${{ runner.os }}-build-cu118
diff --git a/.github/workflows/windows_release_cu118_package.yml b/.github/workflows/windows_release_cu118_package.yml
deleted file mode 100644
index 0f0fbf280..000000000
--- a/.github/workflows/windows_release_cu118_package.yml
+++ /dev/null
@@ -1,79 +0,0 @@
-name: "Windows Release cu118 packaging"
-
-on:
- workflow_dispatch:
-# push:
-# branches:
-# - master
-
-jobs:
- package_comfyui:
- permissions:
- contents: "write"
- packages: "write"
- pull-requests: "read"
- runs-on: windows-latest
- steps:
- - uses: actions/cache/restore@v3
- id: cache
- with:
- path: cu118_python_deps.tar
- key: ${{ runner.os }}-build-cu118
- - shell: bash
- run: |
- mv cu118_python_deps.tar ../
- cd ..
- tar xf cu118_python_deps.tar
- pwd
- ls
-
- - uses: actions/checkout@v3
- with:
- fetch-depth: 0
- persist-credentials: false
- - shell: bash
- run: |
- cd ..
- cp -r ComfyUI ComfyUI_copy
- curl https://www.python.org/ftp/python/3.10.9/python-3.10.9-embed-amd64.zip -o python_embeded.zip
- unzip python_embeded.zip -d python_embeded
- cd python_embeded
- echo 'import site' >> ./python310._pth
- curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py
- ./python.exe get-pip.py
- ./python.exe -s -m pip install ../cu118_python_deps/*
- sed -i '1i../ComfyUI' ./python310._pth
- cd ..
-
- git clone https://github.com/comfyanonymous/taesd
- cp taesd/*.pth ./ComfyUI_copy/models/vae_approx/
-
- mkdir ComfyUI_windows_portable
- mv python_embeded ComfyUI_windows_portable
- mv ComfyUI_copy ComfyUI_windows_portable/ComfyUI
-
- cd ComfyUI_windows_portable
-
- mkdir update
- cp -r ComfyUI/.ci/update_windows/* ./update/
- cp -r ComfyUI/.ci/update_windows_cu118/* ./update/
- cp -r ComfyUI/.ci/windows_base_files/* ./
-
- cd ..
-
- "C:\Program Files\7-Zip\7z.exe" a -t7z -m0=lzma -mx=8 -mfb=64 -md=32m -ms=on -mf=BCJ2 ComfyUI_windows_portable.7z ComfyUI_windows_portable
- mv ComfyUI_windows_portable.7z ComfyUI/new_ComfyUI_windows_portable_nvidia_cu118_or_cpu.7z
-
- cd ComfyUI_windows_portable
- python_embeded/python.exe -s ComfyUI/main.py --quick-test-for-ci --cpu
-
- ls
-
- - name: Upload binaries to release
- uses: svenstaro/upload-release-action@v2
- with:
- repo_token: ${{ secrets.GITHUB_TOKEN }}
- file: new_ComfyUI_windows_portable_nvidia_cu118_or_cpu.7z
- tag: "latest"
- overwrite: true
-
diff --git a/.github/workflows/windows_release_dependencies.yml b/.github/workflows/windows_release_dependencies.yml
index aafe8a214..ffd3e2216 100644
--- a/.github/workflows/windows_release_dependencies.yml
+++ b/.github/workflows/windows_release_dependencies.yml
@@ -24,7 +24,7 @@ on:
description: 'python patch version'
required: true
type: string
- default: "6"
+ default: "8"
# push:
# branches:
# - master
@@ -41,10 +41,9 @@ jobs:
- shell: bash
run: |
echo "@echo off
- ..\python_embeded\python.exe .\update.py ..\ComfyUI\\
+ call update_comfyui.bat nopause
echo -
- echo This will try to update pytorch and all python dependencies, if you get an error wait for pytorch/xformers to fix their stuff
- echo You should not be running this anyways unless you really have to
+ echo This will try to update pytorch and all python dependencies.
echo -
echo If you just want to update normally, close this and run update_comfyui.bat instead.
echo -
diff --git a/.github/workflows/windows_release_nightly_pytorch.yml b/.github/workflows/windows_release_nightly_pytorch.yml
index 90e09d27a..672a7f220 100644
--- a/.github/workflows/windows_release_nightly_pytorch.yml
+++ b/.github/workflows/windows_release_nightly_pytorch.yml
@@ -19,7 +19,7 @@ on:
description: 'python patch version'
required: true
type: string
- default: "1"
+ default: "2"
# push:
# branches:
# - master
@@ -49,7 +49,7 @@ jobs:
echo 'import site' >> ./python3${{ inputs.python_minor }}._pth
curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py
./python.exe get-pip.py
- python -m pip wheel torch torchvision torchaudio --pre --extra-index-url https://download.pytorch.org/whl/nightly/cu${{ inputs.cu }} -r ../ComfyUI/requirements.txt pygit2 -w ../temp_wheel_dir
+ python -m pip wheel torch torchvision torchaudio mpmath==1.3.0 --pre --extra-index-url https://download.pytorch.org/whl/nightly/cu${{ inputs.cu }} -r ../ComfyUI/requirements.txt pygit2 -w ../temp_wheel_dir
ls ../temp_wheel_dir
./python.exe -s -m pip install --pre ../temp_wheel_dir/*
sed -i '1i../ComfyUI' ./python3${{ inputs.python_minor }}._pth
@@ -68,7 +68,7 @@ jobs:
cp -r ComfyUI/.ci/update_windows/* ./update/
cp -r ComfyUI/.ci/windows_base_files/* ./
- echo "..\python_embeded\python.exe .\update.py ..\ComfyUI\\
+ echo "call update_comfyui.bat nopause
..\python_embeded\python.exe -s -m pip install --upgrade --pre torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/nightly/cu${{ inputs.cu }} -r ../ComfyUI/requirements.txt pygit2
pause" > ./update/update_comfyui_and_python_dependencies.bat
cd ..
diff --git a/.github/workflows/windows_release_package.yml b/.github/workflows/windows_release_package.yml
index 87d37c24d..4e3cdabd2 100644
--- a/.github/workflows/windows_release_package.yml
+++ b/.github/workflows/windows_release_package.yml
@@ -19,7 +19,7 @@ on:
description: 'python patch version'
required: true
type: string
- default: "6"
+ default: "8"
# push:
# branches:
# - master
diff --git a/README.md b/README.md
index 093873921..ba1e844b3 100644
--- a/README.md
+++ b/README.md
@@ -11,7 +11,7 @@ This ui will let you design and execute advanced stable diffusion pipelines usin
## Features
- Nodes/graph/flowchart interface to experiment and create complex Stable Diffusion workflows without needing to code anything.
-- Fully supports SD1.x, SD2.x, [SDXL](https://comfyanonymous.github.io/ComfyUI_examples/sdxl/) and [Stable Video Diffusion](https://comfyanonymous.github.io/ComfyUI_examples/video/)
+- Fully supports SD1.x, SD2.x, [SDXL](https://comfyanonymous.github.io/ComfyUI_examples/sdxl/), [Stable Video Diffusion](https://comfyanonymous.github.io/ComfyUI_examples/video/) and [Stable Cascade](https://comfyanonymous.github.io/ComfyUI_examples/stable_cascade/)
- Asynchronous Queue system
- Many optimizations: Only re-executes the parts of the workflow that changes between executions.
- Command line option: ```--lowvram``` to make it work on GPUs with less than 3GB vram (enabled automatically on GPUs with low vram)
@@ -95,16 +95,15 @@ Put your SD checkpoints (the huge ckpt/safetensors files) in: models/checkpoints
Put your VAE in: models/vae
-Note: pytorch stable does not support python 3.12 yet. If you have python 3.12 you will have to use the nightly version of pytorch. If you run into issues you should try python 3.11 instead.
### AMD GPUs (Linux only)
AMD users can install rocm and pytorch with pip if you don't have it already installed, this is the command to install the stable version:
-```pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/rocm5.6```
+```pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/rocm5.7```
-This is the command to install the nightly with ROCm 5.7 which has a python 3.12 package and might have some performance improvements:
+This is the command to install the nightly with ROCm 6.0 which might have some performance improvements:
-```pip install --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/rocm5.7```
+```pip install --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/rocm6.0```
### NVIDIA
@@ -112,7 +111,7 @@ Nvidia users should install stable pytorch using this command:
```pip install torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cu121```
-This is the command to install pytorch nightly instead which has a python 3.12 package and might have performance improvements:
+This is the command to install pytorch nightly instead which might have performance improvements:
```pip install --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/cu121```
@@ -143,7 +142,7 @@ You can install ComfyUI in Apple Mac silicon (M1 or M2) with any recent macOS ve
1. Install pytorch nightly. For instructions, read the [Accelerated PyTorch training on Mac](https://developer.apple.com/metal/pytorch/) Apple Developer guide (make sure to install the latest pytorch nightly).
1. Follow the [ComfyUI manual installation](#manual-install-windows-linux) instructions for Windows and Linux.
1. Install the ComfyUI [dependencies](#dependencies). If you have another Stable Diffusion UI [you might be able to reuse the dependencies](#i-already-have-another-ui-for-stable-diffusion-installed-do-i-really-have-to-install-all-of-these-dependencies).
-1. Launch ComfyUI by running `python main.py --force-fp16`. Note that --force-fp16 will only work if you installed the latest pytorch nightly.
+1. Launch ComfyUI by running `python main.py`
> **Note**: Remember to add your models, VAE, LoRAs etc. to the corresponding Comfy folders, as discussed in [ComfyUI manual installation](#manual-install-windows-linux).
diff --git a/comfy/cli_args.py b/comfy/cli_args.py
index b4bbfbfab..353bb51e7 100644
--- a/comfy/cli_args.py
+++ b/comfy/cli_args.py
@@ -114,6 +114,9 @@ parser.add_argument("--disable-metadata", action="store_true", help="Disable sav
parser.add_argument("--multi-user", action="store_true", help="Enables per-user storage.")
+parser.add_argument("--verbose", action="store_true", help="Enables more debug prints.")
+
+
if comfy.options.args_parsing:
args = parser.parse_args()
else:
@@ -124,3 +127,10 @@ if args.windows_standalone_build:
if args.disable_auto_launch:
args.auto_launch = False
+
+import logging
+logging_level = logging.INFO
+if args.verbose:
+ logging_level = logging.DEBUG
+
+logging.basicConfig(format="%(message)s", level=logging_level)
diff --git a/comfy/clip_model.py b/comfy/clip_model.py
index 09e7bbca1..14f43c568 100644
--- a/comfy/clip_model.py
+++ b/comfy/clip_model.py
@@ -97,7 +97,7 @@ class CLIPTextModel_(torch.nn.Module):
x = self.embeddings(input_tokens)
mask = None
if attention_mask is not None:
- mask = 1.0 - attention_mask.to(x.dtype).unsqueeze(1).unsqueeze(1).expand(attention_mask.shape[0], 1, attention_mask.shape[-1], attention_mask.shape[-1])
+ mask = 1.0 - attention_mask.to(x.dtype).reshape((attention_mask.shape[0], 1, -1, attention_mask.shape[-1])).expand(attention_mask.shape[0], 1, attention_mask.shape[-1], attention_mask.shape[-1])
mask = mask.masked_fill(mask.to(torch.bool), float("-inf"))
causal_mask = torch.empty(x.shape[1], x.shape[1], dtype=x.dtype, device=x.device).fill_(float("-inf")).triu_(1)
@@ -119,6 +119,9 @@ class CLIPTextModel(torch.nn.Module):
super().__init__()
self.num_layers = config_dict["num_hidden_layers"]
self.text_model = CLIPTextModel_(config_dict, dtype, device, operations)
+ embed_dim = config_dict["hidden_size"]
+ self.text_projection = operations.Linear(embed_dim, embed_dim, bias=False, dtype=dtype, device=device)
+ self.text_projection.weight.copy_(torch.eye(embed_dim))
self.dtype = dtype
def get_input_embeddings(self):
@@ -128,7 +131,10 @@ class CLIPTextModel(torch.nn.Module):
self.text_model.embeddings.token_embedding = embeddings
def forward(self, *args, **kwargs):
- return self.text_model(*args, **kwargs)
+ x = self.text_model(*args, **kwargs)
+ out = self.text_projection(x[2])
+ return (x[0], x[1], out, x[2])
+
class CLIPVisionEmbeddings(torch.nn.Module):
def __init__(self, embed_dim, num_channels=3, patch_size=14, image_size=224, dtype=None, device=None, operations=None):
diff --git a/comfy/clip_vision.py b/comfy/clip_vision.py
index 200e1c6ef..acc86be85 100644
--- a/comfy/clip_vision.py
+++ b/comfy/clip_vision.py
@@ -1,8 +1,8 @@
-from .utils import load_torch_file, transformers_convert, common_upscale, state_dict_prefix_replace
+from .utils import load_torch_file, transformers_convert, state_dict_prefix_replace
import os
import torch
-import contextlib
import json
+import logging
import comfy.ops
import comfy.model_patcher
@@ -100,7 +100,7 @@ def load_clipvision_from_sd(sd, prefix="", convert_keys=False):
clip = ClipVisionModel(json_config)
m, u = clip.load_sd(sd)
if len(m) > 0:
- print("missing clip vision:", m)
+ logging.warning("missing clip vision: {}".format(m))
u = set(u)
keys = list(sd.keys())
for k in keys:
diff --git a/comfy/conds.py b/comfy/conds.py
index 6cff25184..23fa48872 100644
--- a/comfy/conds.py
+++ b/comfy/conds.py
@@ -1,4 +1,3 @@
-import enum
import torch
import math
import comfy.utils
diff --git a/comfy/controlnet.py b/comfy/controlnet.py
index df474201c..8cf4a61a6 100644
--- a/comfy/controlnet.py
+++ b/comfy/controlnet.py
@@ -1,7 +1,7 @@
import torch
import math
import os
-import contextlib
+import logging
import comfy.utils
import comfy.model_management
import comfy.model_detection
@@ -10,6 +10,7 @@ import comfy.ops
import comfy.cldm.cldm
import comfy.t2i_adapter.adapter
+import comfy.ldm.cascade.controlnet
def broadcast_image_to(tensor, target_batch_size, batched_number):
@@ -38,6 +39,8 @@ class ControlBase:
self.timestep_percent_range = (0.0, 1.0)
self.global_average_pooling = False
self.timestep_range = None
+ self.compression_ratio = 8
+ self.upscale_algorithm = 'nearest-exact'
if device is None:
device = comfy.model_management.get_torch_device()
@@ -78,6 +81,8 @@ class ControlBase:
c.strength = self.strength
c.timestep_percent_range = self.timestep_percent_range
c.global_average_pooling = self.global_average_pooling
+ c.compression_ratio = self.compression_ratio
+ c.upscale_algorithm = self.upscale_algorithm
def inference_memory_requirements(self, dtype):
if self.previous_controlnet is not None:
@@ -133,11 +138,13 @@ class ControlBase:
return out
class ControlNet(ControlBase):
- def __init__(self, control_model, global_average_pooling=False, device=None, load_device=None, manual_cast_dtype=None):
+ def __init__(self, control_model=None, global_average_pooling=False, device=None, load_device=None, manual_cast_dtype=None):
super().__init__(device)
self.control_model = control_model
self.load_device = load_device
- self.control_model_wrapped = comfy.model_patcher.ModelPatcher(self.control_model, load_device=load_device, offload_device=comfy.model_management.unet_offload_device())
+ if control_model is not None:
+ self.control_model_wrapped = comfy.model_patcher.ModelPatcher(self.control_model, load_device=load_device, offload_device=comfy.model_management.unet_offload_device())
+
self.global_average_pooling = global_average_pooling
self.model_sampling_current = None
self.manual_cast_dtype = manual_cast_dtype
@@ -159,15 +166,15 @@ class ControlNet(ControlBase):
dtype = self.manual_cast_dtype
output_dtype = x_noisy.dtype
- if self.cond_hint is None or x_noisy.shape[2] * 8 != self.cond_hint.shape[2] or x_noisy.shape[3] * 8 != self.cond_hint.shape[3]:
+ if self.cond_hint is None or x_noisy.shape[2] * self.compression_ratio != self.cond_hint.shape[2] or x_noisy.shape[3] * self.compression_ratio != self.cond_hint.shape[3]:
if self.cond_hint is not None:
del self.cond_hint
self.cond_hint = None
- self.cond_hint = comfy.utils.common_upscale(self.cond_hint_original, x_noisy.shape[3] * 8, x_noisy.shape[2] * 8, 'nearest-exact', "center").to(dtype).to(self.device)
+ self.cond_hint = comfy.utils.common_upscale(self.cond_hint_original, x_noisy.shape[3] * self.compression_ratio, x_noisy.shape[2] * self.compression_ratio, self.upscale_algorithm, "center").to(dtype).to(self.device)
if x_noisy.shape[0] != self.cond_hint.shape[0]:
self.cond_hint = broadcast_image_to(self.cond_hint, x_noisy.shape[0], batched_number)
- context = cond['c_crossattn']
+ context = cond.get('crossattn_controlnet', cond['c_crossattn'])
y = cond.get('y', None)
if y is not None:
y = y.to(dtype)
@@ -178,7 +185,9 @@ class ControlNet(ControlBase):
return self.control_merge(None, control, control_prev, output_dtype)
def copy(self):
- c = ControlNet(self.control_model, global_average_pooling=self.global_average_pooling, load_device=self.load_device, manual_cast_dtype=self.manual_cast_dtype)
+ c = ControlNet(None, global_average_pooling=self.global_average_pooling, load_device=self.load_device, manual_cast_dtype=self.manual_cast_dtype)
+ c.control_model = self.control_model
+ c.control_model_wrapped = self.control_model_wrapped
self.copy_to(c)
return c
@@ -196,7 +205,7 @@ class ControlNet(ControlBase):
super().cleanup()
class ControlLoraOps:
- class Linear(torch.nn.Module):
+ class Linear(torch.nn.Module, comfy.ops.CastWeightBiasOp):
def __init__(self, in_features: int, out_features: int, bias: bool = True,
device=None, dtype=None) -> None:
factory_kwargs = {'device': device, 'dtype': dtype}
@@ -215,7 +224,7 @@ class ControlLoraOps:
else:
return torch.nn.functional.linear(input, weight, bias)
- class Conv2d(torch.nn.Module):
+ class Conv2d(torch.nn.Module, comfy.ops.CastWeightBiasOp):
def __init__(
self,
in_channels,
@@ -288,13 +297,13 @@ class ControlLora(ControlNet):
for k in sd:
weight = sd[k]
try:
- comfy.utils.set_attr(self.control_model, k, weight)
+ comfy.utils.set_attr_param(self.control_model, k, weight)
except:
pass
for k in self.control_weights:
if k not in {"lora_controlnet"}:
- comfy.utils.set_attr(self.control_model, k, self.control_weights[k].to(dtype).to(comfy.model_management.get_torch_device()))
+ comfy.utils.set_attr_param(self.control_model, k, self.control_weights[k].to(dtype).to(comfy.model_management.get_torch_device()))
def copy(self):
c = ControlLora(self.control_weights, global_average_pooling=self.global_average_pooling)
@@ -319,9 +328,10 @@ def load_controlnet(ckpt_path, model=None):
return ControlLora(controlnet_data)
controlnet_config = None
+ supported_inference_dtypes = None
+
if "controlnet_cond_embedding.conv_in.weight" in controlnet_data: #diffusers format
- unet_dtype = comfy.model_management.unet_dtype()
- controlnet_config = comfy.model_detection.unet_config_from_diffusers_unet(controlnet_data, unet_dtype)
+ controlnet_config = comfy.model_detection.unet_config_from_diffusers_unet(controlnet_data)
diffusers_keys = comfy.utils.unet_to_diffusers(controlnet_config)
diffusers_keys["controlnet_mid_block.weight"] = "middle_block_out.0.weight"
diffusers_keys["controlnet_mid_block.bias"] = "middle_block_out.0.bias"
@@ -362,7 +372,7 @@ def load_controlnet(ckpt_path, model=None):
leftover_keys = controlnet_data.keys()
if len(leftover_keys) > 0:
- print("leftover keys:", leftover_keys)
+ logging.warning("leftover keys: {}".format(leftover_keys))
controlnet_data = new_sd
pth_key = 'control_model.zero_convs.0.0.weight'
@@ -377,16 +387,24 @@ def load_controlnet(ckpt_path, model=None):
else:
net = load_t2i_adapter(controlnet_data)
if net is None:
- print("error checkpoint does not contain controlnet or t2i adapter data", ckpt_path)
+ logging.error("error checkpoint does not contain controlnet or t2i adapter data {}".format(ckpt_path))
return net
if controlnet_config is None:
- unet_dtype = comfy.model_management.unet_dtype()
- controlnet_config = comfy.model_detection.model_config_from_unet(controlnet_data, prefix, unet_dtype, True).unet_config
+ model_config = comfy.model_detection.model_config_from_unet(controlnet_data, prefix, True)
+ supported_inference_dtypes = model_config.supported_inference_dtypes
+ controlnet_config = model_config.unet_config
+
load_device = comfy.model_management.get_torch_device()
+ if supported_inference_dtypes is None:
+ unet_dtype = comfy.model_management.unet_dtype()
+ else:
+ unet_dtype = comfy.model_management.unet_dtype(supported_dtypes=supported_inference_dtypes)
+
manual_cast_dtype = comfy.model_management.unet_manual_cast(unet_dtype, load_device)
if manual_cast_dtype is not None:
controlnet_config["operations"] = comfy.ops.manual_cast
+ controlnet_config["dtype"] = unet_dtype
controlnet_config.pop("out_channels")
controlnet_config["hint_channels"] = controlnet_data["{}input_hint_block.0.weight".format(prefix)].shape[1]
control_model = comfy.cldm.cldm.ControlNet(**controlnet_config)
@@ -404,7 +422,7 @@ def load_controlnet(ckpt_path, model=None):
cd = controlnet_data[x]
cd += model_sd[sd_key].type(cd.dtype).to(cd.device)
else:
- print("WARNING: Loaded a diff controlnet without a model. It will very likely not work.")
+ logging.warning("WARNING: Loaded a diff controlnet without a model. It will very likely not work.")
class WeightsLoader(torch.nn.Module):
pass
@@ -413,7 +431,12 @@ def load_controlnet(ckpt_path, model=None):
missing, unexpected = w.load_state_dict(controlnet_data, strict=False)
else:
missing, unexpected = control_model.load_state_dict(controlnet_data, strict=False)
- print(missing, unexpected)
+
+ if len(missing) > 0:
+ logging.warning("missing controlnet keys: {}".format(missing))
+
+ if len(unexpected) > 0:
+ logging.debug("unexpected controlnet keys: {}".format(unexpected))
global_average_pooling = False
filename = os.path.splitext(ckpt_path)[0]
@@ -424,11 +447,13 @@ def load_controlnet(ckpt_path, model=None):
return control
class T2IAdapter(ControlBase):
- def __init__(self, t2i_model, channels_in, device=None):
+ def __init__(self, t2i_model, channels_in, compression_ratio, upscale_algorithm, device=None):
super().__init__(device)
self.t2i_model = t2i_model
self.channels_in = channels_in
self.control_input = None
+ self.compression_ratio = compression_ratio
+ self.upscale_algorithm = upscale_algorithm
def scale_image_to(self, width, height):
unshuffle_amount = self.t2i_model.unshuffle_amount
@@ -448,13 +473,13 @@ class T2IAdapter(ControlBase):
else:
return None
- if self.cond_hint is None or x_noisy.shape[2] * 8 != self.cond_hint.shape[2] or x_noisy.shape[3] * 8 != self.cond_hint.shape[3]:
+ if self.cond_hint is None or x_noisy.shape[2] * self.compression_ratio != self.cond_hint.shape[2] or x_noisy.shape[3] * self.compression_ratio != self.cond_hint.shape[3]:
if self.cond_hint is not None:
del self.cond_hint
self.control_input = None
self.cond_hint = None
- width, height = self.scale_image_to(x_noisy.shape[3] * 8, x_noisy.shape[2] * 8)
- self.cond_hint = comfy.utils.common_upscale(self.cond_hint_original, width, height, 'nearest-exact', "center").float().to(self.device)
+ width, height = self.scale_image_to(x_noisy.shape[3] * self.compression_ratio, x_noisy.shape[2] * self.compression_ratio)
+ self.cond_hint = comfy.utils.common_upscale(self.cond_hint_original, width, height, self.upscale_algorithm, "center").float().to(self.device)
if self.channels_in == 1 and self.cond_hint.shape[1] > 1:
self.cond_hint = torch.mean(self.cond_hint, 1, keepdim=True)
if x_noisy.shape[0] != self.cond_hint.shape[0]:
@@ -473,11 +498,14 @@ class T2IAdapter(ControlBase):
return self.control_merge(control_input, mid, control_prev, x_noisy.dtype)
def copy(self):
- c = T2IAdapter(self.t2i_model, self.channels_in)
+ c = T2IAdapter(self.t2i_model, self.channels_in, self.compression_ratio, self.upscale_algorithm)
self.copy_to(c)
return c
def load_t2i_adapter(t2i_data):
+ compression_ratio = 8
+ upscale_algorithm = 'nearest-exact'
+
if 'adapter' in t2i_data:
t2i_data = t2i_data['adapter']
if 'adapter.body.0.resnets.0.block1.weight' in t2i_data: #diffusers format
@@ -505,13 +533,22 @@ def load_t2i_adapter(t2i_data):
if cin == 256 or cin == 768:
xl = True
model_ad = comfy.t2i_adapter.adapter.Adapter(cin=cin, channels=[channel, channel*2, channel*4, channel*4][:4], nums_rb=2, ksize=ksize, sk=True, use_conv=use_conv, xl=xl)
+ elif "backbone.0.0.weight" in keys:
+ model_ad = comfy.ldm.cascade.controlnet.ControlNet(c_in=t2i_data['backbone.0.0.weight'].shape[1], proj_blocks=[0, 4, 8, 12, 51, 55, 59, 63])
+ compression_ratio = 32
+ upscale_algorithm = 'bilinear'
+ elif "backbone.10.blocks.0.weight" in keys:
+ model_ad = comfy.ldm.cascade.controlnet.ControlNet(c_in=t2i_data['backbone.0.weight'].shape[1], bottleneck_mode="large", proj_blocks=[0, 4, 8, 12, 51, 55, 59, 63])
+ compression_ratio = 1
+ upscale_algorithm = 'nearest-exact'
else:
return None
+
missing, unexpected = model_ad.load_state_dict(t2i_data)
if len(missing) > 0:
- print("t2i missing", missing)
+ logging.warning("t2i missing {}".format(missing))
if len(unexpected) > 0:
- print("t2i unexpected", unexpected)
+ logging.debug("t2i unexpected {}".format(unexpected))
- return T2IAdapter(model_ad, model_ad.input_channels)
+ return T2IAdapter(model_ad, model_ad.input_channels, compression_ratio, upscale_algorithm)
diff --git a/comfy/diffusers_convert.py b/comfy/diffusers_convert.py
index a9eb9302f..ed2a45fea 100644
--- a/comfy/diffusers_convert.py
+++ b/comfy/diffusers_convert.py
@@ -1,5 +1,6 @@
import re
import torch
+import logging
# conversion code from https://github.com/huggingface/diffusers/blob/main/scripts/convert_diffusers_to_original_stable_diffusion.py
@@ -177,7 +178,7 @@ def convert_vae_state_dict(vae_state_dict):
for k, v in new_state_dict.items():
for weight_name in weights_to_convert:
if f"mid.attn_1.{weight_name}.weight" in k:
- print(f"Reshaping {k} for SD format")
+ logging.debug(f"Reshaping {k} for SD format")
new_state_dict[k] = reshape_weight_for_sd(v)
return new_state_dict
@@ -205,6 +206,21 @@ textenc_pattern = re.compile("|".join(protected.keys()))
# Ordering is from https://github.com/pytorch/pytorch/blob/master/test/cpp/api/modules.cpp
code2idx = {"q": 0, "k": 1, "v": 2}
+# This function exists because at the time of writing torch.cat can't do fp8 with cuda
+def cat_tensors(tensors):
+ x = 0
+ for t in tensors:
+ x += t.shape[0]
+
+ shape = [x] + list(tensors[0].shape)[1:]
+ out = torch.empty(shape, device=tensors[0].device, dtype=tensors[0].dtype)
+
+ x = 0
+ for t in tensors:
+ out[x:x + t.shape[0]] = t
+ x += t.shape[0]
+
+ return out
def convert_text_enc_state_dict_v20(text_enc_dict, prefix=""):
new_state_dict = {}
@@ -237,20 +253,24 @@ def convert_text_enc_state_dict_v20(text_enc_dict, prefix=""):
capture_qkv_bias[k_pre][code2idx[k_code]] = v
continue
- relabelled_key = textenc_pattern.sub(lambda m: protected[re.escape(m.group(0))], k)
- new_state_dict[relabelled_key] = v
+ text_proj = "transformer.text_projection.weight"
+ if k.endswith(text_proj):
+ new_state_dict[k.replace(text_proj, "text_projection")] = v.transpose(0, 1).contiguous()
+ else:
+ relabelled_key = textenc_pattern.sub(lambda m: protected[re.escape(m.group(0))], k)
+ new_state_dict[relabelled_key] = v
for k_pre, tensors in capture_qkv_weight.items():
if None in tensors:
raise Exception("CORRUPTED MODEL: one of the q-k-v values for the text encoder was missing")
relabelled_key = textenc_pattern.sub(lambda m: protected[re.escape(m.group(0))], k_pre)
- new_state_dict[relabelled_key + ".in_proj_weight"] = torch.cat(tensors)
+ new_state_dict[relabelled_key + ".in_proj_weight"] = cat_tensors(tensors)
for k_pre, tensors in capture_qkv_bias.items():
if None in tensors:
raise Exception("CORRUPTED MODEL: one of the q-k-v values for the text encoder was missing")
relabelled_key = textenc_pattern.sub(lambda m: protected[re.escape(m.group(0))], k_pre)
- new_state_dict[relabelled_key + ".in_proj_bias"] = torch.cat(tensors)
+ new_state_dict[relabelled_key + ".in_proj_bias"] = cat_tensors(tensors)
return new_state_dict
diff --git a/comfy/diffusers_load.py b/comfy/diffusers_load.py
index c0b420e79..98b888a19 100644
--- a/comfy/diffusers_load.py
+++ b/comfy/diffusers_load.py
@@ -1,4 +1,3 @@
-import json
import os
import comfy.sd
diff --git a/comfy/extra_samplers/uni_pc.py b/comfy/extra_samplers/uni_pc.py
index 08bf0fc9e..a30d1d03f 100644
--- a/comfy/extra_samplers/uni_pc.py
+++ b/comfy/extra_samplers/uni_pc.py
@@ -358,9 +358,6 @@ class UniPC:
thresholding=False,
max_val=1.,
variant='bh1',
- noise_mask=None,
- masked_image=None,
- noise=None,
):
"""Construct a UniPC.
@@ -372,9 +369,6 @@ class UniPC:
self.predict_x0 = predict_x0
self.thresholding = thresholding
self.max_val = max_val
- self.noise_mask = noise_mask
- self.masked_image = masked_image
- self.noise = noise
def dynamic_thresholding_fn(self, x0, t=None):
"""
@@ -391,10 +385,7 @@ class UniPC:
"""
Return the noise prediction model.
"""
- if self.noise_mask is not None:
- return self.model(x, t) * self.noise_mask
- else:
- return self.model(x, t)
+ return self.model(x, t)
def data_prediction_fn(self, x, t):
"""
@@ -409,8 +400,6 @@ class UniPC:
s = torch.quantile(torch.abs(x0).reshape((x0.shape[0], -1)), p, dim=1)
s = expand_dims(torch.maximum(s, self.max_val * torch.ones_like(s).to(s.device)), dims)
x0 = torch.clamp(x0, -s, s) / s
- if self.noise_mask is not None:
- x0 = x0 * self.noise_mask + (1. - self.noise_mask) * self.masked_image
return x0
def model_fn(self, x, t):
@@ -723,8 +712,6 @@ class UniPC:
assert timesteps.shape[0] - 1 == steps
# with torch.no_grad():
for step_index in trange(steps, disable=disable_pbar):
- if self.noise_mask is not None:
- x = x * self.noise_mask + (1. - self.noise_mask) * (self.masked_image * self.noise_schedule.marginal_alpha(timesteps[step_index]) + self.noise * self.noise_schedule.marginal_std(timesteps[step_index]))
if step_index == 0:
vec_t = timesteps[0].expand((x.shape[0]))
model_prev_list = [self.model_fn(x, vec_t)]
@@ -766,7 +753,7 @@ class UniPC:
model_x = self.model_fn(x, vec_t)
model_prev_list[-1] = model_x
if callback is not None:
- callback(step_index, model_prev_list[-1], x, steps)
+ callback({'x': x, 'i': step_index, 'denoised': model_prev_list[-1]})
else:
raise NotImplementedError()
# if denoise_to_zero:
@@ -858,7 +845,7 @@ def predict_eps_sigma(model, input, sigma_in, **kwargs):
return (input - model(input, sigma_in, **kwargs)) / sigma
-def sample_unipc(model, noise, image, sigmas, max_denoise, extra_args=None, callback=None, disable=False, noise_mask=None, variant='bh1'):
+def sample_unipc(model, noise, sigmas, extra_args=None, callback=None, disable=False, variant='bh1'):
timesteps = sigmas.clone()
if sigmas[-1] == 0:
timesteps = sigmas[:]
@@ -867,16 +854,7 @@ def sample_unipc(model, noise, image, sigmas, max_denoise, extra_args=None, call
timesteps = sigmas.clone()
ns = SigmaConvert()
- if image is not None:
- img = image * ns.marginal_alpha(timesteps[0])
- if max_denoise:
- noise_mult = 1.0
- else:
- noise_mult = ns.marginal_std(timesteps[0])
- img += noise * noise_mult
- else:
- img = noise
-
+ noise = noise / torch.sqrt(1.0 + timesteps[0] ** 2.0)
model_type = "noise"
model_fn = model_wrapper(
@@ -888,7 +866,10 @@ def sample_unipc(model, noise, image, sigmas, max_denoise, extra_args=None, call
)
order = min(3, len(timesteps) - 2)
- uni_pc = UniPC(model_fn, ns, predict_x0=True, thresholding=False, noise_mask=noise_mask, masked_image=image, noise=noise, variant=variant)
- x = uni_pc.sample(img, timesteps=timesteps, skip_type="time_uniform", method="multistep", order=order, lower_order_final=True, callback=callback, disable_pbar=disable)
+ uni_pc = UniPC(model_fn, ns, predict_x0=True, thresholding=False, variant=variant)
+ x = uni_pc.sample(noise, timesteps=timesteps, skip_type="time_uniform", method="multistep", order=order, lower_order_final=True, callback=callback, disable_pbar=disable)
x /= ns.marginal_alpha(timesteps[-1])
return x
+
+def sample_unipc_bh2(model, noise, sigmas, extra_args=None, callback=None, disable=False):
+ return sample_unipc(model, noise, sigmas, extra_args, callback, disable, variant='bh2')
\ No newline at end of file
diff --git a/comfy/gligen.py b/comfy/gligen.py
index 8d182839e..592522767 100644
--- a/comfy/gligen.py
+++ b/comfy/gligen.py
@@ -1,8 +1,9 @@
import torch
-from torch import nn, einsum
+from torch import nn
from .ldm.modules.attention import CrossAttention
from inspect import isfunction
-
+import comfy.ops
+ops = comfy.ops.manual_cast
def exists(val):
return val is not None
@@ -22,7 +23,7 @@ def default(val, d):
class GEGLU(nn.Module):
def __init__(self, dim_in, dim_out):
super().__init__()
- self.proj = nn.Linear(dim_in, dim_out * 2)
+ self.proj = ops.Linear(dim_in, dim_out * 2)
def forward(self, x):
x, gate = self.proj(x).chunk(2, dim=-1)
@@ -35,14 +36,14 @@ class FeedForward(nn.Module):
inner_dim = int(dim * mult)
dim_out = default(dim_out, dim)
project_in = nn.Sequential(
- nn.Linear(dim, inner_dim),
+ ops.Linear(dim, inner_dim),
nn.GELU()
) if not glu else GEGLU(dim, inner_dim)
self.net = nn.Sequential(
project_in,
nn.Dropout(dropout),
- nn.Linear(inner_dim, dim_out)
+ ops.Linear(inner_dim, dim_out)
)
def forward(self, x):
@@ -57,11 +58,12 @@ class GatedCrossAttentionDense(nn.Module):
query_dim=query_dim,
context_dim=context_dim,
heads=n_heads,
- dim_head=d_head)
+ dim_head=d_head,
+ operations=ops)
self.ff = FeedForward(query_dim, glu=True)
- self.norm1 = nn.LayerNorm(query_dim)
- self.norm2 = nn.LayerNorm(query_dim)
+ self.norm1 = ops.LayerNorm(query_dim)
+ self.norm2 = ops.LayerNorm(query_dim)
self.register_parameter('alpha_attn', nn.Parameter(torch.tensor(0.)))
self.register_parameter('alpha_dense', nn.Parameter(torch.tensor(0.)))
@@ -87,17 +89,18 @@ class GatedSelfAttentionDense(nn.Module):
# we need a linear projection since we need cat visual feature and obj
# feature
- self.linear = nn.Linear(context_dim, query_dim)
+ self.linear = ops.Linear(context_dim, query_dim)
self.attn = CrossAttention(
query_dim=query_dim,
context_dim=query_dim,
heads=n_heads,
- dim_head=d_head)
+ dim_head=d_head,
+ operations=ops)
self.ff = FeedForward(query_dim, glu=True)
- self.norm1 = nn.LayerNorm(query_dim)
- self.norm2 = nn.LayerNorm(query_dim)
+ self.norm1 = ops.LayerNorm(query_dim)
+ self.norm2 = ops.LayerNorm(query_dim)
self.register_parameter('alpha_attn', nn.Parameter(torch.tensor(0.)))
self.register_parameter('alpha_dense', nn.Parameter(torch.tensor(0.)))
@@ -126,14 +129,14 @@ class GatedSelfAttentionDense2(nn.Module):
# we need a linear projection since we need cat visual feature and obj
# feature
- self.linear = nn.Linear(context_dim, query_dim)
+ self.linear = ops.Linear(context_dim, query_dim)
self.attn = CrossAttention(
- query_dim=query_dim, context_dim=query_dim, dim_head=d_head)
+ query_dim=query_dim, context_dim=query_dim, dim_head=d_head, operations=ops)
self.ff = FeedForward(query_dim, glu=True)
- self.norm1 = nn.LayerNorm(query_dim)
- self.norm2 = nn.LayerNorm(query_dim)
+ self.norm1 = ops.LayerNorm(query_dim)
+ self.norm2 = ops.LayerNorm(query_dim)
self.register_parameter('alpha_attn', nn.Parameter(torch.tensor(0.)))
self.register_parameter('alpha_dense', nn.Parameter(torch.tensor(0.)))
@@ -201,11 +204,11 @@ class PositionNet(nn.Module):
self.position_dim = fourier_freqs * 2 * 4 # 2 is sin&cos, 4 is xyxy
self.linears = nn.Sequential(
- nn.Linear(self.in_dim + self.position_dim, 512),
+ ops.Linear(self.in_dim + self.position_dim, 512),
nn.SiLU(),
- nn.Linear(512, 512),
+ ops.Linear(512, 512),
nn.SiLU(),
- nn.Linear(512, out_dim),
+ ops.Linear(512, out_dim),
)
self.null_positive_feature = torch.nn.Parameter(
@@ -215,16 +218,15 @@ class PositionNet(nn.Module):
def forward(self, boxes, masks, positive_embeddings):
B, N, _ = boxes.shape
- dtype = self.linears[0].weight.dtype
- masks = masks.unsqueeze(-1).to(dtype)
- positive_embeddings = positive_embeddings.to(dtype)
+ masks = masks.unsqueeze(-1)
+ positive_embeddings = positive_embeddings
# embedding position (it may includes padding as placeholder)
- xyxy_embedding = self.fourier_embedder(boxes.to(dtype)) # B*N*4 --> B*N*C
+ xyxy_embedding = self.fourier_embedder(boxes) # B*N*4 --> B*N*C
# learnable null embedding
- positive_null = self.null_positive_feature.view(1, 1, -1)
- xyxy_null = self.null_position_feature.view(1, 1, -1)
+ positive_null = self.null_positive_feature.to(device=boxes.device, dtype=boxes.dtype).view(1, 1, -1)
+ xyxy_null = self.null_position_feature.to(device=boxes.device, dtype=boxes.dtype).view(1, 1, -1)
# replace padding with learnable null embedding
positive_embeddings = positive_embeddings * \
@@ -251,7 +253,7 @@ class Gligen(nn.Module):
def func(x, extra_options):
key = extra_options["transformer_index"]
module = self.module_list[key]
- return module(x, objs)
+ return module(x, objs.to(device=x.device, dtype=x.dtype))
return func
def set_position(self, latent_image_shape, position_params, device):
diff --git a/comfy/k_diffusion/sampling.py b/comfy/k_diffusion/sampling.py
index 761c2e0ef..7af016829 100644
--- a/comfy/k_diffusion/sampling.py
+++ b/comfy/k_diffusion/sampling.py
@@ -748,7 +748,7 @@ def sample_lcm(model, x, sigmas, extra_args=None, callback=None, disable=None, n
x = denoised
if sigmas[i + 1] > 0:
- x += sigmas[i + 1] * noise_sampler(sigmas[i], sigmas[i + 1])
+ x = model.inner_model.inner_model.model_sampling.noise_scaling(sigmas[i + 1], noise_sampler(sigmas[i], sigmas[i + 1]), x)
return x
diff --git a/comfy/latent_formats.py b/comfy/latent_formats.py
index 2252a075e..4ca466d9a 100644
--- a/comfy/latent_formats.py
+++ b/comfy/latent_formats.py
@@ -1,3 +1,4 @@
+import torch
class LatentFormat:
scale_factor = 1.0
@@ -34,6 +35,70 @@ class SDXL(LatentFormat):
]
self.taesd_decoder_name = "taesdxl_decoder"
+class SDXL_Playground_2_5(LatentFormat):
+ def __init__(self):
+ self.scale_factor = 0.5
+ self.latents_mean = torch.tensor([-1.6574, 1.886, -1.383, 2.5155]).view(1, 4, 1, 1)
+ self.latents_std = torch.tensor([8.4927, 5.9022, 6.5498, 5.2299]).view(1, 4, 1, 1)
+
+ self.latent_rgb_factors = [
+ # R G B
+ [ 0.3920, 0.4054, 0.4549],
+ [-0.2634, -0.0196, 0.0653],
+ [ 0.0568, 0.1687, -0.0755],
+ [-0.3112, -0.2359, -0.2076]
+ ]
+ self.taesd_decoder_name = "taesdxl_decoder"
+
+ def process_in(self, latent):
+ latents_mean = self.latents_mean.to(latent.device, latent.dtype)
+ latents_std = self.latents_std.to(latent.device, latent.dtype)
+ return (latent - latents_mean) * self.scale_factor / latents_std
+
+ def process_out(self, latent):
+ latents_mean = self.latents_mean.to(latent.device, latent.dtype)
+ latents_std = self.latents_std.to(latent.device, latent.dtype)
+ return latent * latents_std / self.scale_factor + latents_mean
+
+
class SD_X4(LatentFormat):
def __init__(self):
self.scale_factor = 0.08333
+ self.latent_rgb_factors = [
+ [-0.2340, -0.3863, -0.3257],
+ [ 0.0994, 0.0885, -0.0908],
+ [-0.2833, -0.2349, -0.3741],
+ [ 0.2523, -0.0055, -0.1651]
+ ]
+
+class SC_Prior(LatentFormat):
+ def __init__(self):
+ self.scale_factor = 1.0
+ self.latent_rgb_factors = [
+ [-0.0326, -0.0204, -0.0127],
+ [-0.1592, -0.0427, 0.0216],
+ [ 0.0873, 0.0638, -0.0020],
+ [-0.0602, 0.0442, 0.1304],
+ [ 0.0800, -0.0313, -0.1796],
+ [-0.0810, -0.0638, -0.1581],
+ [ 0.1791, 0.1180, 0.0967],
+ [ 0.0740, 0.1416, 0.0432],
+ [-0.1745, -0.1888, -0.1373],
+ [ 0.2412, 0.1577, 0.0928],
+ [ 0.1908, 0.0998, 0.0682],
+ [ 0.0209, 0.0365, -0.0092],
+ [ 0.0448, -0.0650, -0.1728],
+ [-0.1658, -0.1045, -0.1308],
+ [ 0.0542, 0.1545, 0.1325],
+ [-0.0352, -0.1672, -0.2541]
+ ]
+
+class SC_B(LatentFormat):
+ def __init__(self):
+ self.scale_factor = 1.0 / 0.43
+ self.latent_rgb_factors = [
+ [ 0.1121, 0.2006, 0.1023],
+ [-0.2093, -0.0222, -0.0195],
+ [-0.3087, -0.1535, 0.0366],
+ [ 0.0290, -0.1574, -0.4078]
+ ]
diff --git a/comfy/ldm/cascade/common.py b/comfy/ldm/cascade/common.py
new file mode 100644
index 000000000..124902c09
--- /dev/null
+++ b/comfy/ldm/cascade/common.py
@@ -0,0 +1,161 @@
+"""
+ This file is part of ComfyUI.
+ Copyright (C) 2024 Stability AI
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program. If not, see .
+"""
+
+import torch
+import torch.nn as nn
+from comfy.ldm.modules.attention import optimized_attention
+
+class Linear(torch.nn.Linear):
+ def reset_parameters(self):
+ return None
+
+class Conv2d(torch.nn.Conv2d):
+ def reset_parameters(self):
+ return None
+
+class OptimizedAttention(nn.Module):
+ def __init__(self, c, nhead, dropout=0.0, dtype=None, device=None, operations=None):
+ super().__init__()
+ self.heads = nhead
+
+ self.to_q = operations.Linear(c, c, bias=True, dtype=dtype, device=device)
+ self.to_k = operations.Linear(c, c, bias=True, dtype=dtype, device=device)
+ self.to_v = operations.Linear(c, c, bias=True, dtype=dtype, device=device)
+
+ self.out_proj = operations.Linear(c, c, bias=True, dtype=dtype, device=device)
+
+ def forward(self, q, k, v):
+ q = self.to_q(q)
+ k = self.to_k(k)
+ v = self.to_v(v)
+
+ out = optimized_attention(q, k, v, self.heads)
+
+ return self.out_proj(out)
+
+class Attention2D(nn.Module):
+ def __init__(self, c, nhead, dropout=0.0, dtype=None, device=None, operations=None):
+ super().__init__()
+ self.attn = OptimizedAttention(c, nhead, dtype=dtype, device=device, operations=operations)
+ # self.attn = nn.MultiheadAttention(c, nhead, dropout=dropout, bias=True, batch_first=True, dtype=dtype, device=device)
+
+ def forward(self, x, kv, self_attn=False):
+ orig_shape = x.shape
+ x = x.view(x.size(0), x.size(1), -1).permute(0, 2, 1) # Bx4xHxW -> Bx(HxW)x4
+ if self_attn:
+ kv = torch.cat([x, kv], dim=1)
+ # x = self.attn(x, kv, kv, need_weights=False)[0]
+ x = self.attn(x, kv, kv)
+ x = x.permute(0, 2, 1).view(*orig_shape)
+ return x
+
+
+def LayerNorm2d_op(operations):
+ class LayerNorm2d(operations.LayerNorm):
+ def __init__(self, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+
+ def forward(self, x):
+ return super().forward(x.permute(0, 2, 3, 1)).permute(0, 3, 1, 2)
+ return LayerNorm2d
+
+class GlobalResponseNorm(nn.Module):
+ "from https://github.com/facebookresearch/ConvNeXt-V2/blob/3608f67cc1dae164790c5d0aead7bf2d73d9719b/models/utils.py#L105"
+ def __init__(self, dim, dtype=None, device=None):
+ super().__init__()
+ self.gamma = nn.Parameter(torch.zeros(1, 1, 1, dim, dtype=dtype, device=device))
+ self.beta = nn.Parameter(torch.zeros(1, 1, 1, dim, dtype=dtype, device=device))
+
+ def forward(self, x):
+ Gx = torch.norm(x, p=2, dim=(1, 2), keepdim=True)
+ Nx = Gx / (Gx.mean(dim=-1, keepdim=True) + 1e-6)
+ return self.gamma.to(device=x.device, dtype=x.dtype) * (x * Nx) + self.beta.to(device=x.device, dtype=x.dtype) + x
+
+
+class ResBlock(nn.Module):
+ def __init__(self, c, c_skip=0, kernel_size=3, dropout=0.0, dtype=None, device=None, operations=None): # , num_heads=4, expansion=2):
+ super().__init__()
+ self.depthwise = operations.Conv2d(c, c, kernel_size=kernel_size, padding=kernel_size // 2, groups=c, dtype=dtype, device=device)
+ # self.depthwise = SAMBlock(c, num_heads, expansion)
+ self.norm = LayerNorm2d_op(operations)(c, elementwise_affine=False, eps=1e-6, dtype=dtype, device=device)
+ self.channelwise = nn.Sequential(
+ operations.Linear(c + c_skip, c * 4, dtype=dtype, device=device),
+ nn.GELU(),
+ GlobalResponseNorm(c * 4, dtype=dtype, device=device),
+ nn.Dropout(dropout),
+ operations.Linear(c * 4, c, dtype=dtype, device=device)
+ )
+
+ def forward(self, x, x_skip=None):
+ x_res = x
+ x = self.norm(self.depthwise(x))
+ if x_skip is not None:
+ x = torch.cat([x, x_skip], dim=1)
+ x = self.channelwise(x.permute(0, 2, 3, 1)).permute(0, 3, 1, 2)
+ return x + x_res
+
+
+class AttnBlock(nn.Module):
+ def __init__(self, c, c_cond, nhead, self_attn=True, dropout=0.0, dtype=None, device=None, operations=None):
+ super().__init__()
+ self.self_attn = self_attn
+ self.norm = LayerNorm2d_op(operations)(c, elementwise_affine=False, eps=1e-6, dtype=dtype, device=device)
+ self.attention = Attention2D(c, nhead, dropout, dtype=dtype, device=device, operations=operations)
+ self.kv_mapper = nn.Sequential(
+ nn.SiLU(),
+ operations.Linear(c_cond, c, dtype=dtype, device=device)
+ )
+
+ def forward(self, x, kv):
+ kv = self.kv_mapper(kv)
+ x = x + self.attention(self.norm(x), kv, self_attn=self.self_attn)
+ return x
+
+
+class FeedForwardBlock(nn.Module):
+ def __init__(self, c, dropout=0.0, dtype=None, device=None, operations=None):
+ super().__init__()
+ self.norm = LayerNorm2d_op(operations)(c, elementwise_affine=False, eps=1e-6, dtype=dtype, device=device)
+ self.channelwise = nn.Sequential(
+ operations.Linear(c, c * 4, dtype=dtype, device=device),
+ nn.GELU(),
+ GlobalResponseNorm(c * 4, dtype=dtype, device=device),
+ nn.Dropout(dropout),
+ operations.Linear(c * 4, c, dtype=dtype, device=device)
+ )
+
+ def forward(self, x):
+ x = x + self.channelwise(self.norm(x).permute(0, 2, 3, 1)).permute(0, 3, 1, 2)
+ return x
+
+
+class TimestepBlock(nn.Module):
+ def __init__(self, c, c_timestep, conds=['sca'], dtype=None, device=None, operations=None):
+ super().__init__()
+ self.mapper = operations.Linear(c_timestep, c * 2, dtype=dtype, device=device)
+ self.conds = conds
+ for cname in conds:
+ setattr(self, f"mapper_{cname}", operations.Linear(c_timestep, c * 2, dtype=dtype, device=device))
+
+ def forward(self, x, t):
+ t = t.chunk(len(self.conds) + 1, dim=1)
+ a, b = self.mapper(t[0])[:, :, None, None].chunk(2, dim=1)
+ for i, c in enumerate(self.conds):
+ ac, bc = getattr(self, f"mapper_{c}")(t[i + 1])[:, :, None, None].chunk(2, dim=1)
+ a, b = a + ac, b + bc
+ return x * (1 + a) + b
diff --git a/comfy/ldm/cascade/controlnet.py b/comfy/ldm/cascade/controlnet.py
new file mode 100644
index 000000000..5dac59394
--- /dev/null
+++ b/comfy/ldm/cascade/controlnet.py
@@ -0,0 +1,93 @@
+"""
+ This file is part of ComfyUI.
+ Copyright (C) 2024 Stability AI
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program. If not, see .
+"""
+
+import torch
+import torchvision
+from torch import nn
+from .common import LayerNorm2d_op
+
+
+class CNetResBlock(nn.Module):
+ def __init__(self, c, dtype=None, device=None, operations=None):
+ super().__init__()
+ self.blocks = nn.Sequential(
+ LayerNorm2d_op(operations)(c, dtype=dtype, device=device),
+ nn.GELU(),
+ operations.Conv2d(c, c, kernel_size=3, padding=1),
+ LayerNorm2d_op(operations)(c, dtype=dtype, device=device),
+ nn.GELU(),
+ operations.Conv2d(c, c, kernel_size=3, padding=1),
+ )
+
+ def forward(self, x):
+ return x + self.blocks(x)
+
+
+class ControlNet(nn.Module):
+ def __init__(self, c_in=3, c_proj=2048, proj_blocks=None, bottleneck_mode=None, dtype=None, device=None, operations=nn):
+ super().__init__()
+ if bottleneck_mode is None:
+ bottleneck_mode = 'effnet'
+ self.proj_blocks = proj_blocks
+ if bottleneck_mode == 'effnet':
+ embd_channels = 1280
+ self.backbone = torchvision.models.efficientnet_v2_s().features.eval()
+ if c_in != 3:
+ in_weights = self.backbone[0][0].weight.data
+ self.backbone[0][0] = operations.Conv2d(c_in, 24, kernel_size=3, stride=2, bias=False, dtype=dtype, device=device)
+ if c_in > 3:
+ # nn.init.constant_(self.backbone[0][0].weight, 0)
+ self.backbone[0][0].weight.data[:, :3] = in_weights[:, :3].clone()
+ else:
+ self.backbone[0][0].weight.data = in_weights[:, :c_in].clone()
+ elif bottleneck_mode == 'simple':
+ embd_channels = c_in
+ self.backbone = nn.Sequential(
+ operations.Conv2d(embd_channels, embd_channels * 4, kernel_size=3, padding=1, dtype=dtype, device=device),
+ nn.LeakyReLU(0.2, inplace=True),
+ operations.Conv2d(embd_channels * 4, embd_channels, kernel_size=3, padding=1, dtype=dtype, device=device),
+ )
+ elif bottleneck_mode == 'large':
+ self.backbone = nn.Sequential(
+ operations.Conv2d(c_in, 4096 * 4, kernel_size=1, dtype=dtype, device=device),
+ nn.LeakyReLU(0.2, inplace=True),
+ operations.Conv2d(4096 * 4, 1024, kernel_size=1, dtype=dtype, device=device),
+ *[CNetResBlock(1024, dtype=dtype, device=device, operations=operations) for _ in range(8)],
+ operations.Conv2d(1024, 1280, kernel_size=1, dtype=dtype, device=device),
+ )
+ embd_channels = 1280
+ else:
+ raise ValueError(f'Unknown bottleneck mode: {bottleneck_mode}')
+ self.projections = nn.ModuleList()
+ for _ in range(len(proj_blocks)):
+ self.projections.append(nn.Sequential(
+ operations.Conv2d(embd_channels, embd_channels, kernel_size=1, bias=False, dtype=dtype, device=device),
+ nn.LeakyReLU(0.2, inplace=True),
+ operations.Conv2d(embd_channels, c_proj, kernel_size=1, bias=False, dtype=dtype, device=device),
+ ))
+ # nn.init.constant_(self.projections[-1][-1].weight, 0) # zero output projection
+ self.xl = False
+ self.input_channels = c_in
+ self.unshuffle_amount = 8
+
+ def forward(self, x):
+ x = self.backbone(x)
+ proj_outputs = [None for _ in range(max(self.proj_blocks) + 1)]
+ for i, idx in enumerate(self.proj_blocks):
+ proj_outputs[idx] = self.projections[i](x)
+ return proj_outputs
diff --git a/comfy/ldm/cascade/stage_a.py b/comfy/ldm/cascade/stage_a.py
new file mode 100644
index 000000000..ca8867eaf
--- /dev/null
+++ b/comfy/ldm/cascade/stage_a.py
@@ -0,0 +1,255 @@
+"""
+ This file is part of ComfyUI.
+ Copyright (C) 2024 Stability AI
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program. If not, see .
+"""
+
+import torch
+from torch import nn
+from torch.autograd import Function
+
+class vector_quantize(Function):
+ @staticmethod
+ def forward(ctx, x, codebook):
+ with torch.no_grad():
+ codebook_sqr = torch.sum(codebook ** 2, dim=1)
+ x_sqr = torch.sum(x ** 2, dim=1, keepdim=True)
+
+ dist = torch.addmm(codebook_sqr + x_sqr, x, codebook.t(), alpha=-2.0, beta=1.0)
+ _, indices = dist.min(dim=1)
+
+ ctx.save_for_backward(indices, codebook)
+ ctx.mark_non_differentiable(indices)
+
+ nn = torch.index_select(codebook, 0, indices)
+ return nn, indices
+
+ @staticmethod
+ def backward(ctx, grad_output, grad_indices):
+ grad_inputs, grad_codebook = None, None
+
+ if ctx.needs_input_grad[0]:
+ grad_inputs = grad_output.clone()
+ if ctx.needs_input_grad[1]:
+ # Gradient wrt. the codebook
+ indices, codebook = ctx.saved_tensors
+
+ grad_codebook = torch.zeros_like(codebook)
+ grad_codebook.index_add_(0, indices, grad_output)
+
+ return (grad_inputs, grad_codebook)
+
+
+class VectorQuantize(nn.Module):
+ def __init__(self, embedding_size, k, ema_decay=0.99, ema_loss=False):
+ """
+ Takes an input of variable size (as long as the last dimension matches the embedding size).
+ Returns one tensor containing the nearest neigbour embeddings to each of the inputs,
+ with the same size as the input, vq and commitment components for the loss as a touple
+ in the second output and the indices of the quantized vectors in the third:
+ quantized, (vq_loss, commit_loss), indices
+ """
+ super(VectorQuantize, self).__init__()
+
+ self.codebook = nn.Embedding(k, embedding_size)
+ self.codebook.weight.data.uniform_(-1./k, 1./k)
+ self.vq = vector_quantize.apply
+
+ self.ema_decay = ema_decay
+ self.ema_loss = ema_loss
+ if ema_loss:
+ self.register_buffer('ema_element_count', torch.ones(k))
+ self.register_buffer('ema_weight_sum', torch.zeros_like(self.codebook.weight))
+
+ def _laplace_smoothing(self, x, epsilon):
+ n = torch.sum(x)
+ return ((x + epsilon) / (n + x.size(0) * epsilon) * n)
+
+ def _updateEMA(self, z_e_x, indices):
+ mask = nn.functional.one_hot(indices, self.ema_element_count.size(0)).float()
+ elem_count = mask.sum(dim=0)
+ weight_sum = torch.mm(mask.t(), z_e_x)
+
+ self.ema_element_count = (self.ema_decay * self.ema_element_count) + ((1-self.ema_decay) * elem_count)
+ self.ema_element_count = self._laplace_smoothing(self.ema_element_count, 1e-5)
+ self.ema_weight_sum = (self.ema_decay * self.ema_weight_sum) + ((1-self.ema_decay) * weight_sum)
+
+ self.codebook.weight.data = self.ema_weight_sum / self.ema_element_count.unsqueeze(-1)
+
+ def idx2vq(self, idx, dim=-1):
+ q_idx = self.codebook(idx)
+ if dim != -1:
+ q_idx = q_idx.movedim(-1, dim)
+ return q_idx
+
+ def forward(self, x, get_losses=True, dim=-1):
+ if dim != -1:
+ x = x.movedim(dim, -1)
+ z_e_x = x.contiguous().view(-1, x.size(-1)) if len(x.shape) > 2 else x
+ z_q_x, indices = self.vq(z_e_x, self.codebook.weight.detach())
+ vq_loss, commit_loss = None, None
+ if self.ema_loss and self.training:
+ self._updateEMA(z_e_x.detach(), indices.detach())
+ # pick the graded embeddings after updating the codebook in order to have a more accurate commitment loss
+ z_q_x_grd = torch.index_select(self.codebook.weight, dim=0, index=indices)
+ if get_losses:
+ vq_loss = (z_q_x_grd - z_e_x.detach()).pow(2).mean()
+ commit_loss = (z_e_x - z_q_x_grd.detach()).pow(2).mean()
+
+ z_q_x = z_q_x.view(x.shape)
+ if dim != -1:
+ z_q_x = z_q_x.movedim(-1, dim)
+ return z_q_x, (vq_loss, commit_loss), indices.view(x.shape[:-1])
+
+
+class ResBlock(nn.Module):
+ def __init__(self, c, c_hidden):
+ super().__init__()
+ # depthwise/attention
+ self.norm1 = nn.LayerNorm(c, elementwise_affine=False, eps=1e-6)
+ self.depthwise = nn.Sequential(
+ nn.ReplicationPad2d(1),
+ nn.Conv2d(c, c, kernel_size=3, groups=c)
+ )
+
+ # channelwise
+ self.norm2 = nn.LayerNorm(c, elementwise_affine=False, eps=1e-6)
+ self.channelwise = nn.Sequential(
+ nn.Linear(c, c_hidden),
+ nn.GELU(),
+ nn.Linear(c_hidden, c),
+ )
+
+ self.gammas = nn.Parameter(torch.zeros(6), requires_grad=True)
+
+ # Init weights
+ def _basic_init(module):
+ if isinstance(module, nn.Linear) or isinstance(module, nn.Conv2d):
+ torch.nn.init.xavier_uniform_(module.weight)
+ if module.bias is not None:
+ nn.init.constant_(module.bias, 0)
+
+ self.apply(_basic_init)
+
+ def _norm(self, x, norm):
+ return norm(x.permute(0, 2, 3, 1)).permute(0, 3, 1, 2)
+
+ def forward(self, x):
+ mods = self.gammas
+
+ x_temp = self._norm(x, self.norm1) * (1 + mods[0]) + mods[1]
+ try:
+ x = x + self.depthwise(x_temp) * mods[2]
+ except: #operation not implemented for bf16
+ x_temp = self.depthwise[0](x_temp.float()).to(x.dtype)
+ x = x + self.depthwise[1](x_temp) * mods[2]
+
+ x_temp = self._norm(x, self.norm2) * (1 + mods[3]) + mods[4]
+ x = x + self.channelwise(x_temp.permute(0, 2, 3, 1)).permute(0, 3, 1, 2) * mods[5]
+
+ return x
+
+
+class StageA(nn.Module):
+ def __init__(self, levels=2, bottleneck_blocks=12, c_hidden=384, c_latent=4, codebook_size=8192):
+ super().__init__()
+ self.c_latent = c_latent
+ c_levels = [c_hidden // (2 ** i) for i in reversed(range(levels))]
+
+ # Encoder blocks
+ self.in_block = nn.Sequential(
+ nn.PixelUnshuffle(2),
+ nn.Conv2d(3 * 4, c_levels[0], kernel_size=1)
+ )
+ down_blocks = []
+ for i in range(levels):
+ if i > 0:
+ down_blocks.append(nn.Conv2d(c_levels[i - 1], c_levels[i], kernel_size=4, stride=2, padding=1))
+ block = ResBlock(c_levels[i], c_levels[i] * 4)
+ down_blocks.append(block)
+ down_blocks.append(nn.Sequential(
+ nn.Conv2d(c_levels[-1], c_latent, kernel_size=1, bias=False),
+ nn.BatchNorm2d(c_latent), # then normalize them to have mean 0 and std 1
+ ))
+ self.down_blocks = nn.Sequential(*down_blocks)
+ self.down_blocks[0]
+
+ self.codebook_size = codebook_size
+ self.vquantizer = VectorQuantize(c_latent, k=codebook_size)
+
+ # Decoder blocks
+ up_blocks = [nn.Sequential(
+ nn.Conv2d(c_latent, c_levels[-1], kernel_size=1)
+ )]
+ for i in range(levels):
+ for j in range(bottleneck_blocks if i == 0 else 1):
+ block = ResBlock(c_levels[levels - 1 - i], c_levels[levels - 1 - i] * 4)
+ up_blocks.append(block)
+ if i < levels - 1:
+ up_blocks.append(
+ nn.ConvTranspose2d(c_levels[levels - 1 - i], c_levels[levels - 2 - i], kernel_size=4, stride=2,
+ padding=1))
+ self.up_blocks = nn.Sequential(*up_blocks)
+ self.out_block = nn.Sequential(
+ nn.Conv2d(c_levels[0], 3 * 4, kernel_size=1),
+ nn.PixelShuffle(2),
+ )
+
+ def encode(self, x, quantize=False):
+ x = self.in_block(x)
+ x = self.down_blocks(x)
+ if quantize:
+ qe, (vq_loss, commit_loss), indices = self.vquantizer.forward(x, dim=1)
+ return qe, x, indices, vq_loss + commit_loss * 0.25
+ else:
+ return x
+
+ def decode(self, x):
+ x = self.up_blocks(x)
+ x = self.out_block(x)
+ return x
+
+ def forward(self, x, quantize=False):
+ qe, x, _, vq_loss = self.encode(x, quantize)
+ x = self.decode(qe)
+ return x, vq_loss
+
+
+class Discriminator(nn.Module):
+ def __init__(self, c_in=3, c_cond=0, c_hidden=512, depth=6):
+ super().__init__()
+ d = max(depth - 3, 3)
+ layers = [
+ nn.utils.spectral_norm(nn.Conv2d(c_in, c_hidden // (2 ** d), kernel_size=3, stride=2, padding=1)),
+ nn.LeakyReLU(0.2),
+ ]
+ for i in range(depth - 1):
+ c_in = c_hidden // (2 ** max((d - i), 0))
+ c_out = c_hidden // (2 ** max((d - 1 - i), 0))
+ layers.append(nn.utils.spectral_norm(nn.Conv2d(c_in, c_out, kernel_size=3, stride=2, padding=1)))
+ layers.append(nn.InstanceNorm2d(c_out))
+ layers.append(nn.LeakyReLU(0.2))
+ self.encoder = nn.Sequential(*layers)
+ self.shuffle = nn.Conv2d((c_hidden + c_cond) if c_cond > 0 else c_hidden, 1, kernel_size=1)
+ self.logits = nn.Sigmoid()
+
+ def forward(self, x, cond=None):
+ x = self.encoder(x)
+ if cond is not None:
+ cond = cond.view(cond.size(0), cond.size(1), 1, 1, ).expand(-1, -1, x.size(-2), x.size(-1))
+ x = torch.cat([x, cond], dim=1)
+ x = self.shuffle(x)
+ x = self.logits(x)
+ return x
diff --git a/comfy/ldm/cascade/stage_b.py b/comfy/ldm/cascade/stage_b.py
new file mode 100644
index 000000000..6d2c22231
--- /dev/null
+++ b/comfy/ldm/cascade/stage_b.py
@@ -0,0 +1,257 @@
+"""
+ This file is part of ComfyUI.
+ Copyright (C) 2024 Stability AI
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program. If not, see .
+"""
+
+import math
+import numpy as np
+import torch
+from torch import nn
+from .common import AttnBlock, LayerNorm2d_op, ResBlock, FeedForwardBlock, TimestepBlock
+
+class StageB(nn.Module):
+ def __init__(self, c_in=4, c_out=4, c_r=64, patch_size=2, c_cond=1280, c_hidden=[320, 640, 1280, 1280],
+ nhead=[-1, -1, 20, 20], blocks=[[2, 6, 28, 6], [6, 28, 6, 2]],
+ block_repeat=[[1, 1, 1, 1], [3, 3, 2, 2]], level_config=['CT', 'CT', 'CTA', 'CTA'], c_clip=1280,
+ c_clip_seq=4, c_effnet=16, c_pixels=3, kernel_size=3, dropout=[0, 0, 0.0, 0.0], self_attn=True,
+ t_conds=['sca'], stable_cascade_stage=None, dtype=None, device=None, operations=None):
+ super().__init__()
+ self.dtype = dtype
+ self.c_r = c_r
+ self.t_conds = t_conds
+ self.c_clip_seq = c_clip_seq
+ if not isinstance(dropout, list):
+ dropout = [dropout] * len(c_hidden)
+ if not isinstance(self_attn, list):
+ self_attn = [self_attn] * len(c_hidden)
+
+ # CONDITIONING
+ self.effnet_mapper = nn.Sequential(
+ operations.Conv2d(c_effnet, c_hidden[0] * 4, kernel_size=1, dtype=dtype, device=device),
+ nn.GELU(),
+ operations.Conv2d(c_hidden[0] * 4, c_hidden[0], kernel_size=1, dtype=dtype, device=device),
+ LayerNorm2d_op(operations)(c_hidden[0], elementwise_affine=False, eps=1e-6, dtype=dtype, device=device)
+ )
+ self.pixels_mapper = nn.Sequential(
+ operations.Conv2d(c_pixels, c_hidden[0] * 4, kernel_size=1, dtype=dtype, device=device),
+ nn.GELU(),
+ operations.Conv2d(c_hidden[0] * 4, c_hidden[0], kernel_size=1, dtype=dtype, device=device),
+ LayerNorm2d_op(operations)(c_hidden[0], elementwise_affine=False, eps=1e-6, dtype=dtype, device=device)
+ )
+ self.clip_mapper = operations.Linear(c_clip, c_cond * c_clip_seq, dtype=dtype, device=device)
+ self.clip_norm = operations.LayerNorm(c_cond, elementwise_affine=False, eps=1e-6, dtype=dtype, device=device)
+
+ self.embedding = nn.Sequential(
+ nn.PixelUnshuffle(patch_size),
+ operations.Conv2d(c_in * (patch_size ** 2), c_hidden[0], kernel_size=1, dtype=dtype, device=device),
+ LayerNorm2d_op(operations)(c_hidden[0], elementwise_affine=False, eps=1e-6, dtype=dtype, device=device)
+ )
+
+ def get_block(block_type, c_hidden, nhead, c_skip=0, dropout=0, self_attn=True):
+ if block_type == 'C':
+ return ResBlock(c_hidden, c_skip, kernel_size=kernel_size, dropout=dropout, dtype=dtype, device=device, operations=operations)
+ elif block_type == 'A':
+ return AttnBlock(c_hidden, c_cond, nhead, self_attn=self_attn, dropout=dropout, dtype=dtype, device=device, operations=operations)
+ elif block_type == 'F':
+ return FeedForwardBlock(c_hidden, dropout=dropout, dtype=dtype, device=device, operations=operations)
+ elif block_type == 'T':
+ return TimestepBlock(c_hidden, c_r, conds=t_conds, dtype=dtype, device=device, operations=operations)
+ else:
+ raise Exception(f'Block type {block_type} not supported')
+
+ # BLOCKS
+ # -- down blocks
+ self.down_blocks = nn.ModuleList()
+ self.down_downscalers = nn.ModuleList()
+ self.down_repeat_mappers = nn.ModuleList()
+ for i in range(len(c_hidden)):
+ if i > 0:
+ self.down_downscalers.append(nn.Sequential(
+ LayerNorm2d_op(operations)(c_hidden[i - 1], elementwise_affine=False, eps=1e-6, dtype=dtype, device=device),
+ operations.Conv2d(c_hidden[i - 1], c_hidden[i], kernel_size=2, stride=2, dtype=dtype, device=device),
+ ))
+ else:
+ self.down_downscalers.append(nn.Identity())
+ down_block = nn.ModuleList()
+ for _ in range(blocks[0][i]):
+ for block_type in level_config[i]:
+ block = get_block(block_type, c_hidden[i], nhead[i], dropout=dropout[i], self_attn=self_attn[i])
+ down_block.append(block)
+ self.down_blocks.append(down_block)
+ if block_repeat is not None:
+ block_repeat_mappers = nn.ModuleList()
+ for _ in range(block_repeat[0][i] - 1):
+ block_repeat_mappers.append(operations.Conv2d(c_hidden[i], c_hidden[i], kernel_size=1, dtype=dtype, device=device))
+ self.down_repeat_mappers.append(block_repeat_mappers)
+
+ # -- up blocks
+ self.up_blocks = nn.ModuleList()
+ self.up_upscalers = nn.ModuleList()
+ self.up_repeat_mappers = nn.ModuleList()
+ for i in reversed(range(len(c_hidden))):
+ if i > 0:
+ self.up_upscalers.append(nn.Sequential(
+ LayerNorm2d_op(operations)(c_hidden[i], elementwise_affine=False, eps=1e-6, dtype=dtype, device=device),
+ operations.ConvTranspose2d(c_hidden[i], c_hidden[i - 1], kernel_size=2, stride=2, dtype=dtype, device=device),
+ ))
+ else:
+ self.up_upscalers.append(nn.Identity())
+ up_block = nn.ModuleList()
+ for j in range(blocks[1][::-1][i]):
+ for k, block_type in enumerate(level_config[i]):
+ c_skip = c_hidden[i] if i < len(c_hidden) - 1 and j == k == 0 else 0
+ block = get_block(block_type, c_hidden[i], nhead[i], c_skip=c_skip, dropout=dropout[i],
+ self_attn=self_attn[i])
+ up_block.append(block)
+ self.up_blocks.append(up_block)
+ if block_repeat is not None:
+ block_repeat_mappers = nn.ModuleList()
+ for _ in range(block_repeat[1][::-1][i] - 1):
+ block_repeat_mappers.append(operations.Conv2d(c_hidden[i], c_hidden[i], kernel_size=1, dtype=dtype, device=device))
+ self.up_repeat_mappers.append(block_repeat_mappers)
+
+ # OUTPUT
+ self.clf = nn.Sequential(
+ LayerNorm2d_op(operations)(c_hidden[0], elementwise_affine=False, eps=1e-6, dtype=dtype, device=device),
+ operations.Conv2d(c_hidden[0], c_out * (patch_size ** 2), kernel_size=1, dtype=dtype, device=device),
+ nn.PixelShuffle(patch_size),
+ )
+
+ # --- WEIGHT INIT ---
+ # self.apply(self._init_weights) # General init
+ # nn.init.normal_(self.clip_mapper.weight, std=0.02) # conditionings
+ # nn.init.normal_(self.effnet_mapper[0].weight, std=0.02) # conditionings
+ # nn.init.normal_(self.effnet_mapper[2].weight, std=0.02) # conditionings
+ # nn.init.normal_(self.pixels_mapper[0].weight, std=0.02) # conditionings
+ # nn.init.normal_(self.pixels_mapper[2].weight, std=0.02) # conditionings
+ # torch.nn.init.xavier_uniform_(self.embedding[1].weight, 0.02) # inputs
+ # nn.init.constant_(self.clf[1].weight, 0) # outputs
+ #
+ # # blocks
+ # for level_block in self.down_blocks + self.up_blocks:
+ # for block in level_block:
+ # if isinstance(block, ResBlock) or isinstance(block, FeedForwardBlock):
+ # block.channelwise[-1].weight.data *= np.sqrt(1 / sum(blocks[0]))
+ # elif isinstance(block, TimestepBlock):
+ # for layer in block.modules():
+ # if isinstance(layer, nn.Linear):
+ # nn.init.constant_(layer.weight, 0)
+ #
+ # def _init_weights(self, m):
+ # if isinstance(m, (nn.Conv2d, nn.Linear)):
+ # torch.nn.init.xavier_uniform_(m.weight)
+ # if m.bias is not None:
+ # nn.init.constant_(m.bias, 0)
+
+ def gen_r_embedding(self, r, max_positions=10000):
+ r = r * max_positions
+ half_dim = self.c_r // 2
+ emb = math.log(max_positions) / (half_dim - 1)
+ emb = torch.arange(half_dim, device=r.device).float().mul(-emb).exp()
+ emb = r[:, None] * emb[None, :]
+ emb = torch.cat([emb.sin(), emb.cos()], dim=1)
+ if self.c_r % 2 == 1: # zero pad
+ emb = nn.functional.pad(emb, (0, 1), mode='constant')
+ return emb
+
+ def gen_c_embeddings(self, clip):
+ if len(clip.shape) == 2:
+ clip = clip.unsqueeze(1)
+ clip = self.clip_mapper(clip).view(clip.size(0), clip.size(1) * self.c_clip_seq, -1)
+ clip = self.clip_norm(clip)
+ return clip
+
+ def _down_encode(self, x, r_embed, clip):
+ level_outputs = []
+ block_group = zip(self.down_blocks, self.down_downscalers, self.down_repeat_mappers)
+ for down_block, downscaler, repmap in block_group:
+ x = downscaler(x)
+ for i in range(len(repmap) + 1):
+ for block in down_block:
+ if isinstance(block, ResBlock) or (
+ hasattr(block, '_fsdp_wrapped_module') and isinstance(block._fsdp_wrapped_module,
+ ResBlock)):
+ x = block(x)
+ elif isinstance(block, AttnBlock) or (
+ hasattr(block, '_fsdp_wrapped_module') and isinstance(block._fsdp_wrapped_module,
+ AttnBlock)):
+ x = block(x, clip)
+ elif isinstance(block, TimestepBlock) or (
+ hasattr(block, '_fsdp_wrapped_module') and isinstance(block._fsdp_wrapped_module,
+ TimestepBlock)):
+ x = block(x, r_embed)
+ else:
+ x = block(x)
+ if i < len(repmap):
+ x = repmap[i](x)
+ level_outputs.insert(0, x)
+ return level_outputs
+
+ def _up_decode(self, level_outputs, r_embed, clip):
+ x = level_outputs[0]
+ block_group = zip(self.up_blocks, self.up_upscalers, self.up_repeat_mappers)
+ for i, (up_block, upscaler, repmap) in enumerate(block_group):
+ for j in range(len(repmap) + 1):
+ for k, block in enumerate(up_block):
+ if isinstance(block, ResBlock) or (
+ hasattr(block, '_fsdp_wrapped_module') and isinstance(block._fsdp_wrapped_module,
+ ResBlock)):
+ skip = level_outputs[i] if k == 0 and i > 0 else None
+ if skip is not None and (x.size(-1) != skip.size(-1) or x.size(-2) != skip.size(-2)):
+ x = torch.nn.functional.interpolate(x, skip.shape[-2:], mode='bilinear',
+ align_corners=True)
+ x = block(x, skip)
+ elif isinstance(block, AttnBlock) or (
+ hasattr(block, '_fsdp_wrapped_module') and isinstance(block._fsdp_wrapped_module,
+ AttnBlock)):
+ x = block(x, clip)
+ elif isinstance(block, TimestepBlock) or (
+ hasattr(block, '_fsdp_wrapped_module') and isinstance(block._fsdp_wrapped_module,
+ TimestepBlock)):
+ x = block(x, r_embed)
+ else:
+ x = block(x)
+ if j < len(repmap):
+ x = repmap[j](x)
+ x = upscaler(x)
+ return x
+
+ def forward(self, x, r, effnet, clip, pixels=None, **kwargs):
+ if pixels is None:
+ pixels = x.new_zeros(x.size(0), 3, 8, 8)
+
+ # Process the conditioning embeddings
+ r_embed = self.gen_r_embedding(r).to(dtype=x.dtype)
+ for c in self.t_conds:
+ t_cond = kwargs.get(c, torch.zeros_like(r))
+ r_embed = torch.cat([r_embed, self.gen_r_embedding(t_cond).to(dtype=x.dtype)], dim=1)
+ clip = self.gen_c_embeddings(clip)
+
+ # Model Blocks
+ x = self.embedding(x)
+ x = x + self.effnet_mapper(
+ nn.functional.interpolate(effnet, size=x.shape[-2:], mode='bilinear', align_corners=True))
+ x = x + nn.functional.interpolate(self.pixels_mapper(pixels), size=x.shape[-2:], mode='bilinear',
+ align_corners=True)
+ level_outputs = self._down_encode(x, r_embed, clip)
+ x = self._up_decode(level_outputs, r_embed, clip)
+ return self.clf(x)
+
+ def update_weights_ema(self, src_model, beta=0.999):
+ for self_params, src_params in zip(self.parameters(), src_model.parameters()):
+ self_params.data = self_params.data * beta + src_params.data.clone().to(self_params.device) * (1 - beta)
+ for self_buffers, src_buffers in zip(self.buffers(), src_model.buffers()):
+ self_buffers.data = self_buffers.data * beta + src_buffers.data.clone().to(self_buffers.device) * (1 - beta)
diff --git a/comfy/ldm/cascade/stage_c.py b/comfy/ldm/cascade/stage_c.py
new file mode 100644
index 000000000..67c1e52b6
--- /dev/null
+++ b/comfy/ldm/cascade/stage_c.py
@@ -0,0 +1,274 @@
+"""
+ This file is part of ComfyUI.
+ Copyright (C) 2024 Stability AI
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program. If not, see .
+"""
+
+import torch
+from torch import nn
+import numpy as np
+import math
+from .common import AttnBlock, LayerNorm2d_op, ResBlock, FeedForwardBlock, TimestepBlock
+# from .controlnet import ControlNetDeliverer
+
+class UpDownBlock2d(nn.Module):
+ def __init__(self, c_in, c_out, mode, enabled=True, dtype=None, device=None, operations=None):
+ super().__init__()
+ assert mode in ['up', 'down']
+ interpolation = nn.Upsample(scale_factor=2 if mode == 'up' else 0.5, mode='bilinear',
+ align_corners=True) if enabled else nn.Identity()
+ mapping = operations.Conv2d(c_in, c_out, kernel_size=1, dtype=dtype, device=device)
+ self.blocks = nn.ModuleList([interpolation, mapping] if mode == 'up' else [mapping, interpolation])
+
+ def forward(self, x):
+ for block in self.blocks:
+ x = block(x)
+ return x
+
+
+class StageC(nn.Module):
+ def __init__(self, c_in=16, c_out=16, c_r=64, patch_size=1, c_cond=2048, c_hidden=[2048, 2048], nhead=[32, 32],
+ blocks=[[8, 24], [24, 8]], block_repeat=[[1, 1], [1, 1]], level_config=['CTA', 'CTA'],
+ c_clip_text=1280, c_clip_text_pooled=1280, c_clip_img=768, c_clip_seq=4, kernel_size=3,
+ dropout=[0.0, 0.0], self_attn=True, t_conds=['sca', 'crp'], switch_level=[False], stable_cascade_stage=None,
+ dtype=None, device=None, operations=None):
+ super().__init__()
+ self.dtype = dtype
+ self.c_r = c_r
+ self.t_conds = t_conds
+ self.c_clip_seq = c_clip_seq
+ if not isinstance(dropout, list):
+ dropout = [dropout] * len(c_hidden)
+ if not isinstance(self_attn, list):
+ self_attn = [self_attn] * len(c_hidden)
+
+ # CONDITIONING
+ self.clip_txt_mapper = operations.Linear(c_clip_text, c_cond, dtype=dtype, device=device)
+ self.clip_txt_pooled_mapper = operations.Linear(c_clip_text_pooled, c_cond * c_clip_seq, dtype=dtype, device=device)
+ self.clip_img_mapper = operations.Linear(c_clip_img, c_cond * c_clip_seq, dtype=dtype, device=device)
+ self.clip_norm = operations.LayerNorm(c_cond, elementwise_affine=False, eps=1e-6, dtype=dtype, device=device)
+
+ self.embedding = nn.Sequential(
+ nn.PixelUnshuffle(patch_size),
+ operations.Conv2d(c_in * (patch_size ** 2), c_hidden[0], kernel_size=1, dtype=dtype, device=device),
+ LayerNorm2d_op(operations)(c_hidden[0], elementwise_affine=False, eps=1e-6)
+ )
+
+ def get_block(block_type, c_hidden, nhead, c_skip=0, dropout=0, self_attn=True):
+ if block_type == 'C':
+ return ResBlock(c_hidden, c_skip, kernel_size=kernel_size, dropout=dropout, dtype=dtype, device=device, operations=operations)
+ elif block_type == 'A':
+ return AttnBlock(c_hidden, c_cond, nhead, self_attn=self_attn, dropout=dropout, dtype=dtype, device=device, operations=operations)
+ elif block_type == 'F':
+ return FeedForwardBlock(c_hidden, dropout=dropout, dtype=dtype, device=device, operations=operations)
+ elif block_type == 'T':
+ return TimestepBlock(c_hidden, c_r, conds=t_conds, dtype=dtype, device=device, operations=operations)
+ else:
+ raise Exception(f'Block type {block_type} not supported')
+
+ # BLOCKS
+ # -- down blocks
+ self.down_blocks = nn.ModuleList()
+ self.down_downscalers = nn.ModuleList()
+ self.down_repeat_mappers = nn.ModuleList()
+ for i in range(len(c_hidden)):
+ if i > 0:
+ self.down_downscalers.append(nn.Sequential(
+ LayerNorm2d_op(operations)(c_hidden[i - 1], elementwise_affine=False, eps=1e-6),
+ UpDownBlock2d(c_hidden[i - 1], c_hidden[i], mode='down', enabled=switch_level[i - 1], dtype=dtype, device=device, operations=operations)
+ ))
+ else:
+ self.down_downscalers.append(nn.Identity())
+ down_block = nn.ModuleList()
+ for _ in range(blocks[0][i]):
+ for block_type in level_config[i]:
+ block = get_block(block_type, c_hidden[i], nhead[i], dropout=dropout[i], self_attn=self_attn[i])
+ down_block.append(block)
+ self.down_blocks.append(down_block)
+ if block_repeat is not None:
+ block_repeat_mappers = nn.ModuleList()
+ for _ in range(block_repeat[0][i] - 1):
+ block_repeat_mappers.append(operations.Conv2d(c_hidden[i], c_hidden[i], kernel_size=1, dtype=dtype, device=device))
+ self.down_repeat_mappers.append(block_repeat_mappers)
+
+ # -- up blocks
+ self.up_blocks = nn.ModuleList()
+ self.up_upscalers = nn.ModuleList()
+ self.up_repeat_mappers = nn.ModuleList()
+ for i in reversed(range(len(c_hidden))):
+ if i > 0:
+ self.up_upscalers.append(nn.Sequential(
+ LayerNorm2d_op(operations)(c_hidden[i], elementwise_affine=False, eps=1e-6),
+ UpDownBlock2d(c_hidden[i], c_hidden[i - 1], mode='up', enabled=switch_level[i - 1], dtype=dtype, device=device, operations=operations)
+ ))
+ else:
+ self.up_upscalers.append(nn.Identity())
+ up_block = nn.ModuleList()
+ for j in range(blocks[1][::-1][i]):
+ for k, block_type in enumerate(level_config[i]):
+ c_skip = c_hidden[i] if i < len(c_hidden) - 1 and j == k == 0 else 0
+ block = get_block(block_type, c_hidden[i], nhead[i], c_skip=c_skip, dropout=dropout[i],
+ self_attn=self_attn[i])
+ up_block.append(block)
+ self.up_blocks.append(up_block)
+ if block_repeat is not None:
+ block_repeat_mappers = nn.ModuleList()
+ for _ in range(block_repeat[1][::-1][i] - 1):
+ block_repeat_mappers.append(operations.Conv2d(c_hidden[i], c_hidden[i], kernel_size=1, dtype=dtype, device=device))
+ self.up_repeat_mappers.append(block_repeat_mappers)
+
+ # OUTPUT
+ self.clf = nn.Sequential(
+ LayerNorm2d_op(operations)(c_hidden[0], elementwise_affine=False, eps=1e-6, dtype=dtype, device=device),
+ operations.Conv2d(c_hidden[0], c_out * (patch_size ** 2), kernel_size=1, dtype=dtype, device=device),
+ nn.PixelShuffle(patch_size),
+ )
+
+ # --- WEIGHT INIT ---
+ # self.apply(self._init_weights) # General init
+ # nn.init.normal_(self.clip_txt_mapper.weight, std=0.02) # conditionings
+ # nn.init.normal_(self.clip_txt_pooled_mapper.weight, std=0.02) # conditionings
+ # nn.init.normal_(self.clip_img_mapper.weight, std=0.02) # conditionings
+ # torch.nn.init.xavier_uniform_(self.embedding[1].weight, 0.02) # inputs
+ # nn.init.constant_(self.clf[1].weight, 0) # outputs
+ #
+ # # blocks
+ # for level_block in self.down_blocks + self.up_blocks:
+ # for block in level_block:
+ # if isinstance(block, ResBlock) or isinstance(block, FeedForwardBlock):
+ # block.channelwise[-1].weight.data *= np.sqrt(1 / sum(blocks[0]))
+ # elif isinstance(block, TimestepBlock):
+ # for layer in block.modules():
+ # if isinstance(layer, nn.Linear):
+ # nn.init.constant_(layer.weight, 0)
+ #
+ # def _init_weights(self, m):
+ # if isinstance(m, (nn.Conv2d, nn.Linear)):
+ # torch.nn.init.xavier_uniform_(m.weight)
+ # if m.bias is not None:
+ # nn.init.constant_(m.bias, 0)
+
+ def gen_r_embedding(self, r, max_positions=10000):
+ r = r * max_positions
+ half_dim = self.c_r // 2
+ emb = math.log(max_positions) / (half_dim - 1)
+ emb = torch.arange(half_dim, device=r.device).float().mul(-emb).exp()
+ emb = r[:, None] * emb[None, :]
+ emb = torch.cat([emb.sin(), emb.cos()], dim=1)
+ if self.c_r % 2 == 1: # zero pad
+ emb = nn.functional.pad(emb, (0, 1), mode='constant')
+ return emb
+
+ def gen_c_embeddings(self, clip_txt, clip_txt_pooled, clip_img):
+ clip_txt = self.clip_txt_mapper(clip_txt)
+ if len(clip_txt_pooled.shape) == 2:
+ clip_txt_pooled = clip_txt_pooled.unsqueeze(1)
+ if len(clip_img.shape) == 2:
+ clip_img = clip_img.unsqueeze(1)
+ clip_txt_pool = self.clip_txt_pooled_mapper(clip_txt_pooled).view(clip_txt_pooled.size(0), clip_txt_pooled.size(1) * self.c_clip_seq, -1)
+ clip_img = self.clip_img_mapper(clip_img).view(clip_img.size(0), clip_img.size(1) * self.c_clip_seq, -1)
+ clip = torch.cat([clip_txt, clip_txt_pool, clip_img], dim=1)
+ clip = self.clip_norm(clip)
+ return clip
+
+ def _down_encode(self, x, r_embed, clip, cnet=None):
+ level_outputs = []
+ block_group = zip(self.down_blocks, self.down_downscalers, self.down_repeat_mappers)
+ for down_block, downscaler, repmap in block_group:
+ x = downscaler(x)
+ for i in range(len(repmap) + 1):
+ for block in down_block:
+ if isinstance(block, ResBlock) or (
+ hasattr(block, '_fsdp_wrapped_module') and isinstance(block._fsdp_wrapped_module,
+ ResBlock)):
+ if cnet is not None:
+ next_cnet = cnet.pop()
+ if next_cnet is not None:
+ x = x + nn.functional.interpolate(next_cnet, size=x.shape[-2:], mode='bilinear',
+ align_corners=True).to(x.dtype)
+ x = block(x)
+ elif isinstance(block, AttnBlock) or (
+ hasattr(block, '_fsdp_wrapped_module') and isinstance(block._fsdp_wrapped_module,
+ AttnBlock)):
+ x = block(x, clip)
+ elif isinstance(block, TimestepBlock) or (
+ hasattr(block, '_fsdp_wrapped_module') and isinstance(block._fsdp_wrapped_module,
+ TimestepBlock)):
+ x = block(x, r_embed)
+ else:
+ x = block(x)
+ if i < len(repmap):
+ x = repmap[i](x)
+ level_outputs.insert(0, x)
+ return level_outputs
+
+ def _up_decode(self, level_outputs, r_embed, clip, cnet=None):
+ x = level_outputs[0]
+ block_group = zip(self.up_blocks, self.up_upscalers, self.up_repeat_mappers)
+ for i, (up_block, upscaler, repmap) in enumerate(block_group):
+ for j in range(len(repmap) + 1):
+ for k, block in enumerate(up_block):
+ if isinstance(block, ResBlock) or (
+ hasattr(block, '_fsdp_wrapped_module') and isinstance(block._fsdp_wrapped_module,
+ ResBlock)):
+ skip = level_outputs[i] if k == 0 and i > 0 else None
+ if skip is not None and (x.size(-1) != skip.size(-1) or x.size(-2) != skip.size(-2)):
+ x = torch.nn.functional.interpolate(x, skip.shape[-2:], mode='bilinear',
+ align_corners=True)
+ if cnet is not None:
+ next_cnet = cnet.pop()
+ if next_cnet is not None:
+ x = x + nn.functional.interpolate(next_cnet, size=x.shape[-2:], mode='bilinear',
+ align_corners=True).to(x.dtype)
+ x = block(x, skip)
+ elif isinstance(block, AttnBlock) or (
+ hasattr(block, '_fsdp_wrapped_module') and isinstance(block._fsdp_wrapped_module,
+ AttnBlock)):
+ x = block(x, clip)
+ elif isinstance(block, TimestepBlock) or (
+ hasattr(block, '_fsdp_wrapped_module') and isinstance(block._fsdp_wrapped_module,
+ TimestepBlock)):
+ x = block(x, r_embed)
+ else:
+ x = block(x)
+ if j < len(repmap):
+ x = repmap[j](x)
+ x = upscaler(x)
+ return x
+
+ def forward(self, x, r, clip_text, clip_text_pooled, clip_img, control=None, **kwargs):
+ # Process the conditioning embeddings
+ r_embed = self.gen_r_embedding(r).to(dtype=x.dtype)
+ for c in self.t_conds:
+ t_cond = kwargs.get(c, torch.zeros_like(r))
+ r_embed = torch.cat([r_embed, self.gen_r_embedding(t_cond).to(dtype=x.dtype)], dim=1)
+ clip = self.gen_c_embeddings(clip_text, clip_text_pooled, clip_img)
+
+ if control is not None:
+ cnet = control.get("input")
+ else:
+ cnet = None
+
+ # Model Blocks
+ x = self.embedding(x)
+ level_outputs = self._down_encode(x, r_embed, clip, cnet)
+ x = self._up_decode(level_outputs, r_embed, clip, cnet)
+ return self.clf(x)
+
+ def update_weights_ema(self, src_model, beta=0.999):
+ for self_params, src_params in zip(self.parameters(), src_model.parameters()):
+ self_params.data = self_params.data * beta + src_params.data.clone().to(self_params.device) * (1 - beta)
+ for self_buffers, src_buffers in zip(self.buffers(), src_model.buffers()):
+ self_buffers.data = self_buffers.data * beta + src_buffers.data.clone().to(self_buffers.device) * (1 - beta)
diff --git a/comfy/ldm/cascade/stage_c_coder.py b/comfy/ldm/cascade/stage_c_coder.py
new file mode 100644
index 000000000..0cb7c49fc
--- /dev/null
+++ b/comfy/ldm/cascade/stage_c_coder.py
@@ -0,0 +1,95 @@
+"""
+ This file is part of ComfyUI.
+ Copyright (C) 2024 Stability AI
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program. If not, see .
+"""
+import torch
+import torchvision
+from torch import nn
+
+
+# EfficientNet
+class EfficientNetEncoder(nn.Module):
+ def __init__(self, c_latent=16):
+ super().__init__()
+ self.backbone = torchvision.models.efficientnet_v2_s().features.eval()
+ self.mapper = nn.Sequential(
+ nn.Conv2d(1280, c_latent, kernel_size=1, bias=False),
+ nn.BatchNorm2d(c_latent, affine=False), # then normalize them to have mean 0 and std 1
+ )
+ self.mean = nn.Parameter(torch.tensor([0.485, 0.456, 0.406]))
+ self.std = nn.Parameter(torch.tensor([0.229, 0.224, 0.225]))
+
+ def forward(self, x):
+ x = x * 0.5 + 0.5
+ x = (x - self.mean.view([3,1,1])) / self.std.view([3,1,1])
+ o = self.mapper(self.backbone(x))
+ return o
+
+
+# Fast Decoder for Stage C latents. E.g. 16 x 24 x 24 -> 3 x 192 x 192
+class Previewer(nn.Module):
+ def __init__(self, c_in=16, c_hidden=512, c_out=3):
+ super().__init__()
+ self.blocks = nn.Sequential(
+ nn.Conv2d(c_in, c_hidden, kernel_size=1), # 16 channels to 512 channels
+ nn.GELU(),
+ nn.BatchNorm2d(c_hidden),
+
+ nn.Conv2d(c_hidden, c_hidden, kernel_size=3, padding=1),
+ nn.GELU(),
+ nn.BatchNorm2d(c_hidden),
+
+ nn.ConvTranspose2d(c_hidden, c_hidden // 2, kernel_size=2, stride=2), # 16 -> 32
+ nn.GELU(),
+ nn.BatchNorm2d(c_hidden // 2),
+
+ nn.Conv2d(c_hidden // 2, c_hidden // 2, kernel_size=3, padding=1),
+ nn.GELU(),
+ nn.BatchNorm2d(c_hidden // 2),
+
+ nn.ConvTranspose2d(c_hidden // 2, c_hidden // 4, kernel_size=2, stride=2), # 32 -> 64
+ nn.GELU(),
+ nn.BatchNorm2d(c_hidden // 4),
+
+ nn.Conv2d(c_hidden // 4, c_hidden // 4, kernel_size=3, padding=1),
+ nn.GELU(),
+ nn.BatchNorm2d(c_hidden // 4),
+
+ nn.ConvTranspose2d(c_hidden // 4, c_hidden // 4, kernel_size=2, stride=2), # 64 -> 128
+ nn.GELU(),
+ nn.BatchNorm2d(c_hidden // 4),
+
+ nn.Conv2d(c_hidden // 4, c_hidden // 4, kernel_size=3, padding=1),
+ nn.GELU(),
+ nn.BatchNorm2d(c_hidden // 4),
+
+ nn.Conv2d(c_hidden // 4, c_out, kernel_size=1),
+ )
+
+ def forward(self, x):
+ return (self.blocks(x) - 0.5) * 2.0
+
+class StageC_coder(nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.previewer = Previewer()
+ self.encoder = EfficientNetEncoder()
+
+ def encode(self, x):
+ return self.encoder(x)
+
+ def decode(self, x):
+ return self.previewer(x)
diff --git a/comfy/ldm/modules/attention.py b/comfy/ldm/modules/attention.py
index fd8888d0e..f116efee3 100644
--- a/comfy/ldm/modules/attention.py
+++ b/comfy/ldm/modules/attention.py
@@ -1,12 +1,10 @@
-from inspect import isfunction
import math
import torch
import torch.nn.functional as F
from torch import nn, einsum
from einops import rearrange, repeat
from typing import Optional, Any
-from functools import partial
-
+import logging
from .diffusionmodules.util import checkpoint, AlphaBlender, timestep_embedding
from .sub_quadratic_attention import efficient_dot_product_attention
@@ -23,7 +21,7 @@ ops = comfy.ops.disable_weight_init
# CrossAttn precision handling
if args.dont_upcast_attention:
- print("disabling upcasting of attention")
+ logging.info("disabling upcasting of attention")
_ATTN_PRECISION = "fp16"
else:
_ATTN_PRECISION = "fp32"
@@ -117,7 +115,12 @@ def attention_basic(q, k, v, heads, mask=None):
mask = repeat(mask, 'b j -> (b h) () j', h=h)
sim.masked_fill_(~mask, max_neg_value)
else:
- sim += mask
+ if len(mask.shape) == 2:
+ bs = 1
+ else:
+ bs = mask.shape[0]
+ mask = mask.reshape(bs, -1, mask.shape[-2], mask.shape[-1]).expand(b, heads, -1, -1).reshape(-1, mask.shape[-2], mask.shape[-1])
+ sim.add_(mask)
# attention, what we cannot get enough of
sim = sim.softmax(dim=-1)
@@ -168,6 +171,13 @@ def attention_sub_quad(query, key, value, heads, mask=None):
if query_chunk_size is None:
query_chunk_size = 512
+ if mask is not None:
+ if len(mask.shape) == 2:
+ bs = 1
+ else:
+ bs = mask.shape[0]
+ mask = mask.reshape(bs, -1, mask.shape[-2], mask.shape[-1]).expand(b, heads, -1, -1).reshape(-1, mask.shape[-2], mask.shape[-1])
+
hidden_states = efficient_dot_product_attention(
query,
key,
@@ -226,6 +236,13 @@ def attention_split(q, k, v, heads, mask=None):
raise RuntimeError(f'Not enough memory, use lower resolution (max approx. {max_res}x{max_res}). '
f'Need: {mem_required/64/gb:0.1f}GB free, Have:{mem_free_total/gb:0.1f}GB free')
+ if mask is not None:
+ if len(mask.shape) == 2:
+ bs = 1
+ else:
+ bs = mask.shape[0]
+ mask = mask.reshape(bs, -1, mask.shape[-2], mask.shape[-1]).expand(b, heads, -1, -1).reshape(-1, mask.shape[-2], mask.shape[-1])
+
# print("steps", steps, mem_required, mem_free_total, modifier, q.element_size(), tensor_size)
first_op_done = False
cleared_cache = False
@@ -258,12 +275,12 @@ def attention_split(q, k, v, heads, mask=None):
model_management.soft_empty_cache(True)
if cleared_cache == False:
cleared_cache = True
- print("out of memory error, emptying cache and trying again")
+ logging.warning("out of memory error, emptying cache and trying again")
continue
steps *= 2
if steps > 64:
raise e
- print("out of memory error, increasing steps and trying again", steps)
+ logging.warning("out of memory error, increasing steps and trying again {}".format(steps))
else:
raise e
@@ -335,17 +352,17 @@ def attention_pytorch(q, k, v, heads, mask=None):
optimized_attention = attention_basic
if model_management.xformers_enabled():
- print("Using xformers cross attention")
+ logging.info("Using xformers cross attention")
optimized_attention = attention_xformers
elif model_management.pytorch_attention_enabled():
- print("Using pytorch cross attention")
+ logging.info("Using pytorch cross attention")
optimized_attention = attention_pytorch
else:
if args.use_split_cross_attention:
- print("Using split optimization for cross attention")
+ logging.info("Using split optimization for cross attention")
optimized_attention = attention_split
else:
- print("Using sub quadratic optimization for cross attention, if you have memory or speed issues try using: --use-split-cross-attention")
+ logging.info("Using sub quadratic optimization for cross attention, if you have memory or speed issues try using: --use-split-cross-attention")
optimized_attention = attention_sub_quad
optimized_attention_masked = optimized_attention
diff --git a/comfy/ldm/modules/diffusionmodules/model.py b/comfy/ldm/modules/diffusionmodules/model.py
index cc81c1f23..fabc5c5e5 100644
--- a/comfy/ldm/modules/diffusionmodules/model.py
+++ b/comfy/ldm/modules/diffusionmodules/model.py
@@ -5,6 +5,7 @@ import torch.nn as nn
import numpy as np
from einops import rearrange
from typing import Optional, Any
+import logging
from comfy import model_management
import comfy.ops
@@ -190,7 +191,7 @@ def slice_attention(q, k, v):
steps *= 2
if steps > 128:
raise e
- print("out of memory error, increasing steps and trying again", steps)
+ logging.warning("out of memory error, increasing steps and trying again {}".format(steps))
return r1
@@ -235,7 +236,7 @@ def pytorch_attention(q, k, v):
out = torch.nn.functional.scaled_dot_product_attention(q, k, v, attn_mask=None, dropout_p=0.0, is_causal=False)
out = out.transpose(2, 3).reshape(B, C, H, W)
except model_management.OOM_EXCEPTION as e:
- print("scaled_dot_product_attention OOMed: switched to slice attention")
+ logging.warning("scaled_dot_product_attention OOMed: switched to slice attention")
out = slice_attention(q.view(B, -1, C), k.view(B, -1, C).transpose(1, 2), v.view(B, -1, C).transpose(1, 2)).reshape(B, C, H, W)
return out
@@ -268,13 +269,13 @@ class AttnBlock(nn.Module):
padding=0)
if model_management.xformers_enabled_vae():
- print("Using xformers attention in VAE")
+ logging.info("Using xformers attention in VAE")
self.optimized_attention = xformers_attention
elif model_management.pytorch_attention_enabled():
- print("Using pytorch attention in VAE")
+ logging.info("Using pytorch attention in VAE")
self.optimized_attention = pytorch_attention
else:
- print("Using split attention in VAE")
+ logging.info("Using split attention in VAE")
self.optimized_attention = normal_attention
def forward(self, x):
@@ -562,7 +563,7 @@ class Decoder(nn.Module):
block_in = ch*ch_mult[self.num_resolutions-1]
curr_res = resolution // 2**(self.num_resolutions-1)
self.z_shape = (1,z_channels,curr_res,curr_res)
- print("Working with z of shape {} = {} dimensions.".format(
+ logging.debug("Working with z of shape {} = {} dimensions.".format(
self.z_shape, np.prod(self.z_shape)))
# z to block_in
diff --git a/comfy/ldm/modules/diffusionmodules/openaimodel.py b/comfy/ldm/modules/diffusionmodules/openaimodel.py
index ea936e066..d782eff31 100644
--- a/comfy/ldm/modules/diffusionmodules/openaimodel.py
+++ b/comfy/ldm/modules/diffusionmodules/openaimodel.py
@@ -1,12 +1,10 @@
from abc import abstractmethod
-import math
-import numpy as np
import torch as th
import torch.nn as nn
import torch.nn.functional as F
from einops import rearrange
-from functools import partial
+import logging
from .util import (
checkpoint,
@@ -362,7 +360,7 @@ def apply_control(h, control, name):
try:
h += ctrl
except:
- print("warning control could not be applied", h.shape, ctrl.shape)
+ logging.warning("warning control could not be applied {} {}".format(h.shape, ctrl.shape))
return h
class UNetModel(nn.Module):
@@ -487,7 +485,6 @@ class UNetModel(nn.Module):
self.predict_codebook_ids = n_embed is not None
self.default_num_video_frames = None
- self.default_image_only_indicator = None
time_embed_dim = model_channels * 4
self.time_embed = nn.Sequential(
@@ -500,7 +497,7 @@ class UNetModel(nn.Module):
if isinstance(self.num_classes, int):
self.label_emb = nn.Embedding(num_classes, time_embed_dim, dtype=self.dtype, device=device)
elif self.num_classes == "continuous":
- print("setting up linear c_adm embedding layer")
+ logging.debug("setting up linear c_adm embedding layer")
self.label_emb = nn.Linear(1, time_embed_dim)
elif self.num_classes == "sequential":
assert adm_in_channels is not None
@@ -711,27 +708,30 @@ class UNetModel(nn.Module):
device=device,
operations=operations
)]
- if transformer_depth_middle >= 0:
- mid_block += [get_attention_layer( # always uses a self-attn
- ch, num_heads, dim_head, depth=transformer_depth_middle, context_dim=context_dim,
- disable_self_attn=disable_middle_self_attn, use_checkpoint=use_checkpoint
- ),
- get_resblock(
- merge_factor=merge_factor,
- merge_strategy=merge_strategy,
- video_kernel_size=video_kernel_size,
- ch=ch,
- time_embed_dim=time_embed_dim,
- dropout=dropout,
- out_channels=None,
- dims=dims,
- use_checkpoint=use_checkpoint,
- use_scale_shift_norm=use_scale_shift_norm,
- dtype=self.dtype,
- device=device,
- operations=operations
- )]
- self.middle_block = TimestepEmbedSequential(*mid_block)
+
+ self.middle_block = None
+ if transformer_depth_middle >= -1:
+ if transformer_depth_middle >= 0:
+ mid_block += [get_attention_layer( # always uses a self-attn
+ ch, num_heads, dim_head, depth=transformer_depth_middle, context_dim=context_dim,
+ disable_self_attn=disable_middle_self_attn, use_checkpoint=use_checkpoint
+ ),
+ get_resblock(
+ merge_factor=merge_factor,
+ merge_strategy=merge_strategy,
+ video_kernel_size=video_kernel_size,
+ ch=ch,
+ time_embed_dim=time_embed_dim,
+ dropout=dropout,
+ out_channels=None,
+ dims=dims,
+ use_checkpoint=use_checkpoint,
+ use_scale_shift_norm=use_scale_shift_norm,
+ dtype=self.dtype,
+ device=device,
+ operations=operations
+ )]
+ self.middle_block = TimestepEmbedSequential(*mid_block)
self._feature_size += ch
self.output_blocks = nn.ModuleList([])
@@ -830,7 +830,7 @@ class UNetModel(nn.Module):
transformer_patches = transformer_options.get("patches", {})
num_video_frames = kwargs.get("num_video_frames", self.default_num_video_frames)
- image_only_indicator = kwargs.get("image_only_indicator", self.default_image_only_indicator)
+ image_only_indicator = kwargs.get("image_only_indicator", None)
time_context = kwargs.get("time_context", None)
assert (y is not None) == (
@@ -861,7 +861,8 @@ class UNetModel(nn.Module):
h = p(h, transformer_options)
transformer_options["block"] = ("middle", 0)
- h = forward_timestep_embed(self.middle_block, h, emb, context, transformer_options, time_context=time_context, num_video_frames=num_video_frames, image_only_indicator=image_only_indicator)
+ if self.middle_block is not None:
+ h = forward_timestep_embed(self.middle_block, h, emb, context, transformer_options, time_context=time_context, num_video_frames=num_video_frames, image_only_indicator=image_only_indicator)
h = apply_control(h, control, 'middle')
diff --git a/comfy/ldm/modules/diffusionmodules/util.py b/comfy/ldm/modules/diffusionmodules/util.py
index ac7e27173..ce14ad5e1 100644
--- a/comfy/ldm/modules/diffusionmodules/util.py
+++ b/comfy/ldm/modules/diffusionmodules/util.py
@@ -46,23 +46,25 @@ class AlphaBlender(nn.Module):
else:
raise ValueError(f"unknown merge strategy {self.merge_strategy}")
- def get_alpha(self, image_only_indicator: torch.Tensor) -> torch.Tensor:
+ def get_alpha(self, image_only_indicator: torch.Tensor, device) -> torch.Tensor:
# skip_time_mix = rearrange(repeat(skip_time_mix, 'b -> (b t) () () ()', t=t), '(b t) 1 ... -> b 1 t ...', t=t)
if self.merge_strategy == "fixed":
# make shape compatible
# alpha = repeat(self.mix_factor, '1 -> b () t () ()', t=t, b=bs)
- alpha = self.mix_factor.to(image_only_indicator.device)
+ alpha = self.mix_factor.to(device)
elif self.merge_strategy == "learned":
- alpha = torch.sigmoid(self.mix_factor.to(image_only_indicator.device))
+ alpha = torch.sigmoid(self.mix_factor.to(device))
# make shape compatible
# alpha = repeat(alpha, '1 -> s () ()', s = t * bs)
elif self.merge_strategy == "learned_with_images":
- assert image_only_indicator is not None, "need image_only_indicator ..."
- alpha = torch.where(
- image_only_indicator.bool(),
- torch.ones(1, 1, device=image_only_indicator.device),
- rearrange(torch.sigmoid(self.mix_factor.to(image_only_indicator.device)), "... -> ... 1"),
- )
+ if image_only_indicator is None:
+ alpha = rearrange(torch.sigmoid(self.mix_factor.to(device)), "... -> ... 1")
+ else:
+ alpha = torch.where(
+ image_only_indicator.bool(),
+ torch.ones(1, 1, device=image_only_indicator.device),
+ rearrange(torch.sigmoid(self.mix_factor.to(image_only_indicator.device)), "... -> ... 1"),
+ )
alpha = rearrange(alpha, self.rearrange_pattern)
# make shape compatible
# alpha = repeat(alpha, '1 -> s () ()', s = t * bs)
@@ -76,7 +78,7 @@ class AlphaBlender(nn.Module):
x_temporal,
image_only_indicator=None,
) -> torch.Tensor:
- alpha = self.get_alpha(image_only_indicator)
+ alpha = self.get_alpha(image_only_indicator, x_spatial.device)
x = (
alpha.to(x_spatial.dtype) * x_spatial
+ (1.0 - alpha).to(x_spatial.dtype) * x_temporal
@@ -98,7 +100,7 @@ def make_beta_schedule(schedule, n_timestep, linear_start=1e-4, linear_end=2e-2,
alphas = torch.cos(alphas).pow(2)
alphas = alphas / alphas[0]
betas = 1 - alphas[1:] / alphas[:-1]
- betas = np.clip(betas, a_min=0, a_max=0.999)
+ betas = torch.clamp(betas, min=0, max=0.999)
elif schedule == "squaredcos_cap_v2": # used for karlo prior
# return early
@@ -113,7 +115,7 @@ def make_beta_schedule(schedule, n_timestep, linear_start=1e-4, linear_end=2e-2,
betas = torch.linspace(linear_start, linear_end, n_timestep, dtype=torch.float64) ** 0.5
else:
raise ValueError(f"schedule '{schedule}' unknown.")
- return betas.numpy()
+ return betas
def make_ddim_timesteps(ddim_discr_method, num_ddim_timesteps, num_ddpm_timesteps, verbose=True):
diff --git a/comfy/ldm/modules/sub_quadratic_attention.py b/comfy/ldm/modules/sub_quadratic_attention.py
index cb0896b0d..1bc4138c3 100644
--- a/comfy/ldm/modules/sub_quadratic_attention.py
+++ b/comfy/ldm/modules/sub_quadratic_attention.py
@@ -14,6 +14,7 @@ import torch
from torch import Tensor
from torch.utils.checkpoint import checkpoint
import math
+import logging
try:
from typing import Optional, NamedTuple, List, Protocol
@@ -170,7 +171,7 @@ def _get_attention_scores_no_kv_chunking(
attn_probs = attn_scores.softmax(dim=-1)
del attn_scores
except model_management.OOM_EXCEPTION:
- print("ran out of memory while running softmax in _get_attention_scores_no_kv_chunking, trying slower in place softmax instead")
+ logging.warning("ran out of memory while running softmax in _get_attention_scores_no_kv_chunking, trying slower in place softmax instead")
attn_scores -= attn_scores.max(dim=-1, keepdim=True).values
torch.exp(attn_scores, out=attn_scores)
summed = torch.sum(attn_scores, dim=-1, keepdim=True)
diff --git a/comfy/lora.py b/comfy/lora.py
index 5e4009b47..096285bba 100644
--- a/comfy/lora.py
+++ b/comfy/lora.py
@@ -1,4 +1,5 @@
import comfy.utils
+import logging
LORA_CLIP_MAP = {
"mlp.fc1": "mlp_fc1",
@@ -20,6 +21,12 @@ def load_lora(lora, to_load):
alpha = lora[alpha_name].item()
loaded_keys.add(alpha_name)
+ dora_scale_name = "{}.dora_scale".format(x)
+ dora_scale = None
+ if dora_scale_name in lora.keys():
+ dora_scale = lora[dora_scale_name]
+ loaded_keys.add(dora_scale_name)
+
regular_lora = "{}.lora_up.weight".format(x)
diffusers_lora = "{}_lora.up.weight".format(x)
transformers_lora = "{}.lora_linear_layer.up.weight".format(x)
@@ -43,7 +50,7 @@ def load_lora(lora, to_load):
if mid_name is not None and mid_name in lora.keys():
mid = lora[mid_name]
loaded_keys.add(mid_name)
- patch_dict[to_load[x]] = ("lora", (lora[A_name], lora[B_name], alpha, mid))
+ patch_dict[to_load[x]] = ("lora", (lora[A_name], lora[B_name], alpha, mid, dora_scale))
loaded_keys.add(A_name)
loaded_keys.add(B_name)
@@ -64,7 +71,7 @@ def load_lora(lora, to_load):
loaded_keys.add(hada_t1_name)
loaded_keys.add(hada_t2_name)
- patch_dict[to_load[x]] = ("loha", (lora[hada_w1_a_name], lora[hada_w1_b_name], alpha, lora[hada_w2_a_name], lora[hada_w2_b_name], hada_t1, hada_t2))
+ patch_dict[to_load[x]] = ("loha", (lora[hada_w1_a_name], lora[hada_w1_b_name], alpha, lora[hada_w2_a_name], lora[hada_w2_b_name], hada_t1, hada_t2, dora_scale))
loaded_keys.add(hada_w1_a_name)
loaded_keys.add(hada_w1_b_name)
loaded_keys.add(hada_w2_a_name)
@@ -116,7 +123,7 @@ def load_lora(lora, to_load):
loaded_keys.add(lokr_t2_name)
if (lokr_w1 is not None) or (lokr_w2 is not None) or (lokr_w1_a is not None) or (lokr_w2_a is not None):
- patch_dict[to_load[x]] = ("lokr", (lokr_w1, lokr_w2, alpha, lokr_w1_a, lokr_w1_b, lokr_w2_a, lokr_w2_b, lokr_t2))
+ patch_dict[to_load[x]] = ("lokr", (lokr_w1, lokr_w2, alpha, lokr_w1_a, lokr_w1_b, lokr_w2_a, lokr_w2_b, lokr_t2, dora_scale))
#glora
a1_name = "{}.a1.weight".format(x)
@@ -124,7 +131,7 @@ def load_lora(lora, to_load):
b1_name = "{}.b1.weight".format(x)
b2_name = "{}.b2.weight".format(x)
if a1_name in lora:
- patch_dict[to_load[x]] = ("glora", (lora[a1_name], lora[a2_name], lora[b1_name], lora[b2_name], alpha))
+ patch_dict[to_load[x]] = ("glora", (lora[a1_name], lora[a2_name], lora[b1_name], lora[b2_name], alpha, dora_scale))
loaded_keys.add(a1_name)
loaded_keys.add(a2_name)
loaded_keys.add(b1_name)
@@ -156,7 +163,7 @@ def load_lora(lora, to_load):
for x in lora.keys():
if x not in loaded_keys:
- print("lora key not loaded", x)
+ logging.warning("lora key not loaded: {}".format(x))
return patch_dict
def model_lora_keys_clip(model, key_map={}):
@@ -197,6 +204,15 @@ def model_lora_keys_clip(model, key_map={}):
key_map[lora_key] = k
lora_key = "text_encoder.text_model.encoder.layers.{}.{}".format(b, c) #diffusers lora
key_map[lora_key] = k
+ lora_key = "lora_prior_te_text_model_encoder_layers_{}_{}".format(b, LORA_CLIP_MAP[c]) #cascade lora: TODO put lora key prefix in the model config
+ key_map[lora_key] = k
+
+
+ k = "clip_g.transformer.text_projection.weight"
+ if k in sdk:
+ key_map["lora_prior_te_text_projection"] = k #cascade lora?
+ # key_map["text_encoder.text_projection"] = k #TODO: check if other lora have the text_projection too
+ # key_map["lora_te_text_projection"] = k
return key_map
@@ -207,6 +223,7 @@ def model_lora_keys_unet(model, key_map={}):
if k.startswith("diffusion_model.") and k.endswith(".weight"):
key_lora = k[len("diffusion_model."):-len(".weight")].replace(".", "_")
key_map["lora_unet_{}".format(key_lora)] = k
+ key_map["lora_prior_unet_{}".format(key_lora)] = k #cascade lora: TODO put lora key prefix in the model config
diffusers_keys = comfy.utils.unet_to_diffusers(model.model_config.unet_config)
for k in diffusers_keys:
diff --git a/comfy/model_base.py b/comfy/model_base.py
index 847687be4..8c89adf5e 100644
--- a/comfy/model_base.py
+++ b/comfy/model_base.py
@@ -1,21 +1,25 @@
import torch
+import logging
from comfy.ldm.modules.diffusionmodules.openaimodel import UNetModel, Timestep
+from comfy.ldm.cascade.stage_c import StageC
+from comfy.ldm.cascade.stage_b import StageB
from comfy.ldm.modules.encoders.noise_aug_modules import CLIPEmbeddingNoiseAugmentation
from comfy.ldm.modules.diffusionmodules.upscaling import ImageConcatWithNoiseAugmentation
import comfy.model_management
import comfy.conds
import comfy.ops
from enum import Enum
-import contextlib
from . import utils
class ModelType(Enum):
EPS = 1
V_PREDICTION = 2
V_PREDICTION_EDM = 3
+ STABLE_CASCADE = 4
+ EDM = 5
-from comfy.model_sampling import EPS, V_PREDICTION, ModelSamplingDiscrete, ModelSamplingContinuousEDM
+from comfy.model_sampling import EPS, V_PREDICTION, EDM, ModelSamplingDiscrete, ModelSamplingContinuousEDM, StableCascadeSampling
def model_sampling(model_config, model_type):
@@ -28,6 +32,12 @@ def model_sampling(model_config, model_type):
elif model_type == ModelType.V_PREDICTION_EDM:
c = V_PREDICTION
s = ModelSamplingContinuousEDM
+ elif model_type == ModelType.STABLE_CASCADE:
+ c = EPS
+ s = StableCascadeSampling
+ elif model_type == ModelType.EDM:
+ c = EDM
+ s = ModelSamplingContinuousEDM
class ModelSampling(s, c):
pass
@@ -36,7 +46,7 @@ def model_sampling(model_config, model_type):
class BaseModel(torch.nn.Module):
- def __init__(self, model_config, model_type=ModelType.EPS, device=None):
+ def __init__(self, model_config, model_type=ModelType.EPS, device=None, unet_model=UNetModel):
super().__init__()
unet_config = model_config.unet_config
@@ -49,16 +59,17 @@ class BaseModel(torch.nn.Module):
operations = comfy.ops.manual_cast
else:
operations = comfy.ops.disable_weight_init
- self.diffusion_model = UNetModel(**unet_config, device=device, operations=operations)
+ self.diffusion_model = unet_model(**unet_config, device=device, operations=operations)
self.model_type = model_type
self.model_sampling = model_sampling(model_config, model_type)
self.adm_channels = unet_config.get("adm_in_channels", None)
if self.adm_channels is None:
self.adm_channels = 0
- self.inpaint_model = False
- print("model_type", model_type.name)
- print("adm", self.adm_channels)
+
+ self.concat_keys = ()
+ logging.info("model_type {}".format(model_type.name))
+ logging.debug("adm {}".format(self.adm_channels))
def apply_model(self, x, t, c_concat=None, c_crossattn=None, control=None, transformer_options={}, **kwargs):
sigma = t
@@ -97,8 +108,7 @@ class BaseModel(torch.nn.Module):
def extra_conds(self, **kwargs):
out = {}
- if self.inpaint_model:
- concat_keys = ("mask", "masked_image")
+ if len(self.concat_keys) > 0:
cond_concat = []
denoise_mask = kwargs.get("concat_mask", kwargs.get("denoise_mask", None))
concat_latent_image = kwargs.get("concat_latent_image", None)
@@ -115,24 +125,16 @@ class BaseModel(torch.nn.Module):
concat_latent_image = utils.resize_to_batch_size(concat_latent_image, noise.shape[0])
- if len(denoise_mask.shape) == len(noise.shape):
- denoise_mask = denoise_mask[:,:1]
+ if denoise_mask is not None:
+ if len(denoise_mask.shape) == len(noise.shape):
+ denoise_mask = denoise_mask[:,:1]
- denoise_mask = denoise_mask.reshape((-1, 1, denoise_mask.shape[-2], denoise_mask.shape[-1]))
- if denoise_mask.shape[-2:] != noise.shape[-2:]:
- denoise_mask = utils.common_upscale(denoise_mask, noise.shape[-1], noise.shape[-2], "bilinear", "center")
- denoise_mask = utils.resize_to_batch_size(denoise_mask.round(), noise.shape[0])
+ denoise_mask = denoise_mask.reshape((-1, 1, denoise_mask.shape[-2], denoise_mask.shape[-1]))
+ if denoise_mask.shape[-2:] != noise.shape[-2:]:
+ denoise_mask = utils.common_upscale(denoise_mask, noise.shape[-1], noise.shape[-2], "bilinear", "center")
+ denoise_mask = utils.resize_to_batch_size(denoise_mask.round(), noise.shape[0])
- def blank_inpaint_image_like(latent_image):
- blank_image = torch.ones_like(latent_image)
- # these are the values for "zero" in pixel space translated to latent space
- blank_image[:,0] *= 0.8223
- blank_image[:,1] *= -0.6876
- blank_image[:,2] *= 0.6364
- blank_image[:,3] *= 0.1380
- return blank_image
-
- for ck in concat_keys:
+ for ck in self.concat_keys:
if denoise_mask is not None:
if ck == "mask":
cond_concat.append(denoise_mask.to(device))
@@ -142,7 +144,7 @@ class BaseModel(torch.nn.Module):
if ck == "mask":
cond_concat.append(torch.ones_like(noise)[:,:1])
elif ck == "masked_image":
- cond_concat.append(blank_inpaint_image_like(noise))
+ cond_concat.append(self.blank_inpaint_image_like(noise))
data = torch.cat(cond_concat, dim=1)
out['c_concat'] = comfy.conds.CONDNoiseShape(data)
@@ -154,6 +156,14 @@ class BaseModel(torch.nn.Module):
if cross_attn is not None:
out['c_crossattn'] = comfy.conds.CONDCrossAttn(cross_attn)
+ cross_attn_cnet = kwargs.get("cross_attn_controlnet", None)
+ if cross_attn_cnet is not None:
+ out['crossattn_controlnet'] = comfy.conds.CONDCrossAttn(cross_attn_cnet)
+
+ c_concat = kwargs.get("noise_concat", None)
+ if c_concat is not None:
+ out['c_concat'] = comfy.conds.CONDNoiseShape(data)
+
return out
def load_model_weights(self, sd, unet_prefix=""):
@@ -166,10 +176,10 @@ class BaseModel(torch.nn.Module):
to_load = self.model_config.process_unet_state_dict(to_load)
m, u = self.diffusion_model.load_state_dict(to_load, strict=False)
if len(m) > 0:
- print("unet missing:", m)
+ logging.warning("unet missing: {}".format(m))
if len(u) > 0:
- print("unet unexpected:", u)
+ logging.warning("unet unexpected: {}".format(u))
del to_load
return self
@@ -203,7 +213,16 @@ class BaseModel(torch.nn.Module):
return unet_state_dict
def set_inpaint(self):
- self.inpaint_model = True
+ self.concat_keys = ("mask", "masked_image")
+ def blank_inpaint_image_like(latent_image):
+ blank_image = torch.ones_like(latent_image)
+ # these are the values for "zero" in pixel space translated to latent space
+ blank_image[:,0] *= 0.8223
+ blank_image[:,1] *= -0.6876
+ blank_image[:,2] *= 0.6364
+ blank_image[:,3] *= 0.1380
+ return blank_image
+ self.blank_inpaint_image_like = blank_inpaint_image_like
def memory_required(self, input_shape):
if comfy.model_management.xformers_enabled() or comfy.model_management.pytorch_attention_flash_attention():
@@ -359,10 +378,39 @@ class SVD_img2vid(BaseModel):
if "time_conditioning" in kwargs:
out["time_context"] = comfy.conds.CONDCrossAttn(kwargs["time_conditioning"])
- out['image_only_indicator'] = comfy.conds.CONDConstant(torch.zeros((1,), device=device))
out['num_video_frames'] = comfy.conds.CONDConstant(noise.shape[0])
return out
+class SV3D_u(SVD_img2vid):
+ def encode_adm(self, **kwargs):
+ augmentation = kwargs.get("augmentation_level", 0)
+
+ out = []
+ out.append(self.embedder(torch.flatten(torch.Tensor([augmentation]))))
+
+ flat = torch.flatten(torch.cat(out)).unsqueeze(dim=0)
+ return flat
+
+class SV3D_p(SVD_img2vid):
+ def __init__(self, model_config, model_type=ModelType.V_PREDICTION_EDM, device=None):
+ super().__init__(model_config, model_type, device=device)
+ self.embedder_512 = Timestep(512)
+
+ def encode_adm(self, **kwargs):
+ augmentation = kwargs.get("augmentation_level", 0)
+ elevation = kwargs.get("elevation", 0) #elevation and azimuth are in degrees here
+ azimuth = kwargs.get("azimuth", 0)
+ noise = kwargs.get("noise", None)
+
+ out = []
+ out.append(self.embedder(torch.flatten(torch.Tensor([augmentation]))))
+ out.append(self.embedder_512(torch.deg2rad(torch.fmod(torch.flatten(90 - torch.Tensor([elevation])), 360.0))))
+ out.append(self.embedder_512(torch.deg2rad(torch.fmod(torch.flatten(torch.Tensor([azimuth])), 360.0))))
+
+ out = list(map(lambda a: utils.resize_to_batch_size(a, noise.shape[0]), out))
+ return torch.cat(out, dim=1)
+
+
class Stable_Zero123(BaseModel):
def __init__(self, model_config, model_type=ModelType.EPS, device=None, cc_projection_weight=None, cc_projection_bias=None):
super().__init__(model_config, model_type, device=device)
@@ -424,3 +472,88 @@ class SD_X4Upscaler(BaseModel):
out['c_concat'] = comfy.conds.CONDNoiseShape(image)
out['y'] = comfy.conds.CONDRegular(noise_level)
return out
+
+class IP2P:
+ def extra_conds(self, **kwargs):
+ out = {}
+
+ image = kwargs.get("concat_latent_image", None)
+ noise = kwargs.get("noise", None)
+ device = kwargs["device"]
+
+ if image is None:
+ image = torch.zeros_like(noise)
+
+ if image.shape[1:] != noise.shape[1:]:
+ image = utils.common_upscale(image.to(device), noise.shape[-1], noise.shape[-2], "bilinear", "center")
+
+ image = utils.resize_to_batch_size(image, noise.shape[0])
+
+ out['c_concat'] = comfy.conds.CONDNoiseShape(self.process_ip2p_image_in(image))
+ adm = self.encode_adm(**kwargs)
+ if adm is not None:
+ out['y'] = comfy.conds.CONDRegular(adm)
+ return out
+
+class SD15_instructpix2pix(IP2P, BaseModel):
+ def __init__(self, model_config, model_type=ModelType.EPS, device=None):
+ super().__init__(model_config, model_type, device=device)
+ self.process_ip2p_image_in = lambda image: image
+
+class SDXL_instructpix2pix(IP2P, SDXL):
+ def __init__(self, model_config, model_type=ModelType.EPS, device=None):
+ super().__init__(model_config, model_type, device=device)
+ if model_type == ModelType.V_PREDICTION_EDM:
+ self.process_ip2p_image_in = lambda image: comfy.latent_formats.SDXL().process_in(image) #cosxl ip2p
+ else:
+ self.process_ip2p_image_in = lambda image: image #diffusers ip2p
+
+
+class StableCascade_C(BaseModel):
+ def __init__(self, model_config, model_type=ModelType.STABLE_CASCADE, device=None):
+ super().__init__(model_config, model_type, device=device, unet_model=StageC)
+ self.diffusion_model.eval().requires_grad_(False)
+
+ def extra_conds(self, **kwargs):
+ out = {}
+ clip_text_pooled = kwargs["pooled_output"]
+ if clip_text_pooled is not None:
+ out['clip_text_pooled'] = comfy.conds.CONDRegular(clip_text_pooled)
+
+ if "unclip_conditioning" in kwargs:
+ embeds = []
+ for unclip_cond in kwargs["unclip_conditioning"]:
+ weight = unclip_cond["strength"]
+ embeds.append(unclip_cond["clip_vision_output"].image_embeds.unsqueeze(0) * weight)
+ clip_img = torch.cat(embeds, dim=1)
+ else:
+ clip_img = torch.zeros((1, 1, 768))
+ out["clip_img"] = comfy.conds.CONDRegular(clip_img)
+ out["sca"] = comfy.conds.CONDRegular(torch.zeros((1,)))
+ out["crp"] = comfy.conds.CONDRegular(torch.zeros((1,)))
+
+ cross_attn = kwargs.get("cross_attn", None)
+ if cross_attn is not None:
+ out['clip_text'] = comfy.conds.CONDCrossAttn(cross_attn)
+ return out
+
+
+class StableCascade_B(BaseModel):
+ def __init__(self, model_config, model_type=ModelType.STABLE_CASCADE, device=None):
+ super().__init__(model_config, model_type, device=device, unet_model=StageB)
+ self.diffusion_model.eval().requires_grad_(False)
+
+ def extra_conds(self, **kwargs):
+ out = {}
+ noise = kwargs.get("noise", None)
+
+ clip_text_pooled = kwargs["pooled_output"]
+ if clip_text_pooled is not None:
+ out['clip'] = comfy.conds.CONDRegular(clip_text_pooled)
+
+ #size of prior doesn't really matter if zeros because it gets resized but I still want it to get batched
+ prior = kwargs.get("stable_cascade_prior", torch.zeros((1, 16, (noise.shape[2] * 4) // 42, (noise.shape[3] * 4) // 42), dtype=noise.dtype, layout=noise.layout, device=noise.device))
+
+ out["effnet"] = comfy.conds.CONDRegular(prior)
+ out["sca"] = comfy.conds.CONDRegular(torch.zeros((1,)))
+ return out
diff --git a/comfy/model_detection.py b/comfy/model_detection.py
index ea824c44c..795af828a 100644
--- a/comfy/model_detection.py
+++ b/comfy/model_detection.py
@@ -1,5 +1,6 @@
import comfy.supported_models
import comfy.supported_models_base
+import logging
def count_blocks(state_dict_keys, prefix_string):
count = 0
@@ -28,9 +29,38 @@ def calculate_transformer_depth(prefix, state_dict_keys, state_dict):
return last_transformer_depth, context_dim, use_linear_in_transformer, time_stack
return None
-def detect_unet_config(state_dict, key_prefix, dtype):
+def detect_unet_config(state_dict, key_prefix):
state_dict_keys = list(state_dict.keys())
+ if '{}clf.1.weight'.format(key_prefix) in state_dict_keys: #stable cascade
+ unet_config = {}
+ text_mapper_name = '{}clip_txt_mapper.weight'.format(key_prefix)
+ if text_mapper_name in state_dict_keys:
+ unet_config['stable_cascade_stage'] = 'c'
+ w = state_dict[text_mapper_name]
+ if w.shape[0] == 1536: #stage c lite
+ unet_config['c_cond'] = 1536
+ unet_config['c_hidden'] = [1536, 1536]
+ unet_config['nhead'] = [24, 24]
+ unet_config['blocks'] = [[4, 12], [12, 4]]
+ elif w.shape[0] == 2048: #stage c full
+ unet_config['c_cond'] = 2048
+ elif '{}clip_mapper.weight'.format(key_prefix) in state_dict_keys:
+ unet_config['stable_cascade_stage'] = 'b'
+ w = state_dict['{}down_blocks.1.0.channelwise.0.weight'.format(key_prefix)]
+ if w.shape[-1] == 640:
+ unet_config['c_hidden'] = [320, 640, 1280, 1280]
+ unet_config['nhead'] = [-1, -1, 20, 20]
+ unet_config['blocks'] = [[2, 6, 28, 6], [6, 28, 6, 2]]
+ unet_config['block_repeat'] = [[1, 1, 1, 1], [3, 3, 2, 2]]
+ elif w.shape[-1] == 576: #stage b lite
+ unet_config['c_hidden'] = [320, 576, 1152, 1152]
+ unet_config['nhead'] = [-1, 9, 18, 18]
+ unet_config['blocks'] = [[2, 4, 14, 4], [4, 14, 4, 2]]
+ unet_config['block_repeat'] = [[1, 1, 1, 1], [2, 2, 2, 2]]
+
+ return unet_config
+
unet_config = {
"use_checkpoint": False,
"image_size": 32,
@@ -45,7 +75,6 @@ def detect_unet_config(state_dict, key_prefix, dtype):
else:
unet_config["adm_in_channels"] = None
- unet_config["dtype"] = dtype
model_channels = state_dict['{}input_blocks.0.0.weight'.format(key_prefix)].shape[0]
in_channels = state_dict['{}input_blocks.0.0.weight'.format(key_prefix)].shape[1]
@@ -123,8 +152,10 @@ def detect_unet_config(state_dict, key_prefix, dtype):
channel_mult.append(last_channel_mult)
if "{}middle_block.1.proj_in.weight".format(key_prefix) in state_dict_keys:
transformer_depth_middle = count_blocks(state_dict_keys, '{}middle_block.1.transformer_blocks.'.format(key_prefix) + '{}')
- else:
+ elif "{}middle_block.0.in_layers.0.weight".format(key_prefix) in state_dict_keys:
transformer_depth_middle = -1
+ else:
+ transformer_depth_middle = -2
unet_config["in_channels"] = in_channels
unet_config["out_channels"] = out_channels
@@ -151,17 +182,17 @@ def detect_unet_config(state_dict, key_prefix, dtype):
return unet_config
-def model_config_from_unet_config(unet_config):
+def model_config_from_unet_config(unet_config, state_dict=None):
for model_config in comfy.supported_models.models:
- if model_config.matches(unet_config):
+ if model_config.matches(unet_config, state_dict):
return model_config(unet_config)
- print("no match", unet_config)
+ logging.error("no match {}".format(unet_config))
return None
-def model_config_from_unet(state_dict, unet_key_prefix, dtype, use_base_if_no_match=False):
- unet_config = detect_unet_config(state_dict, unet_key_prefix, dtype)
- model_config = model_config_from_unet_config(unet_config)
+def model_config_from_unet(state_dict, unet_key_prefix, use_base_if_no_match=False):
+ unet_config = detect_unet_config(state_dict, unet_key_prefix)
+ model_config = model_config_from_unet_config(unet_config, state_dict)
if model_config is None and use_base_if_no_match:
return comfy.supported_models_base.BASE(unet_config)
else:
@@ -206,7 +237,7 @@ def convert_config(unet_config):
return new_config
-def unet_config_from_diffusers_unet(state_dict, dtype):
+def unet_config_from_diffusers_unet(state_dict, dtype=None):
match = {}
transformer_depth = []
@@ -214,6 +245,7 @@ def unet_config_from_diffusers_unet(state_dict, dtype):
down_blocks = count_blocks(state_dict, "down_blocks.{}")
for i in range(down_blocks):
attn_blocks = count_blocks(state_dict, "down_blocks.{}.attentions.".format(i) + '{}')
+ res_blocks = count_blocks(state_dict, "down_blocks.{}.resnets.".format(i) + '{}')
for ab in range(attn_blocks):
transformer_count = count_blocks(state_dict, "down_blocks.{}.attentions.{}.transformer_blocks.".format(i, ab) + '{}')
transformer_depth.append(transformer_count)
@@ -222,8 +254,8 @@ def unet_config_from_diffusers_unet(state_dict, dtype):
attn_res *= 2
if attn_blocks == 0:
- transformer_depth.append(0)
- transformer_depth.append(0)
+ for i in range(res_blocks):
+ transformer_depth.append(0)
match["transformer_depth"] = transformer_depth
@@ -289,6 +321,12 @@ def unet_config_from_diffusers_unet(state_dict, dtype):
'use_linear_in_transformer': True, 'context_dim': 2048, 'num_head_channels': 64, 'transformer_depth_output': [0, 0, 0, 2, 2, 2, 10, 10, 10],
'use_temporal_attention': False, 'use_temporal_resblock': False}
+ SDXL_diffusers_ip2p = {'use_checkpoint': False, 'image_size': 32, 'out_channels': 4, 'use_spatial_transformer': True, 'legacy': False,
+ 'num_classes': 'sequential', 'adm_in_channels': 2816, 'dtype': dtype, 'in_channels': 8, 'model_channels': 320,
+ 'num_res_blocks': [2, 2, 2], 'transformer_depth': [0, 0, 2, 2, 10, 10], 'channel_mult': [1, 2, 4], 'transformer_depth_middle': 10,
+ 'use_linear_in_transformer': True, 'context_dim': 2048, 'num_head_channels': 64, 'transformer_depth_output': [0, 0, 0, 2, 2, 2, 10, 10, 10],
+ 'use_temporal_attention': False, 'use_temporal_resblock': False}
+
SSD_1B = {'use_checkpoint': False, 'image_size': 32, 'out_channels': 4, 'use_spatial_transformer': True, 'legacy': False,
'num_classes': 'sequential', 'adm_in_channels': 2816, 'dtype': dtype, 'in_channels': 4, 'model_channels': 320,
'num_res_blocks': [2, 2, 2], 'transformer_depth': [0, 0, 2, 2, 4, 4], 'transformer_depth_output': [0, 0, 0, 1, 1, 2, 10, 4, 4],
@@ -301,7 +339,25 @@ def unet_config_from_diffusers_unet(state_dict, dtype):
'channel_mult': [1, 2, 4], 'transformer_depth_middle': -1, 'use_linear_in_transformer': True, 'context_dim': 2048, 'num_head_channels': 64,
'use_temporal_attention': False, 'use_temporal_resblock': False}
- supported_models = [SDXL, SDXL_refiner, SD21, SD15, SD21_uncliph, SD21_unclipl, SDXL_mid_cnet, SDXL_small_cnet, SDXL_diffusers_inpaint, SSD_1B, Segmind_Vega]
+ KOALA_700M = {'use_checkpoint': False, 'image_size': 32, 'out_channels': 4, 'use_spatial_transformer': True, 'legacy': False,
+ 'num_classes': 'sequential', 'adm_in_channels': 2816, 'dtype': dtype, 'in_channels': 4, 'model_channels': 320,
+ 'num_res_blocks': [1, 1, 1], 'transformer_depth': [0, 2, 5], 'transformer_depth_output': [0, 0, 2, 2, 5, 5],
+ 'channel_mult': [1, 2, 4], 'transformer_depth_middle': -2, 'use_linear_in_transformer': True, 'context_dim': 2048, 'num_head_channels': 64,
+ 'use_temporal_attention': False, 'use_temporal_resblock': False}
+
+ KOALA_1B = {'use_checkpoint': False, 'image_size': 32, 'out_channels': 4, 'use_spatial_transformer': True, 'legacy': False,
+ 'num_classes': 'sequential', 'adm_in_channels': 2816, 'dtype': dtype, 'in_channels': 4, 'model_channels': 320,
+ 'num_res_blocks': [1, 1, 1], 'transformer_depth': [0, 2, 6], 'transformer_depth_output': [0, 0, 2, 2, 6, 6],
+ 'channel_mult': [1, 2, 4], 'transformer_depth_middle': 6, 'use_linear_in_transformer': True, 'context_dim': 2048, 'num_head_channels': 64,
+ 'use_temporal_attention': False, 'use_temporal_resblock': False}
+
+ SD09_XS = {'use_checkpoint': False, 'image_size': 32, 'out_channels': 4, 'use_spatial_transformer': True, 'legacy': False,
+ 'adm_in_channels': None, 'dtype': dtype, 'in_channels': 4, 'model_channels': 320, 'num_res_blocks': [1, 1, 1],
+ 'transformer_depth': [1, 1, 1], 'channel_mult': [1, 2, 4], 'transformer_depth_middle': -2, 'use_linear_in_transformer': True,
+ 'context_dim': 1024, 'num_head_channels': 64, 'transformer_depth_output': [1, 1, 1, 1, 1, 1],
+ 'use_temporal_attention': False, 'use_temporal_resblock': False, 'disable_self_attentions': [True, False, False]}
+
+ supported_models = [SDXL, SDXL_refiner, SD21, SD15, SD21_uncliph, SD21_unclipl, SDXL_mid_cnet, SDXL_small_cnet, SDXL_diffusers_inpaint, SSD_1B, Segmind_Vega, KOALA_700M, KOALA_1B, SD09_XS, SDXL_diffusers_ip2p]
for unet_config in supported_models:
matches = True
@@ -313,8 +369,8 @@ def unet_config_from_diffusers_unet(state_dict, dtype):
return convert_config(unet_config)
return None
-def model_config_from_diffusers_unet(state_dict, dtype):
- unet_config = unet_config_from_diffusers_unet(state_dict, dtype)
+def model_config_from_diffusers_unet(state_dict):
+ unet_config = unet_config_from_diffusers_unet(state_dict)
if unet_config is not None:
return model_config_from_unet_config(unet_config)
return None
diff --git a/comfy/model_management.py b/comfy/model_management.py
index e12146d11..310ec2537 100644
--- a/comfy/model_management.py
+++ b/comfy/model_management.py
@@ -1,4 +1,5 @@
import psutil
+import logging
from enum import Enum
from comfy.cli_args import args
import comfy.utils
@@ -29,7 +30,7 @@ lowvram_available = True
xpu_available = False
if args.deterministic:
- print("Using deterministic algorithms for pytorch")
+ logging.info("Using deterministic algorithms for pytorch")
torch.use_deterministic_algorithms(True, warn_only=True)
directml_enabled = False
@@ -41,7 +42,7 @@ if args.directml is not None:
directml_device = torch_directml.device()
else:
directml_device = torch_directml.device(device_index)
- print("Using directml with device:", torch_directml.device_name(device_index))
+ logging.info("Using directml with device: {}".format(torch_directml.device_name(device_index)))
# torch_directml.disable_tiled_resources(True)
lowvram_available = False #TODO: need to find a way to get free memory in directml before this can be enabled by default.
@@ -117,10 +118,10 @@ def get_total_memory(dev=None, torch_total_too=False):
total_vram = get_total_memory(get_torch_device()) / (1024 * 1024)
total_ram = psutil.virtual_memory().total / (1024 * 1024)
-print("Total VRAM {:0.0f} MB, total RAM {:0.0f} MB".format(total_vram, total_ram))
+logging.info("Total VRAM {:0.0f} MB, total RAM {:0.0f} MB".format(total_vram, total_ram))
if not args.normalvram and not args.cpu:
if lowvram_available and total_vram <= 4096:
- print("Trying to enable lowvram mode because your GPU seems to have 4GB or less. If you don't want this use: --normalvram")
+ logging.warning("Trying to enable lowvram mode because your GPU seems to have 4GB or less. If you don't want this use: --normalvram")
set_vram_to = VRAMState.LOW_VRAM
try:
@@ -143,12 +144,10 @@ else:
pass
try:
XFORMERS_VERSION = xformers.version.__version__
- print("xformers version:", XFORMERS_VERSION)
+ logging.info("xformers version: {}".format(XFORMERS_VERSION))
if XFORMERS_VERSION.startswith("0.0.18"):
- print()
- print("WARNING: This version of xformers has a major bug where you will get black images when generating high resolution images.")
- print("Please downgrade or upgrade xformers to a different version.")
- print()
+ logging.warning("\nWARNING: This version of xformers has a major bug where you will get black images when generating high resolution images.")
+ logging.warning("Please downgrade or upgrade xformers to a different version.\n")
XFORMERS_ENABLED_VAE = False
except:
pass
@@ -213,11 +212,11 @@ elif args.highvram or args.gpu_only:
FORCE_FP32 = False
FORCE_FP16 = False
if args.force_fp32:
- print("Forcing FP32, if this improves things please report it.")
+ logging.info("Forcing FP32, if this improves things please report it.")
FORCE_FP32 = True
if args.force_fp16:
- print("Forcing FP16.")
+ logging.info("Forcing FP16.")
FORCE_FP16 = True
if lowvram_available:
@@ -231,12 +230,12 @@ if cpu_state != CPUState.GPU:
if cpu_state == CPUState.MPS:
vram_state = VRAMState.SHARED
-print(f"Set vram state to: {vram_state.name}")
+logging.info(f"Set vram state to: {vram_state.name}")
DISABLE_SMART_MEMORY = args.disable_smart_memory
if DISABLE_SMART_MEMORY:
- print("Disabling smart memory management")
+ logging.info("Disabling smart memory management")
def get_torch_device_name(device):
if hasattr(device, 'type'):
@@ -254,11 +253,11 @@ def get_torch_device_name(device):
return "CUDA {}: {}".format(device, torch.cuda.get_device_name(device))
try:
- print("Device:", get_torch_device_name(get_torch_device()))
+ logging.info("Device: {}".format(get_torch_device_name(get_torch_device())))
except:
- print("Could not pick default device.")
+ logging.warning("Could not pick default device.")
-print("VAE dtype:", VAE_DTYPE)
+logging.info("VAE dtype: {}".format(VAE_DTYPE))
current_loaded_models = []
@@ -273,8 +272,9 @@ def module_size(module):
class LoadedModel:
def __init__(self, model):
self.model = model
- self.model_accelerated = False
self.device = model.load_device
+ self.weights_loaded = False
+ self.real_model = None
def model_memory(self):
return self.model.model_size()
@@ -286,54 +286,34 @@ class LoadedModel:
return self.model_memory()
def model_load(self, lowvram_model_memory=0):
- patch_model_to = None
- if lowvram_model_memory == 0:
- patch_model_to = self.device
+ patch_model_to = self.device
self.model.model_patches_to(self.device)
self.model.model_patches_to(self.model.model_dtype())
+ load_weights = not self.weights_loaded
+
try:
- self.real_model = self.model.patch_model(device_to=patch_model_to) #TODO: do something with loras and offloading to CPU
+ if lowvram_model_memory > 0 and load_weights:
+ self.real_model = self.model.patch_model_lowvram(device_to=patch_model_to, lowvram_model_memory=lowvram_model_memory)
+ else:
+ self.real_model = self.model.patch_model(device_to=patch_model_to, patch_weights=load_weights)
except Exception as e:
self.model.unpatch_model(self.model.offload_device)
self.model_unload()
raise e
- if lowvram_model_memory > 0:
- print("loading in lowvram mode", lowvram_model_memory/(1024 * 1024))
- mem_counter = 0
- for m in self.real_model.modules():
- if hasattr(m, "comfy_cast_weights"):
- m.prev_comfy_cast_weights = m.comfy_cast_weights
- m.comfy_cast_weights = True
- module_mem = module_size(m)
- if mem_counter + module_mem < lowvram_model_memory:
- m.to(self.device)
- mem_counter += module_mem
- elif hasattr(m, "weight"): #only modules with comfy_cast_weights can be set to lowvram mode
- m.to(self.device)
- mem_counter += module_size(m)
- print("lowvram: loaded module regularly", m)
-
- self.model_accelerated = True
-
if is_intel_xpu() and not args.disable_ipex_optimize:
self.real_model = torch.xpu.optimize(self.real_model.eval(), inplace=True, auto_kernel_selection=True, graph_mode=True)
+ self.weights_loaded = True
return self.real_model
- def model_unload(self):
- if self.model_accelerated:
- for m in self.real_model.modules():
- if hasattr(m, "prev_comfy_cast_weights"):
- m.comfy_cast_weights = m.prev_comfy_cast_weights
- del m.prev_comfy_cast_weights
-
- self.model_accelerated = False
-
- self.model.unpatch_model(self.model.offload_device)
+ def model_unload(self, unpatch_weights=True):
+ self.model.unpatch_model(self.model.offload_device, unpatch_weights=unpatch_weights)
self.model.model_patches_to(self.model.offload_device)
+ self.weights_loaded = self.weights_loaded and not unpatch_weights
+ self.real_model = None
def __eq__(self, other):
return self.model is other.model
@@ -341,31 +321,57 @@ class LoadedModel:
def minimum_inference_memory():
return (1024 * 1024 * 1024)
-def unload_model_clones(model):
+def unload_model_clones(model, unload_weights_only=True, force_unload=True):
to_unload = []
for i in range(len(current_loaded_models)):
if model.is_clone(current_loaded_models[i].model):
to_unload = [i] + to_unload
+ if len(to_unload) == 0:
+ return True
+
+ same_weights = 0
for i in to_unload:
- print("unload clone", i)
- current_loaded_models.pop(i).model_unload()
+ if model.clone_has_same_weights(current_loaded_models[i].model):
+ same_weights += 1
+
+ if same_weights == len(to_unload):
+ unload_weight = False
+ else:
+ unload_weight = True
+
+ if not force_unload:
+ if unload_weights_only and unload_weight == False:
+ return None
+
+ for i in to_unload:
+ logging.debug("unload clone {} {}".format(i, unload_weight))
+ current_loaded_models.pop(i).model_unload(unpatch_weights=unload_weight)
+
+ return unload_weight
def free_memory(memory_required, device, keep_loaded=[]):
- unloaded_model = False
+ unloaded_model = []
+ can_unload = []
+
for i in range(len(current_loaded_models) -1, -1, -1):
- if not DISABLE_SMART_MEMORY:
- if get_free_memory(device) > memory_required:
- break
shift_model = current_loaded_models[i]
if shift_model.device == device:
if shift_model not in keep_loaded:
- m = current_loaded_models.pop(i)
- m.model_unload()
- del m
- unloaded_model = True
+ can_unload.append((sys.getrefcount(shift_model.model), shift_model.model_memory(), i))
- if unloaded_model:
+ for x in sorted(can_unload):
+ i = x[-1]
+ if not DISABLE_SMART_MEMORY:
+ if get_free_memory(device) > memory_required:
+ break
+ current_loaded_models[i].model_unload()
+ unloaded_model.append(i)
+
+ for i in sorted(unloaded_model, reverse=True):
+ current_loaded_models.pop(i)
+
+ if len(unloaded_model) > 0:
soft_empty_cache()
else:
if vram_state != VRAMState.HIGH_VRAM:
@@ -379,6 +385,8 @@ def load_models_gpu(models, memory_required=0):
inference_memory = minimum_inference_memory()
extra_mem = max(inference_memory, memory_required)
+ models = set(models)
+
models_to_load = []
models_already_loaded = []
for x in models:
@@ -390,7 +398,7 @@ def load_models_gpu(models, memory_required=0):
models_already_loaded.append(loaded_model)
else:
if hasattr(x, "model"):
- print(f"Requested to load {x.model.__class__.__name__}")
+ logging.info(f"Requested to load {x.model.__class__.__name__}")
models_to_load.append(loaded_model)
if len(models_to_load) == 0:
@@ -400,17 +408,22 @@ def load_models_gpu(models, memory_required=0):
free_memory(extra_mem, d, models_already_loaded)
return
- print(f"Loading {len(models_to_load)} new model{'s' if len(models_to_load) > 1 else ''}")
+ logging.info(f"Loading {len(models_to_load)} new model{'s' if len(models_to_load) > 1 else ''}")
total_memory_required = {}
for loaded_model in models_to_load:
- unload_model_clones(loaded_model.model)
- total_memory_required[loaded_model.device] = total_memory_required.get(loaded_model.device, 0) + loaded_model.model_memory_required(loaded_model.device)
+ if unload_model_clones(loaded_model.model, unload_weights_only=True, force_unload=False) == True:#unload clones where the weights are different
+ total_memory_required[loaded_model.device] = total_memory_required.get(loaded_model.device, 0) + loaded_model.model_memory_required(loaded_model.device)
for device in total_memory_required:
if device != torch.device("cpu"):
free_memory(total_memory_required[device] * 1.3 + extra_mem, device, models_already_loaded)
+ for loaded_model in models_to_load:
+ weights_unloaded = unload_model_clones(loaded_model.model, unload_weights_only=False, force_unload=False) #unload the rest of the clones where the weights can stay loaded
+ if weights_unloaded is not None:
+ loaded_model.weights_loaded = not weights_unloaded
+
for loaded_model in models_to_load:
model = loaded_model.model
torch_dev = model.load_device
@@ -439,11 +452,15 @@ def load_models_gpu(models, memory_required=0):
def load_model_gpu(model):
return load_models_gpu([model])
-def cleanup_models():
+def cleanup_models(keep_clone_weights_loaded=False):
to_delete = []
for i in range(len(current_loaded_models)):
if sys.getrefcount(current_loaded_models[i].model) <= 2:
- to_delete = [i] + to_delete
+ if not keep_clone_weights_loaded:
+ to_delete = [i] + to_delete
+ #TODO: find a less fragile way to do this.
+ elif sys.getrefcount(current_loaded_models[i].real_model) <= 3: #references from .real_model + the .model
+ to_delete = [i] + to_delete
for i in to_delete:
x = current_loaded_models.pop(i)
@@ -487,7 +504,7 @@ def unet_inital_load_device(parameters, dtype):
else:
return cpu_dev
-def unet_dtype(device=None, model_params=0):
+def unet_dtype(device=None, model_params=0, supported_dtypes=[torch.float16, torch.bfloat16, torch.float32]):
if args.bf16_unet:
return torch.bfloat16
if args.fp16_unet:
@@ -496,21 +513,32 @@ def unet_dtype(device=None, model_params=0):
return torch.float8_e4m3fn
if args.fp8_e5m2_unet:
return torch.float8_e5m2
- if should_use_fp16(device=device, model_params=model_params):
- return torch.float16
+ if should_use_fp16(device=device, model_params=model_params, manual_cast=True):
+ if torch.float16 in supported_dtypes:
+ return torch.float16
+ if should_use_bf16(device, model_params=model_params, manual_cast=True):
+ if torch.bfloat16 in supported_dtypes:
+ return torch.bfloat16
return torch.float32
# None means no manual cast
-def unet_manual_cast(weight_dtype, inference_device):
+def unet_manual_cast(weight_dtype, inference_device, supported_dtypes=[torch.float16, torch.bfloat16, torch.float32]):
if weight_dtype == torch.float32:
return None
- fp16_supported = comfy.model_management.should_use_fp16(inference_device, prioritize_performance=False)
+ fp16_supported = should_use_fp16(inference_device, prioritize_performance=False)
if fp16_supported and weight_dtype == torch.float16:
return None
- if fp16_supported:
+ bf16_supported = should_use_bf16(inference_device)
+ if bf16_supported and weight_dtype == torch.bfloat16:
+ return None
+
+ if fp16_supported and torch.float16 in supported_dtypes:
return torch.float16
+
+ elif bf16_supported and torch.bfloat16 in supported_dtypes:
+ return torch.bfloat16
else:
return torch.float32
@@ -546,10 +574,8 @@ def text_encoder_dtype(device=None):
if is_device_cpu(device):
return torch.float16
- if should_use_fp16(device, prioritize_performance=False):
- return torch.float16
- else:
- return torch.float32
+ return torch.float16
+
def intermediate_device():
if args.gpu_only:
@@ -686,19 +712,22 @@ def mps_mode():
global cpu_state
return cpu_state == CPUState.MPS
-def is_device_cpu(device):
+def is_device_type(device, type):
if hasattr(device, 'type'):
- if (device.type == 'cpu'):
+ if (device.type == type):
return True
return False
+def is_device_cpu(device):
+ return is_device_type(device, 'cpu')
+
def is_device_mps(device):
- if hasattr(device, 'type'):
- if (device.type == 'mps'):
- return True
- return False
+ return is_device_type(device, 'mps')
-def should_use_fp16(device=None, model_params=0, prioritize_performance=True):
+def is_device_cuda(device):
+ return is_device_type(device, 'cuda')
+
+def should_use_fp16(device=None, model_params=0, prioritize_performance=True, manual_cast=False):
global directml_enabled
if device is not None:
@@ -708,9 +737,9 @@ def should_use_fp16(device=None, model_params=0, prioritize_performance=True):
if FORCE_FP16:
return True
- if device is not None: #TODO
+ if device is not None:
if is_device_mps(device):
- return False
+ return True
if FORCE_FP32:
return False
@@ -718,16 +747,22 @@ def should_use_fp16(device=None, model_params=0, prioritize_performance=True):
if directml_enabled:
return False
- if cpu_mode() or mps_mode():
- return False #TODO ?
+ if mps_mode():
+ return True
+
+ if cpu_mode():
+ return False
if is_intel_xpu():
return True
- if torch.cuda.is_bf16_supported():
+ if torch.version.hip:
return True
props = torch.cuda.get_device_properties("cuda")
+ if props.major >= 8:
+ return True
+
if props.major < 6:
return False
@@ -735,12 +770,12 @@ def should_use_fp16(device=None, model_params=0, prioritize_performance=True):
#FP16 is confirmed working on a 1080 (GP104) but it's a bit slower than FP32 so it should only be enabled
#when the model doesn't actually fit on the card
#TODO: actually test if GP106 and others have the same type of behavior
- nvidia_10_series = ["1080", "1070", "titan x", "p3000", "p3200", "p4000", "p4200", "p5000", "p5200", "p6000", "1060", "1050"]
+ nvidia_10_series = ["1080", "1070", "titan x", "p3000", "p3200", "p4000", "p4200", "p5000", "p5200", "p6000", "1060", "1050", "p40", "p100", "p6", "p4"]
for x in nvidia_10_series:
if x in props.name.lower():
fp16_works = True
- if fp16_works:
+ if fp16_works or manual_cast:
free_model_memory = (get_free_memory() * 0.9 - minimum_inference_memory())
if (not prioritize_performance) or model_params * 4 > free_model_memory:
return True
@@ -756,6 +791,43 @@ def should_use_fp16(device=None, model_params=0, prioritize_performance=True):
return True
+def should_use_bf16(device=None, model_params=0, prioritize_performance=True, manual_cast=False):
+ if device is not None:
+ if is_device_cpu(device): #TODO ? bf16 works on CPU but is extremely slow
+ return False
+
+ if device is not None: #TODO not sure about mps bf16 support
+ if is_device_mps(device):
+ return False
+
+ if FORCE_FP32:
+ return False
+
+ if directml_enabled:
+ return False
+
+ if cpu_mode() or mps_mode():
+ return False
+
+ if is_intel_xpu():
+ return True
+
+ if device is None:
+ device = torch.device("cuda")
+
+ props = torch.cuda.get_device_properties(device)
+ if props.major >= 8:
+ return True
+
+ bf16_works = torch.cuda.is_bf16_supported()
+
+ if bf16_works or manual_cast:
+ free_model_memory = (get_free_memory() * 0.9 - minimum_inference_memory())
+ if (not prioritize_performance) or model_params * 4 > free_model_memory:
+ return True
+
+ return False
+
def soft_empty_cache(force=False):
global cpu_state
if cpu_state == CPUState.MPS:
diff --git a/comfy/model_patcher.py b/comfy/model_patcher.py
index a88b737cc..657ddf843 100644
--- a/comfy/model_patcher.py
+++ b/comfy/model_patcher.py
@@ -1,10 +1,24 @@
import torch
import copy
import inspect
+import logging
+import uuid
import comfy.utils
import comfy.model_management
+def apply_weight_decompose(dora_scale, weight):
+ weight_norm = (
+ weight.transpose(0, 1)
+ .reshape(weight.shape[1], -1)
+ .norm(dim=1, keepdim=True)
+ .reshape(weight.shape[1], *[1] * (weight.dim() - 1))
+ .transpose(0, 1)
+ )
+
+ return weight * (dora_scale / weight_norm)
+
+
class ModelPatcher:
def __init__(self, model, load_device, offload_device, size=0, current_device=None, weight_inplace_update=False):
self.size = size
@@ -23,6 +37,8 @@ class ModelPatcher:
self.current_device = current_device
self.weight_inplace_update = weight_inplace_update
+ self.model_lowvram = False
+ self.patches_uuid = uuid.uuid4()
def model_size(self):
if self.size > 0:
@@ -37,10 +53,13 @@ class ModelPatcher:
n.patches = {}
for k in self.patches:
n.patches[k] = self.patches[k][:]
+ n.patches_uuid = self.patches_uuid
n.object_patches = self.object_patches.copy()
n.model_options = copy.deepcopy(self.model_options)
n.model_keys = self.model_keys
+ n.backup = self.backup
+ n.object_patches_backup = self.object_patches_backup
return n
def is_clone(self, other):
@@ -48,6 +67,19 @@ class ModelPatcher:
return True
return False
+ def clone_has_same_weights(self, clone):
+ if not self.is_clone(clone):
+ return False
+
+ if len(self.patches) == 0 and len(clone.patches) == 0:
+ return True
+
+ if self.patches_uuid == clone.patches_uuid:
+ if len(self.patches) != len(clone.patches):
+ logging.warning("WARNING: something went wrong, same patch uuid but different length of patches.")
+ else:
+ return True
+
def memory_required(self, input_shape):
return self.model.memory_required(input_shape=input_shape)
@@ -67,6 +99,9 @@ class ModelPatcher:
def set_model_unet_function_wrapper(self, unet_wrapper_function):
self.model_options["model_function_wrapper"] = unet_wrapper_function
+ def set_model_denoise_mask_function(self, denoise_mask_function):
+ self.model_options["denoise_mask_function"] = denoise_mask_function
+
def set_model_patch(self, patch, name):
to = self.model_options["transformer_options"]
if "patches" not in to:
@@ -115,6 +150,15 @@ class ModelPatcher:
def add_object_patch(self, name, obj):
self.object_patches[name] = obj
+ def get_model_object(self, name):
+ if name in self.object_patches:
+ return self.object_patches[name]
+ else:
+ if name in self.object_patches_backup:
+ return self.object_patches_backup[name]
+ else:
+ return comfy.utils.get_attr(self.model, name)
+
def model_patches_to(self, device):
to = self.model_options["transformer_options"]
if "patches" in to:
@@ -149,6 +193,7 @@ class ModelPatcher:
current_patches.append((strength_patch, patches[k], strength_model))
self.patches[k] = current_patches
+ self.patches_uuid = uuid.uuid4()
return list(p)
def get_key_patches(self, filter_prefix=None):
@@ -174,37 +219,41 @@ class ModelPatcher:
sd.pop(k)
return sd
+ def patch_weight_to_device(self, key, device_to=None):
+ if key not in self.patches:
+ return
+
+ weight = comfy.utils.get_attr(self.model, key)
+
+ inplace_update = self.weight_inplace_update
+
+ if key not in self.backup:
+ self.backup[key] = weight.to(device=self.offload_device, copy=inplace_update)
+
+ if device_to is not None:
+ temp_weight = comfy.model_management.cast_to_device(weight, device_to, torch.float32, copy=True)
+ else:
+ temp_weight = weight.to(torch.float32, copy=True)
+ out_weight = self.calculate_weight(self.patches[key], temp_weight, key).to(weight.dtype)
+ if inplace_update:
+ comfy.utils.copy_to_param(self.model, key, out_weight)
+ else:
+ comfy.utils.set_attr_param(self.model, key, out_weight)
+
def patch_model(self, device_to=None, patch_weights=True):
for k in self.object_patches:
- old = getattr(self.model, k)
+ old = comfy.utils.set_attr(self.model, k, self.object_patches[k])
if k not in self.object_patches_backup:
self.object_patches_backup[k] = old
- setattr(self.model, k, self.object_patches[k])
if patch_weights:
model_sd = self.model_state_dict()
for key in self.patches:
if key not in model_sd:
- print("could not patch. key doesn't exist in model:", key)
+ logging.warning("could not patch. key doesn't exist in model: {}".format(key))
continue
- weight = model_sd[key]
-
- inplace_update = self.weight_inplace_update
-
- if key not in self.backup:
- self.backup[key] = weight.to(device=self.offload_device, copy=inplace_update)
-
- if device_to is not None:
- temp_weight = comfy.model_management.cast_to_device(weight, device_to, torch.float32, copy=True)
- else:
- temp_weight = weight.to(torch.float32, copy=True)
- out_weight = self.calculate_weight(self.patches[key], temp_weight, key).to(weight.dtype)
- if inplace_update:
- comfy.utils.copy_to_param(self.model, key, out_weight)
- else:
- comfy.utils.set_attr(self.model, key, out_weight)
- del temp_weight
+ self.patch_weight_to_device(key, device_to)
if device_to is not None:
self.model.to(device_to)
@@ -212,6 +261,47 @@ class ModelPatcher:
return self.model
+ def patch_model_lowvram(self, device_to=None, lowvram_model_memory=0):
+ self.patch_model(device_to, patch_weights=False)
+
+ logging.info("loading in lowvram mode {}".format(lowvram_model_memory/(1024 * 1024)))
+ class LowVramPatch:
+ def __init__(self, key, model_patcher):
+ self.key = key
+ self.model_patcher = model_patcher
+ def __call__(self, weight):
+ return self.model_patcher.calculate_weight(self.model_patcher.patches[self.key], weight, self.key)
+
+ mem_counter = 0
+ for n, m in self.model.named_modules():
+ lowvram_weight = False
+ if hasattr(m, "comfy_cast_weights"):
+ module_mem = comfy.model_management.module_size(m)
+ if mem_counter + module_mem >= lowvram_model_memory:
+ lowvram_weight = True
+
+ weight_key = "{}.weight".format(n)
+ bias_key = "{}.bias".format(n)
+
+ if lowvram_weight:
+ if weight_key in self.patches:
+ m.weight_function = LowVramPatch(weight_key, self)
+ if bias_key in self.patches:
+ m.bias_function = LowVramPatch(bias_key, self)
+
+ m.prev_comfy_cast_weights = m.comfy_cast_weights
+ m.comfy_cast_weights = True
+ else:
+ if hasattr(m, "weight"):
+ self.patch_weight_to_device(weight_key, device_to)
+ self.patch_weight_to_device(bias_key, device_to)
+ m.to(device_to)
+ mem_counter += comfy.model_management.module_size(m)
+ logging.debug("lowvram: loaded module regularly {}".format(m))
+
+ self.model_lowvram = True
+ return self.model
+
def calculate_weight(self, patches, weight, key):
for p in patches:
alpha = p[0]
@@ -234,12 +324,13 @@ class ModelPatcher:
w1 = v[0]
if alpha != 0.0:
if w1.shape != weight.shape:
- print("WARNING SHAPE MISMATCH {} WEIGHT NOT MERGED {} != {}".format(key, w1.shape, weight.shape))
+ logging.warning("WARNING SHAPE MISMATCH {} WEIGHT NOT MERGED {} != {}".format(key, w1.shape, weight.shape))
else:
weight += alpha * comfy.model_management.cast_to_device(w1, weight.device, weight.dtype)
elif patch_type == "lora": #lora/locon
mat1 = comfy.model_management.cast_to_device(v[0], weight.device, torch.float32)
mat2 = comfy.model_management.cast_to_device(v[1], weight.device, torch.float32)
+ dora_scale = v[4]
if v[2] is not None:
alpha *= v[2] / mat2.shape[0]
if v[3] is not None:
@@ -249,8 +340,10 @@ class ModelPatcher:
mat2 = torch.mm(mat2.transpose(0, 1).flatten(start_dim=1), mat3.transpose(0, 1).flatten(start_dim=1)).reshape(final_shape).transpose(0, 1)
try:
weight += (alpha * torch.mm(mat1.flatten(start_dim=1), mat2.flatten(start_dim=1))).reshape(weight.shape).type(weight.dtype)
+ if dora_scale is not None:
+ weight = apply_weight_decompose(comfy.model_management.cast_to_device(dora_scale, weight.device, torch.float32), weight)
except Exception as e:
- print("ERROR", key, e)
+ logging.error("ERROR {} {} {}".format(patch_type, key, e))
elif patch_type == "lokr":
w1 = v[0]
w2 = v[1]
@@ -259,6 +352,7 @@ class ModelPatcher:
w2_a = v[5]
w2_b = v[6]
t2 = v[7]
+ dora_scale = v[8]
dim = None
if w1 is None:
@@ -288,8 +382,10 @@ class ModelPatcher:
try:
weight += alpha * torch.kron(w1, w2).reshape(weight.shape).type(weight.dtype)
+ if dora_scale is not None:
+ weight = apply_weight_decompose(comfy.model_management.cast_to_device(dora_scale, weight.device, torch.float32), weight)
except Exception as e:
- print("ERROR", key, e)
+ logging.error("ERROR {} {} {}".format(patch_type, key, e))
elif patch_type == "loha":
w1a = v[0]
w1b = v[1]
@@ -297,6 +393,7 @@ class ModelPatcher:
alpha *= v[2] / w1b.shape[0]
w2a = v[3]
w2b = v[4]
+ dora_scale = v[7]
if v[5] is not None: #cp decomposition
t1 = v[5]
t2 = v[6]
@@ -317,41 +414,61 @@ class ModelPatcher:
try:
weight += (alpha * m1 * m2).reshape(weight.shape).type(weight.dtype)
+ if dora_scale is not None:
+ weight = apply_weight_decompose(comfy.model_management.cast_to_device(dora_scale, weight.device, torch.float32), weight)
except Exception as e:
- print("ERROR", key, e)
+ logging.error("ERROR {} {} {}".format(patch_type, key, e))
elif patch_type == "glora":
if v[4] is not None:
alpha *= v[4] / v[0].shape[0]
+ dora_scale = v[5]
+
a1 = comfy.model_management.cast_to_device(v[0].flatten(start_dim=1), weight.device, torch.float32)
a2 = comfy.model_management.cast_to_device(v[1].flatten(start_dim=1), weight.device, torch.float32)
b1 = comfy.model_management.cast_to_device(v[2].flatten(start_dim=1), weight.device, torch.float32)
b2 = comfy.model_management.cast_to_device(v[3].flatten(start_dim=1), weight.device, torch.float32)
- weight += ((torch.mm(b2, b1) + torch.mm(torch.mm(weight.flatten(start_dim=1), a2), a1)) * alpha).reshape(weight.shape).type(weight.dtype)
+ try:
+ weight += ((torch.mm(b2, b1) + torch.mm(torch.mm(weight.flatten(start_dim=1), a2), a1)) * alpha).reshape(weight.shape).type(weight.dtype)
+ if dora_scale is not None:
+ weight = apply_weight_decompose(comfy.model_management.cast_to_device(dora_scale, weight.device, torch.float32), weight)
+ except Exception as e:
+ logging.error("ERROR {} {} {}".format(patch_type, key, e))
else:
- print("patch type not recognized", patch_type, key)
+ logging.warning("patch type not recognized {} {}".format(patch_type, key))
return weight
- def unpatch_model(self, device_to=None):
- keys = list(self.backup.keys())
+ def unpatch_model(self, device_to=None, unpatch_weights=True):
+ if unpatch_weights:
+ if self.model_lowvram:
+ for m in self.model.modules():
+ if hasattr(m, "prev_comfy_cast_weights"):
+ m.comfy_cast_weights = m.prev_comfy_cast_weights
+ del m.prev_comfy_cast_weights
+ m.weight_function = None
+ m.bias_function = None
- if self.weight_inplace_update:
- for k in keys:
- comfy.utils.copy_to_param(self.model, k, self.backup[k])
- else:
- for k in keys:
- comfy.utils.set_attr(self.model, k, self.backup[k])
+ self.model_lowvram = False
- self.backup = {}
+ keys = list(self.backup.keys())
- if device_to is not None:
- self.model.to(device_to)
- self.current_device = device_to
+ if self.weight_inplace_update:
+ for k in keys:
+ comfy.utils.copy_to_param(self.model, k, self.backup[k])
+ else:
+ for k in keys:
+ comfy.utils.set_attr_param(self.model, k, self.backup[k])
+
+ self.backup.clear()
+
+ if device_to is not None:
+ self.model.to(device_to)
+ self.current_device = device_to
keys = list(self.object_patches_backup.keys())
for k in keys:
- setattr(self.model, k, self.object_patches_backup[k])
+ comfy.utils.set_attr(self.model, k, self.object_patches_backup[k])
- self.object_patches_backup = {}
+ self.object_patches_backup.clear()
diff --git a/comfy/model_sampling.py b/comfy/model_sampling.py
index cc8745c10..37976b326 100644
--- a/comfy/model_sampling.py
+++ b/comfy/model_sampling.py
@@ -1,5 +1,4 @@
import torch
-import numpy as np
from comfy.ldm.modules.diffusionmodules.util import make_beta_schedule
import math
@@ -12,12 +11,28 @@ class EPS:
sigma = sigma.view(sigma.shape[:1] + (1,) * (model_output.ndim - 1))
return model_input - model_output * sigma
+ def noise_scaling(self, sigma, noise, latent_image, max_denoise=False):
+ if max_denoise:
+ noise = noise * torch.sqrt(1.0 + sigma ** 2.0)
+ else:
+ noise = noise * sigma
+
+ noise += latent_image
+ return noise
+
+ def inverse_noise_scaling(self, sigma, latent):
+ return latent
class V_PREDICTION(EPS):
def calculate_denoised(self, sigma, model_output, model_input):
sigma = sigma.view(sigma.shape[:1] + (1,) * (model_output.ndim - 1))
return model_input * self.sigma_data ** 2 / (sigma ** 2 + self.sigma_data ** 2) - model_output * sigma * self.sigma_data / (sigma ** 2 + self.sigma_data ** 2) ** 0.5
+class EDM(V_PREDICTION):
+ def calculate_denoised(self, sigma, model_output, model_input):
+ sigma = sigma.view(sigma.shape[:1] + (1,) * (model_output.ndim - 1))
+ return model_input * self.sigma_data ** 2 / (sigma ** 2 + self.sigma_data ** 2) + model_output * sigma * self.sigma_data / (sigma ** 2 + self.sigma_data ** 2) ** 0.5
+
class ModelSamplingDiscrete(torch.nn.Module):
def __init__(self, model_config=None):
@@ -42,8 +57,7 @@ class ModelSamplingDiscrete(torch.nn.Module):
else:
betas = make_beta_schedule(beta_schedule, timesteps, linear_start=linear_start, linear_end=linear_end, cosine_s=cosine_s)
alphas = 1. - betas
- alphas_cumprod = torch.tensor(np.cumprod(alphas, axis=0), dtype=torch.float32)
- # alphas_cumprod_prev = np.append(1., alphas_cumprod[:-1])
+ alphas_cumprod = torch.cumprod(alphas, dim=0)
timesteps, = betas.shape
self.num_timesteps = int(timesteps)
@@ -58,8 +72,8 @@ class ModelSamplingDiscrete(torch.nn.Module):
self.set_sigmas(sigmas)
def set_sigmas(self, sigmas):
- self.register_buffer('sigmas', sigmas)
- self.register_buffer('log_sigmas', sigmas.log())
+ self.register_buffer('sigmas', sigmas.float())
+ self.register_buffer('log_sigmas', sigmas.log().float())
@property
def sigma_min(self):
@@ -94,8 +108,6 @@ class ModelSamplingDiscrete(torch.nn.Module):
class ModelSamplingContinuousEDM(torch.nn.Module):
def __init__(self, model_config=None):
super().__init__()
- self.sigma_data = 1.0
-
if model_config is not None:
sampling_settings = model_config.sampling_settings
else:
@@ -103,9 +115,11 @@ class ModelSamplingContinuousEDM(torch.nn.Module):
sigma_min = sampling_settings.get("sigma_min", 0.002)
sigma_max = sampling_settings.get("sigma_max", 120.0)
- self.set_sigma_range(sigma_min, sigma_max)
+ sigma_data = sampling_settings.get("sigma_data", 1.0)
+ self.set_parameters(sigma_min, sigma_max, sigma_data)
- def set_sigma_range(self, sigma_min, sigma_max):
+ def set_parameters(self, sigma_min, sigma_max, sigma_data):
+ self.sigma_data = sigma_data
sigmas = torch.linspace(math.log(sigma_min), math.log(sigma_max), 1000).exp()
self.register_buffer('sigmas', sigmas) #for compatibility with some schedulers
@@ -134,3 +148,56 @@ class ModelSamplingContinuousEDM(torch.nn.Module):
log_sigma_min = math.log(self.sigma_min)
return math.exp((math.log(self.sigma_max) - log_sigma_min) * percent + log_sigma_min)
+
+class StableCascadeSampling(ModelSamplingDiscrete):
+ def __init__(self, model_config=None):
+ super().__init__()
+
+ if model_config is not None:
+ sampling_settings = model_config.sampling_settings
+ else:
+ sampling_settings = {}
+
+ self.set_parameters(sampling_settings.get("shift", 1.0))
+
+ def set_parameters(self, shift=1.0, cosine_s=8e-3):
+ self.shift = shift
+ self.cosine_s = torch.tensor(cosine_s)
+ self._init_alpha_cumprod = torch.cos(self.cosine_s / (1 + self.cosine_s) * torch.pi * 0.5) ** 2
+
+ #This part is just for compatibility with some schedulers in the codebase
+ self.num_timesteps = 10000
+ sigmas = torch.empty((self.num_timesteps), dtype=torch.float32)
+ for x in range(self.num_timesteps):
+ t = (x + 1) / self.num_timesteps
+ sigmas[x] = self.sigma(t)
+
+ self.set_sigmas(sigmas)
+
+ def sigma(self, timestep):
+ alpha_cumprod = (torch.cos((timestep + self.cosine_s) / (1 + self.cosine_s) * torch.pi * 0.5) ** 2 / self._init_alpha_cumprod)
+
+ if self.shift != 1.0:
+ var = alpha_cumprod
+ logSNR = (var/(1-var)).log()
+ logSNR += 2 * torch.log(1.0 / torch.tensor(self.shift))
+ alpha_cumprod = logSNR.sigmoid()
+
+ alpha_cumprod = alpha_cumprod.clamp(0.0001, 0.9999)
+ return ((1 - alpha_cumprod) / alpha_cumprod) ** 0.5
+
+ def timestep(self, sigma):
+ var = 1 / ((sigma * sigma) + 1)
+ var = var.clamp(0, 1.0)
+ s, min_var = self.cosine_s.to(var.device), self._init_alpha_cumprod.to(var.device)
+ t = (((var * min_var) ** 0.5).acos() / (torch.pi * 0.5)) * (1 + s) - s
+ return t
+
+ def percent_to_sigma(self, percent):
+ if percent <= 0.0:
+ return 999999999.9
+ if percent >= 1.0:
+ return 0.0
+
+ percent = 1.0 - percent
+ return self.sigma(torch.tensor(percent))
diff --git a/comfy/ops.py b/comfy/ops.py
index f674b47f7..eb6507682 100644
--- a/comfy/ops.py
+++ b/comfy/ops.py
@@ -1,3 +1,21 @@
+"""
+ This file is part of ComfyUI.
+ Copyright (C) 2024 Stability AI
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program. If not, see .
+"""
+
import torch
import comfy.model_management
@@ -6,13 +24,20 @@ def cast_bias_weight(s, input):
non_blocking = comfy.model_management.device_supports_non_blocking(input.device)
if s.bias is not None:
bias = s.bias.to(device=input.device, dtype=input.dtype, non_blocking=non_blocking)
+ if s.bias_function is not None:
+ bias = s.bias_function(bias)
weight = s.weight.to(device=input.device, dtype=input.dtype, non_blocking=non_blocking)
+ if s.weight_function is not None:
+ weight = s.weight_function(weight)
return weight, bias
+class CastWeightBiasOp:
+ comfy_cast_weights = False
+ weight_function = None
+ bias_function = None
class disable_weight_init:
- class Linear(torch.nn.Linear):
- comfy_cast_weights = False
+ class Linear(torch.nn.Linear, CastWeightBiasOp):
def reset_parameters(self):
return None
@@ -26,8 +51,7 @@ class disable_weight_init:
else:
return super().forward(*args, **kwargs)
- class Conv2d(torch.nn.Conv2d):
- comfy_cast_weights = False
+ class Conv2d(torch.nn.Conv2d, CastWeightBiasOp):
def reset_parameters(self):
return None
@@ -41,8 +65,7 @@ class disable_weight_init:
else:
return super().forward(*args, **kwargs)
- class Conv3d(torch.nn.Conv3d):
- comfy_cast_weights = False
+ class Conv3d(torch.nn.Conv3d, CastWeightBiasOp):
def reset_parameters(self):
return None
@@ -56,8 +79,7 @@ class disable_weight_init:
else:
return super().forward(*args, **kwargs)
- class GroupNorm(torch.nn.GroupNorm):
- comfy_cast_weights = False
+ class GroupNorm(torch.nn.GroupNorm, CastWeightBiasOp):
def reset_parameters(self):
return None
@@ -72,13 +94,16 @@ class disable_weight_init:
return super().forward(*args, **kwargs)
- class LayerNorm(torch.nn.LayerNorm):
- comfy_cast_weights = False
+ class LayerNorm(torch.nn.LayerNorm, CastWeightBiasOp):
def reset_parameters(self):
return None
def forward_comfy_cast_weights(self, input):
- weight, bias = cast_bias_weight(self, input)
+ if self.weight is not None:
+ weight, bias = cast_bias_weight(self, input)
+ else:
+ weight = None
+ bias = None
return torch.nn.functional.layer_norm(input, self.normalized_shape, weight, bias, self.eps)
def forward(self, *args, **kwargs):
@@ -87,6 +112,27 @@ class disable_weight_init:
else:
return super().forward(*args, **kwargs)
+ class ConvTranspose2d(torch.nn.ConvTranspose2d, CastWeightBiasOp):
+ def reset_parameters(self):
+ return None
+
+ def forward_comfy_cast_weights(self, input, output_size=None):
+ num_spatial_dims = 2
+ output_padding = self._output_padding(
+ input, output_size, self.stride, self.padding, self.kernel_size,
+ num_spatial_dims, self.dilation)
+
+ weight, bias = cast_bias_weight(self, input)
+ return torch.nn.functional.conv_transpose2d(
+ input, weight, bias, self.stride, self.padding,
+ output_padding, self.groups, self.dilation)
+
+ def forward(self, *args, **kwargs):
+ if self.comfy_cast_weights:
+ return self.forward_comfy_cast_weights(*args, **kwargs)
+ else:
+ return super().forward(*args, **kwargs)
+
@classmethod
def conv_nd(s, dims, *args, **kwargs):
if dims == 2:
@@ -112,3 +158,6 @@ class manual_cast(disable_weight_init):
class LayerNorm(disable_weight_init.LayerNorm):
comfy_cast_weights = True
+
+ class ConvTranspose2d(disable_weight_init.ConvTranspose2d):
+ comfy_cast_weights = True
diff --git a/comfy/sample.py b/comfy/sample.py
index 5c8a7d130..e51bd67d6 100644
--- a/comfy/sample.py
+++ b/comfy/sample.py
@@ -1,10 +1,9 @@
import torch
import comfy.model_management
import comfy.samplers
-import comfy.conds
import comfy.utils
-import math
import numpy as np
+import logging
def prepare_noise(latent_image, seed, noise_inds=None):
"""
@@ -25,94 +24,21 @@ def prepare_noise(latent_image, seed, noise_inds=None):
noises = torch.cat(noises, axis=0)
return noises
-def prepare_mask(noise_mask, shape, device):
- """ensures noise mask is of proper dimensions"""
- noise_mask = torch.nn.functional.interpolate(noise_mask.reshape((-1, 1, noise_mask.shape[-2], noise_mask.shape[-1])), size=(shape[2], shape[3]), mode="bilinear")
- noise_mask = torch.cat([noise_mask] * shape[1], dim=1)
- noise_mask = comfy.utils.repeat_to_batch_size(noise_mask, shape[0])
- noise_mask = noise_mask.to(device)
- return noise_mask
-
-def get_models_from_cond(cond, model_type):
- models = []
- for c in cond:
- if model_type in c:
- models += [c[model_type]]
- return models
-
-def convert_cond(cond):
- out = []
- for c in cond:
- temp = c[1].copy()
- model_conds = temp.get("model_conds", {})
- if c[0] is not None:
- model_conds["c_crossattn"] = comfy.conds.CONDCrossAttn(c[0]) #TODO: remove
- temp["cross_attn"] = c[0]
- temp["model_conds"] = model_conds
- out.append(temp)
- return out
-
-def get_additional_models(positive, negative, dtype):
- """loads additional models in positive and negative conditioning"""
- control_nets = set(get_models_from_cond(positive, "control") + get_models_from_cond(negative, "control"))
-
- inference_memory = 0
- control_models = []
- for m in control_nets:
- control_models += m.get_models()
- inference_memory += m.inference_memory_requirements(dtype)
-
- gligen = get_models_from_cond(positive, "gligen") + get_models_from_cond(negative, "gligen")
- gligen = [x[1] for x in gligen]
- models = control_models + gligen
- return models, inference_memory
+def prepare_sampling(model, noise_shape, positive, negative, noise_mask):
+ logging.warning("Warning: comfy.sample.prepare_sampling isn't used anymore and can be removed")
+ return model, positive, negative, noise_mask, []
def cleanup_additional_models(models):
- """cleanup additional models that were loaded"""
- for m in models:
- if hasattr(m, 'cleanup'):
- m.cleanup()
-
-def prepare_sampling(model, noise_shape, positive, negative, noise_mask):
- device = model.load_device
- positive = convert_cond(positive)
- negative = convert_cond(negative)
-
- if noise_mask is not None:
- noise_mask = prepare_mask(noise_mask, noise_shape, device)
-
- real_model = None
- models, inference_memory = get_additional_models(positive, negative, model.model_dtype())
- comfy.model_management.load_models_gpu([model] + models, model.memory_required([noise_shape[0] * 2] + list(noise_shape[1:])) + inference_memory)
- real_model = model.model
-
- return real_model, positive, negative, noise_mask, models
-
+ logging.warning("Warning: comfy.sample.cleanup_additional_models isn't used anymore and can be removed")
def sample(model, noise, steps, cfg, sampler_name, scheduler, positive, negative, latent_image, denoise=1.0, disable_noise=False, start_step=None, last_step=None, force_full_denoise=False, noise_mask=None, sigmas=None, callback=None, disable_pbar=False, seed=None):
- real_model, positive_copy, negative_copy, noise_mask, models = prepare_sampling(model, noise.shape, positive, negative, noise_mask)
+ sampler = comfy.samplers.KSampler(model, steps=steps, device=model.load_device, sampler=sampler_name, scheduler=scheduler, denoise=denoise, model_options=model.model_options)
- noise = noise.to(model.load_device)
- latent_image = latent_image.to(model.load_device)
-
- sampler = comfy.samplers.KSampler(real_model, steps=steps, device=model.load_device, sampler=sampler_name, scheduler=scheduler, denoise=denoise, model_options=model.model_options)
-
- samples = sampler.sample(noise, positive_copy, negative_copy, cfg=cfg, latent_image=latent_image, start_step=start_step, last_step=last_step, force_full_denoise=force_full_denoise, denoise_mask=noise_mask, sigmas=sigmas, callback=callback, disable_pbar=disable_pbar, seed=seed)
+ samples = sampler.sample(noise, positive, negative, cfg=cfg, latent_image=latent_image, start_step=start_step, last_step=last_step, force_full_denoise=force_full_denoise, denoise_mask=noise_mask, sigmas=sigmas, callback=callback, disable_pbar=disable_pbar, seed=seed)
samples = samples.to(comfy.model_management.intermediate_device())
-
- cleanup_additional_models(models)
- cleanup_additional_models(set(get_models_from_cond(positive_copy, "control") + get_models_from_cond(negative_copy, "control")))
return samples
def sample_custom(model, noise, cfg, sampler, sigmas, positive, negative, latent_image, noise_mask=None, callback=None, disable_pbar=False, seed=None):
- real_model, positive_copy, negative_copy, noise_mask, models = prepare_sampling(model, noise.shape, positive, negative, noise_mask)
- noise = noise.to(model.load_device)
- latent_image = latent_image.to(model.load_device)
- sigmas = sigmas.to(model.load_device)
-
- samples = comfy.samplers.sample(real_model, noise, positive_copy, negative_copy, cfg, model.load_device, sampler, sigmas, model_options=model.model_options, latent_image=latent_image, denoise_mask=noise_mask, callback=callback, disable_pbar=disable_pbar, seed=seed)
+ samples = comfy.samplers.sample(model, noise, positive, negative, cfg, model.load_device, sampler, sigmas, model_options=model.model_options, latent_image=latent_image, denoise_mask=noise_mask, callback=callback, disable_pbar=disable_pbar, seed=seed)
samples = samples.to(comfy.model_management.intermediate_device())
- cleanup_additional_models(models)
- cleanup_additional_models(set(get_models_from_cond(positive_copy, "control") + get_models_from_cond(negative_copy, "control")))
return samples
-
diff --git a/comfy/sampler_helpers.py b/comfy/sampler_helpers.py
new file mode 100644
index 000000000..a18abd9e9
--- /dev/null
+++ b/comfy/sampler_helpers.py
@@ -0,0 +1,76 @@
+import torch
+import comfy.model_management
+import comfy.conds
+
+def prepare_mask(noise_mask, shape, device):
+ """ensures noise mask is of proper dimensions"""
+ noise_mask = torch.nn.functional.interpolate(noise_mask.reshape((-1, 1, noise_mask.shape[-2], noise_mask.shape[-1])), size=(shape[2], shape[3]), mode="bilinear")
+ noise_mask = torch.cat([noise_mask] * shape[1], dim=1)
+ noise_mask = comfy.utils.repeat_to_batch_size(noise_mask, shape[0])
+ noise_mask = noise_mask.to(device)
+ return noise_mask
+
+def get_models_from_cond(cond, model_type):
+ models = []
+ for c in cond:
+ if model_type in c:
+ models += [c[model_type]]
+ return models
+
+def convert_cond(cond):
+ out = []
+ for c in cond:
+ temp = c[1].copy()
+ model_conds = temp.get("model_conds", {})
+ if c[0] is not None:
+ model_conds["c_crossattn"] = comfy.conds.CONDCrossAttn(c[0]) #TODO: remove
+ temp["cross_attn"] = c[0]
+ temp["model_conds"] = model_conds
+ out.append(temp)
+ return out
+
+def get_additional_models(conds, dtype):
+ """loads additional models in conditioning"""
+ cnets = []
+ gligen = []
+
+ for k in conds:
+ cnets += get_models_from_cond(conds[k], "control")
+ gligen += get_models_from_cond(conds[k], "gligen")
+
+ control_nets = set(cnets)
+
+ inference_memory = 0
+ control_models = []
+ for m in control_nets:
+ control_models += m.get_models()
+ inference_memory += m.inference_memory_requirements(dtype)
+
+ gligen = [x[1] for x in gligen]
+ models = control_models + gligen
+ return models, inference_memory
+
+def cleanup_additional_models(models):
+ """cleanup additional models that were loaded"""
+ for m in models:
+ if hasattr(m, 'cleanup'):
+ m.cleanup()
+
+
+def prepare_sampling(model, noise_shape, conds):
+ device = model.load_device
+ real_model = None
+ models, inference_memory = get_additional_models(conds, model.model_dtype())
+ comfy.model_management.load_models_gpu([model] + models, model.memory_required([noise_shape[0] * 2] + list(noise_shape[1:])) + inference_memory)
+ real_model = model.model
+
+ return real_model, conds, models
+
+def cleanup_models(conds, models):
+ cleanup_additional_models(models)
+
+ control_cleanup = []
+ for k in conds:
+ control_cleanup += get_models_from_cond(conds[k], "control")
+
+ cleanup_additional_models(set(control_cleanup))
diff --git a/comfy/samplers.py b/comfy/samplers.py
index 89d8d4f28..415a35cc3 100644
--- a/comfy/samplers.py
+++ b/comfy/samplers.py
@@ -1,13 +1,11 @@
from .k_diffusion import sampling as k_diffusion_sampling
from .extra_samplers import uni_pc
import torch
-import enum
import collections
from comfy import model_management
import math
-from comfy import model_base
-import comfy.utils
-import comfy.conds
+import logging
+import comfy.sampler_helpers
def get_area_and_mult(conds, x_in, timestep_in):
area = (x_in.shape[2], x_in.shape[3], 0, 0)
@@ -130,30 +128,23 @@ def cond_cat(c_list):
return out
-def calc_cond_uncond_batch(model, cond, uncond, x_in, timestep, model_options):
- out_cond = torch.zeros_like(x_in)
- out_count = torch.ones_like(x_in) * 1e-37
-
- out_uncond = torch.zeros_like(x_in)
- out_uncond_count = torch.ones_like(x_in) * 1e-37
-
- COND = 0
- UNCOND = 1
-
+def calc_cond_batch(model, conds, x_in, timestep, model_options):
+ out_conds = []
+ out_counts = []
to_run = []
- for x in cond:
- p = get_area_and_mult(x, x_in, timestep)
- if p is None:
- continue
- to_run += [(p, COND)]
- if uncond is not None:
- for x in uncond:
- p = get_area_and_mult(x, x_in, timestep)
- if p is None:
- continue
+ for i in range(len(conds)):
+ out_conds.append(torch.zeros_like(x_in))
+ out_counts.append(torch.ones_like(x_in) * 1e-37)
- to_run += [(p, UNCOND)]
+ cond = conds[i]
+ if cond is not None:
+ for x in cond:
+ p = get_area_and_mult(x, x_in, timestep)
+ if p is None:
+ continue
+
+ to_run += [(p, i)]
while len(to_run) > 0:
first = to_run[0]
@@ -212,6 +203,7 @@ def calc_cond_uncond_batch(model, cond, uncond, x_in, timestep, model_options):
cur_patches[p] = cur_patches[p] + patches[p]
else:
cur_patches[p] = patches[p]
+ transformer_options["patches"] = cur_patches
else:
transformer_options["patches"] = patches
@@ -224,71 +216,66 @@ def calc_cond_uncond_batch(model, cond, uncond, x_in, timestep, model_options):
output = model_options['model_function_wrapper'](model.apply_model, {"input": input_x, "timestep": timestep_, "c": c, "cond_or_uncond": cond_or_uncond}).chunk(batch_chunks)
else:
output = model.apply_model(input_x, timestep_, **c).chunk(batch_chunks)
- del input_x
for o in range(batch_chunks):
- if cond_or_uncond[o] == COND:
- out_cond[:,:,area[o][2]:area[o][0] + area[o][2],area[o][3]:area[o][1] + area[o][3]] += output[o] * mult[o]
- out_count[:,:,area[o][2]:area[o][0] + area[o][2],area[o][3]:area[o][1] + area[o][3]] += mult[o]
- else:
- out_uncond[:,:,area[o][2]:area[o][0] + area[o][2],area[o][3]:area[o][1] + area[o][3]] += output[o] * mult[o]
- out_uncond_count[:,:,area[o][2]:area[o][0] + area[o][2],area[o][3]:area[o][1] + area[o][3]] += mult[o]
- del mult
+ cond_index = cond_or_uncond[o]
+ out_conds[cond_index][:,:,area[o][2]:area[o][0] + area[o][2],area[o][3]:area[o][1] + area[o][3]] += output[o] * mult[o]
+ out_counts[cond_index][:,:,area[o][2]:area[o][0] + area[o][2],area[o][3]:area[o][1] + area[o][3]] += mult[o]
- out_cond /= out_count
- del out_count
- out_uncond /= out_uncond_count
- del out_uncond_count
- return out_cond, out_uncond
+ for i in range(len(out_conds)):
+ out_conds[i] /= out_counts[i]
+
+ return out_conds
+
+def calc_cond_uncond_batch(model, cond, uncond, x_in, timestep, model_options): #TODO: remove
+ logging.warning("WARNING: The comfy.samplers.calc_cond_uncond_batch function is deprecated please use the calc_cond_batch one instead.")
+ return tuple(calc_cond_batch(model, [cond, uncond], x_in, timestep, model_options))
+
+def cfg_function(model, cond_pred, uncond_pred, cond_scale, x, timestep, model_options={}, cond=None, uncond=None):
+ if "sampler_cfg_function" in model_options:
+ args = {"cond": x - cond_pred, "uncond": x - uncond_pred, "cond_scale": cond_scale, "timestep": timestep, "input": x, "sigma": timestep,
+ "cond_denoised": cond_pred, "uncond_denoised": uncond_pred, "model": model, "model_options": model_options}
+ cfg_result = x - model_options["sampler_cfg_function"](args)
+ else:
+ cfg_result = uncond_pred + (cond_pred - uncond_pred) * cond_scale
+
+ for fn in model_options.get("sampler_post_cfg_function", []):
+ args = {"denoised": cfg_result, "cond": cond, "uncond": uncond, "model": model, "uncond_denoised": uncond_pred, "cond_denoised": cond_pred,
+ "sigma": timestep, "model_options": model_options, "input": x}
+ cfg_result = fn(args)
+
+ return cfg_result
#The main sampling function shared by all the samplers
#Returns denoised
def sampling_function(model, x, timestep, uncond, cond, cond_scale, model_options={}, seed=None):
- if math.isclose(cond_scale, 1.0) and model_options.get("disable_cfg1_optimization", False) == False:
- uncond_ = None
- else:
- uncond_ = uncond
+ if math.isclose(cond_scale, 1.0) and model_options.get("disable_cfg1_optimization", False) == False:
+ uncond_ = None
+ else:
+ uncond_ = uncond
- cond_pred, uncond_pred = calc_cond_uncond_batch(model, cond, uncond_, x, timestep, model_options)
- if "sampler_cfg_function" in model_options:
- args = {"cond": x - cond_pred, "uncond": x - uncond_pred, "cond_scale": cond_scale, "timestep": timestep, "input": x, "sigma": timestep,
- "cond_denoised": cond_pred, "uncond_denoised": uncond_pred, "model": model, "model_options": model_options}
- cfg_result = x - model_options["sampler_cfg_function"](args)
- else:
- cfg_result = uncond_pred + (cond_pred - uncond_pred) * cond_scale
+ conds = [cond, uncond_]
+ out = calc_cond_batch(model, conds, x, timestep, model_options)
+ return cfg_function(model, out[0], out[1], cond_scale, x, timestep, model_options=model_options, cond=cond, uncond=uncond_)
- for fn in model_options.get("sampler_post_cfg_function", []):
- args = {"denoised": cfg_result, "cond": cond, "uncond": uncond, "model": model, "uncond_denoised": uncond_pred, "cond_denoised": cond_pred,
- "sigma": timestep, "model_options": model_options, "input": x}
- cfg_result = fn(args)
- return cfg_result
-
-class CFGNoisePredictor(torch.nn.Module):
- def __init__(self, model):
- super().__init__()
+class KSamplerX0Inpaint:
+ def __init__(self, model, sigmas):
self.inner_model = model
- def apply_model(self, x, timestep, cond, uncond, cond_scale, model_options={}, seed=None):
- out = sampling_function(self.inner_model, x, timestep, uncond, cond, cond_scale, model_options=model_options, seed=seed)
- return out
- def forward(self, *args, **kwargs):
- return self.apply_model(*args, **kwargs)
-
-class KSamplerX0Inpaint(torch.nn.Module):
- def __init__(self, model):
- super().__init__()
- self.inner_model = model
- def forward(self, x, sigma, uncond, cond, cond_scale, denoise_mask, model_options={}, seed=None):
+ self.sigmas = sigmas
+ def __call__(self, x, sigma, denoise_mask, model_options={}, seed=None):
if denoise_mask is not None:
+ if "denoise_mask_function" in model_options:
+ denoise_mask = model_options["denoise_mask_function"](sigma, denoise_mask, extra_options={"model": self.inner_model, "sigmas": self.sigmas})
latent_mask = 1. - denoise_mask
- x = x * denoise_mask + (self.latent_image + self.noise * sigma.reshape([sigma.shape[0]] + [1] * (len(self.noise.shape) - 1))) * latent_mask
- out = self.inner_model(x, sigma, cond=cond, uncond=uncond, cond_scale=cond_scale, model_options=model_options, seed=seed)
+ x = x * denoise_mask + self.inner_model.inner_model.model_sampling.noise_scaling(sigma.reshape([sigma.shape[0]] + [1] * (len(self.noise.shape) - 1)), self.noise, self.latent_image) * latent_mask
+ out = self.inner_model(x, sigma, model_options=model_options, seed=seed)
if denoise_mask is not None:
out = out * denoise_mask + self.latent_image * latent_mask
return out
-def simple_scheduler(model, steps):
- s = model.model_sampling
+def simple_scheduler(model_sampling, steps):
+ s = model_sampling
sigs = []
ss = len(s.sigmas) / steps
for x in range(steps):
@@ -296,10 +283,10 @@ def simple_scheduler(model, steps):
sigs += [0.0]
return torch.FloatTensor(sigs)
-def ddim_scheduler(model, steps):
- s = model.model_sampling
+def ddim_scheduler(model_sampling, steps):
+ s = model_sampling
sigs = []
- ss = len(s.sigmas) // steps
+ ss = max(len(s.sigmas) // steps, 1)
x = 1
while x < len(s.sigmas):
sigs += [float(s.sigmas[x])]
@@ -308,8 +295,8 @@ def ddim_scheduler(model, steps):
sigs += [0.0]
return torch.FloatTensor(sigs)
-def normal_scheduler(model, steps, sgm=False, floor=False):
- s = model.model_sampling
+def normal_scheduler(model_sampling, steps, sgm=False, floor=False):
+ s = model_sampling
start = s.timestep(s.sigma_max)
end = s.timestep(s.sigma_min)
@@ -517,14 +504,6 @@ class Sampler:
sigma = float(sigmas[0])
return math.isclose(max_sigma, sigma, rel_tol=1e-05) or sigma > max_sigma
-class UNIPC(Sampler):
- def sample(self, model_wrap, sigmas, extra_args, callback, noise, latent_image=None, denoise_mask=None, disable_pbar=False):
- return uni_pc.sample_unipc(model_wrap, noise, latent_image, sigmas, max_denoise=self.max_denoise(model_wrap, sigmas), extra_args=extra_args, noise_mask=denoise_mask, callback=callback, disable=disable_pbar)
-
-class UNIPCBH2(Sampler):
- def sample(self, model_wrap, sigmas, extra_args, callback, noise, latent_image=None, denoise_mask=None, disable_pbar=False):
- return uni_pc.sample_unipc(model_wrap, noise, latent_image, sigmas, max_denoise=self.max_denoise(model_wrap, sigmas), extra_args=extra_args, noise_mask=denoise_mask, callback=callback, variant='bh2', disable=disable_pbar)
-
KSAMPLER_NAMES = ["euler", "euler_ancestral", "heun", "heunpp2","dpm_2", "dpm_2_ancestral",
"lms", "dpm_fast", "dpm_adaptive", "dpmpp_2s_ancestral", "dpmpp_sde", "dpmpp_sde_gpu",
"dpmpp_2m", "dpmpp_2m_sde", "dpmpp_2m_sde_gpu", "dpmpp_3m_sde", "dpmpp_3m_sde_gpu", "ddpm", "lcm"]
@@ -537,7 +516,7 @@ class KSAMPLER(Sampler):
def sample(self, model_wrap, sigmas, extra_args, callback, noise, latent_image=None, denoise_mask=None, disable_pbar=False):
extra_args["denoise_mask"] = denoise_mask
- model_k = KSamplerX0Inpaint(model_wrap)
+ model_k = KSamplerX0Inpaint(model_wrap, sigmas)
model_k.latent_image = latent_image
if self.inpaint_options.get("random", False): #TODO: Should this be the default?
generator = torch.manual_seed(extra_args.get("seed", 41) + 1)
@@ -545,20 +524,15 @@ class KSAMPLER(Sampler):
else:
model_k.noise = noise
- if self.max_denoise(model_wrap, sigmas):
- noise = noise * torch.sqrt(1.0 + sigmas[0] ** 2.0)
- else:
- noise = noise * sigmas[0]
+ noise = model_wrap.inner_model.model_sampling.noise_scaling(sigmas[0], noise, latent_image, self.max_denoise(model_wrap, sigmas))
k_callback = None
total_steps = len(sigmas) - 1
if callback is not None:
k_callback = lambda x: callback(x["i"], x["denoised"], x["x"], total_steps)
- if latent_image is not None:
- noise += latent_image
-
samples = self.sampler_function(model_k, noise, sigmas, extra_args=extra_args, callback=k_callback, disable=disable_pbar, **self.extra_options)
+ samples = model_wrap.inner_model.model_sampling.inverse_noise_scaling(sigmas[-1], samples)
return samples
@@ -572,81 +546,142 @@ def ksampler(sampler_name, extra_options={}, inpaint_options={}):
return k_diffusion_sampling.sample_dpm_fast(model, noise, sigma_min, sigmas[0], total_steps, extra_args=extra_args, callback=callback, disable=disable)
sampler_function = dpm_fast_function
elif sampler_name == "dpm_adaptive":
- def dpm_adaptive_function(model, noise, sigmas, extra_args, callback, disable):
+ def dpm_adaptive_function(model, noise, sigmas, extra_args, callback, disable, **extra_options):
sigma_min = sigmas[-1]
if sigma_min == 0:
sigma_min = sigmas[-2]
- return k_diffusion_sampling.sample_dpm_adaptive(model, noise, sigma_min, sigmas[0], extra_args=extra_args, callback=callback, disable=disable)
+ return k_diffusion_sampling.sample_dpm_adaptive(model, noise, sigma_min, sigmas[0], extra_args=extra_args, callback=callback, disable=disable, **extra_options)
sampler_function = dpm_adaptive_function
else:
sampler_function = getattr(k_diffusion_sampling, "sample_{}".format(sampler_name))
return KSAMPLER(sampler_function, extra_options, inpaint_options)
-def wrap_model(model):
- model_denoise = CFGNoisePredictor(model)
- return model_denoise
-def sample(model, noise, positive, negative, cfg, device, sampler, sigmas, model_options={}, latent_image=None, denoise_mask=None, callback=None, disable_pbar=False, seed=None):
- positive = positive[:]
- negative = negative[:]
+def process_conds(model, noise, conds, device, latent_image=None, denoise_mask=None, seed=None):
+ for k in conds:
+ conds[k] = conds[k][:]
+ resolve_areas_and_cond_masks(conds[k], noise.shape[2], noise.shape[3], device)
- resolve_areas_and_cond_masks(positive, noise.shape[2], noise.shape[3], device)
- resolve_areas_and_cond_masks(negative, noise.shape[2], noise.shape[3], device)
-
- model_wrap = wrap_model(model)
-
- calculate_start_end_timesteps(model, negative)
- calculate_start_end_timesteps(model, positive)
-
- if latent_image is not None:
- latent_image = model.process_latent_in(latent_image)
+ for k in conds:
+ calculate_start_end_timesteps(model, conds[k])
if hasattr(model, 'extra_conds'):
- positive = encode_model_conds(model.extra_conds, positive, noise, device, "positive", latent_image=latent_image, denoise_mask=denoise_mask, seed=seed)
- negative = encode_model_conds(model.extra_conds, negative, noise, device, "negative", latent_image=latent_image, denoise_mask=denoise_mask, seed=seed)
+ for k in conds:
+ conds[k] = encode_model_conds(model.extra_conds, conds[k], noise, device, k, latent_image=latent_image, denoise_mask=denoise_mask, seed=seed)
#make sure each cond area has an opposite one with the same area
- for c in positive:
- create_cond_with_same_area_if_none(negative, c)
- for c in negative:
- create_cond_with_same_area_if_none(positive, c)
+ for k in conds:
+ for c in conds[k]:
+ for kk in conds:
+ if k != kk:
+ create_cond_with_same_area_if_none(conds[kk], c)
- pre_run_control(model, negative + positive)
+ for k in conds:
+ pre_run_control(model, conds[k])
- apply_empty_x_to_equal_area(list(filter(lambda c: c.get('control_apply_to_uncond', False) == True, positive)), negative, 'control', lambda cond_cnets, x: cond_cnets[x])
- apply_empty_x_to_equal_area(positive, negative, 'gligen', lambda cond_cnets, x: cond_cnets[x])
+ if "positive" in conds:
+ positive = conds["positive"]
+ for k in conds:
+ if k != "positive":
+ apply_empty_x_to_equal_area(list(filter(lambda c: c.get('control_apply_to_uncond', False) == True, positive)), conds[k], 'control', lambda cond_cnets, x: cond_cnets[x])
+ apply_empty_x_to_equal_area(positive, conds[k], 'gligen', lambda cond_cnets, x: cond_cnets[x])
- extra_args = {"cond":positive, "uncond":negative, "cond_scale": cfg, "model_options": model_options, "seed":seed}
+ return conds
+
+class CFGGuider:
+ def __init__(self, model_patcher):
+ self.model_patcher = model_patcher
+ self.model_options = model_patcher.model_options
+ self.original_conds = {}
+ self.cfg = 1.0
+
+ def set_conds(self, positive, negative):
+ self.inner_set_conds({"positive": positive, "negative": negative})
+
+ def set_cfg(self, cfg):
+ self.cfg = cfg
+
+ def inner_set_conds(self, conds):
+ for k in conds:
+ self.original_conds[k] = comfy.sampler_helpers.convert_cond(conds[k])
+
+ def __call__(self, *args, **kwargs):
+ return self.predict_noise(*args, **kwargs)
+
+ def predict_noise(self, x, timestep, model_options={}, seed=None):
+ return sampling_function(self.inner_model, x, timestep, self.conds.get("negative", None), self.conds.get("positive", None), self.cfg, model_options=model_options, seed=seed)
+
+ def inner_sample(self, noise, latent_image, device, sampler, sigmas, denoise_mask, callback, disable_pbar, seed):
+ if latent_image is not None and torch.count_nonzero(latent_image) > 0: #Don't shift the empty latent image.
+ latent_image = self.inner_model.process_latent_in(latent_image)
+
+ self.conds = process_conds(self.inner_model, noise, self.conds, device, latent_image, denoise_mask, seed)
+
+ extra_args = {"model_options": self.model_options, "seed":seed}
+
+ samples = sampler.sample(self, sigmas, extra_args, callback, noise, latent_image, denoise_mask, disable_pbar)
+ return self.inner_model.process_latent_out(samples.to(torch.float32))
+
+ def sample(self, noise, latent_image, sampler, sigmas, denoise_mask=None, callback=None, disable_pbar=False, seed=None):
+ if sigmas.shape[-1] == 0:
+ return latent_image
+
+ self.conds = {}
+ for k in self.original_conds:
+ self.conds[k] = list(map(lambda a: a.copy(), self.original_conds[k]))
+
+ self.inner_model, self.conds, self.loaded_models = comfy.sampler_helpers.prepare_sampling(self.model_patcher, noise.shape, self.conds)
+ device = self.model_patcher.load_device
+
+ if denoise_mask is not None:
+ denoise_mask = comfy.sampler_helpers.prepare_mask(denoise_mask, noise.shape, device)
+
+ noise = noise.to(device)
+ latent_image = latent_image.to(device)
+ sigmas = sigmas.to(device)
+
+ output = self.inner_sample(noise, latent_image, device, sampler, sigmas, denoise_mask, callback, disable_pbar, seed)
+
+ comfy.sampler_helpers.cleanup_models(self.conds, self.loaded_models)
+ del self.inner_model
+ del self.conds
+ del self.loaded_models
+ return output
+
+
+def sample(model, noise, positive, negative, cfg, device, sampler, sigmas, model_options={}, latent_image=None, denoise_mask=None, callback=None, disable_pbar=False, seed=None):
+ cfg_guider = CFGGuider(model)
+ cfg_guider.set_conds(positive, negative)
+ cfg_guider.set_cfg(cfg)
+ return cfg_guider.sample(noise, latent_image, sampler, sigmas, denoise_mask, callback, disable_pbar, seed)
- samples = sampler.sample(model_wrap, sigmas, extra_args, callback, noise, latent_image, denoise_mask, disable_pbar)
- return model.process_latent_out(samples.to(torch.float32))
SCHEDULER_NAMES = ["normal", "karras", "exponential", "sgm_uniform", "simple", "ddim_uniform"]
SAMPLER_NAMES = KSAMPLER_NAMES + ["ddim", "uni_pc", "uni_pc_bh2"]
-def calculate_sigmas_scheduler(model, scheduler_name, steps):
+def calculate_sigmas(model_sampling, scheduler_name, steps):
if scheduler_name == "karras":
- sigmas = k_diffusion_sampling.get_sigmas_karras(n=steps, sigma_min=float(model.model_sampling.sigma_min), sigma_max=float(model.model_sampling.sigma_max))
+ sigmas = k_diffusion_sampling.get_sigmas_karras(n=steps, sigma_min=float(model_sampling.sigma_min), sigma_max=float(model_sampling.sigma_max))
elif scheduler_name == "exponential":
- sigmas = k_diffusion_sampling.get_sigmas_exponential(n=steps, sigma_min=float(model.model_sampling.sigma_min), sigma_max=float(model.model_sampling.sigma_max))
+ sigmas = k_diffusion_sampling.get_sigmas_exponential(n=steps, sigma_min=float(model_sampling.sigma_min), sigma_max=float(model_sampling.sigma_max))
elif scheduler_name == "normal":
- sigmas = normal_scheduler(model, steps)
+ sigmas = normal_scheduler(model_sampling, steps)
elif scheduler_name == "simple":
- sigmas = simple_scheduler(model, steps)
+ sigmas = simple_scheduler(model_sampling, steps)
elif scheduler_name == "ddim_uniform":
- sigmas = ddim_scheduler(model, steps)
+ sigmas = ddim_scheduler(model_sampling, steps)
elif scheduler_name == "sgm_uniform":
- sigmas = normal_scheduler(model, steps, sgm=True)
+ sigmas = normal_scheduler(model_sampling, steps, sgm=True)
else:
- print("error invalid scheduler", self.scheduler)
+ logging.error("error invalid scheduler {}".format(scheduler_name))
return sigmas
def sampler_object(name):
if name == "uni_pc":
- sampler = UNIPC()
+ sampler = KSAMPLER(uni_pc.sample_unipc)
elif name == "uni_pc_bh2":
- sampler = UNIPCBH2()
+ sampler = KSAMPLER(uni_pc.sample_unipc_bh2)
elif name == "ddim":
sampler = ksampler("euler", inpaint_options={"random": True})
else:
@@ -656,6 +691,7 @@ def sampler_object(name):
class KSampler:
SCHEDULERS = SCHEDULER_NAMES
SAMPLERS = SAMPLER_NAMES
+ DISCARD_PENULTIMATE_SIGMA_SAMPLERS = set(('dpm_2', 'dpm_2_ancestral', 'uni_pc', 'uni_pc_bh2'))
def __init__(self, model, steps, device, sampler=None, scheduler=None, denoise=None, model_options={}):
self.model = model
@@ -674,11 +710,11 @@ class KSampler:
sigmas = None
discard_penultimate_sigma = False
- if self.sampler in ['dpm_2', 'dpm_2_ancestral', 'uni_pc', 'uni_pc_bh2']:
+ if self.sampler in self.DISCARD_PENULTIMATE_SIGMA_SAMPLERS:
steps += 1
discard_penultimate_sigma = True
- sigmas = calculate_sigmas_scheduler(self.model, self.scheduler, steps)
+ sigmas = calculate_sigmas(self.model.get_model_object("model_sampling"), self.scheduler, steps)
if discard_penultimate_sigma:
sigmas = torch.cat([sigmas[:-2], sigmas[-1:]])
@@ -689,9 +725,12 @@ class KSampler:
if denoise is None or denoise > 0.9999:
self.sigmas = self.calculate_sigmas(steps).to(self.device)
else:
- new_steps = int(steps/denoise)
- sigmas = self.calculate_sigmas(new_steps).to(self.device)
- self.sigmas = sigmas[-(steps + 1):]
+ if denoise <= 0.0:
+ self.sigmas = torch.FloatTensor([])
+ else:
+ new_steps = int(steps/denoise)
+ sigmas = self.calculate_sigmas(new_steps).to(self.device)
+ self.sigmas = sigmas[-(steps + 1):]
def sample(self, noise, positive, negative, cfg, latent_image=None, start_step=None, last_step=None, force_full_denoise=False, denoise_mask=None, sigmas=None, callback=None, disable_pbar=False, seed=None):
if sigmas is None:
diff --git a/comfy/sd.py b/comfy/sd.py
index f49e87b15..3919f4bfa 100644
--- a/comfy/sd.py
+++ b/comfy/sd.py
@@ -1,10 +1,12 @@
import torch
-import contextlib
-import math
+from enum import Enum
+import logging
from comfy import model_management
-from .ldm.util import instantiate_from_config
from .ldm.models.autoencoder import AutoencoderKL, AutoencodingEngine
+from .ldm.cascade.stage_a import StageA
+from .ldm.cascade.stage_c_coder import StageC_coder
+
import yaml
import comfy.utils
@@ -36,7 +38,7 @@ def load_model_weights(model, sd):
w = sd.pop(x)
del w
if len(m) > 0:
- print("missing", m)
+ logging.warning("missing {}".format(m))
return model
def load_clip_weights(model, sd):
@@ -51,7 +53,7 @@ def load_clip_weights(model, sd):
if ids.dtype == torch.float32:
sd['cond_stage_model.transformer.text_model.embeddings.position_ids'] = ids.round()
- sd = comfy.utils.transformers_convert(sd, "cond_stage_model.model.", "cond_stage_model.transformer.text_model.", 24)
+ sd = comfy.utils.clip_text_transformers_convert(sd, "cond_stage_model.model.", "cond_stage_model.transformer.")
return load_model_weights(model, sd)
@@ -80,7 +82,7 @@ def load_lora_for_models(model, clip, lora, strength_model, strength_clip):
k1 = set(k1)
for x in loaded:
if (x not in k) and (x not in k1):
- print("NOT LOADED", x)
+ logging.warning("NOT LOADED {}".format(x))
return (new_modelpatcher, new_clip)
@@ -122,10 +124,13 @@ class CLIP:
return self.tokenizer.tokenize_with_weights(text, return_word_ids)
def encode_from_tokens(self, tokens, return_pooled=False):
+ self.cond_stage_model.reset_clip_options()
+
if self.layer_idx is not None:
- self.cond_stage_model.clip_layer(self.layer_idx)
- else:
- self.cond_stage_model.reset_clip_layer()
+ self.cond_stage_model.set_clip_options({"layer": self.layer_idx})
+
+ if return_pooled == "unprojected":
+ self.cond_stage_model.set_clip_options({"projected_pooled": False})
self.load_model()
cond, pooled = self.cond_stage_model.encode_token_weights(tokens)
@@ -137,8 +142,11 @@ class CLIP:
tokens = self.tokenize(text)
return self.encode_from_tokens(tokens)
- def load_sd(self, sd):
- return self.cond_stage_model.load_sd(sd)
+ def load_sd(self, sd, full_model=False):
+ if full_model:
+ return self.cond_stage_model.load_state_dict(sd, strict=False)
+ else:
+ return self.cond_stage_model.load_sd(sd)
def get_sd(self):
return self.cond_stage_model.state_dict()
@@ -158,7 +166,10 @@ class VAE:
self.memory_used_encode = lambda shape, dtype: (1767 * shape[2] * shape[3]) * model_management.dtype_size(dtype) #These are for AutoencoderKL and need tweaking (should be lower)
self.memory_used_decode = lambda shape, dtype: (2178 * shape[2] * shape[3] * 64) * model_management.dtype_size(dtype)
self.downscale_ratio = 8
+ self.upscale_ratio = 8
self.latent_channels = 4
+ self.process_input = lambda image: image * 2.0 - 1.0
+ self.process_output = lambda image: torch.clamp((image + 1.0) / 2.0, min=0.0, max=1.0)
if config is None:
if "decoder.mid.block_1.mix_factor" in sd:
@@ -171,6 +182,34 @@ class VAE:
decoder_config={'target': "comfy.ldm.modules.temporal_ae.VideoDecoder", 'params': decoder_config})
elif "taesd_decoder.1.weight" in sd:
self.first_stage_model = comfy.taesd.taesd.TAESD()
+ elif "vquantizer.codebook.weight" in sd: #VQGan: stage a of stable cascade
+ self.first_stage_model = StageA()
+ self.downscale_ratio = 4
+ self.upscale_ratio = 4
+ #TODO
+ #self.memory_used_encode
+ #self.memory_used_decode
+ self.process_input = lambda image: image
+ self.process_output = lambda image: image
+ elif "backbone.1.0.block.0.1.num_batches_tracked" in sd: #effnet: encoder for stage c latent of stable cascade
+ self.first_stage_model = StageC_coder()
+ self.downscale_ratio = 32
+ self.latent_channels = 16
+ new_sd = {}
+ for k in sd:
+ new_sd["encoder.{}".format(k)] = sd[k]
+ sd = new_sd
+ elif "blocks.11.num_batches_tracked" in sd: #previewer: decoder for stage c latent of stable cascade
+ self.first_stage_model = StageC_coder()
+ self.latent_channels = 16
+ new_sd = {}
+ for k in sd:
+ new_sd["previewer.{}".format(k)] = sd[k]
+ sd = new_sd
+ elif "encoder.backbone.1.0.block.0.1.num_batches_tracked" in sd: #combined effnet and previewer for stable cascade
+ self.first_stage_model = StageC_coder()
+ self.downscale_ratio = 32
+ self.latent_channels = 16
else:
#default SD1.x/SD2.x VAE parameters
ddconfig = {'double_z': True, 'z_channels': 4, 'resolution': 256, 'in_channels': 3, 'out_ch': 3, 'ch': 128, 'ch_mult': [1, 2, 4, 4], 'num_res_blocks': 2, 'attn_resolutions': [], 'dropout': 0.0}
@@ -178,6 +217,7 @@ class VAE:
if 'encoder.down.2.downsample.conv.weight' not in sd: #Stable diffusion x4 upscaler VAE
ddconfig['ch_mult'] = [1, 2, 4]
self.downscale_ratio = 4
+ self.upscale_ratio = 4
self.first_stage_model = AutoencoderKL(ddconfig=ddconfig, embed_dim=4)
else:
@@ -186,10 +226,10 @@ class VAE:
m, u = self.first_stage_model.load_state_dict(sd, strict=False)
if len(m) > 0:
- print("Missing VAE keys", m)
+ logging.warning("Missing VAE keys {}".format(m))
if len(u) > 0:
- print("Leftover VAE keys", u)
+ logging.debug("Leftover VAE keys {}".format(u))
if device is None:
device = model_management.vae_device()
@@ -203,18 +243,27 @@ class VAE:
self.patcher = comfy.model_patcher.ModelPatcher(self.first_stage_model, load_device=self.device, offload_device=offload_device)
+ def vae_encode_crop_pixels(self, pixels):
+ x = (pixels.shape[1] // self.downscale_ratio) * self.downscale_ratio
+ y = (pixels.shape[2] // self.downscale_ratio) * self.downscale_ratio
+ if pixels.shape[1] != x or pixels.shape[2] != y:
+ x_offset = (pixels.shape[1] % self.downscale_ratio) // 2
+ y_offset = (pixels.shape[2] % self.downscale_ratio) // 2
+ pixels = pixels[:, x_offset:x + x_offset, y_offset:y + y_offset, :]
+ return pixels
+
def decode_tiled_(self, samples, tile_x=64, tile_y=64, overlap = 16):
steps = samples.shape[0] * comfy.utils.get_tiled_scale_steps(samples.shape[3], samples.shape[2], tile_x, tile_y, overlap)
steps += samples.shape[0] * comfy.utils.get_tiled_scale_steps(samples.shape[3], samples.shape[2], tile_x // 2, tile_y * 2, overlap)
steps += samples.shape[0] * comfy.utils.get_tiled_scale_steps(samples.shape[3], samples.shape[2], tile_x * 2, tile_y // 2, overlap)
pbar = comfy.utils.ProgressBar(steps)
- decode_fn = lambda a: (self.first_stage_model.decode(a.to(self.vae_dtype).to(self.device)) + 1.0).float()
- output = torch.clamp((
- (comfy.utils.tiled_scale(samples, decode_fn, tile_x // 2, tile_y * 2, overlap, upscale_amount = self.downscale_ratio, output_device=self.output_device, pbar = pbar) +
- comfy.utils.tiled_scale(samples, decode_fn, tile_x * 2, tile_y // 2, overlap, upscale_amount = self.downscale_ratio, output_device=self.output_device, pbar = pbar) +
- comfy.utils.tiled_scale(samples, decode_fn, tile_x, tile_y, overlap, upscale_amount = self.downscale_ratio, output_device=self.output_device, pbar = pbar))
- / 3.0) / 2.0, min=0.0, max=1.0)
+ decode_fn = lambda a: self.first_stage_model.decode(a.to(self.vae_dtype).to(self.device)).float()
+ output = self.process_output(
+ (comfy.utils.tiled_scale(samples, decode_fn, tile_x // 2, tile_y * 2, overlap, upscale_amount = self.upscale_ratio, output_device=self.output_device, pbar = pbar) +
+ comfy.utils.tiled_scale(samples, decode_fn, tile_x * 2, tile_y // 2, overlap, upscale_amount = self.upscale_ratio, output_device=self.output_device, pbar = pbar) +
+ comfy.utils.tiled_scale(samples, decode_fn, tile_x, tile_y, overlap, upscale_amount = self.upscale_ratio, output_device=self.output_device, pbar = pbar))
+ / 3.0)
return output
def encode_tiled_(self, pixel_samples, tile_x=512, tile_y=512, overlap = 64):
@@ -223,7 +272,7 @@ class VAE:
steps += pixel_samples.shape[0] * comfy.utils.get_tiled_scale_steps(pixel_samples.shape[3], pixel_samples.shape[2], tile_x * 2, tile_y // 2, overlap)
pbar = comfy.utils.ProgressBar(steps)
- encode_fn = lambda a: self.first_stage_model.encode((2. * a - 1.).to(self.vae_dtype).to(self.device)).float()
+ encode_fn = lambda a: self.first_stage_model.encode((self.process_input(a)).to(self.vae_dtype).to(self.device)).float()
samples = comfy.utils.tiled_scale(pixel_samples, encode_fn, tile_x, tile_y, overlap, upscale_amount = (1/self.downscale_ratio), out_channels=self.latent_channels, output_device=self.output_device, pbar=pbar)
samples += comfy.utils.tiled_scale(pixel_samples, encode_fn, tile_x * 2, tile_y // 2, overlap, upscale_amount = (1/self.downscale_ratio), out_channels=self.latent_channels, output_device=self.output_device, pbar=pbar)
samples += comfy.utils.tiled_scale(pixel_samples, encode_fn, tile_x // 2, tile_y * 2, overlap, upscale_amount = (1/self.downscale_ratio), out_channels=self.latent_channels, output_device=self.output_device, pbar=pbar)
@@ -238,12 +287,12 @@ class VAE:
batch_number = int(free_memory / memory_used)
batch_number = max(1, batch_number)
- pixel_samples = torch.empty((samples_in.shape[0], 3, round(samples_in.shape[2] * self.downscale_ratio), round(samples_in.shape[3] * self.downscale_ratio)), device=self.output_device)
+ pixel_samples = torch.empty((samples_in.shape[0], 3, round(samples_in.shape[2] * self.upscale_ratio), round(samples_in.shape[3] * self.upscale_ratio)), device=self.output_device)
for x in range(0, samples_in.shape[0], batch_number):
samples = samples_in[x:x+batch_number].to(self.vae_dtype).to(self.device)
- pixel_samples[x:x+batch_number] = torch.clamp((self.first_stage_model.decode(samples).to(self.output_device).float() + 1.0) / 2.0, min=0.0, max=1.0)
+ pixel_samples[x:x+batch_number] = self.process_output(self.first_stage_model.decode(samples).to(self.output_device).float())
except model_management.OOM_EXCEPTION as e:
- print("Warning: Ran out of memory when regular VAE decoding, retrying with tiled VAE decoding.")
+ logging.warning("Warning: Ran out of memory when regular VAE decoding, retrying with tiled VAE decoding.")
pixel_samples = self.decode_tiled_(samples_in)
pixel_samples = pixel_samples.to(self.output_device).movedim(1,-1)
@@ -255,6 +304,7 @@ class VAE:
return output.movedim(1,-1)
def encode(self, pixel_samples):
+ pixel_samples = self.vae_encode_crop_pixels(pixel_samples)
pixel_samples = pixel_samples.movedim(-1,1)
try:
memory_used = self.memory_used_encode(pixel_samples.shape, self.vae_dtype)
@@ -264,16 +314,17 @@ class VAE:
batch_number = max(1, batch_number)
samples = torch.empty((pixel_samples.shape[0], self.latent_channels, round(pixel_samples.shape[2] // self.downscale_ratio), round(pixel_samples.shape[3] // self.downscale_ratio)), device=self.output_device)
for x in range(0, pixel_samples.shape[0], batch_number):
- pixels_in = (2. * pixel_samples[x:x+batch_number] - 1.).to(self.vae_dtype).to(self.device)
+ pixels_in = self.process_input(pixel_samples[x:x+batch_number]).to(self.vae_dtype).to(self.device)
samples[x:x+batch_number] = self.first_stage_model.encode(pixels_in).to(self.output_device).float()
except model_management.OOM_EXCEPTION as e:
- print("Warning: Ran out of memory when regular VAE encoding, retrying with tiled VAE encoding.")
+ logging.warning("Warning: Ran out of memory when regular VAE encoding, retrying with tiled VAE encoding.")
samples = self.encode_tiled_(pixel_samples)
return samples
def encode_tiled(self, pixel_samples, tile_x=512, tile_y=512, overlap = 64):
+ pixel_samples = self.vae_encode_crop_pixels(pixel_samples)
model_management.load_model_gpu(self.patcher)
pixel_samples = pixel_samples.movedim(-1,1)
samples = self.encode_tiled_(pixel_samples, tile_x=tile_x, tile_y=tile_y, overlap=overlap)
@@ -300,8 +351,11 @@ def load_style_model(ckpt_path):
model.load_state_dict(model_data)
return StyleModel(model)
+class CLIPType(Enum):
+ STABLE_DIFFUSION = 1
+ STABLE_CASCADE = 2
-def load_clip(ckpt_paths, embedding_directory=None):
+def load_clip(ckpt_paths, embedding_directory=None, clip_type=CLIPType.STABLE_DIFFUSION):
clip_data = []
for p in ckpt_paths:
clip_data.append(comfy.utils.load_torch_file(p, safe_load=True))
@@ -311,14 +365,21 @@ def load_clip(ckpt_paths, embedding_directory=None):
for i in range(len(clip_data)):
if "transformer.resblocks.0.ln_1.weight" in clip_data[i]:
- clip_data[i] = comfy.utils.transformers_convert(clip_data[i], "", "text_model.", 32)
+ clip_data[i] = comfy.utils.clip_text_transformers_convert(clip_data[i], "", "")
+ else:
+ if "text_projection" in clip_data[i]:
+ clip_data[i]["text_projection.weight"] = clip_data[i]["text_projection"].transpose(0, 1) #old models saved with the CLIPSave node
clip_target = EmptyClass()
clip_target.params = {}
if len(clip_data) == 1:
if "text_model.encoder.layers.30.mlp.fc1.weight" in clip_data[0]:
- clip_target.clip = sdxl_clip.SDXLRefinerClipModel
- clip_target.tokenizer = sdxl_clip.SDXLTokenizer
+ if clip_type == CLIPType.STABLE_CASCADE:
+ clip_target.clip = sdxl_clip.StableCascadeClipModel
+ clip_target.tokenizer = sdxl_clip.StableCascadeTokenizer
+ else:
+ clip_target.clip = sdxl_clip.SDXLRefinerClipModel
+ clip_target.tokenizer = sdxl_clip.SDXLTokenizer
elif "text_model.encoder.layers.22.mlp.fc1.weight" in clip_data[0]:
clip_target.clip = sd2_clip.SD2ClipModel
clip_target.tokenizer = sd2_clip.SD2Tokenizer
@@ -333,10 +394,10 @@ def load_clip(ckpt_paths, embedding_directory=None):
for c in clip_data:
m, u = clip.load_sd(c)
if len(m) > 0:
- print("clip missing:", m)
+ logging.warning("clip missing: {}".format(m))
if len(u) > 0:
- print("clip unexpected:", u)
+ logging.debug("clip unexpected: {}".format(u))
return clip
def load_gligen(ckpt_path):
@@ -441,15 +502,12 @@ def load_checkpoint_guess_config(ckpt_path, output_vae=True, output_clip=True, o
clip_target = None
parameters = comfy.utils.calculate_parameters(sd, "model.diffusion_model.")
- unet_dtype = model_management.unet_dtype(model_params=parameters)
load_device = model_management.get_torch_device()
- manual_cast_dtype = model_management.unet_manual_cast(unet_dtype, load_device)
- class WeightsLoader(torch.nn.Module):
- pass
-
- model_config = model_detection.model_config_from_unet(sd, "model.diffusion_model.", unet_dtype)
- model_config.set_manual_cast(manual_cast_dtype)
+ model_config = model_detection.model_config_from_unet(sd, "model.diffusion_model.")
+ unet_dtype = model_management.unet_dtype(model_params=parameters, supported_dtypes=model_config.supported_inference_dtypes)
+ manual_cast_dtype = model_management.unet_manual_cast(unet_dtype, load_device, model_config.supported_inference_dtypes)
+ model_config.set_inference_dtype(unet_dtype, manual_cast_dtype)
if model_config is None:
raise RuntimeError("ERROR: Could not detect model type of: {}".format(ckpt_path))
@@ -465,27 +523,33 @@ def load_checkpoint_guess_config(ckpt_path, output_vae=True, output_clip=True, o
model.load_model_weights(sd, "model.diffusion_model.")
if output_vae:
- vae_sd = comfy.utils.state_dict_prefix_replace(sd, {"first_stage_model.": ""}, filter_keys=True)
+ vae_sd = comfy.utils.state_dict_prefix_replace(sd, {k: "" for k in model_config.vae_key_prefix}, filter_keys=True)
vae_sd = model_config.process_vae_state_dict(vae_sd)
vae = VAE(sd=vae_sd)
if output_clip:
- w = WeightsLoader()
clip_target = model_config.clip_target()
if clip_target is not None:
- clip = CLIP(clip_target, embedding_directory=embedding_directory)
- w.cond_stage_model = clip.cond_stage_model
- sd = model_config.process_clip_state_dict(sd)
- load_model_weights(w, sd)
+ clip_sd = model_config.process_clip_state_dict(sd)
+ if len(clip_sd) > 0:
+ clip = CLIP(clip_target, embedding_directory=embedding_directory)
+ m, u = clip.load_sd(clip_sd, full_model=True)
+ if len(m) > 0:
+ logging.warning("clip missing: {}".format(m))
+
+ if len(u) > 0:
+ logging.debug("clip unexpected {}:".format(u))
+ else:
+ logging.warning("no CLIP/text encoder weights in checkpoint, the text encoder model will not be loaded.")
left_over = sd.keys()
if len(left_over) > 0:
- print("left over keys:", left_over)
+ logging.debug("left over keys: {}".format(left_over))
if output_model:
model_patcher = comfy.model_patcher.ModelPatcher(model, load_device=load_device, offload_device=model_management.unet_offload_device(), current_device=inital_load_device)
if inital_load_device != torch.device("cpu"):
- print("loaded straight to GPU")
+ logging.info("loaded straight to GPU")
model_management.load_model_gpu(model_patcher)
return (model_patcher, clip, vae, clipvision)
@@ -495,16 +559,15 @@ def load_unet_state_dict(sd): #load unet in diffusers format
parameters = comfy.utils.calculate_parameters(sd)
unet_dtype = model_management.unet_dtype(model_params=parameters)
load_device = model_management.get_torch_device()
- manual_cast_dtype = model_management.unet_manual_cast(unet_dtype, load_device)
- if "input_blocks.0.0.weight" in sd: #ldm
- model_config = model_detection.model_config_from_unet(sd, "", unet_dtype)
+ if "input_blocks.0.0.weight" in sd or 'clf.1.weight' in sd: #ldm or stable cascade
+ model_config = model_detection.model_config_from_unet(sd, "")
if model_config is None:
return None
new_sd = sd
else: #diffusers
- model_config = model_detection.model_config_from_diffusers_unet(sd, unet_dtype)
+ model_config = model_detection.model_config_from_diffusers_unet(sd)
if model_config is None:
return None
@@ -515,26 +578,29 @@ def load_unet_state_dict(sd): #load unet in diffusers format
if k in sd:
new_sd[diffusers_keys[k]] = sd.pop(k)
else:
- print(diffusers_keys[k], k)
+ logging.warning("{} {}".format(diffusers_keys[k], k))
+
offload_device = model_management.unet_offload_device()
- model_config.set_manual_cast(manual_cast_dtype)
+ unet_dtype = model_management.unet_dtype(model_params=parameters, supported_dtypes=model_config.supported_inference_dtypes)
+ manual_cast_dtype = model_management.unet_manual_cast(unet_dtype, load_device, model_config.supported_inference_dtypes)
+ model_config.set_inference_dtype(unet_dtype, manual_cast_dtype)
model = model_config.get_model(new_sd, "")
model = model.to(offload_device)
model.load_model_weights(new_sd, "")
left_over = sd.keys()
if len(left_over) > 0:
- print("left over keys in unet:", left_over)
+ logging.info("left over keys in unet: {}".format(left_over))
return comfy.model_patcher.ModelPatcher(model, load_device=load_device, offload_device=offload_device)
def load_unet(unet_path):
sd = comfy.utils.load_torch_file(unet_path)
model = load_unet_state_dict(sd)
if model is None:
- print("ERROR UNSUPPORTED UNET", unet_path)
+ logging.error("ERROR UNSUPPORTED UNET {}".format(unet_path))
raise RuntimeError("ERROR: Could not detect model type of: {}".format(unet_path))
return model
-def save_checkpoint(output_path, model, clip=None, vae=None, clip_vision=None, metadata=None):
+def save_checkpoint(output_path, model, clip=None, vae=None, clip_vision=None, metadata=None, extra_keys={}):
clip_sd = None
load_models = [model]
if clip is not None:
@@ -544,4 +610,7 @@ def save_checkpoint(output_path, model, clip=None, vae=None, clip_vision=None, m
model_management.load_models_gpu(load_models)
clip_vision_sd = clip_vision.get_sd() if clip_vision is not None else None
sd = model.model.state_dict_for_saving(clip_sd, vae.get_sd(), clip_vision_sd)
+ for k in extra_keys:
+ sd[k] = extra_keys[k]
+
comfy.utils.save_torch_file(sd, output_path, metadata=metadata)
diff --git a/comfy/sd1_clip.py b/comfy/sd1_clip.py
index 6ffef515e..ff6db0d20 100644
--- a/comfy/sd1_clip.py
+++ b/comfy/sd1_clip.py
@@ -6,9 +6,9 @@ import torch
import traceback
import zipfile
from . import model_management
-import contextlib
import comfy.clip_model
import json
+import logging
def gen_empty_tokens(special_tokens, length):
start_token = special_tokens.get("start", None)
@@ -68,7 +68,7 @@ class SDClipModel(torch.nn.Module, ClipTokenWeightEncoder):
]
def __init__(self, version="openai/clip-vit-large-patch14", device="cpu", max_length=77,
freeze=True, layer="last", layer_idx=None, textmodel_json_config=None, dtype=None, model_class=comfy.clip_model.CLIPTextModel,
- special_tokens={"start": 49406, "end": 49407, "pad": 49407}, layer_norm_hidden_state=True): # clip-vit-base-patch32
+ special_tokens={"start": 49406, "end": 49407, "pad": 49407}, layer_norm_hidden_state=True, enable_attention_masks=False, return_projected_pooled=True): # clip-vit-base-patch32
super().__init__()
assert layer in self.LAYERS
@@ -87,16 +87,18 @@ class SDClipModel(torch.nn.Module, ClipTokenWeightEncoder):
self.layer = layer
self.layer_idx = None
self.special_tokens = special_tokens
- self.text_projection = torch.nn.Parameter(torch.eye(self.transformer.get_input_embeddings().weight.shape[1]))
+
self.logit_scale = torch.nn.Parameter(torch.tensor(4.6055))
- self.enable_attention_masks = False
+ self.enable_attention_masks = enable_attention_masks
self.layer_norm_hidden_state = layer_norm_hidden_state
+ self.return_projected_pooled = return_projected_pooled
+
if layer == "hidden":
assert layer_idx is not None
assert abs(layer_idx) < self.num_layers
- self.clip_layer(layer_idx)
- self.layer_default = (self.layer, self.layer_idx)
+ self.set_clip_options({"layer": layer_idx})
+ self.options_default = (self.layer, self.layer_idx, self.return_projected_pooled)
def freeze(self):
self.transformer = self.transformer.eval()
@@ -104,16 +106,19 @@ class SDClipModel(torch.nn.Module, ClipTokenWeightEncoder):
for param in self.parameters():
param.requires_grad = False
- def clip_layer(self, layer_idx):
- if abs(layer_idx) > self.num_layers:
+ def set_clip_options(self, options):
+ layer_idx = options.get("layer", self.layer_idx)
+ self.return_projected_pooled = options.get("projected_pooled", self.return_projected_pooled)
+ if layer_idx is None or abs(layer_idx) > self.num_layers:
self.layer = "last"
else:
self.layer = "hidden"
self.layer_idx = layer_idx
- def reset_clip_layer(self):
- self.layer = self.layer_default[0]
- self.layer_idx = self.layer_default[1]
+ def reset_clip_options(self):
+ self.layer = self.options_default[0]
+ self.layer_idx = self.options_default[1]
+ self.return_projected_pooled = self.options_default[2]
def set_up_textual_embeddings(self, tokens, current_embeds):
out_tokens = []
@@ -133,7 +138,7 @@ class SDClipModel(torch.nn.Module, ClipTokenWeightEncoder):
tokens_temp += [next_new_token]
next_new_token += 1
else:
- print("WARNING: shape mismatch when trying to apply embedding, embedding will be ignored", y.shape[0], current_embeds.weight.shape[1])
+ logging.warning("WARNING: shape mismatch when trying to apply embedding, embedding will be ignored {} != {}".format(y.shape[0], current_embeds.weight.shape[1]))
while len(tokens_temp) < len(x):
tokens_temp += [self.special_tokens["pad"]]
out_tokens += [tokens_temp]
@@ -178,23 +183,19 @@ class SDClipModel(torch.nn.Module, ClipTokenWeightEncoder):
else:
z = outputs[1]
- if outputs[2] is not None:
- pooled_output = outputs[2].float()
- else:
- pooled_output = None
+ pooled_output = None
+ if len(outputs) >= 3:
+ if not self.return_projected_pooled and len(outputs) >= 4 and outputs[3] is not None:
+ pooled_output = outputs[3].float()
+ elif outputs[2] is not None:
+ pooled_output = outputs[2].float()
- if self.text_projection is not None and pooled_output is not None:
- pooled_output = pooled_output.float().to(self.text_projection.device) @ self.text_projection.float()
return z.float(), pooled_output
def encode(self, tokens):
return self(tokens)
def load_sd(self, sd):
- if "text_projection" in sd:
- self.text_projection[:] = sd.pop("text_projection")
- if "text_projection.weight" in sd:
- self.text_projection[:] = sd.pop("text_projection.weight").transpose(0, 1)
return self.transformer.load_state_dict(sd, strict=False)
def parse_parentheses(string):
@@ -329,9 +330,7 @@ def load_embed(embedding_name, embedding_directory, embedding_size, embed_key=No
else:
embed = torch.load(embed_path, map_location="cpu")
except Exception as e:
- print(traceback.format_exc())
- print()
- print("error loading embedding, skipping loading:", embedding_name)
+ logging.warning("{}\n\nerror loading embedding, skipping loading: {}".format(traceback.format_exc(), embedding_name))
return None
if embed_out is None:
@@ -355,11 +354,12 @@ def load_embed(embedding_name, embedding_directory, embedding_size, embed_key=No
return embed_out
class SDTokenizer:
- def __init__(self, tokenizer_path=None, max_length=77, pad_with_end=True, embedding_directory=None, embedding_size=768, embedding_key='clip_l', tokenizer_class=CLIPTokenizer, has_start_token=True, pad_to_max_length=True):
+ def __init__(self, tokenizer_path=None, max_length=77, pad_with_end=True, embedding_directory=None, embedding_size=768, embedding_key='clip_l', tokenizer_class=CLIPTokenizer, has_start_token=True, pad_to_max_length=True, min_length=None):
if tokenizer_path is None:
tokenizer_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "sd1_tokenizer")
self.tokenizer = tokenizer_class.from_pretrained(tokenizer_path)
self.max_length = max_length
+ self.min_length = min_length
empty = self.tokenizer('')["input_ids"]
if has_start_token:
@@ -421,7 +421,7 @@ class SDTokenizer:
embedding_name = word[len(self.embedding_identifier):].strip('\n')
embed, leftover = self._try_get_embedding(embedding_name)
if embed is None:
- print(f"warning, embedding:{embedding_name} does not exist, ignoring")
+ logging.warning(f"warning, embedding:{embedding_name} does not exist, ignoring")
else:
if len(embed.shape) == 1:
tokens.append([(embed, weight)])
@@ -471,6 +471,8 @@ class SDTokenizer:
batch.append((self.end_token, 1.0, 0))
if self.pad_to_max_length:
batch.extend([(pad_token, 1.0, 0)] * (self.max_length - len(batch)))
+ if self.min_length is not None and len(batch) < self.min_length:
+ batch.extend([(pad_token, 1.0, 0)] * (self.min_length - len(batch)))
if not return_word_ids:
batched_tokens = [[(t, w) for t, w,_ in x] for x in batched_tokens]
@@ -504,11 +506,11 @@ class SD1ClipModel(torch.nn.Module):
self.clip = "clip_{}".format(self.clip_name)
setattr(self, self.clip, clip_model(device=device, dtype=dtype, **kwargs))
- def clip_layer(self, layer_idx):
- getattr(self, self.clip).clip_layer(layer_idx)
+ def set_clip_options(self, options):
+ getattr(self, self.clip).set_clip_options(options)
- def reset_clip_layer(self):
- getattr(self, self.clip).reset_clip_layer()
+ def reset_clip_options(self):
+ getattr(self, self.clip).reset_clip_options()
def encode_token_weights(self, token_weight_pairs):
token_weight_pairs = token_weight_pairs[self.clip_name]
diff --git a/comfy/sdxl_clip.py b/comfy/sdxl_clip.py
index b35056bb9..e62d1ed86 100644
--- a/comfy/sdxl_clip.py
+++ b/comfy/sdxl_clip.py
@@ -40,13 +40,13 @@ class SDXLClipModel(torch.nn.Module):
self.clip_l = sd1_clip.SDClipModel(layer="hidden", layer_idx=-2, device=device, dtype=dtype, layer_norm_hidden_state=False)
self.clip_g = SDXLClipG(device=device, dtype=dtype)
- def clip_layer(self, layer_idx):
- self.clip_l.clip_layer(layer_idx)
- self.clip_g.clip_layer(layer_idx)
+ def set_clip_options(self, options):
+ self.clip_l.set_clip_options(options)
+ self.clip_g.set_clip_options(options)
- def reset_clip_layer(self):
- self.clip_g.reset_clip_layer()
- self.clip_l.reset_clip_layer()
+ def reset_clip_options(self):
+ self.clip_g.reset_clip_options()
+ self.clip_l.reset_clip_options()
def encode_token_weights(self, token_weight_pairs):
token_weight_pairs_g = token_weight_pairs["g"]
@@ -64,3 +64,25 @@ class SDXLClipModel(torch.nn.Module):
class SDXLRefinerClipModel(sd1_clip.SD1ClipModel):
def __init__(self, device="cpu", dtype=None):
super().__init__(device=device, dtype=dtype, clip_name="g", clip_model=SDXLClipG)
+
+
+class StableCascadeClipGTokenizer(sd1_clip.SDTokenizer):
+ def __init__(self, tokenizer_path=None, embedding_directory=None):
+ super().__init__(tokenizer_path, pad_with_end=True, embedding_directory=embedding_directory, embedding_size=1280, embedding_key='clip_g')
+
+class StableCascadeTokenizer(sd1_clip.SD1Tokenizer):
+ def __init__(self, embedding_directory=None):
+ super().__init__(embedding_directory=embedding_directory, clip_name="g", tokenizer=StableCascadeClipGTokenizer)
+
+class StableCascadeClipG(sd1_clip.SDClipModel):
+ def __init__(self, device="cpu", max_length=77, freeze=True, layer="hidden", layer_idx=-1, dtype=None):
+ textmodel_json_config = os.path.join(os.path.dirname(os.path.realpath(__file__)), "clip_config_bigg.json")
+ super().__init__(device=device, freeze=freeze, layer=layer, layer_idx=layer_idx, textmodel_json_config=textmodel_json_config, dtype=dtype,
+ special_tokens={"start": 49406, "end": 49407, "pad": 49407}, layer_norm_hidden_state=False, enable_attention_masks=True)
+
+ def load_sd(self, sd):
+ return super().load_sd(sd)
+
+class StableCascadeClipModel(sd1_clip.SD1ClipModel):
+ def __init__(self, device="cpu", dtype=None):
+ super().__init__(device=device, dtype=dtype, clip_name="g", clip_model=StableCascadeClipG)
diff --git a/comfy/supported_models.py b/comfy/supported_models.py
index 1d442d4dd..b3b69e05b 100644
--- a/comfy/supported_models.py
+++ b/comfy/supported_models.py
@@ -40,11 +40,16 @@ class SD15(supported_models_base.BASE):
state_dict['cond_stage_model.transformer.text_model.embeddings.position_ids'] = ids.round()
replace_prefix = {}
- replace_prefix["cond_stage_model."] = "cond_stage_model.clip_l."
- state_dict = utils.state_dict_prefix_replace(state_dict, replace_prefix)
+ replace_prefix["cond_stage_model."] = "clip_l."
+ state_dict = utils.state_dict_prefix_replace(state_dict, replace_prefix, filter_keys=True)
return state_dict
def process_clip_state_dict_for_saving(self, state_dict):
+ pop_keys = ["clip_l.transformer.text_projection.weight", "clip_l.logit_scale"]
+ for p in pop_keys:
+ if p in state_dict:
+ state_dict.pop(p)
+
replace_prefix = {"clip_l.": "cond_stage_model."}
return utils.state_dict_prefix_replace(state_dict, replace_prefix)
@@ -65,17 +70,17 @@ class SD20(supported_models_base.BASE):
def model_type(self, state_dict, prefix=""):
if self.unet_config["in_channels"] == 4: #SD2.0 inpainting models are not v prediction
k = "{}output_blocks.11.1.transformer_blocks.0.norm1.bias".format(prefix)
- out = state_dict[k]
- if torch.std(out, unbiased=False) > 0.09: # not sure how well this will actually work. I guess we will find out.
+ out = state_dict.get(k, None)
+ if out is not None and torch.std(out, unbiased=False) > 0.09: # not sure how well this will actually work. I guess we will find out.
return model_base.ModelType.V_PREDICTION
return model_base.ModelType.EPS
def process_clip_state_dict(self, state_dict):
replace_prefix = {}
- replace_prefix["conditioner.embedders.0.model."] = "cond_stage_model.model." #SD2 in sgm format
- state_dict = utils.state_dict_prefix_replace(state_dict, replace_prefix)
-
- state_dict = utils.transformers_convert(state_dict, "cond_stage_model.model.", "cond_stage_model.clip_h.transformer.text_model.", 24)
+ replace_prefix["conditioner.embedders.0.model."] = "clip_h." #SD2 in sgm format
+ replace_prefix["cond_stage_model.model."] = "clip_h."
+ state_dict = utils.state_dict_prefix_replace(state_dict, replace_prefix, filter_keys=True)
+ state_dict = utils.clip_text_transformers_convert(state_dict, "clip_h.", "clip_h.transformer.")
return state_dict
def process_clip_state_dict_for_saving(self, state_dict):
@@ -131,11 +136,10 @@ class SDXLRefiner(supported_models_base.BASE):
def process_clip_state_dict(self, state_dict):
keys_to_replace = {}
replace_prefix = {}
+ replace_prefix["conditioner.embedders.0.model."] = "clip_g."
+ state_dict = utils.state_dict_prefix_replace(state_dict, replace_prefix, filter_keys=True)
- state_dict = utils.transformers_convert(state_dict, "conditioner.embedders.0.model.", "cond_stage_model.clip_g.transformer.text_model.", 32)
- keys_to_replace["conditioner.embedders.0.model.text_projection"] = "cond_stage_model.clip_g.text_projection"
- keys_to_replace["conditioner.embedders.0.model.logit_scale"] = "cond_stage_model.clip_g.logit_scale"
-
+ state_dict = utils.clip_text_transformers_convert(state_dict, "clip_g.", "clip_g.transformer.")
state_dict = utils.state_dict_key_replace(state_dict, keys_to_replace)
return state_dict
@@ -164,7 +168,18 @@ class SDXL(supported_models_base.BASE):
latent_format = latent_formats.SDXL
def model_type(self, state_dict, prefix=""):
- if "v_pred" in state_dict:
+ if 'edm_mean' in state_dict and 'edm_std' in state_dict: #Playground V2.5
+ self.latent_format = latent_formats.SDXL_Playground_2_5()
+ self.sampling_settings["sigma_data"] = 0.5
+ self.sampling_settings["sigma_max"] = 80.0
+ self.sampling_settings["sigma_min"] = 0.002
+ return model_base.ModelType.EDM
+ elif "edm_vpred.sigma_max" in state_dict:
+ self.sampling_settings["sigma_max"] = float(state_dict["edm_vpred.sigma_max"].item())
+ if "edm_vpred.sigma_min" in state_dict:
+ self.sampling_settings["sigma_min"] = float(state_dict["edm_vpred.sigma_min"].item())
+ return model_base.ModelType.V_PREDICTION_EDM
+ elif "v_pred" in state_dict:
return model_base.ModelType.V_PREDICTION
else:
return model_base.ModelType.EPS
@@ -179,26 +194,28 @@ class SDXL(supported_models_base.BASE):
keys_to_replace = {}
replace_prefix = {}
- replace_prefix["conditioner.embedders.0.transformer.text_model"] = "cond_stage_model.clip_l.transformer.text_model"
- state_dict = utils.transformers_convert(state_dict, "conditioner.embedders.1.model.", "cond_stage_model.clip_g.transformer.text_model.", 32)
- keys_to_replace["conditioner.embedders.1.model.text_projection"] = "cond_stage_model.clip_g.text_projection"
- keys_to_replace["conditioner.embedders.1.model.text_projection.weight"] = "cond_stage_model.clip_g.text_projection"
- keys_to_replace["conditioner.embedders.1.model.logit_scale"] = "cond_stage_model.clip_g.logit_scale"
+ replace_prefix["conditioner.embedders.0.transformer.text_model"] = "clip_l.transformer.text_model"
+ replace_prefix["conditioner.embedders.1.model."] = "clip_g."
+ state_dict = utils.state_dict_prefix_replace(state_dict, replace_prefix, filter_keys=True)
- state_dict = utils.state_dict_prefix_replace(state_dict, replace_prefix)
state_dict = utils.state_dict_key_replace(state_dict, keys_to_replace)
+ state_dict = utils.clip_text_transformers_convert(state_dict, "clip_g.", "clip_g.transformer.")
return state_dict
def process_clip_state_dict_for_saving(self, state_dict):
replace_prefix = {}
keys_to_replace = {}
state_dict_g = diffusers_convert.convert_text_enc_state_dict_v20(state_dict, "clip_g")
- if "clip_g.transformer.text_model.embeddings.position_ids" in state_dict_g:
- state_dict_g.pop("clip_g.transformer.text_model.embeddings.position_ids")
for k in state_dict:
if k.startswith("clip_l"):
state_dict_g[k] = state_dict[k]
+ state_dict_g["clip_l.transformer.text_model.embeddings.position_ids"] = torch.arange(77).expand((1, -1))
+ pop_keys = ["clip_l.transformer.text_projection.weight", "clip_l.logit_scale"]
+ for p in pop_keys:
+ if p in state_dict_g:
+ state_dict_g.pop(p)
+
replace_prefix["clip_g"] = "conditioner.embedders.1.model"
replace_prefix["clip_l"] = "conditioner.embedders.0"
state_dict_g = utils.state_dict_prefix_replace(state_dict_g, replace_prefix)
@@ -227,6 +244,26 @@ class Segmind_Vega(SDXL):
"use_temporal_attention": False,
}
+class KOALA_700M(SDXL):
+ unet_config = {
+ "model_channels": 320,
+ "use_linear_in_transformer": True,
+ "transformer_depth": [0, 2, 5],
+ "context_dim": 2048,
+ "adm_in_channels": 2816,
+ "use_temporal_attention": False,
+ }
+
+class KOALA_1B(SDXL):
+ unet_config = {
+ "model_channels": 320,
+ "use_linear_in_transformer": True,
+ "transformer_depth": [0, 2, 6],
+ "context_dim": 2048,
+ "adm_in_channels": 2816,
+ "use_temporal_attention": False,
+ }
+
class SVD_img2vid(supported_models_base.BASE):
unet_config = {
"model_channels": 320,
@@ -252,6 +289,41 @@ class SVD_img2vid(supported_models_base.BASE):
def clip_target(self):
return None
+class SV3D_u(SVD_img2vid):
+ unet_config = {
+ "model_channels": 320,
+ "in_channels": 8,
+ "use_linear_in_transformer": True,
+ "transformer_depth": [1, 1, 1, 1, 1, 1, 0, 0],
+ "context_dim": 1024,
+ "adm_in_channels": 256,
+ "use_temporal_attention": True,
+ "use_temporal_resblock": True
+ }
+
+ vae_key_prefix = ["conditioner.embedders.1.encoder."]
+
+ def get_model(self, state_dict, prefix="", device=None):
+ out = model_base.SV3D_u(self, device=device)
+ return out
+
+class SV3D_p(SV3D_u):
+ unet_config = {
+ "model_channels": 320,
+ "in_channels": 8,
+ "use_linear_in_transformer": True,
+ "transformer_depth": [1, 1, 1, 1, 1, 1, 0, 0],
+ "context_dim": 1024,
+ "adm_in_channels": 1280,
+ "use_temporal_attention": True,
+ "use_temporal_resblock": True
+ }
+
+
+ def get_model(self, state_dict, prefix="", device=None):
+ out = model_base.SV3D_p(self, device=device)
+ return out
+
class Stable_Zero123(supported_models_base.BASE):
unet_config = {
"context_dim": 768,
@@ -267,6 +339,11 @@ class Stable_Zero123(supported_models_base.BASE):
"num_head_channels": -1,
}
+ required_keys = {
+ "cc_projection.weight": None,
+ "cc_projection.bias": None,
+ }
+
clip_vision_prefix = "cond_stage_model.model.visual."
latent_format = latent_formats.SD15
@@ -306,5 +383,99 @@ class SD_X4Upscaler(SD20):
out = model_base.SD_X4Upscaler(self, device=device)
return out
-models = [Stable_Zero123, SD15, SD20, SD21UnclipL, SD21UnclipH, SDXLRefiner, SDXL, SSD1B, Segmind_Vega, SD_X4Upscaler]
+class Stable_Cascade_C(supported_models_base.BASE):
+ unet_config = {
+ "stable_cascade_stage": 'c',
+ }
+
+ unet_extra_config = {}
+
+ latent_format = latent_formats.SC_Prior
+ supported_inference_dtypes = [torch.bfloat16, torch.float32]
+
+ sampling_settings = {
+ "shift": 2.0,
+ }
+
+ vae_key_prefix = ["vae."]
+ text_encoder_key_prefix = ["text_encoder."]
+ clip_vision_prefix = "clip_l_vision."
+
+ def process_unet_state_dict(self, state_dict):
+ key_list = list(state_dict.keys())
+ for y in ["weight", "bias"]:
+ suffix = "in_proj_{}".format(y)
+ keys = filter(lambda a: a.endswith(suffix), key_list)
+ for k_from in keys:
+ weights = state_dict.pop(k_from)
+ prefix = k_from[:-(len(suffix) + 1)]
+ shape_from = weights.shape[0] // 3
+ for x in range(3):
+ p = ["to_q", "to_k", "to_v"]
+ k_to = "{}.{}.{}".format(prefix, p[x], y)
+ state_dict[k_to] = weights[shape_from*x:shape_from*(x + 1)]
+ return state_dict
+
+ def process_clip_state_dict(self, state_dict):
+ state_dict = utils.state_dict_prefix_replace(state_dict, {k: "" for k in self.text_encoder_key_prefix}, filter_keys=True)
+ if "clip_g.text_projection" in state_dict:
+ state_dict["clip_g.transformer.text_projection.weight"] = state_dict.pop("clip_g.text_projection").transpose(0, 1)
+ return state_dict
+
+ def get_model(self, state_dict, prefix="", device=None):
+ out = model_base.StableCascade_C(self, device=device)
+ return out
+
+ def clip_target(self):
+ return supported_models_base.ClipTarget(sdxl_clip.StableCascadeTokenizer, sdxl_clip.StableCascadeClipModel)
+
+class Stable_Cascade_B(Stable_Cascade_C):
+ unet_config = {
+ "stable_cascade_stage": 'b',
+ }
+
+ unet_extra_config = {}
+
+ latent_format = latent_formats.SC_B
+ supported_inference_dtypes = [torch.float16, torch.bfloat16, torch.float32]
+
+ sampling_settings = {
+ "shift": 1.0,
+ }
+
+ clip_vision_prefix = None
+
+ def get_model(self, state_dict, prefix="", device=None):
+ out = model_base.StableCascade_B(self, device=device)
+ return out
+
+class SD15_instructpix2pix(SD15):
+ unet_config = {
+ "context_dim": 768,
+ "model_channels": 320,
+ "use_linear_in_transformer": False,
+ "adm_in_channels": None,
+ "use_temporal_attention": False,
+ "in_channels": 8,
+ }
+
+ def get_model(self, state_dict, prefix="", device=None):
+ return model_base.SD15_instructpix2pix(self, device=device)
+
+class SDXL_instructpix2pix(SDXL):
+ unet_config = {
+ "model_channels": 320,
+ "use_linear_in_transformer": True,
+ "transformer_depth": [0, 0, 2, 2, 10, 10],
+ "context_dim": 2048,
+ "adm_in_channels": 2816,
+ "use_temporal_attention": False,
+ "in_channels": 8,
+ }
+
+ def get_model(self, state_dict, prefix="", device=None):
+ return model_base.SDXL_instructpix2pix(self, model_type=self.model_type(state_dict, prefix), device=device)
+
+models = [Stable_Zero123, SD15_instructpix2pix, SD15, SD20, SD21UnclipL, SD21UnclipH, SDXL_instructpix2pix, SDXLRefiner, SDXL, SSD1B, KOALA_700M, KOALA_1B, Segmind_Vega, SD_X4Upscaler, Stable_Cascade_C, Stable_Cascade_B, SV3D_u, SV3D_p]
+
models += [SVD_img2vid]
diff --git a/comfy/supported_models_base.py b/comfy/supported_models_base.py
index 5baf4bca6..cf7cdff34 100644
--- a/comfy/supported_models_base.py
+++ b/comfy/supported_models_base.py
@@ -16,19 +16,28 @@ class BASE:
"num_head_channels": 64,
}
+ required_keys = {}
+
clip_prefix = []
clip_vision_prefix = None
noise_aug_config = None
sampling_settings = {}
latent_format = latent_formats.LatentFormat
+ vae_key_prefix = ["first_stage_model."]
+ text_encoder_key_prefix = ["cond_stage_model."]
+ supported_inference_dtypes = [torch.float16, torch.bfloat16, torch.float32]
manual_cast_dtype = None
@classmethod
- def matches(s, unet_config):
+ def matches(s, unet_config, state_dict=None):
for k in s.unet_config:
- if s.unet_config[k] != unet_config[k]:
+ if k not in unet_config or s.unet_config[k] != unet_config[k]:
return False
+ if state_dict is not None:
+ for k in s.required_keys:
+ if k not in state_dict:
+ return False
return True
def model_type(self, state_dict, prefix=""):
@@ -38,7 +47,8 @@ class BASE:
return self.unet_config["in_channels"] > 4
def __init__(self, unet_config):
- self.unet_config = unet_config
+ self.unet_config = unet_config.copy()
+ self.sampling_settings = self.sampling_settings.copy()
self.latent_format = self.latent_format()
for x in self.unet_extra_config:
self.unet_config[x] = self.unet_extra_config[x]
@@ -53,6 +63,7 @@ class BASE:
return out
def process_clip_state_dict(self, state_dict):
+ state_dict = utils.state_dict_prefix_replace(state_dict, {k: "" for k in self.text_encoder_key_prefix}, filter_keys=True)
return state_dict
def process_unet_state_dict(self, state_dict):
@@ -62,7 +73,7 @@ class BASE:
return state_dict
def process_clip_state_dict_for_saving(self, state_dict):
- replace_prefix = {"": "cond_stage_model."}
+ replace_prefix = {"": self.text_encoder_key_prefix[0]}
return utils.state_dict_prefix_replace(state_dict, replace_prefix)
def process_clip_vision_state_dict_for_saving(self, state_dict):
@@ -76,8 +87,9 @@ class BASE:
return utils.state_dict_prefix_replace(state_dict, replace_prefix)
def process_vae_state_dict_for_saving(self, state_dict):
- replace_prefix = {"": "first_stage_model."}
+ replace_prefix = {"": self.vae_key_prefix[0]}
return utils.state_dict_prefix_replace(state_dict, replace_prefix)
- def set_manual_cast(self, manual_cast_dtype):
+ def set_inference_dtype(self, dtype, manual_cast_dtype):
+ self.unet_config['dtype'] = dtype
self.manual_cast_dtype = manual_cast_dtype
diff --git a/comfy/utils.py b/comfy/utils.py
index f8026ddab..ab47b8f28 100644
--- a/comfy/utils.py
+++ b/comfy/utils.py
@@ -5,6 +5,7 @@ import comfy.checkpoint_pickle
import safetensors.torch
import numpy as np
from PIL import Image
+import logging
def load_torch_file(ckpt, safe_load=False, device=None):
if device is None:
@@ -14,14 +15,14 @@ def load_torch_file(ckpt, safe_load=False, device=None):
else:
if safe_load:
if not 'weights_only' in torch.load.__code__.co_varnames:
- print("Warning torch.load doesn't support weights_only on this pytorch version, loading unsafely.")
+ logging.warning("Warning torch.load doesn't support weights_only on this pytorch version, loading unsafely.")
safe_load = False
if safe_load:
pl_sd = torch.load(ckpt, map_location=device, weights_only=True)
else:
pl_sd = torch.load(ckpt, map_location=device, pickle_module=comfy.checkpoint_pickle)
if "global_step" in pl_sd:
- print(f"Global Step: {pl_sd['global_step']}")
+ logging.debug(f"Global Step: {pl_sd['global_step']}")
if "state_dict" in pl_sd:
sd = pl_sd["state_dict"]
else:
@@ -98,8 +99,22 @@ def transformers_convert(sd, prefix_from, prefix_to, number):
p = ["self_attn.q_proj", "self_attn.k_proj", "self_attn.v_proj"]
k_to = "{}encoder.layers.{}.{}.{}".format(prefix_to, resblock, p[x], y)
sd[k_to] = weights[shape_from*x:shape_from*(x + 1)]
+
return sd
+def clip_text_transformers_convert(sd, prefix_from, prefix_to):
+ sd = transformers_convert(sd, prefix_from, "{}text_model.".format(prefix_to), 32)
+
+ tp = "{}text_projection.weight".format(prefix_from)
+ if tp in sd:
+ sd["{}text_projection.weight".format(prefix_to)] = sd.pop(tp)
+
+ tp = "{}text_projection".format(prefix_from)
+ if tp in sd:
+ sd["{}text_projection.weight".format(prefix_to)] = sd.pop(tp).transpose(0, 1).contiguous()
+ return sd
+
+
UNET_MAP_ATTENTIONS = {
"proj_in.weight",
"proj_in.bias",
@@ -169,6 +184,8 @@ UNET_MAP_BASIC = {
}
def unet_to_diffusers(unet_config):
+ if "num_res_blocks" not in unet_config:
+ return {}
num_res_blocks = unet_config["num_res_blocks"]
channel_mult = unet_config["channel_mult"]
transformer_depth = unet_config["transformer_depth"][:]
@@ -278,8 +295,11 @@ def set_attr(obj, attr, value):
for name in attrs[:-1]:
obj = getattr(obj, name)
prev = getattr(obj, attrs[-1])
- setattr(obj, attrs[-1], torch.nn.Parameter(value, requires_grad=False))
- del prev
+ setattr(obj, attrs[-1], value)
+ return prev
+
+def set_attr_param(obj, attr, value):
+ return set_attr(obj, attr, torch.nn.Parameter(value, requires_grad=False))
def copy_to_param(obj, attr, value):
# inplace update tensor instead of replacing it
@@ -413,6 +433,8 @@ def tiled_scale(samples, function, tile_x=64, tile_y=64, overlap = 8, upscale_am
out_div = torch.zeros((s.shape[0], out_channels, round(s.shape[2] * upscale_amount), round(s.shape[3] * upscale_amount)), device=output_device)
for y in range(0, s.shape[2], tile_y - overlap):
for x in range(0, s.shape[3], tile_x - overlap):
+ x = max(0, min(s.shape[-1] - overlap, x))
+ y = max(0, min(s.shape[-2] - overlap, y))
s_in = s[:,:,y:y+tile_y,x:x+tile_x]
ps = function(s_in).to(output_device)
diff --git a/comfy_extras/nodes_canny.py b/comfy_extras/nodes_canny.py
index 730dded5f..8138b5f73 100644
--- a/comfy_extras/nodes_canny.py
+++ b/comfy_extras/nodes_canny.py
@@ -5,275 +5,7 @@ import torch
import torch.nn.functional as F
import comfy.model_management
-def get_canny_nms_kernel(device=None, dtype=None):
- """Utility function that returns 3x3 kernels for the Canny Non-maximal suppression."""
- return torch.tensor(
- [
- [[[0.0, 0.0, 0.0], [0.0, 1.0, -1.0], [0.0, 0.0, 0.0]]],
- [[[0.0, 0.0, 0.0], [0.0, 1.0, 0.0], [0.0, 0.0, -1.0]]],
- [[[0.0, 0.0, 0.0], [0.0, 1.0, 0.0], [0.0, -1.0, 0.0]]],
- [[[0.0, 0.0, 0.0], [0.0, 1.0, 0.0], [-1.0, 0.0, 0.0]]],
- [[[0.0, 0.0, 0.0], [-1.0, 1.0, 0.0], [0.0, 0.0, 0.0]]],
- [[[-1.0, 0.0, 0.0], [0.0, 1.0, 0.0], [0.0, 0.0, 0.0]]],
- [[[0.0, -1.0, 0.0], [0.0, 1.0, 0.0], [0.0, 0.0, 0.0]]],
- [[[0.0, 0.0, -1.0], [0.0, 1.0, 0.0], [0.0, 0.0, 0.0]]],
- ],
- device=device,
- dtype=dtype,
- )
-
-
-def get_hysteresis_kernel(device=None, dtype=None):
- """Utility function that returns the 3x3 kernels for the Canny hysteresis."""
- return torch.tensor(
- [
- [[[0.0, 0.0, 0.0], [0.0, 0.0, 1.0], [0.0, 0.0, 0.0]]],
- [[[0.0, 0.0, 0.0], [0.0, 0.0, 0.0], [0.0, 0.0, 1.0]]],
- [[[0.0, 0.0, 0.0], [0.0, 0.0, 0.0], [0.0, 1.0, 0.0]]],
- [[[0.0, 0.0, 0.0], [0.0, 0.0, 0.0], [1.0, 0.0, 0.0]]],
- [[[0.0, 0.0, 0.0], [1.0, 0.0, 0.0], [0.0, 0.0, 0.0]]],
- [[[1.0, 0.0, 0.0], [0.0, 0.0, 0.0], [0.0, 0.0, 0.0]]],
- [[[0.0, 1.0, 0.0], [0.0, 0.0, 0.0], [0.0, 0.0, 0.0]]],
- [[[0.0, 0.0, 1.0], [0.0, 0.0, 0.0], [0.0, 0.0, 0.0]]],
- ],
- device=device,
- dtype=dtype,
- )
-
-def gaussian_blur_2d(img, kernel_size, sigma):
- ksize_half = (kernel_size - 1) * 0.5
-
- x = torch.linspace(-ksize_half, ksize_half, steps=kernel_size)
-
- pdf = torch.exp(-0.5 * (x / sigma).pow(2))
-
- x_kernel = pdf / pdf.sum()
- x_kernel = x_kernel.to(device=img.device, dtype=img.dtype)
-
- kernel2d = torch.mm(x_kernel[:, None], x_kernel[None, :])
- kernel2d = kernel2d.expand(img.shape[-3], 1, kernel2d.shape[0], kernel2d.shape[1])
-
- padding = [kernel_size // 2, kernel_size // 2, kernel_size // 2, kernel_size // 2]
-
- img = torch.nn.functional.pad(img, padding, mode="reflect")
- img = torch.nn.functional.conv2d(img, kernel2d, groups=img.shape[-3])
-
- return img
-
-def get_sobel_kernel2d(device=None, dtype=None):
- kernel_x = torch.tensor([[-1.0, 0.0, 1.0], [-2.0, 0.0, 2.0], [-1.0, 0.0, 1.0]], device=device, dtype=dtype)
- kernel_y = kernel_x.transpose(0, 1)
- return torch.stack([kernel_x, kernel_y])
-
-def spatial_gradient(input, normalized: bool = True):
- r"""Compute the first order image derivative in both x and y using a Sobel operator.
- .. image:: _static/img/spatial_gradient.png
- Args:
- input: input image tensor with shape :math:`(B, C, H, W)`.
- mode: derivatives modality, can be: `sobel` or `diff`.
- order: the order of the derivatives.
- normalized: whether the output is normalized.
- Return:
- the derivatives of the input feature map. with shape :math:`(B, C, 2, H, W)`.
- .. note::
- See a working example `here `__.
- Examples:
- >>> input = torch.rand(1, 3, 4, 4)
- >>> output = spatial_gradient(input) # 1x3x2x4x4
- >>> output.shape
- torch.Size([1, 3, 2, 4, 4])
- """
- # KORNIA_CHECK_IS_TENSOR(input)
- # KORNIA_CHECK_SHAPE(input, ['B', 'C', 'H', 'W'])
-
- # allocate kernel
- kernel = get_sobel_kernel2d(device=input.device, dtype=input.dtype)
- if normalized:
- kernel = normalize_kernel2d(kernel)
-
- # prepare kernel
- b, c, h, w = input.shape
- tmp_kernel = kernel[:, None, ...]
-
- # Pad with "replicate for spatial dims, but with zeros for channel
- spatial_pad = [kernel.size(1) // 2, kernel.size(1) // 2, kernel.size(2) // 2, kernel.size(2) // 2]
- out_channels: int = 2
- padded_inp = torch.nn.functional.pad(input.reshape(b * c, 1, h, w), spatial_pad, 'replicate')
- out = F.conv2d(padded_inp, tmp_kernel, groups=1, padding=0, stride=1)
- return out.reshape(b, c, out_channels, h, w)
-
-def rgb_to_grayscale(image, rgb_weights = None):
- r"""Convert a RGB image to grayscale version of image.
-
- .. image:: _static/img/rgb_to_grayscale.png
-
- The image data is assumed to be in the range of (0, 1).
-
- Args:
- image: RGB image to be converted to grayscale with shape :math:`(*,3,H,W)`.
- rgb_weights: Weights that will be applied on each channel (RGB).
- The sum of the weights should add up to one.
- Returns:
- grayscale version of the image with shape :math:`(*,1,H,W)`.
-
- .. note::
- See a working example `here `__.
-
- Example:
- >>> input = torch.rand(2, 3, 4, 5)
- >>> gray = rgb_to_grayscale(input) # 2x1x4x5
- """
-
- if len(image.shape) < 3 or image.shape[-3] != 3:
- raise ValueError(f"Input size must have a shape of (*, 3, H, W). Got {image.shape}")
-
- if rgb_weights is None:
- # 8 bit images
- if image.dtype == torch.uint8:
- rgb_weights = torch.tensor([76, 150, 29], device=image.device, dtype=torch.uint8)
- # floating point images
- elif image.dtype in (torch.float16, torch.float32, torch.float64):
- rgb_weights = torch.tensor([0.299, 0.587, 0.114], device=image.device, dtype=image.dtype)
- else:
- raise TypeError(f"Unknown data type: {image.dtype}")
- else:
- # is tensor that we make sure is in the same device/dtype
- rgb_weights = rgb_weights.to(image)
-
- # unpack the color image channels with RGB order
- r: Tensor = image[..., 0:1, :, :]
- g: Tensor = image[..., 1:2, :, :]
- b: Tensor = image[..., 2:3, :, :]
-
- w_r, w_g, w_b = rgb_weights.unbind()
- return w_r * r + w_g * g + w_b * b
-
-def canny(
- input,
- low_threshold = 0.1,
- high_threshold = 0.2,
- kernel_size = 5,
- sigma = 1,
- hysteresis = True,
- eps = 1e-6,
-):
- r"""Find edges of the input image and filters them using the Canny algorithm.
- .. image:: _static/img/canny.png
- Args:
- input: input image tensor with shape :math:`(B,C,H,W)`.
- low_threshold: lower threshold for the hysteresis procedure.
- high_threshold: upper threshold for the hysteresis procedure.
- kernel_size: the size of the kernel for the gaussian blur.
- sigma: the standard deviation of the kernel for the gaussian blur.
- hysteresis: if True, applies the hysteresis edge tracking.
- Otherwise, the edges are divided between weak (0.5) and strong (1) edges.
- eps: regularization number to avoid NaN during backprop.
- Returns:
- - the canny edge magnitudes map, shape of :math:`(B,1,H,W)`.
- - the canny edge detection filtered by thresholds and hysteresis, shape of :math:`(B,1,H,W)`.
- .. note::
- See a working example `here `__.
- Example:
- >>> input = torch.rand(5, 3, 4, 4)
- >>> magnitude, edges = canny(input) # 5x3x4x4
- >>> magnitude.shape
- torch.Size([5, 1, 4, 4])
- >>> edges.shape
- torch.Size([5, 1, 4, 4])
- """
- # KORNIA_CHECK_IS_TENSOR(input)
- # KORNIA_CHECK_SHAPE(input, ['B', 'C', 'H', 'W'])
- # KORNIA_CHECK(
- # low_threshold <= high_threshold,
- # "Invalid input thresholds. low_threshold should be smaller than the high_threshold. Got: "
- # f"{low_threshold}>{high_threshold}",
- # )
- # KORNIA_CHECK(0 < low_threshold < 1, f'Invalid low threshold. Should be in range (0, 1). Got: {low_threshold}')
- # KORNIA_CHECK(0 < high_threshold < 1, f'Invalid high threshold. Should be in range (0, 1). Got: {high_threshold}')
-
- device = input.device
- dtype = input.dtype
-
- # To Grayscale
- if input.shape[1] == 3:
- input = rgb_to_grayscale(input)
-
- # Gaussian filter
- blurred: Tensor = gaussian_blur_2d(input, kernel_size, sigma)
-
- # Compute the gradients
- gradients: Tensor = spatial_gradient(blurred, normalized=False)
-
- # Unpack the edges
- gx: Tensor = gradients[:, :, 0]
- gy: Tensor = gradients[:, :, 1]
-
- # Compute gradient magnitude and angle
- magnitude: Tensor = torch.sqrt(gx * gx + gy * gy + eps)
- angle: Tensor = torch.atan2(gy, gx)
-
- # Radians to Degrees
- angle = 180.0 * angle / math.pi
-
- # Round angle to the nearest 45 degree
- angle = torch.round(angle / 45) * 45
-
- # Non-maximal suppression
- nms_kernels: Tensor = get_canny_nms_kernel(device, dtype)
- nms_magnitude: Tensor = F.conv2d(magnitude, nms_kernels, padding=nms_kernels.shape[-1] // 2)
-
- # Get the indices for both directions
- positive_idx: Tensor = (angle / 45) % 8
- positive_idx = positive_idx.long()
-
- negative_idx: Tensor = ((angle / 45) + 4) % 8
- negative_idx = negative_idx.long()
-
- # Apply the non-maximum suppression to the different directions
- channel_select_filtered_positive: Tensor = torch.gather(nms_magnitude, 1, positive_idx)
- channel_select_filtered_negative: Tensor = torch.gather(nms_magnitude, 1, negative_idx)
-
- channel_select_filtered: Tensor = torch.stack(
- [channel_select_filtered_positive, channel_select_filtered_negative], 1
- )
-
- is_max: Tensor = channel_select_filtered.min(dim=1)[0] > 0.0
-
- magnitude = magnitude * is_max
-
- # Threshold
- edges: Tensor = F.threshold(magnitude, low_threshold, 0.0)
-
- low: Tensor = magnitude > low_threshold
- high: Tensor = magnitude > high_threshold
-
- edges = low * 0.5 + high * 0.5
- edges = edges.to(dtype)
-
- # Hysteresis
- if hysteresis:
- edges_old: Tensor = -torch.ones(edges.shape, device=edges.device, dtype=dtype)
- hysteresis_kernels: Tensor = get_hysteresis_kernel(device, dtype)
-
- while ((edges_old - edges).abs() != 0).any():
- weak: Tensor = (edges == 0.5).float()
- strong: Tensor = (edges == 1).float()
-
- hysteresis_magnitude: Tensor = F.conv2d(
- edges, hysteresis_kernels, padding=hysteresis_kernels.shape[-1] // 2
- )
- hysteresis_magnitude = (hysteresis_magnitude == 1).any(1, keepdim=True).to(dtype)
- hysteresis_magnitude = hysteresis_magnitude * weak + strong
-
- edges_old = edges.clone()
- edges = hysteresis_magnitude + (hysteresis_magnitude == 0) * weak * 0.5
-
- edges = hysteresis_magnitude
-
- return magnitude, edges
+from kornia.filters import canny
class Canny:
diff --git a/comfy_extras/nodes_cond.py b/comfy_extras/nodes_cond.py
new file mode 100644
index 000000000..646fefa17
--- /dev/null
+++ b/comfy_extras/nodes_cond.py
@@ -0,0 +1,25 @@
+
+
+class CLIPTextEncodeControlnet:
+ @classmethod
+ def INPUT_TYPES(s):
+ return {"required": {"clip": ("CLIP", ), "conditioning": ("CONDITIONING", ), "text": ("STRING", {"multiline": True})}}
+ RETURN_TYPES = ("CONDITIONING",)
+ FUNCTION = "encode"
+
+ CATEGORY = "_for_testing/conditioning"
+
+ def encode(self, clip, conditioning, text):
+ tokens = clip.tokenize(text)
+ cond, pooled = clip.encode_from_tokens(tokens, return_pooled=True)
+ c = []
+ for t in conditioning:
+ n = [t[0], t[1].copy()]
+ n[1]['cross_attn_controlnet'] = cond
+ n[1]['pooled_output_controlnet'] = pooled
+ c.append(n)
+ return (c, )
+
+NODE_CLASS_MAPPINGS = {
+ "CLIPTextEncodeControlnet": CLIPTextEncodeControlnet
+}
diff --git a/comfy_extras/nodes_custom_sampler.py b/comfy_extras/nodes_custom_sampler.py
index f02cb5ef7..06238f892 100644
--- a/comfy_extras/nodes_custom_sampler.py
+++ b/comfy_extras/nodes_custom_sampler.py
@@ -4,6 +4,7 @@ from comfy.k_diffusion import sampling as k_diffusion_sampling
import latent_preview
import torch
import comfy.utils
+import node_helpers
class BasicScheduler:
@@ -24,10 +25,11 @@ class BasicScheduler:
def get_sigmas(self, model, scheduler, steps, denoise):
total_steps = steps
if denoise < 1.0:
+ if denoise <= 0.0:
+ return (torch.FloatTensor([]),)
total_steps = int(steps/denoise)
- comfy.model_management.load_models_gpu([model])
- sigmas = comfy.samplers.calculate_sigmas_scheduler(model.model, scheduler, total_steps).cpu()
+ sigmas = comfy.samplers.calculate_sigmas(model.get_model_object("model_sampling"), scheduler, total_steps).cpu()
sigmas = sigmas[-(steps + 1):]
return (sigmas, )
@@ -105,9 +107,8 @@ class SDTurboScheduler:
def get_sigmas(self, model, steps, denoise):
start_step = 10 - int(10 * denoise)
timesteps = torch.flip(torch.arange(1, 11) * 100 - 1, (0,))[start_step:start_step + steps]
- inner_model = model.patch_model(patch_weights=False)
- sigmas = inner_model.model_sampling.sigma(timesteps)
- model.unpatch_model()
+ comfy.model_management.load_models_gpu([model])
+ sigmas = model.model.model_sampling.sigma(timesteps)
sigmas = torch.cat([sigmas, sigmas.new_zeros([1])])
return (sigmas, )
@@ -161,6 +162,9 @@ class FlipSigmas:
FUNCTION = "get_sigmas"
def get_sigmas(self, sigmas):
+ if len(sigmas) == 0:
+ return (sigmas,)
+
sigmas = sigmas.flip(0)
if sigmas[0] == 0:
sigmas[0] = 0.0001
@@ -182,6 +186,28 @@ class KSamplerSelect:
sampler = comfy.samplers.sampler_object(sampler_name)
return (sampler, )
+class SamplerDPMPP_3M_SDE:
+ @classmethod
+ def INPUT_TYPES(s):
+ return {"required":
+ {"eta": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 100.0, "step":0.01, "round": False}),
+ "s_noise": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 100.0, "step":0.01, "round": False}),
+ "noise_device": (['gpu', 'cpu'], ),
+ }
+ }
+ RETURN_TYPES = ("SAMPLER",)
+ CATEGORY = "sampling/custom_sampling/samplers"
+
+ FUNCTION = "get_sampler"
+
+ def get_sampler(self, eta, s_noise, noise_device):
+ if noise_device == 'cpu':
+ sampler_name = "dpmpp_3m_sde"
+ else:
+ sampler_name = "dpmpp_3m_sde_gpu"
+ sampler = comfy.samplers.ksampler(sampler_name, {"eta": eta, "s_noise": s_noise})
+ return (sampler, )
+
class SamplerDPMPP_2M_SDE:
@classmethod
def INPUT_TYPES(s):
@@ -229,6 +255,84 @@ class SamplerDPMPP_SDE:
sampler = comfy.samplers.ksampler(sampler_name, {"eta": eta, "s_noise": s_noise, "r": r})
return (sampler, )
+class SamplerEulerAncestral:
+ @classmethod
+ def INPUT_TYPES(s):
+ return {"required":
+ {"eta": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 100.0, "step":0.01, "round": False}),
+ "s_noise": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 100.0, "step":0.01, "round": False}),
+ }
+ }
+ RETURN_TYPES = ("SAMPLER",)
+ CATEGORY = "sampling/custom_sampling/samplers"
+
+ FUNCTION = "get_sampler"
+
+ def get_sampler(self, eta, s_noise):
+ sampler = comfy.samplers.ksampler("euler_ancestral", {"eta": eta, "s_noise": s_noise})
+ return (sampler, )
+
+class SamplerLMS:
+ @classmethod
+ def INPUT_TYPES(s):
+ return {"required":
+ {"order": ("INT", {"default": 4, "min": 1, "max": 100}),
+ }
+ }
+ RETURN_TYPES = ("SAMPLER",)
+ CATEGORY = "sampling/custom_sampling/samplers"
+
+ FUNCTION = "get_sampler"
+
+ def get_sampler(self, order):
+ sampler = comfy.samplers.ksampler("lms", {"order": order})
+ return (sampler, )
+
+class SamplerDPMAdaptative:
+ @classmethod
+ def INPUT_TYPES(s):
+ return {"required":
+ {"order": ("INT", {"default": 3, "min": 2, "max": 3}),
+ "rtol": ("FLOAT", {"default": 0.05, "min": 0.0, "max": 100.0, "step":0.01, "round": False}),
+ "atol": ("FLOAT", {"default": 0.0078, "min": 0.0, "max": 100.0, "step":0.01, "round": False}),
+ "h_init": ("FLOAT", {"default": 0.05, "min": 0.0, "max": 100.0, "step":0.01, "round": False}),
+ "pcoeff": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 100.0, "step":0.01, "round": False}),
+ "icoeff": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 100.0, "step":0.01, "round": False}),
+ "dcoeff": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 100.0, "step":0.01, "round": False}),
+ "accept_safety": ("FLOAT", {"default": 0.81, "min": 0.0, "max": 100.0, "step":0.01, "round": False}),
+ "eta": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 100.0, "step":0.01, "round": False}),
+ "s_noise": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 100.0, "step":0.01, "round": False}),
+ }
+ }
+ RETURN_TYPES = ("SAMPLER",)
+ CATEGORY = "sampling/custom_sampling/samplers"
+
+ FUNCTION = "get_sampler"
+
+ def get_sampler(self, order, rtol, atol, h_init, pcoeff, icoeff, dcoeff, accept_safety, eta, s_noise):
+ sampler = comfy.samplers.ksampler("dpm_adaptive", {"order": order, "rtol": rtol, "atol": atol, "h_init": h_init, "pcoeff": pcoeff,
+ "icoeff": icoeff, "dcoeff": dcoeff, "accept_safety": accept_safety, "eta": eta,
+ "s_noise":s_noise })
+ return (sampler, )
+
+class Noise_EmptyNoise:
+ def __init__(self):
+ self.seed = 0
+
+ def generate_noise(self, input_latent):
+ latent_image = input_latent["samples"]
+ return torch.zeros(latent_image.shape, dtype=latent_image.dtype, layout=latent_image.layout, device="cpu")
+
+
+class Noise_RandomNoise:
+ def __init__(self, seed):
+ self.seed = seed
+
+ def generate_noise(self, input_latent):
+ latent_image = input_latent["samples"]
+ batch_inds = input_latent["batch_index"] if "batch_index" in input_latent else None
+ return comfy.sample.prepare_noise(latent_image, self.seed, batch_inds)
+
class SamplerCustom:
@classmethod
def INPUT_TYPES(s):
@@ -256,10 +360,9 @@ class SamplerCustom:
latent = latent_image
latent_image = latent["samples"]
if not add_noise:
- noise = torch.zeros(latent_image.size(), dtype=latent_image.dtype, layout=latent_image.layout, device="cpu")
+ noise = Noise_EmptyNoise().generate_noise(latent)
else:
- batch_inds = latent["batch_index"] if "batch_index" in latent else None
- noise = comfy.sample.prepare_noise(latent_image, noise_seed, batch_inds)
+ noise = Noise_RandomNoise(noise_seed).generate_noise(latent)
noise_mask = None
if "noise_mask" in latent:
@@ -280,6 +383,207 @@ class SamplerCustom:
out_denoised = out
return (out, out_denoised)
+class Guider_Basic(comfy.samplers.CFGGuider):
+ def set_conds(self, positive):
+ self.inner_set_conds({"positive": positive})
+
+class BasicGuider:
+ @classmethod
+ def INPUT_TYPES(s):
+ return {"required":
+ {"model": ("MODEL",),
+ "conditioning": ("CONDITIONING", ),
+ }
+ }
+
+ RETURN_TYPES = ("GUIDER",)
+
+ FUNCTION = "get_guider"
+ CATEGORY = "sampling/custom_sampling/guiders"
+
+ def get_guider(self, model, conditioning):
+ guider = Guider_Basic(model)
+ guider.set_conds(conditioning)
+ return (guider,)
+
+class CFGGuider:
+ @classmethod
+ def INPUT_TYPES(s):
+ return {"required":
+ {"model": ("MODEL",),
+ "positive": ("CONDITIONING", ),
+ "negative": ("CONDITIONING", ),
+ "cfg": ("FLOAT", {"default": 8.0, "min": 0.0, "max": 100.0, "step":0.1, "round": 0.01}),
+ }
+ }
+
+ RETURN_TYPES = ("GUIDER",)
+
+ FUNCTION = "get_guider"
+ CATEGORY = "sampling/custom_sampling/guiders"
+
+ def get_guider(self, model, positive, negative, cfg):
+ guider = comfy.samplers.CFGGuider(model)
+ guider.set_conds(positive, negative)
+ guider.set_cfg(cfg)
+ return (guider,)
+
+class Guider_DualCFG(comfy.samplers.CFGGuider):
+ def set_cfg(self, cfg1, cfg2):
+ self.cfg1 = cfg1
+ self.cfg2 = cfg2
+
+ def set_conds(self, positive, middle, negative):
+ middle = node_helpers.conditioning_set_values(middle, {"prompt_type": "negative"})
+ self.inner_set_conds({"positive": positive, "middle": middle, "negative": negative})
+
+ def predict_noise(self, x, timestep, model_options={}, seed=None):
+ negative_cond = self.conds.get("negative", None)
+ middle_cond = self.conds.get("middle", None)
+
+ out = comfy.samplers.calc_cond_batch(self.inner_model, [negative_cond, middle_cond, self.conds.get("positive", None)], x, timestep, model_options)
+ return comfy.samplers.cfg_function(self.inner_model, out[1], out[0], self.cfg2, x, timestep, model_options=model_options, cond=middle_cond, uncond=negative_cond) + (out[2] - out[1]) * self.cfg1
+
+class DualCFGGuider:
+ @classmethod
+ def INPUT_TYPES(s):
+ return {"required":
+ {"model": ("MODEL",),
+ "cond1": ("CONDITIONING", ),
+ "cond2": ("CONDITIONING", ),
+ "negative": ("CONDITIONING", ),
+ "cfg_conds": ("FLOAT", {"default": 8.0, "min": 0.0, "max": 100.0, "step":0.1, "round": 0.01}),
+ "cfg_cond2_negative": ("FLOAT", {"default": 8.0, "min": 0.0, "max": 100.0, "step":0.1, "round": 0.01}),
+ }
+ }
+
+ RETURN_TYPES = ("GUIDER",)
+
+ FUNCTION = "get_guider"
+ CATEGORY = "sampling/custom_sampling/guiders"
+
+ def get_guider(self, model, cond1, cond2, negative, cfg_conds, cfg_cond2_negative):
+ guider = Guider_DualCFG(model)
+ guider.set_conds(cond1, cond2, negative)
+ guider.set_cfg(cfg_conds, cfg_cond2_negative)
+ return (guider,)
+
+class DisableNoise:
+ @classmethod
+ def INPUT_TYPES(s):
+ return {"required":{
+ }
+ }
+
+ RETURN_TYPES = ("NOISE",)
+ FUNCTION = "get_noise"
+ CATEGORY = "sampling/custom_sampling/noise"
+
+ def get_noise(self):
+ return (Noise_EmptyNoise(),)
+
+
+class RandomNoise(DisableNoise):
+ @classmethod
+ def INPUT_TYPES(s):
+ return {"required":{
+ "noise_seed": ("INT", {"default": 0, "min": 0, "max": 0xffffffffffffffff}),
+ }
+ }
+
+ def get_noise(self, noise_seed):
+ return (Noise_RandomNoise(noise_seed),)
+
+
+class SamplerCustomAdvanced:
+ @classmethod
+ def INPUT_TYPES(s):
+ return {"required":
+ {"noise": ("NOISE", ),
+ "guider": ("GUIDER", ),
+ "sampler": ("SAMPLER", ),
+ "sigmas": ("SIGMAS", ),
+ "latent_image": ("LATENT", ),
+ }
+ }
+
+ RETURN_TYPES = ("LATENT","LATENT")
+ RETURN_NAMES = ("output", "denoised_output")
+
+ FUNCTION = "sample"
+
+ CATEGORY = "sampling/custom_sampling"
+
+ def sample(self, noise, guider, sampler, sigmas, latent_image):
+ latent = latent_image
+ latent_image = latent["samples"]
+
+ noise_mask = None
+ if "noise_mask" in latent:
+ noise_mask = latent["noise_mask"]
+
+ x0_output = {}
+ callback = latent_preview.prepare_callback(guider.model_patcher, sigmas.shape[-1] - 1, x0_output)
+
+ disable_pbar = not comfy.utils.PROGRESS_BAR_ENABLED
+ samples = guider.sample(noise.generate_noise(latent), latent_image, sampler, sigmas, denoise_mask=noise_mask, callback=callback, disable_pbar=disable_pbar, seed=noise.seed)
+ samples = samples.to(comfy.model_management.intermediate_device())
+
+ out = latent.copy()
+ out["samples"] = samples
+ if "x0" in x0_output:
+ out_denoised = latent.copy()
+ out_denoised["samples"] = guider.model_patcher.model.process_latent_out(x0_output["x0"].cpu())
+ else:
+ out_denoised = out
+ return (out, out_denoised)
+
+class AddNoise:
+ @classmethod
+ def INPUT_TYPES(s):
+ return {"required":
+ {"model": ("MODEL",),
+ "noise": ("NOISE", ),
+ "sigmas": ("SIGMAS", ),
+ "latent_image": ("LATENT", ),
+ }
+ }
+
+ RETURN_TYPES = ("LATENT",)
+
+ FUNCTION = "add_noise"
+
+ CATEGORY = "_for_testing/custom_sampling/noise"
+
+ def add_noise(self, model, noise, sigmas, latent_image):
+ if len(sigmas) == 0:
+ return latent_image
+
+ latent = latent_image
+ latent_image = latent["samples"]
+
+ noisy = noise.generate_noise(latent)
+
+ model_sampling = model.get_model_object("model_sampling")
+ process_latent_out = model.get_model_object("process_latent_out")
+ process_latent_in = model.get_model_object("process_latent_in")
+
+ if len(sigmas) > 1:
+ scale = torch.abs(sigmas[0] - sigmas[-1])
+ else:
+ scale = sigmas[0]
+
+ if torch.count_nonzero(latent_image) > 0: #Don't shift the empty latent image.
+ latent_image = process_latent_in(latent_image)
+ noisy = model_sampling.noise_scaling(scale, noisy, latent_image)
+ noisy = process_latent_out(noisy)
+ noisy = torch.nan_to_num(noisy, nan=0.0, posinf=0.0, neginf=0.0)
+
+ out = latent.copy()
+ out["samples"] = noisy
+ return (out,)
+
+
NODE_CLASS_MAPPINGS = {
"SamplerCustom": SamplerCustom,
"BasicScheduler": BasicScheduler,
@@ -289,8 +593,20 @@ NODE_CLASS_MAPPINGS = {
"VPScheduler": VPScheduler,
"SDTurboScheduler": SDTurboScheduler,
"KSamplerSelect": KSamplerSelect,
+ "SamplerEulerAncestral": SamplerEulerAncestral,
+ "SamplerLMS": SamplerLMS,
+ "SamplerDPMPP_3M_SDE": SamplerDPMPP_3M_SDE,
"SamplerDPMPP_2M_SDE": SamplerDPMPP_2M_SDE,
"SamplerDPMPP_SDE": SamplerDPMPP_SDE,
+ "SamplerDPMAdaptative": SamplerDPMAdaptative,
"SplitSigmas": SplitSigmas,
"FlipSigmas": FlipSigmas,
+
+ "CFGGuider": CFGGuider,
+ "DualCFGGuider": DualCFGGuider,
+ "BasicGuider": BasicGuider,
+ "RandomNoise": RandomNoise,
+ "DisableNoise": DisableNoise,
+ "AddNoise": AddNoise,
+ "SamplerCustomAdvanced": SamplerCustomAdvanced,
}
diff --git a/comfy_extras/nodes_differential_diffusion.py b/comfy_extras/nodes_differential_diffusion.py
new file mode 100644
index 000000000..98dbbf102
--- /dev/null
+++ b/comfy_extras/nodes_differential_diffusion.py
@@ -0,0 +1,42 @@
+# code adapted from https://github.com/exx8/differential-diffusion
+
+import torch
+
+class DifferentialDiffusion():
+ @classmethod
+ def INPUT_TYPES(s):
+ return {"required": {"model": ("MODEL", ),
+ }}
+ RETURN_TYPES = ("MODEL",)
+ FUNCTION = "apply"
+ CATEGORY = "_for_testing"
+ INIT = False
+
+ def apply(self, model):
+ model = model.clone()
+ model.set_model_denoise_mask_function(self.forward)
+ return (model,)
+
+ def forward(self, sigma: torch.Tensor, denoise_mask: torch.Tensor, extra_options: dict):
+ model = extra_options["model"]
+ step_sigmas = extra_options["sigmas"]
+ sigma_to = model.inner_model.model_sampling.sigma_min
+ if step_sigmas[-1] > sigma_to:
+ sigma_to = step_sigmas[-1]
+ sigma_from = step_sigmas[0]
+
+ ts_from = model.inner_model.model_sampling.timestep(sigma_from)
+ ts_to = model.inner_model.model_sampling.timestep(sigma_to)
+ current_ts = model.inner_model.model_sampling.timestep(sigma[0])
+
+ threshold = (current_ts - ts_to) / (ts_from - ts_to)
+
+ return (denoise_mask >= threshold).to(denoise_mask.dtype)
+
+
+NODE_CLASS_MAPPINGS = {
+ "DifferentialDiffusion": DifferentialDiffusion,
+}
+NODE_DISPLAY_NAME_MAPPINGS = {
+ "DifferentialDiffusion": "Differential Diffusion",
+}
diff --git a/comfy_extras/nodes_freelunch.py b/comfy_extras/nodes_freelunch.py
index 7764aa0b0..6f1d87bf3 100644
--- a/comfy_extras/nodes_freelunch.py
+++ b/comfy_extras/nodes_freelunch.py
@@ -1,7 +1,7 @@
#code originally taken from: https://github.com/ChenyangSi/FreeU (under MIT License)
import torch
-
+import logging
def Fourier_filter(x, threshold, scale):
# FFT
@@ -49,7 +49,7 @@ class FreeU:
try:
hsp = Fourier_filter(hsp, threshold=1, scale=scale[1])
except:
- print("Device", hsp.device, "does not support the torch.fft functions used in the FreeU node, switching to CPU.")
+ logging.warning("Device {} does not support the torch.fft functions used in the FreeU node, switching to CPU.".format(hsp.device))
on_cpu_devices[hsp.device] = True
hsp = Fourier_filter(hsp.cpu(), threshold=1, scale=scale[1]).to(hsp.device)
else:
@@ -95,7 +95,7 @@ class FreeU_V2:
try:
hsp = Fourier_filter(hsp, threshold=1, scale=scale[1])
except:
- print("Device", hsp.device, "does not support the torch.fft functions used in the FreeU node, switching to CPU.")
+ logging.warning("Device {} does not support the torch.fft functions used in the FreeU node, switching to CPU.".format(hsp.device))
on_cpu_devices[hsp.device] = True
hsp = Fourier_filter(hsp.cpu(), threshold=1, scale=scale[1]).to(hsp.device)
else:
diff --git a/comfy_extras/nodes_hypernetwork.py b/comfy_extras/nodes_hypernetwork.py
index f692945a8..cafafa6ab 100644
--- a/comfy_extras/nodes_hypernetwork.py
+++ b/comfy_extras/nodes_hypernetwork.py
@@ -1,6 +1,7 @@
import comfy.utils
import folder_paths
import torch
+import logging
def load_hypernetwork_patch(path, strength):
sd = comfy.utils.load_torch_file(path, safe_load=True)
@@ -23,7 +24,7 @@ def load_hypernetwork_patch(path, strength):
}
if activation_func not in valid_activation:
- print("Unsupported Hypernetwork format, if you report it I might implement it.", path, " ", activation_func, is_layer_norm, use_dropout, activate_output, last_layer_dropout)
+ logging.error("Unsupported Hypernetwork format, if you report it I might implement it. {} {} {} {} {} {}".format(path, activation_func, is_layer_norm, use_dropout, activate_output, last_layer_dropout))
return None
out = {}
diff --git a/comfy_extras/nodes_images.py b/comfy_extras/nodes_images.py
index aa80f5269..af37666b2 100644
--- a/comfy_extras/nodes_images.py
+++ b/comfy_extras/nodes_images.py
@@ -37,7 +37,7 @@ class RepeatImageBatch:
@classmethod
def INPUT_TYPES(s):
return {"required": { "image": ("IMAGE",),
- "amount": ("INT", {"default": 1, "min": 1, "max": 64}),
+ "amount": ("INT", {"default": 1, "min": 1, "max": 4096}),
}}
RETURN_TYPES = ("IMAGE",)
FUNCTION = "repeat"
@@ -48,6 +48,25 @@ class RepeatImageBatch:
s = image.repeat((amount, 1,1,1))
return (s,)
+class ImageFromBatch:
+ @classmethod
+ def INPUT_TYPES(s):
+ return {"required": { "image": ("IMAGE",),
+ "batch_index": ("INT", {"default": 0, "min": 0, "max": 4095}),
+ "length": ("INT", {"default": 1, "min": 1, "max": 4096}),
+ }}
+ RETURN_TYPES = ("IMAGE",)
+ FUNCTION = "frombatch"
+
+ CATEGORY = "image/batch"
+
+ def frombatch(self, image, batch_index, length):
+ s_in = image
+ batch_index = min(s_in.shape[0] - 1, batch_index)
+ length = min(s_in.shape[0] - batch_index, length)
+ s = s_in[batch_index:batch_index + length].clone()
+ return (s,)
+
class SaveAnimatedWEBP:
def __init__(self):
self.output_dir = folder_paths.get_output_directory()
@@ -170,6 +189,7 @@ class SaveAnimatedPNG:
NODE_CLASS_MAPPINGS = {
"ImageCrop": ImageCrop,
"RepeatImageBatch": RepeatImageBatch,
+ "ImageFromBatch": ImageFromBatch,
"SaveAnimatedWEBP": SaveAnimatedWEBP,
"SaveAnimatedPNG": SaveAnimatedPNG,
}
diff --git a/comfy_extras/nodes_ip2p.py b/comfy_extras/nodes_ip2p.py
new file mode 100644
index 000000000..c2e70a84c
--- /dev/null
+++ b/comfy_extras/nodes_ip2p.py
@@ -0,0 +1,45 @@
+import torch
+
+class InstructPixToPixConditioning:
+ @classmethod
+ def INPUT_TYPES(s):
+ return {"required": {"positive": ("CONDITIONING", ),
+ "negative": ("CONDITIONING", ),
+ "vae": ("VAE", ),
+ "pixels": ("IMAGE", ),
+ }}
+
+ RETURN_TYPES = ("CONDITIONING","CONDITIONING","LATENT")
+ RETURN_NAMES = ("positive", "negative", "latent")
+ FUNCTION = "encode"
+
+ CATEGORY = "conditioning/instructpix2pix"
+
+ def encode(self, positive, negative, pixels, vae):
+ x = (pixels.shape[1] // 8) * 8
+ y = (pixels.shape[2] // 8) * 8
+
+ if pixels.shape[1] != x or pixels.shape[2] != y:
+ x_offset = (pixels.shape[1] % 8) // 2
+ y_offset = (pixels.shape[2] % 8) // 2
+ pixels = pixels[:,x_offset:x + x_offset, y_offset:y + y_offset,:]
+
+ concat_latent = vae.encode(pixels)
+
+ out_latent = {}
+ out_latent["samples"] = torch.zeros_like(concat_latent)
+
+ out = []
+ for conditioning in [positive, negative]:
+ c = []
+ for t in conditioning:
+ d = t[1].copy()
+ d["concat_latent_image"] = concat_latent
+ n = [t[0], d]
+ c.append(n)
+ out.append(c)
+ return (out[0], out[1], out_latent)
+
+NODE_CLASS_MAPPINGS = {
+ "InstructPixToPixConditioning": InstructPixToPixConditioning,
+}
diff --git a/comfy_extras/nodes_latent.py b/comfy_extras/nodes_latent.py
index 2eefc4c55..eabae0885 100644
--- a/comfy_extras/nodes_latent.py
+++ b/comfy_extras/nodes_latent.py
@@ -122,10 +122,34 @@ class LatentBatch:
samples_out["batch_index"] = samples1.get("batch_index", [x for x in range(0, s1.shape[0])]) + samples2.get("batch_index", [x for x in range(0, s2.shape[0])])
return (samples_out,)
+class LatentBatchSeedBehavior:
+ @classmethod
+ def INPUT_TYPES(s):
+ return {"required": { "samples": ("LATENT",),
+ "seed_behavior": (["random", "fixed"],{"default": "fixed"}),}}
+
+ RETURN_TYPES = ("LATENT",)
+ FUNCTION = "op"
+
+ CATEGORY = "latent/advanced"
+
+ def op(self, samples, seed_behavior):
+ samples_out = samples.copy()
+ latent = samples["samples"]
+ if seed_behavior == "random":
+ if 'batch_index' in samples_out:
+ samples_out.pop('batch_index')
+ elif seed_behavior == "fixed":
+ batch_number = samples_out.get("batch_index", [0])[0]
+ samples_out["batch_index"] = [batch_number] * latent.shape[0]
+
+ return (samples_out,)
+
NODE_CLASS_MAPPINGS = {
"LatentAdd": LatentAdd,
"LatentSubtract": LatentSubtract,
"LatentMultiply": LatentMultiply,
"LatentInterpolate": LatentInterpolate,
"LatentBatch": LatentBatch,
+ "LatentBatchSeedBehavior": LatentBatchSeedBehavior,
}
diff --git a/comfy_extras/nodes_mask.py b/comfy_extras/nodes_mask.py
index a7d164bf7..29589b4ab 100644
--- a/comfy_extras/nodes_mask.py
+++ b/comfy_extras/nodes_mask.py
@@ -341,6 +341,24 @@ class GrowMask:
out.append(output)
return (torch.stack(out, dim=0),)
+class ThresholdMask:
+ @classmethod
+ def INPUT_TYPES(s):
+ return {
+ "required": {
+ "mask": ("MASK",),
+ "value": ("FLOAT", {"default": 0.5, "min": 0.0, "max": 1.0, "step": 0.01}),
+ }
+ }
+
+ CATEGORY = "mask"
+
+ RETURN_TYPES = ("MASK",)
+ FUNCTION = "image_to_mask"
+
+ def image_to_mask(self, mask, value):
+ mask = (mask > value).float()
+ return (mask,)
NODE_CLASS_MAPPINGS = {
@@ -355,6 +373,7 @@ NODE_CLASS_MAPPINGS = {
"MaskComposite": MaskComposite,
"FeatherMask": FeatherMask,
"GrowMask": GrowMask,
+ "ThresholdMask": ThresholdMask,
}
NODE_DISPLAY_NAME_MAPPINGS = {
diff --git a/comfy_extras/nodes_model_advanced.py b/comfy_extras/nodes_model_advanced.py
index 541ce8fa5..21af4b733 100644
--- a/comfy_extras/nodes_model_advanced.py
+++ b/comfy_extras/nodes_model_advanced.py
@@ -1,6 +1,7 @@
import folder_paths
import comfy.sd
import comfy.model_sampling
+import comfy.latent_formats
import torch
class LCM(comfy.model_sampling.EPS):
@@ -17,6 +18,10 @@ class LCM(comfy.model_sampling.EPS):
return c_out * x0 + c_skip * model_input
+class X0(comfy.model_sampling.EPS):
+ def calculate_denoised(self, sigma, model_output, model_input):
+ return model_output
+
class ModelSamplingDiscreteDistilled(comfy.model_sampling.ModelSamplingDiscrete):
original_timesteps = 50
@@ -68,7 +73,7 @@ class ModelSamplingDiscrete:
@classmethod
def INPUT_TYPES(s):
return {"required": { "model": ("MODEL",),
- "sampling": (["eps", "v_prediction", "lcm"],),
+ "sampling": (["eps", "v_prediction", "lcm", "x0"],),
"zsnr": ("BOOLEAN", {"default": False}),
}}
@@ -88,6 +93,8 @@ class ModelSamplingDiscrete:
elif sampling == "lcm":
sampling_type = LCM
sampling_base = ModelSamplingDiscreteDistilled
+ elif sampling == "x0":
+ sampling_type = X0
class ModelSamplingAdvanced(sampling_base, sampling_type):
pass
@@ -99,11 +106,37 @@ class ModelSamplingDiscrete:
m.add_object_patch("model_sampling", model_sampling)
return (m, )
+class ModelSamplingStableCascade:
+ @classmethod
+ def INPUT_TYPES(s):
+ return {"required": { "model": ("MODEL",),
+ "shift": ("FLOAT", {"default": 2.0, "min": 0.0, "max": 100.0, "step":0.01}),
+ }}
+
+ RETURN_TYPES = ("MODEL",)
+ FUNCTION = "patch"
+
+ CATEGORY = "advanced/model"
+
+ def patch(self, model, shift):
+ m = model.clone()
+
+ sampling_base = comfy.model_sampling.StableCascadeSampling
+ sampling_type = comfy.model_sampling.EPS
+
+ class ModelSamplingAdvanced(sampling_base, sampling_type):
+ pass
+
+ model_sampling = ModelSamplingAdvanced(model.model.model_config)
+ model_sampling.set_parameters(shift)
+ m.add_object_patch("model_sampling", model_sampling)
+ return (m, )
+
class ModelSamplingContinuousEDM:
@classmethod
def INPUT_TYPES(s):
return {"required": { "model": ("MODEL",),
- "sampling": (["v_prediction", "eps"],),
+ "sampling": (["v_prediction", "edm_playground_v2.5", "eps"],),
"sigma_max": ("FLOAT", {"default": 120.0, "min": 0.0, "max": 1000.0, "step":0.001, "round": False}),
"sigma_min": ("FLOAT", {"default": 0.002, "min": 0.0, "max": 1000.0, "step":0.001, "round": False}),
}}
@@ -116,17 +149,25 @@ class ModelSamplingContinuousEDM:
def patch(self, model, sampling, sigma_max, sigma_min):
m = model.clone()
+ latent_format = None
+ sigma_data = 1.0
if sampling == "eps":
sampling_type = comfy.model_sampling.EPS
elif sampling == "v_prediction":
sampling_type = comfy.model_sampling.V_PREDICTION
+ elif sampling == "edm_playground_v2.5":
+ sampling_type = comfy.model_sampling.EDM
+ sigma_data = 0.5
+ latent_format = comfy.latent_formats.SDXL_Playground_2_5()
class ModelSamplingAdvanced(comfy.model_sampling.ModelSamplingContinuousEDM, sampling_type):
pass
model_sampling = ModelSamplingAdvanced(model.model.model_config)
- model_sampling.set_sigma_range(sigma_min, sigma_max)
+ model_sampling.set_parameters(sigma_min, sigma_max, sigma_data)
m.add_object_patch("model_sampling", model_sampling)
+ if latent_format is not None:
+ m.add_object_patch("latent_format", latent_format)
return (m, )
class RescaleCFG:
@@ -171,5 +212,6 @@ class RescaleCFG:
NODE_CLASS_MAPPINGS = {
"ModelSamplingDiscrete": ModelSamplingDiscrete,
"ModelSamplingContinuousEDM": ModelSamplingContinuousEDM,
+ "ModelSamplingStableCascade": ModelSamplingStableCascade,
"RescaleCFG": RescaleCFG,
}
diff --git a/comfy_extras/nodes_model_merging.py b/comfy_extras/nodes_model_merging.py
index d594cf490..2a431f65d 100644
--- a/comfy_extras/nodes_model_merging.py
+++ b/comfy_extras/nodes_model_merging.py
@@ -2,7 +2,9 @@ import comfy.sd
import comfy.utils
import comfy.model_base
import comfy.model_management
+import comfy.model_sampling
+import torch
import folder_paths
import json
import os
@@ -87,6 +89,50 @@ class CLIPMergeSimple:
m.add_patches({k: kp[k]}, 1.0 - ratio, ratio)
return (m, )
+
+class CLIPSubtract:
+ @classmethod
+ def INPUT_TYPES(s):
+ return {"required": { "clip1": ("CLIP",),
+ "clip2": ("CLIP",),
+ "multiplier": ("FLOAT", {"default": 1.0, "min": -10.0, "max": 10.0, "step": 0.01}),
+ }}
+ RETURN_TYPES = ("CLIP",)
+ FUNCTION = "merge"
+
+ CATEGORY = "advanced/model_merging"
+
+ def merge(self, clip1, clip2, multiplier):
+ m = clip1.clone()
+ kp = clip2.get_key_patches()
+ for k in kp:
+ if k.endswith(".position_ids") or k.endswith(".logit_scale"):
+ continue
+ m.add_patches({k: kp[k]}, - multiplier, multiplier)
+ return (m, )
+
+
+class CLIPAdd:
+ @classmethod
+ def INPUT_TYPES(s):
+ return {"required": { "clip1": ("CLIP",),
+ "clip2": ("CLIP",),
+ }}
+ RETURN_TYPES = ("CLIP",)
+ FUNCTION = "merge"
+
+ CATEGORY = "advanced/model_merging"
+
+ def merge(self, clip1, clip2):
+ m = clip1.clone()
+ kp = clip2.get_key_patches()
+ for k in kp:
+ if k.endswith(".position_ids") or k.endswith(".logit_scale"):
+ continue
+ m.add_patches({k: kp[k]}, 1.0, 1.0)
+ return (m, )
+
+
class ModelMergeBlocks:
@classmethod
def INPUT_TYPES(s):
@@ -145,6 +191,13 @@ def save_checkpoint(model, clip=None, vae=None, clip_vision=None, filename_prefi
# "stable-diffusion-v2-768-v", "stable-diffusion-v2-unclip-l", "stable-diffusion-v2-unclip-h",
# "v2-inpainting"
+ extra_keys = {}
+ model_sampling = model.get_model_object("model_sampling")
+ if isinstance(model_sampling, comfy.model_sampling.ModelSamplingContinuousEDM):
+ if isinstance(model_sampling, comfy.model_sampling.V_PREDICTION):
+ extra_keys["edm_vpred.sigma_max"] = torch.tensor(model_sampling.sigma_max).float()
+ extra_keys["edm_vpred.sigma_min"] = torch.tensor(model_sampling.sigma_min).float()
+
if model.model.model_type == comfy.model_base.ModelType.EPS:
metadata["modelspec.predict_key"] = "epsilon"
elif model.model.model_type == comfy.model_base.ModelType.V_PREDICTION:
@@ -159,7 +212,7 @@ def save_checkpoint(model, clip=None, vae=None, clip_vision=None, filename_prefi
output_checkpoint = f"{filename}_{counter:05}_.safetensors"
output_checkpoint = os.path.join(full_output_folder, output_checkpoint)
- comfy.sd.save_checkpoint(output_checkpoint, model, clip, vae, clip_vision, metadata=metadata)
+ comfy.sd.save_checkpoint(output_checkpoint, model, clip, vae, clip_vision, metadata=metadata, extra_keys=extra_keys)
class CheckpointSave:
def __init__(self):
@@ -279,6 +332,8 @@ NODE_CLASS_MAPPINGS = {
"ModelMergeAdd": ModelAdd,
"CheckpointSave": CheckpointSave,
"CLIPMergeSimple": CLIPMergeSimple,
+ "CLIPMergeSubtract": CLIPSubtract,
+ "CLIPMergeAdd": CLIPAdd,
"CLIPSave": CLIPSave,
"VAESave": VAESave,
}
diff --git a/comfy_extras/nodes_model_merging_model_specific.py b/comfy_extras/nodes_model_merging_model_specific.py
new file mode 100644
index 000000000..f2d008d8b
--- /dev/null
+++ b/comfy_extras/nodes_model_merging_model_specific.py
@@ -0,0 +1,60 @@
+import comfy_extras.nodes_model_merging
+
+class ModelMergeSD1(comfy_extras.nodes_model_merging.ModelMergeBlocks):
+ CATEGORY = "advanced/model_merging/model_specific"
+ @classmethod
+ def INPUT_TYPES(s):
+ arg_dict = { "model1": ("MODEL",),
+ "model2": ("MODEL",)}
+
+ argument = ("FLOAT", {"default": 1.0, "min": 0.0, "max": 1.0, "step": 0.01})
+
+ arg_dict["time_embed."] = argument
+ arg_dict["label_emb."] = argument
+
+ for i in range(12):
+ arg_dict["input_blocks.{}.".format(i)] = argument
+
+ for i in range(3):
+ arg_dict["middle_block.{}.".format(i)] = argument
+
+ for i in range(12):
+ arg_dict["output_blocks.{}.".format(i)] = argument
+
+ arg_dict["out."] = argument
+
+ return {"required": arg_dict}
+
+
+class ModelMergeSDXL(comfy_extras.nodes_model_merging.ModelMergeBlocks):
+ CATEGORY = "advanced/model_merging/model_specific"
+
+ @classmethod
+ def INPUT_TYPES(s):
+ arg_dict = { "model1": ("MODEL",),
+ "model2": ("MODEL",)}
+
+ argument = ("FLOAT", {"default": 1.0, "min": 0.0, "max": 1.0, "step": 0.01})
+
+ arg_dict["time_embed."] = argument
+ arg_dict["label_emb."] = argument
+
+ for i in range(9):
+ arg_dict["input_blocks.{}".format(i)] = argument
+
+ for i in range(3):
+ arg_dict["middle_block.{}".format(i)] = argument
+
+ for i in range(9):
+ arg_dict["output_blocks.{}".format(i)] = argument
+
+ arg_dict["out."] = argument
+
+ return {"required": arg_dict}
+
+
+NODE_CLASS_MAPPINGS = {
+ "ModelMergeSD1": ModelMergeSD1,
+ "ModelMergeSD2": ModelMergeSD1, #SD1 and SD2 have the same blocks
+ "ModelMergeSDXL": ModelMergeSDXL,
+}
diff --git a/comfy_extras/nodes_morphology.py b/comfy_extras/nodes_morphology.py
new file mode 100644
index 000000000..071521d87
--- /dev/null
+++ b/comfy_extras/nodes_morphology.py
@@ -0,0 +1,49 @@
+import torch
+import comfy.model_management
+
+from kornia.morphology import dilation, erosion, opening, closing, gradient, top_hat, bottom_hat
+
+
+class Morphology:
+ @classmethod
+ def INPUT_TYPES(s):
+ return {"required": {"image": ("IMAGE",),
+ "operation": (["erode", "dilate", "open", "close", "gradient", "bottom_hat", "top_hat"],),
+ "kernel_size": ("INT", {"default": 3, "min": 3, "max": 999, "step": 1}),
+ }}
+
+ RETURN_TYPES = ("IMAGE",)
+ FUNCTION = "process"
+
+ CATEGORY = "image/postprocessing"
+
+ def process(self, image, operation, kernel_size):
+ device = comfy.model_management.get_torch_device()
+ kernel = torch.ones(kernel_size, kernel_size, device=device)
+ image_k = image.to(device).movedim(-1, 1)
+ if operation == "erode":
+ output = erosion(image_k, kernel)
+ elif operation == "dilate":
+ output = dilation(image_k, kernel)
+ elif operation == "open":
+ output = opening(image_k, kernel)
+ elif operation == "close":
+ output = closing(image_k, kernel)
+ elif operation == "gradient":
+ output = gradient(image_k, kernel)
+ elif operation == "top_hat":
+ output = top_hat(image_k, kernel)
+ elif operation == "bottom_hat":
+ output = bottom_hat(image_k, kernel)
+ else:
+ raise ValueError(f"Invalid operation {operation} for morphology. Must be one of 'erode', 'dilate', 'open', 'close', 'gradient', 'tophat', 'bottomhat'")
+ img_out = output.to(comfy.model_management.intermediate_device()).movedim(1, -1)
+ return (img_out,)
+
+NODE_CLASS_MAPPINGS = {
+ "Morphology": Morphology,
+}
+
+NODE_DISPLAY_NAME_MAPPINGS = {
+ "Morphology": "ImageMorphology",
+}
\ No newline at end of file
diff --git a/comfy_extras/nodes_perpneg.py b/comfy_extras/nodes_perpneg.py
index 45e4d418f..992e8ab62 100644
--- a/comfy_extras/nodes_perpneg.py
+++ b/comfy_extras/nodes_perpneg.py
@@ -1,16 +1,17 @@
import torch
import comfy.model_management
-import comfy.sample
+import comfy.sampler_helpers
import comfy.samplers
import comfy.utils
+#TODO: This node should be removed and replaced with one that uses the new Guider/SamplerCustomAdvanced.
class PerpNeg:
@classmethod
def INPUT_TYPES(s):
return {"required": {"model": ("MODEL", ),
"empty_conditioning": ("CONDITIONING", ),
- "neg_scale": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 100.0}),
+ "neg_scale": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 100.0, "step": 0.01}),
}}
RETURN_TYPES = ("MODEL",)
FUNCTION = "patch"
@@ -19,7 +20,7 @@ class PerpNeg:
def patch(self, model, empty_conditioning, neg_scale):
m = model.clone()
- nocond = comfy.sample.convert_cond(empty_conditioning)
+ nocond = comfy.sampler_helpers.convert_cond(empty_conditioning)
def cfg_function(args):
model = args["model"]
@@ -31,11 +32,11 @@ class PerpNeg:
model_options = args["model_options"]
nocond_processed = comfy.samplers.encode_model_conds(model.extra_conds, nocond, x, x.device, "negative")
- (noise_pred_nocond, _) = comfy.samplers.calc_cond_uncond_batch(model, nocond_processed, None, x, sigma, model_options)
+ (noise_pred_nocond,) = comfy.samplers.calc_cond_batch(model, [nocond_processed], x, sigma, model_options)
pos = noise_pred_pos - noise_pred_nocond
neg = noise_pred_neg - noise_pred_nocond
- perp = ((torch.mul(pos, neg).sum())/(torch.norm(neg)**2)) * neg
+ perp = neg - ((torch.mul(neg, pos).sum())/(torch.norm(pos)**2)) * pos
perp_neg = perp * neg_scale
cfg_result = noise_pred_nocond + cond_scale*(pos - perp_neg)
cfg_result = x - cfg_result
diff --git a/comfy_extras/nodes_photomaker.py b/comfy_extras/nodes_photomaker.py
new file mode 100644
index 000000000..90130142b
--- /dev/null
+++ b/comfy_extras/nodes_photomaker.py
@@ -0,0 +1,187 @@
+import torch
+import torch.nn as nn
+import folder_paths
+import comfy.clip_model
+import comfy.clip_vision
+import comfy.ops
+
+# code for model from: https://github.com/TencentARC/PhotoMaker/blob/main/photomaker/model.py under Apache License Version 2.0
+VISION_CONFIG_DICT = {
+ "hidden_size": 1024,
+ "image_size": 224,
+ "intermediate_size": 4096,
+ "num_attention_heads": 16,
+ "num_channels": 3,
+ "num_hidden_layers": 24,
+ "patch_size": 14,
+ "projection_dim": 768,
+ "hidden_act": "quick_gelu",
+}
+
+class MLP(nn.Module):
+ def __init__(self, in_dim, out_dim, hidden_dim, use_residual=True, operations=comfy.ops):
+ super().__init__()
+ if use_residual:
+ assert in_dim == out_dim
+ self.layernorm = operations.LayerNorm(in_dim)
+ self.fc1 = operations.Linear(in_dim, hidden_dim)
+ self.fc2 = operations.Linear(hidden_dim, out_dim)
+ self.use_residual = use_residual
+ self.act_fn = nn.GELU()
+
+ def forward(self, x):
+ residual = x
+ x = self.layernorm(x)
+ x = self.fc1(x)
+ x = self.act_fn(x)
+ x = self.fc2(x)
+ if self.use_residual:
+ x = x + residual
+ return x
+
+
+class FuseModule(nn.Module):
+ def __init__(self, embed_dim, operations):
+ super().__init__()
+ self.mlp1 = MLP(embed_dim * 2, embed_dim, embed_dim, use_residual=False, operations=operations)
+ self.mlp2 = MLP(embed_dim, embed_dim, embed_dim, use_residual=True, operations=operations)
+ self.layer_norm = operations.LayerNorm(embed_dim)
+
+ def fuse_fn(self, prompt_embeds, id_embeds):
+ stacked_id_embeds = torch.cat([prompt_embeds, id_embeds], dim=-1)
+ stacked_id_embeds = self.mlp1(stacked_id_embeds) + prompt_embeds
+ stacked_id_embeds = self.mlp2(stacked_id_embeds)
+ stacked_id_embeds = self.layer_norm(stacked_id_embeds)
+ return stacked_id_embeds
+
+ def forward(
+ self,
+ prompt_embeds,
+ id_embeds,
+ class_tokens_mask,
+ ) -> torch.Tensor:
+ # id_embeds shape: [b, max_num_inputs, 1, 2048]
+ id_embeds = id_embeds.to(prompt_embeds.dtype)
+ num_inputs = class_tokens_mask.sum().unsqueeze(0) # TODO: check for training case
+ batch_size, max_num_inputs = id_embeds.shape[:2]
+ # seq_length: 77
+ seq_length = prompt_embeds.shape[1]
+ # flat_id_embeds shape: [b*max_num_inputs, 1, 2048]
+ flat_id_embeds = id_embeds.view(
+ -1, id_embeds.shape[-2], id_embeds.shape[-1]
+ )
+ # valid_id_mask [b*max_num_inputs]
+ valid_id_mask = (
+ torch.arange(max_num_inputs, device=flat_id_embeds.device)[None, :]
+ < num_inputs[:, None]
+ )
+ valid_id_embeds = flat_id_embeds[valid_id_mask.flatten()]
+
+ prompt_embeds = prompt_embeds.view(-1, prompt_embeds.shape[-1])
+ class_tokens_mask = class_tokens_mask.view(-1)
+ valid_id_embeds = valid_id_embeds.view(-1, valid_id_embeds.shape[-1])
+ # slice out the image token embeddings
+ image_token_embeds = prompt_embeds[class_tokens_mask]
+ stacked_id_embeds = self.fuse_fn(image_token_embeds, valid_id_embeds)
+ assert class_tokens_mask.sum() == stacked_id_embeds.shape[0], f"{class_tokens_mask.sum()} != {stacked_id_embeds.shape[0]}"
+ prompt_embeds.masked_scatter_(class_tokens_mask[:, None], stacked_id_embeds.to(prompt_embeds.dtype))
+ updated_prompt_embeds = prompt_embeds.view(batch_size, seq_length, -1)
+ return updated_prompt_embeds
+
+class PhotoMakerIDEncoder(comfy.clip_model.CLIPVisionModelProjection):
+ def __init__(self):
+ self.load_device = comfy.model_management.text_encoder_device()
+ offload_device = comfy.model_management.text_encoder_offload_device()
+ dtype = comfy.model_management.text_encoder_dtype(self.load_device)
+
+ super().__init__(VISION_CONFIG_DICT, dtype, offload_device, comfy.ops.manual_cast)
+ self.visual_projection_2 = comfy.ops.manual_cast.Linear(1024, 1280, bias=False)
+ self.fuse_module = FuseModule(2048, comfy.ops.manual_cast)
+
+ def forward(self, id_pixel_values, prompt_embeds, class_tokens_mask):
+ b, num_inputs, c, h, w = id_pixel_values.shape
+ id_pixel_values = id_pixel_values.view(b * num_inputs, c, h, w)
+
+ shared_id_embeds = self.vision_model(id_pixel_values)[2]
+ id_embeds = self.visual_projection(shared_id_embeds)
+ id_embeds_2 = self.visual_projection_2(shared_id_embeds)
+
+ id_embeds = id_embeds.view(b, num_inputs, 1, -1)
+ id_embeds_2 = id_embeds_2.view(b, num_inputs, 1, -1)
+
+ id_embeds = torch.cat((id_embeds, id_embeds_2), dim=-1)
+ updated_prompt_embeds = self.fuse_module(prompt_embeds, id_embeds, class_tokens_mask)
+
+ return updated_prompt_embeds
+
+
+class PhotoMakerLoader:
+ @classmethod
+ def INPUT_TYPES(s):
+ return {"required": { "photomaker_model_name": (folder_paths.get_filename_list("photomaker"), )}}
+
+ RETURN_TYPES = ("PHOTOMAKER",)
+ FUNCTION = "load_photomaker_model"
+
+ CATEGORY = "_for_testing/photomaker"
+
+ def load_photomaker_model(self, photomaker_model_name):
+ photomaker_model_path = folder_paths.get_full_path("photomaker", photomaker_model_name)
+ photomaker_model = PhotoMakerIDEncoder()
+ data = comfy.utils.load_torch_file(photomaker_model_path, safe_load=True)
+ if "id_encoder" in data:
+ data = data["id_encoder"]
+ photomaker_model.load_state_dict(data)
+ return (photomaker_model,)
+
+
+class PhotoMakerEncode:
+ @classmethod
+ def INPUT_TYPES(s):
+ return {"required": { "photomaker": ("PHOTOMAKER",),
+ "image": ("IMAGE",),
+ "clip": ("CLIP", ),
+ "text": ("STRING", {"multiline": True, "default": "photograph of photomaker"}),
+ }}
+
+ RETURN_TYPES = ("CONDITIONING",)
+ FUNCTION = "apply_photomaker"
+
+ CATEGORY = "_for_testing/photomaker"
+
+ def apply_photomaker(self, photomaker, image, clip, text):
+ special_token = "photomaker"
+ pixel_values = comfy.clip_vision.clip_preprocess(image.to(photomaker.load_device)).float()
+ try:
+ index = text.split(" ").index(special_token) + 1
+ except ValueError:
+ index = -1
+ tokens = clip.tokenize(text, return_word_ids=True)
+ out_tokens = {}
+ for k in tokens:
+ out_tokens[k] = []
+ for t in tokens[k]:
+ f = list(filter(lambda x: x[2] != index, t))
+ while len(f) < len(t):
+ f.append(t[-1])
+ out_tokens[k].append(f)
+
+ cond, pooled = clip.encode_from_tokens(out_tokens, return_pooled=True)
+
+ if index > 0:
+ token_index = index - 1
+ num_id_images = 1
+ class_tokens_mask = [True if token_index <= i < token_index+num_id_images else False for i in range(77)]
+ out = photomaker(id_pixel_values=pixel_values.unsqueeze(0), prompt_embeds=cond.to(photomaker.load_device),
+ class_tokens_mask=torch.tensor(class_tokens_mask, dtype=torch.bool, device=photomaker.load_device).unsqueeze(0))
+ else:
+ out = cond
+
+ return ([[out, {"pooled_output": pooled}]], )
+
+
+NODE_CLASS_MAPPINGS = {
+ "PhotoMakerLoader": PhotoMakerLoader,
+ "PhotoMakerEncode": PhotoMakerEncode,
+}
+
diff --git a/comfy_extras/nodes_post_processing.py b/comfy_extras/nodes_post_processing.py
index 71660f8a5..0110b472f 100644
--- a/comfy_extras/nodes_post_processing.py
+++ b/comfy_extras/nodes_post_processing.py
@@ -33,6 +33,7 @@ class Blend:
CATEGORY = "image/postprocessing"
def blend_images(self, image1: torch.Tensor, image2: torch.Tensor, blend_factor: float, blend_mode: str):
+ image2 = image2.to(image1.device)
if image1.shape != image2.shape:
image2 = image2.permute(0, 3, 1, 2)
image2 = comfy.utils.common_upscale(image2, image1.shape[2], image1.shape[1], upscale_method='bicubic', crop='center')
@@ -203,13 +204,13 @@ class Sharpen:
"default": 1.0,
"min": 0.1,
"max": 10.0,
- "step": 0.1
+ "step": 0.01
}),
"alpha": ("FLOAT", {
"default": 1.0,
"min": 0.0,
"max": 5.0,
- "step": 0.1
+ "step": 0.01
}),
},
}
diff --git a/comfy_extras/nodes_sag.py b/comfy_extras/nodes_sag.py
index bbd380807..69084e91d 100644
--- a/comfy_extras/nodes_sag.py
+++ b/comfy_extras/nodes_sag.py
@@ -150,7 +150,7 @@ class SelfAttentionGuidance:
degraded = create_blur_map(uncond_pred, uncond_attn, sag_sigma, sag_threshold)
degraded_noised = degraded + x - uncond_pred
# call into the UNet
- (sag, _) = comfy.samplers.calc_cond_uncond_batch(model, uncond, None, degraded_noised, sigma, model_options)
+ (sag,) = comfy.samplers.calc_cond_batch(model, [uncond], degraded_noised, sigma, model_options)
return cfg_result + (degraded - sag) * sag_scale
m.set_model_sampler_post_cfg_function(post_cfg_function, disable_cfg1_optimization=True)
diff --git a/comfy_extras/nodes_stable3d.py b/comfy_extras/nodes_stable3d.py
index e02a9875a..be2e34c28 100644
--- a/comfy_extras/nodes_stable3d.py
+++ b/comfy_extras/nodes_stable3d.py
@@ -29,8 +29,8 @@ class StableZero123_Conditioning:
"width": ("INT", {"default": 256, "min": 16, "max": nodes.MAX_RESOLUTION, "step": 8}),
"height": ("INT", {"default": 256, "min": 16, "max": nodes.MAX_RESOLUTION, "step": 8}),
"batch_size": ("INT", {"default": 1, "min": 1, "max": 4096}),
- "elevation": ("FLOAT", {"default": 0.0, "min": -180.0, "max": 180.0}),
- "azimuth": ("FLOAT", {"default": 0.0, "min": -180.0, "max": 180.0}),
+ "elevation": ("FLOAT", {"default": 0.0, "min": -180.0, "max": 180.0, "step": 0.1, "round": False}),
+ "azimuth": ("FLOAT", {"default": 0.0, "min": -180.0, "max": 180.0, "step": 0.1, "round": False}),
}}
RETURN_TYPES = ("CONDITIONING", "CONDITIONING", "LATENT")
RETURN_NAMES = ("positive", "negative", "latent")
@@ -46,7 +46,7 @@ class StableZero123_Conditioning:
encode_pixels = pixels[:,:,:,:3]
t = vae.encode(encode_pixels)
cam_embeds = camera_embeddings(elevation, azimuth)
- cond = torch.cat([pooled, cam_embeds.repeat((pooled.shape[0], 1, 1))], dim=-1)
+ cond = torch.cat([pooled, cam_embeds.to(pooled.device).repeat((pooled.shape[0], 1, 1))], dim=-1)
positive = [[cond, {"concat_latent_image": t}]]
negative = [[torch.zeros_like(pooled), {"concat_latent_image": torch.zeros_like(t)}]]
@@ -62,10 +62,10 @@ class StableZero123_Conditioning_Batched:
"width": ("INT", {"default": 256, "min": 16, "max": nodes.MAX_RESOLUTION, "step": 8}),
"height": ("INT", {"default": 256, "min": 16, "max": nodes.MAX_RESOLUTION, "step": 8}),
"batch_size": ("INT", {"default": 1, "min": 1, "max": 4096}),
- "elevation": ("FLOAT", {"default": 0.0, "min": -180.0, "max": 180.0}),
- "azimuth": ("FLOAT", {"default": 0.0, "min": -180.0, "max": 180.0}),
- "elevation_batch_increment": ("FLOAT", {"default": 0.0, "min": -180.0, "max": 180.0}),
- "azimuth_batch_increment": ("FLOAT", {"default": 0.0, "min": -180.0, "max": 180.0}),
+ "elevation": ("FLOAT", {"default": 0.0, "min": -180.0, "max": 180.0, "step": 0.1, "round": False}),
+ "azimuth": ("FLOAT", {"default": 0.0, "min": -180.0, "max": 180.0, "step": 0.1, "round": False}),
+ "elevation_batch_increment": ("FLOAT", {"default": 0.0, "min": -180.0, "max": 180.0, "step": 0.1, "round": False}),
+ "azimuth_batch_increment": ("FLOAT", {"default": 0.0, "min": -180.0, "max": 180.0, "step": 0.1, "round": False}),
}}
RETURN_TYPES = ("CONDITIONING", "CONDITIONING", "LATENT")
RETURN_NAMES = ("positive", "negative", "latent")
@@ -95,8 +95,49 @@ class StableZero123_Conditioning_Batched:
latent = torch.zeros([batch_size, 4, height // 8, width // 8])
return (positive, negative, {"samples":latent, "batch_index": [0] * batch_size})
+class SV3D_Conditioning:
+ @classmethod
+ def INPUT_TYPES(s):
+ return {"required": { "clip_vision": ("CLIP_VISION",),
+ "init_image": ("IMAGE",),
+ "vae": ("VAE",),
+ "width": ("INT", {"default": 576, "min": 16, "max": nodes.MAX_RESOLUTION, "step": 8}),
+ "height": ("INT", {"default": 576, "min": 16, "max": nodes.MAX_RESOLUTION, "step": 8}),
+ "video_frames": ("INT", {"default": 21, "min": 1, "max": 4096}),
+ "elevation": ("FLOAT", {"default": 0.0, "min": -90.0, "max": 90.0, "step": 0.1, "round": False}),
+ }}
+ RETURN_TYPES = ("CONDITIONING", "CONDITIONING", "LATENT")
+ RETURN_NAMES = ("positive", "negative", "latent")
+
+ FUNCTION = "encode"
+
+ CATEGORY = "conditioning/3d_models"
+
+ def encode(self, clip_vision, init_image, vae, width, height, video_frames, elevation):
+ output = clip_vision.encode_image(init_image)
+ pooled = output.image_embeds.unsqueeze(0)
+ pixels = comfy.utils.common_upscale(init_image.movedim(-1,1), width, height, "bilinear", "center").movedim(1,-1)
+ encode_pixels = pixels[:,:,:,:3]
+ t = vae.encode(encode_pixels)
+
+ azimuth = 0
+ azimuth_increment = 360 / (max(video_frames, 2) - 1)
+
+ elevations = []
+ azimuths = []
+ for i in range(video_frames):
+ elevations.append(elevation)
+ azimuths.append(azimuth)
+ azimuth += azimuth_increment
+
+ positive = [[pooled, {"concat_latent_image": t, "elevation": elevations, "azimuth": azimuths}]]
+ negative = [[torch.zeros_like(pooled), {"concat_latent_image": torch.zeros_like(t), "elevation": elevations, "azimuth": azimuths}]]
+ latent = torch.zeros([video_frames, 4, height // 8, width // 8])
+ return (positive, negative, {"samples":latent})
+
NODE_CLASS_MAPPINGS = {
"StableZero123_Conditioning": StableZero123_Conditioning,
"StableZero123_Conditioning_Batched": StableZero123_Conditioning_Batched,
+ "SV3D_Conditioning": SV3D_Conditioning,
}
diff --git a/comfy_extras/nodes_stable_cascade.py b/comfy_extras/nodes_stable_cascade.py
new file mode 100644
index 000000000..fcbbeb27f
--- /dev/null
+++ b/comfy_extras/nodes_stable_cascade.py
@@ -0,0 +1,140 @@
+"""
+ This file is part of ComfyUI.
+ Copyright (C) 2024 Stability AI
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program. If not, see .
+"""
+
+import torch
+import nodes
+import comfy.utils
+
+
+class StableCascade_EmptyLatentImage:
+ def __init__(self, device="cpu"):
+ self.device = device
+
+ @classmethod
+ def INPUT_TYPES(s):
+ return {"required": {
+ "width": ("INT", {"default": 1024, "min": 256, "max": nodes.MAX_RESOLUTION, "step": 8}),
+ "height": ("INT", {"default": 1024, "min": 256, "max": nodes.MAX_RESOLUTION, "step": 8}),
+ "compression": ("INT", {"default": 42, "min": 4, "max": 128, "step": 1}),
+ "batch_size": ("INT", {"default": 1, "min": 1, "max": 4096})
+ }}
+ RETURN_TYPES = ("LATENT", "LATENT")
+ RETURN_NAMES = ("stage_c", "stage_b")
+ FUNCTION = "generate"
+
+ CATEGORY = "latent/stable_cascade"
+
+ def generate(self, width, height, compression, batch_size=1):
+ c_latent = torch.zeros([batch_size, 16, height // compression, width // compression])
+ b_latent = torch.zeros([batch_size, 4, height // 4, width // 4])
+ return ({
+ "samples": c_latent,
+ }, {
+ "samples": b_latent,
+ })
+
+class StableCascade_StageC_VAEEncode:
+ def __init__(self, device="cpu"):
+ self.device = device
+
+ @classmethod
+ def INPUT_TYPES(s):
+ return {"required": {
+ "image": ("IMAGE",),
+ "vae": ("VAE", ),
+ "compression": ("INT", {"default": 42, "min": 4, "max": 128, "step": 1}),
+ }}
+ RETURN_TYPES = ("LATENT", "LATENT")
+ RETURN_NAMES = ("stage_c", "stage_b")
+ FUNCTION = "generate"
+
+ CATEGORY = "latent/stable_cascade"
+
+ def generate(self, image, vae, compression):
+ width = image.shape[-2]
+ height = image.shape[-3]
+ out_width = (width // compression) * vae.downscale_ratio
+ out_height = (height // compression) * vae.downscale_ratio
+
+ s = comfy.utils.common_upscale(image.movedim(-1,1), out_width, out_height, "bicubic", "center").movedim(1,-1)
+
+ c_latent = vae.encode(s[:,:,:,:3])
+ b_latent = torch.zeros([c_latent.shape[0], 4, (height // 8) * 2, (width // 8) * 2])
+ return ({
+ "samples": c_latent,
+ }, {
+ "samples": b_latent,
+ })
+
+class StableCascade_StageB_Conditioning:
+ @classmethod
+ def INPUT_TYPES(s):
+ return {"required": { "conditioning": ("CONDITIONING",),
+ "stage_c": ("LATENT",),
+ }}
+ RETURN_TYPES = ("CONDITIONING",)
+
+ FUNCTION = "set_prior"
+
+ CATEGORY = "conditioning/stable_cascade"
+
+ def set_prior(self, conditioning, stage_c):
+ c = []
+ for t in conditioning:
+ d = t[1].copy()
+ d['stable_cascade_prior'] = stage_c['samples']
+ n = [t[0], d]
+ c.append(n)
+ return (c, )
+
+class StableCascade_SuperResolutionControlnet:
+ def __init__(self, device="cpu"):
+ self.device = device
+
+ @classmethod
+ def INPUT_TYPES(s):
+ return {"required": {
+ "image": ("IMAGE",),
+ "vae": ("VAE", ),
+ }}
+ RETURN_TYPES = ("IMAGE", "LATENT", "LATENT")
+ RETURN_NAMES = ("controlnet_input", "stage_c", "stage_b")
+ FUNCTION = "generate"
+
+ CATEGORY = "_for_testing/stable_cascade"
+
+ def generate(self, image, vae):
+ width = image.shape[-2]
+ height = image.shape[-3]
+ batch_size = image.shape[0]
+ controlnet_input = vae.encode(image[:,:,:,:3]).movedim(1, -1)
+
+ c_latent = torch.zeros([batch_size, 16, height // 16, width // 16])
+ b_latent = torch.zeros([batch_size, 4, height // 2, width // 2])
+ return (controlnet_input, {
+ "samples": c_latent,
+ }, {
+ "samples": b_latent,
+ })
+
+NODE_CLASS_MAPPINGS = {
+ "StableCascade_EmptyLatentImage": StableCascade_EmptyLatentImage,
+ "StableCascade_StageB_Conditioning": StableCascade_StageB_Conditioning,
+ "StableCascade_StageC_VAEEncode": StableCascade_StageC_VAEEncode,
+ "StableCascade_SuperResolutionControlnet": StableCascade_SuperResolutionControlnet,
+}
diff --git a/comfy_extras/nodes_video_model.py b/comfy_extras/nodes_video_model.py
index a52625652..1a0189ed4 100644
--- a/comfy_extras/nodes_video_model.py
+++ b/comfy_extras/nodes_video_model.py
@@ -79,6 +79,33 @@ class VideoLinearCFGGuidance:
m.set_model_sampler_cfg_function(linear_cfg)
return (m, )
+class VideoTriangleCFGGuidance:
+ @classmethod
+ def INPUT_TYPES(s):
+ return {"required": { "model": ("MODEL",),
+ "min_cfg": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 100.0, "step":0.5, "round": 0.01}),
+ }}
+ RETURN_TYPES = ("MODEL",)
+ FUNCTION = "patch"
+
+ CATEGORY = "sampling/video_models"
+
+ def patch(self, model, min_cfg):
+ def linear_cfg(args):
+ cond = args["cond"]
+ uncond = args["uncond"]
+ cond_scale = args["cond_scale"]
+ period = 1.0
+ values = torch.linspace(0, 1, cond.shape[0], device=cond.device)
+ values = 2 * (values / period - torch.floor(values / period + 0.5)).abs()
+ scale = (values * (cond_scale - min_cfg) + min_cfg).reshape((cond.shape[0], 1, 1, 1))
+
+ return uncond + scale * (cond - uncond)
+
+ m = model.clone()
+ m.set_model_sampler_cfg_function(linear_cfg)
+ return (m, )
+
class ImageOnlyCheckpointSave(comfy_extras.nodes_model_merging.CheckpointSave):
CATEGORY = "_for_testing"
@@ -98,6 +125,7 @@ NODE_CLASS_MAPPINGS = {
"ImageOnlyCheckpointLoader": ImageOnlyCheckpointLoader,
"SVD_img2vid_Conditioning": SVD_img2vid_Conditioning,
"VideoLinearCFGGuidance": VideoLinearCFGGuidance,
+ "VideoTriangleCFGGuidance": VideoTriangleCFGGuidance,
"ImageOnlyCheckpointSave": ImageOnlyCheckpointSave,
}
diff --git a/cuda_malloc.py b/cuda_malloc.py
index 144cdacd3..eb2857c5f 100644
--- a/cuda_malloc.py
+++ b/cuda_malloc.py
@@ -1,6 +1,7 @@
import os
import importlib.util
from comfy.cli_args import args
+import subprocess
#Can't use pytorch to get the GPU names because the cuda malloc has to be set before the first import.
def get_gpu_names():
@@ -34,14 +35,19 @@ def get_gpu_names():
return gpu_names
return enum_display_devices()
else:
- return set()
+ gpu_names = set()
+ out = subprocess.check_output(['nvidia-smi', '-L'])
+ for l in out.split(b'\n'):
+ if len(l) > 0:
+ gpu_names.add(l.decode('utf-8').split(' (UUID')[0])
+ return gpu_names
blacklist = {"GeForce GTX TITAN X", "GeForce GTX 980", "GeForce GTX 970", "GeForce GTX 960", "GeForce GTX 950", "GeForce 945M",
"GeForce 940M", "GeForce 930M", "GeForce 920M", "GeForce 910M", "GeForce GTX 750", "GeForce GTX 745", "Quadro K620",
"Quadro K1200", "Quadro K2200", "Quadro M500", "Quadro M520", "Quadro M600", "Quadro M620", "Quadro M1000",
"Quadro M1200", "Quadro M2000", "Quadro M2200", "Quadro M3000", "Quadro M4000", "Quadro M5000", "Quadro M5500", "Quadro M6000",
"GeForce MX110", "GeForce MX130", "GeForce 830M", "GeForce 840M", "GeForce GTX 850M", "GeForce GTX 860M",
- "GeForce GTX 1650", "GeForce GTX 1630"
+ "GeForce GTX 1650", "GeForce GTX 1630", "Tesla M4", "Tesla M6", "Tesla M10", "Tesla M40", "Tesla M60"
}
def cuda_malloc_supported():
diff --git a/custom_nodes/example_node.py.example b/custom_nodes/example_node.py.example
index 733014f3c..f06632593 100644
--- a/custom_nodes/example_node.py.example
+++ b/custom_nodes/example_node.py.example
@@ -6,6 +6,8 @@ class Example:
-------------
INPUT_TYPES (dict):
Tell the main program input parameters of nodes.
+ IS_CHANGED:
+ optional method to control when the node is re executed.
Attributes
----------
@@ -89,6 +91,20 @@ class Example:
image = 1.0 - image
return (image,)
+ """
+ The node will always be re executed if any of the inputs change but
+ this method can be used to force the node to execute again even when the inputs don't change.
+ You can make this node return a number or a string. This value will be compared to the one returned the last time the node was
+ executed, if it is different the node will be executed again.
+ This method is used in the core repo for the LoadImage node where they return the image hash as a string, if the image hash
+ changes between executions the LoadImage node is executed again.
+ """
+ #@classmethod
+ #def IS_CHANGED(s, image, string_field, int_field, float_field, print_to_screen):
+ # return ""
+
+# Set the web directory, any .js file in that directory will be loaded by the frontend as a frontend extension
+# WEB_DIRECTORY = "./somejs"
# A dictionary that contains all nodes you want to export with their names
# NOTE: names should be globally unique
diff --git a/custom_nodes/websocket_image_save.py b/custom_nodes/websocket_image_save.py
new file mode 100644
index 000000000..5aa573642
--- /dev/null
+++ b/custom_nodes/websocket_image_save.py
@@ -0,0 +1,45 @@
+from PIL import Image, ImageOps
+from io import BytesIO
+import numpy as np
+import struct
+import comfy.utils
+import time
+
+#You can use this node to save full size images through the websocket, the
+#images will be sent in exactly the same format as the image previews: as
+#binary images on the websocket with a 8 byte header indicating the type
+#of binary message (first 4 bytes) and the image format (next 4 bytes).
+
+#Note that no metadata will be put in the images saved with this node.
+
+class SaveImageWebsocket:
+ @classmethod
+ def INPUT_TYPES(s):
+ return {"required":
+ {"images": ("IMAGE", ),}
+ }
+
+ RETURN_TYPES = ()
+ FUNCTION = "save_images"
+
+ OUTPUT_NODE = True
+
+ CATEGORY = "api/image"
+
+ def save_images(self, images):
+ pbar = comfy.utils.ProgressBar(images.shape[0])
+ step = 0
+ for image in images:
+ i = 255. * image.cpu().numpy()
+ img = Image.fromarray(np.clip(i, 0, 255).astype(np.uint8))
+ pbar.update_absolute(step, images.shape[0], ("PNG", img, None))
+ step += 1
+
+ return {}
+
+ def IS_CHANGED(s, images):
+ return time.time()
+
+NODE_CLASS_MAPPINGS = {
+ "SaveImageWebsocket": SaveImageWebsocket,
+}
diff --git a/execution.py b/execution.py
index bc5cfe55c..35bdb77a6 100644
--- a/execution.py
+++ b/execution.py
@@ -1,12 +1,9 @@
-import os
import sys
import copy
-import json
import logging
import threading
import heapq
import traceback
-import gc
import inspect
from typing import List, Literal, NamedTuple, Optional
@@ -38,8 +35,7 @@ def get_input_data(inputs, class_def, unique_id, outputs={}, prompt={}, extra_da
if h[x] == "PROMPT":
input_data_all[x] = [prompt]
if h[x] == "EXTRA_PNGINFO":
- if "extra_pnginfo" in extra_data:
- input_data_all[x] = [extra_data['extra_pnginfo']]
+ input_data_all[x] = [extra_data.get('extra_pnginfo', None)]
if h[x] == "UNIQUE_ID":
input_data_all[x] = [unique_id]
return input_data_all
@@ -197,8 +193,12 @@ def recursive_execute(server, prompt, outputs, current_item, extra_data, execute
return (True, None, None)
-def recursive_will_execute(prompt, outputs, current_item):
+def recursive_will_execute(prompt, outputs, current_item, memo={}):
unique_id = current_item
+
+ if unique_id in memo:
+ return memo[unique_id]
+
inputs = prompt[unique_id]['inputs']
will_execute = []
if unique_id in outputs:
@@ -210,9 +210,10 @@ def recursive_will_execute(prompt, outputs, current_item):
input_unique_id = input_data[0]
output_index = input_data[1]
if input_unique_id not in outputs:
- will_execute += recursive_will_execute(prompt, outputs, input_unique_id)
+ will_execute += recursive_will_execute(prompt, outputs, input_unique_id, memo)
- return will_execute + [unique_id]
+ memo[unique_id] = will_execute + [unique_id]
+ return memo[unique_id]
def recursive_output_delete_if_changed(prompt, old_prompt, outputs, current_item):
unique_id = current_item
@@ -367,7 +368,7 @@ class PromptExecutor:
d = self.outputs_ui.pop(x)
del d
- comfy.model_management.cleanup_models()
+ comfy.model_management.cleanup_models(keep_clone_weights_loaded=True)
self.add_message("execution_cached",
{ "nodes": list(current_outputs) , "prompt_id": prompt_id},
broadcast=False)
@@ -380,7 +381,8 @@ class PromptExecutor:
while len(to_execute) > 0:
#always execute the output that depends on the least amount of unexecuted nodes first
- to_execute = sorted(list(map(lambda a: (len(recursive_will_execute(prompt, self.outputs, a[-1])), a[-1]), to_execute)))
+ memo = {}
+ to_execute = sorted(list(map(lambda a: (len(recursive_will_execute(prompt, self.outputs, a[-1], memo)), a[-1]), to_execute)))
output_node_id = to_execute.pop(0)[-1]
# This call shouldn't raise anything if there's an error deep in
diff --git a/folder_paths.py b/folder_paths.py
index ef9b8ccfa..a84524d9f 100644
--- a/folder_paths.py
+++ b/folder_paths.py
@@ -1,5 +1,6 @@
import os
import time
+import logging
supported_pt_extensions = set(['.ckpt', '.pt', '.bin', '.pth', '.safetensors'])
@@ -29,6 +30,8 @@ folder_names_and_paths["custom_nodes"] = ([os.path.join(base_path, "custom_nodes
folder_names_and_paths["hypernetworks"] = ([os.path.join(models_dir, "hypernetworks")], supported_pt_extensions)
+folder_names_and_paths["photomaker"] = ([os.path.join(models_dir, "photomaker")], supported_pt_extensions)
+
folder_names_and_paths["classifiers"] = ([os.path.join(models_dir, "classifiers")], {""})
output_directory = os.path.join(os.path.dirname(os.path.realpath(__file__)), "output")
@@ -42,7 +45,7 @@ if not os.path.exists(input_directory):
try:
os.makedirs(input_directory)
except:
- print("Failed to create input directory")
+ logging.error("Failed to create input directory")
def set_output_directory(output_dir):
global output_directory
@@ -144,21 +147,23 @@ def recursive_search(directory, excluded_dir_names=None):
try:
dirs[directory] = os.path.getmtime(directory)
except FileNotFoundError:
- print(f"Warning: Unable to access {directory}. Skipping this path.")
-
+ logging.warning(f"Warning: Unable to access {directory}. Skipping this path.")
+
+ logging.debug("recursive file list on directory {}".format(directory))
for dirpath, subdirs, filenames in os.walk(directory, followlinks=True, topdown=True):
subdirs[:] = [d for d in subdirs if d not in excluded_dir_names]
for file_name in filenames:
relative_path = os.path.relpath(os.path.join(dirpath, file_name), directory)
result.append(relative_path)
-
+
for d in subdirs:
path = os.path.join(dirpath, d)
try:
dirs[path] = os.path.getmtime(path)
except FileNotFoundError:
- print(f"Warning: Unable to access {path}. Skipping this path.")
+ logging.warning(f"Warning: Unable to access {path}. Skipping this path.")
continue
+ logging.debug("found {} files".format(len(result)))
return result, dirs
def filter_files_extensions(files, extensions):
@@ -246,8 +251,8 @@ def get_save_image_path(filename_prefix, output_dir, image_width=0, image_height
err = "**** ERROR: Saving image outside the output folder is not allowed." + \
"\n full_output_folder: " + os.path.abspath(full_output_folder) + \
"\n output_dir: " + output_dir + \
- "\n commonpath: " + os.path.commonpath((output_dir, os.path.abspath(full_output_folder)))
- print(err)
+ "\n commonpath: " + os.path.commonpath((output_dir, os.path.abspath(full_output_folder)))
+ logging.error(err)
raise Exception(err)
try:
diff --git a/latent_preview.py b/latent_preview.py
index 61754751e..4dbcbf455 100644
--- a/latent_preview.py
+++ b/latent_preview.py
@@ -6,6 +6,7 @@ from comfy.cli_args import args, LatentPreviewMethod
from comfy.taesd.taesd import TAESD
import folder_paths
import comfy.utils
+import logging
MAX_PREVIEW_RESOLUTION = 512
@@ -70,7 +71,7 @@ def get_previewer(device, latent_format):
taesd = TAESD(None, taesd_decoder_path).to(device)
previewer = TAESDPreviewerImpl(taesd)
else:
- print("Warning: TAESD previews enabled, but could not find models/vae_approx/{}".format(latent_format.taesd_decoder_name))
+ logging.warning("Warning: TAESD previews enabled, but could not find models/vae_approx/{}".format(latent_format.taesd_decoder_name))
if previewer is None:
if latent_format.latent_rgb_factors is not None:
diff --git a/main.py b/main.py
index 69d9bce6c..b3a3ebea8 100644
--- a/main.py
+++ b/main.py
@@ -54,15 +54,15 @@ import threading
import gc
from comfy.cli_args import args
+import logging
if os.name == "nt":
- import logging
logging.getLogger("xformers").addFilter(lambda record: 'A matching Triton is not available' not in record.getMessage())
if __name__ == "__main__":
if args.cuda_device is not None:
os.environ['CUDA_VISIBLE_DEVICES'] = str(args.cuda_device)
- print("Set cuda device to:", args.cuda_device)
+ logging.info("Set cuda device to: {}".format(args.cuda_device))
if args.deterministic:
if 'CUBLAS_WORKSPACE_CONFIG' not in os.environ:
@@ -88,7 +88,7 @@ def cuda_malloc_warning():
if b in device_name:
cuda_malloc_warning = True
if cuda_malloc_warning:
- print("\nWARNING: this card most likely does not support cuda-malloc, if you get \"CUDA error\" please run ComfyUI with: --disable-cuda-malloc\n")
+ logging.warning("\nWARNING: this card most likely does not support cuda-malloc, if you get \"CUDA error\" please run ComfyUI with: --disable-cuda-malloc\n")
def prompt_worker(q, server):
e = execution.PromptExecutor(server)
@@ -121,7 +121,7 @@ def prompt_worker(q, server):
current_time = time.perf_counter()
execution_time = current_time - execution_start_time
- print("Prompt executed in {:.2f} seconds".format(execution_time))
+ logging.info("Prompt executed in {:.2f} seconds".format(execution_time))
flags = q.get_flags()
free_memory = flags.get("free_memory", False)
@@ -139,6 +139,7 @@ def prompt_worker(q, server):
if need_gc:
current_time = time.perf_counter()
if (current_time - last_gc_collect) > gc_collect_interval:
+ comfy.model_management.cleanup_models()
gc.collect()
comfy.model_management.soft_empty_cache()
last_gc_collect = current_time
@@ -182,17 +183,24 @@ def load_extra_path_config(yaml_path):
full_path = y
if base_path is not None:
full_path = os.path.join(base_path, full_path)
- print("Adding extra search path", x, full_path)
+ logging.info("Adding extra search path {} {}".format(x, full_path))
folder_paths.add_model_folder_path(x, full_path)
if __name__ == "__main__":
if args.temp_directory:
temp_dir = os.path.join(os.path.abspath(args.temp_directory), "temp")
- print(f"Setting temp directory to: {temp_dir}")
+ logging.info(f"Setting temp directory to: {temp_dir}")
folder_paths.set_temp_directory(temp_dir)
cleanup_temp()
+ if args.windows_standalone_build:
+ try:
+ import new_updater
+ new_updater.update_windows_updater()
+ except:
+ pass
+
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
server = server.PromptServer(loop)
@@ -217,7 +225,7 @@ if __name__ == "__main__":
if args.output_directory:
output_dir = os.path.abspath(args.output_directory)
- print(f"Setting output directory to: {output_dir}")
+ logging.info(f"Setting output directory to: {output_dir}")
folder_paths.set_output_directory(output_dir)
#These are the default folders that checkpoints, clip and vae models will be saved to when using CheckpointSave, etc.. nodes
@@ -227,7 +235,7 @@ if __name__ == "__main__":
if args.input_directory:
input_dir = os.path.abspath(args.input_directory)
- print(f"Setting input directory to: {input_dir}")
+ logging.info(f"Setting input directory to: {input_dir}")
folder_paths.set_input_directory(input_dir)
if args.quick_test_for_ci:
@@ -245,6 +253,6 @@ if __name__ == "__main__":
try:
loop.run_until_complete(run(server, address=args.listen, port=args.port, verbose=not args.dont_print_server, call_on_start=call_on_start))
except KeyboardInterrupt:
- print("\nStopped server")
+ logging.info("\nStopped server")
cleanup_temp()
diff --git a/models/photomaker/put_photomaker_models_here b/models/photomaker/put_photomaker_models_here
new file mode 100644
index 000000000..e69de29bb
diff --git a/new_updater.py b/new_updater.py
new file mode 100644
index 000000000..a49e0877c
--- /dev/null
+++ b/new_updater.py
@@ -0,0 +1,35 @@
+import os
+import shutil
+
+base_path = os.path.dirname(os.path.realpath(__file__))
+
+
+def update_windows_updater():
+ top_path = os.path.dirname(base_path)
+ updater_path = os.path.join(base_path, ".ci/update_windows/update.py")
+ bat_path = os.path.join(base_path, ".ci/update_windows/update_comfyui.bat")
+
+ dest_updater_path = os.path.join(top_path, "update/update.py")
+ dest_bat_path = os.path.join(top_path, "update/update_comfyui.bat")
+ dest_bat_deps_path = os.path.join(top_path, "update/update_comfyui_and_python_dependencies.bat")
+
+ try:
+ with open(dest_bat_path, 'rb') as f:
+ contents = f.read()
+ except:
+ return
+
+ if not contents.startswith(b"..\\python_embeded\\python.exe .\\update.py"):
+ return
+
+ shutil.copy(updater_path, dest_updater_path)
+ try:
+ with open(dest_bat_deps_path, 'rb') as f:
+ contents = f.read()
+ contents = contents.replace(b'..\\python_embeded\\python.exe .\\update.py ..\\ComfyUI\\', b'call update_comfyui.bat nopause')
+ with open(dest_bat_deps_path, 'wb') as f:
+ f.write(contents)
+ except:
+ pass
+ shutil.copy(bat_path, dest_bat_path)
+ print("Updated the windows standalone package updater.")
diff --git a/node_helpers.py b/node_helpers.py
new file mode 100644
index 000000000..8828a4ec9
--- /dev/null
+++ b/node_helpers.py
@@ -0,0 +1,10 @@
+
+def conditioning_set_values(conditioning, values={}):
+ c = []
+ for t in conditioning:
+ n = [t[0], t[1].copy()]
+ for k in values:
+ n[1][k] = values[k]
+ c.append(n)
+
+ return c
diff --git a/nodes.py b/nodes.py
index 6c7317b69..78e0cf116 100644
--- a/nodes.py
+++ b/nodes.py
@@ -8,6 +8,7 @@ import traceback
import math
import time
import random
+import logging
from PIL import Image, ImageOps, ImageSequence
from PIL.PngImagePlugin import PngInfo
@@ -33,6 +34,7 @@ import importlib
import folder_paths
import latent_preview
+import node_helpers
def before_node_execution():
comfy.model_management.throw_exception_if_processing_interrupted()
@@ -40,7 +42,7 @@ def before_node_execution():
def interrupt_processing(value=True):
comfy.model_management.interrupt_current_processing(value)
-MAX_RESOLUTION=8192
+MAX_RESOLUTION=16384
class CLIPTextEncode:
@classmethod
@@ -83,7 +85,7 @@ class ConditioningAverage :
out = []
if len(conditioning_from) > 1:
- print("Warning: ConditioningAverage conditioning_from contains more than 1 cond, only the first one will actually be applied to conditioning_to.")
+ logging.warning("Warning: ConditioningAverage conditioning_from contains more than 1 cond, only the first one will actually be applied to conditioning_to.")
cond_from = conditioning_from[0][0]
pooled_output_from = conditioning_from[0][1].get("pooled_output", None)
@@ -122,7 +124,7 @@ class ConditioningConcat:
out = []
if len(conditioning_from) > 1:
- print("Warning: ConditioningConcat conditioning_from contains more than 1 cond, only the first one will actually be applied to conditioning_to.")
+ logging.warning("Warning: ConditioningConcat conditioning_from contains more than 1 cond, only the first one will actually be applied to conditioning_to.")
cond_from = conditioning_from[0][0]
@@ -150,13 +152,9 @@ class ConditioningSetArea:
CATEGORY = "conditioning"
def append(self, conditioning, width, height, x, y, strength):
- c = []
- for t in conditioning:
- n = [t[0], t[1].copy()]
- n[1]['area'] = (height // 8, width // 8, y // 8, x // 8)
- n[1]['strength'] = strength
- n[1]['set_area_to_bounds'] = False
- c.append(n)
+ c = node_helpers.conditioning_set_values(conditioning, {"area": (height // 8, width // 8, y // 8, x // 8),
+ "strength": strength,
+ "set_area_to_bounds": False})
return (c, )
class ConditioningSetAreaPercentage:
@@ -175,15 +173,27 @@ class ConditioningSetAreaPercentage:
CATEGORY = "conditioning"
def append(self, conditioning, width, height, x, y, strength):
- c = []
- for t in conditioning:
- n = [t[0], t[1].copy()]
- n[1]['area'] = ("percentage", height, width, y, x)
- n[1]['strength'] = strength
- n[1]['set_area_to_bounds'] = False
- c.append(n)
+ c = node_helpers.conditioning_set_values(conditioning, {"area": ("percentage", height, width, y, x),
+ "strength": strength,
+ "set_area_to_bounds": False})
return (c, )
+class ConditioningSetAreaStrength:
+ @classmethod
+ def INPUT_TYPES(s):
+ return {"required": {"conditioning": ("CONDITIONING", ),
+ "strength": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 10.0, "step": 0.01}),
+ }}
+ RETURN_TYPES = ("CONDITIONING",)
+ FUNCTION = "append"
+
+ CATEGORY = "conditioning"
+
+ def append(self, conditioning, strength):
+ c = node_helpers.conditioning_set_values(conditioning, {"strength": strength})
+ return (c, )
+
+
class ConditioningSetMask:
@classmethod
def INPUT_TYPES(s):
@@ -198,19 +208,15 @@ class ConditioningSetMask:
CATEGORY = "conditioning"
def append(self, conditioning, mask, set_cond_area, strength):
- c = []
set_area_to_bounds = False
if set_cond_area != "default":
set_area_to_bounds = True
if len(mask.shape) < 3:
mask = mask.unsqueeze(0)
- for t in conditioning:
- n = [t[0], t[1].copy()]
- _, h, w = mask.shape
- n[1]['mask'] = mask
- n[1]['set_area_to_bounds'] = set_area_to_bounds
- n[1]['mask_strength'] = strength
- c.append(n)
+
+ c = node_helpers.conditioning_set_values(conditioning, {"mask": mask,
+ "set_area_to_bounds": set_area_to_bounds,
+ "mask_strength": strength})
return (c, )
class ConditioningZeroOut:
@@ -245,13 +251,8 @@ class ConditioningSetTimestepRange:
CATEGORY = "advanced/conditioning"
def set_range(self, conditioning, start, end):
- c = []
- for t in conditioning:
- d = t[1].copy()
- d['start_percent'] = start
- d['end_percent'] = end
- n = [t[0], d]
- c.append(n)
+ c = node_helpers.conditioning_set_values(conditioning, {"start_percent": start,
+ "end_percent": end})
return (c, )
class VAEDecode:
@@ -289,18 +290,7 @@ class VAEEncode:
CATEGORY = "latent"
- @staticmethod
- def vae_encode_crop_pixels(pixels):
- x = (pixels.shape[1] // 8) * 8
- y = (pixels.shape[2] // 8) * 8
- if pixels.shape[1] != x or pixels.shape[2] != y:
- x_offset = (pixels.shape[1] % 8) // 2
- y_offset = (pixels.shape[2] % 8) // 2
- pixels = pixels[:, x_offset:x + x_offset, y_offset:y + y_offset, :]
- return pixels
-
def encode(self, vae, pixels):
- pixels = self.vae_encode_crop_pixels(pixels)
t = vae.encode(pixels[:,:,:,:3])
return ({"samples":t}, )
@@ -316,7 +306,6 @@ class VAEEncodeTiled:
CATEGORY = "_for_testing"
def encode(self, vae, pixels, tile_size):
- pixels = VAEEncode.vae_encode_crop_pixels(pixels)
t = vae.encode_tiled(pixels[:,:,:,:3], tile_x=tile_size, tile_y=tile_size, )
return ({"samples":t}, )
@@ -330,14 +319,14 @@ class VAEEncodeForInpaint:
CATEGORY = "latent/inpaint"
def encode(self, vae, pixels, mask, grow_mask_by=6):
- x = (pixels.shape[1] // 8) * 8
- y = (pixels.shape[2] // 8) * 8
+ x = (pixels.shape[1] // vae.downscale_ratio) * vae.downscale_ratio
+ y = (pixels.shape[2] // vae.downscale_ratio) * vae.downscale_ratio
mask = torch.nn.functional.interpolate(mask.reshape((-1, 1, mask.shape[-2], mask.shape[-1])), size=(pixels.shape[1], pixels.shape[2]), mode="bilinear")
pixels = pixels.clone()
if pixels.shape[1] != x or pixels.shape[2] != y:
- x_offset = (pixels.shape[1] % 8) // 2
- y_offset = (pixels.shape[2] % 8) // 2
+ x_offset = (pixels.shape[1] % vae.downscale_ratio) // 2
+ y_offset = (pixels.shape[2] % vae.downscale_ratio) // 2
pixels = pixels[:,x_offset:x + x_offset, y_offset:y + y_offset,:]
mask = mask[:,:,x_offset:x + x_offset, y_offset:y + y_offset]
@@ -404,13 +393,8 @@ class InpaintModelConditioning:
out = []
for conditioning in [positive, negative]:
- c = []
- for t in conditioning:
- d = t[1].copy()
- d["concat_latent_image"] = concat_latent
- d["concat_mask"] = mask
- n = [t[0], d]
- c.append(n)
+ c = node_helpers.conditioning_set_values(conditioning, {"concat_latent_image": concat_latent,
+ "concat_mask": mask})
out.append(c)
return (out[0], out[1], out_latent)
@@ -834,15 +818,20 @@ class CLIPLoader:
@classmethod
def INPUT_TYPES(s):
return {"required": { "clip_name": (folder_paths.get_filename_list("clip"), ),
+ "type": (["stable_diffusion", "stable_cascade"], ),
}}
RETURN_TYPES = ("CLIP",)
FUNCTION = "load_clip"
CATEGORY = "advanced/loaders"
- def load_clip(self, clip_name):
+ def load_clip(self, clip_name, type="stable_diffusion"):
+ clip_type = comfy.sd.CLIPType.STABLE_DIFFUSION
+ if type == "stable_cascade":
+ clip_type = comfy.sd.CLIPType.STABLE_CASCADE
+
clip_path = folder_paths.get_full_path("clip", clip_name)
- clip = comfy.sd.load_clip(ckpt_paths=[clip_path], embedding_directory=folder_paths.get_folder_paths("embeddings"))
+ clip = comfy.sd.load_clip(ckpt_paths=[clip_path], embedding_directory=folder_paths.get_folder_paths("embeddings"), clip_type=clip_type)
return (clip,)
class DualCLIPLoader:
@@ -990,7 +979,7 @@ class GLIGENTextBoxApply:
def append(self, conditioning_to, clip, gligen_textbox_model, text, width, height, x, y):
c = []
- cond, cond_pooled = clip.encode_from_tokens(clip.tokenize(text), return_pooled=True)
+ cond, cond_pooled = clip.encode_from_tokens(clip.tokenize(text), return_pooled="unprojected")
for t in conditioning_to:
n = [t[0], t[1].copy()]
position_params = [(cond_pooled, height // 8, width // 8, y // 8, x // 8)]
@@ -1414,7 +1403,7 @@ class SaveImage:
filename_prefix += self.prefix_append
full_output_folder, filename, counter, subfolder, filename_prefix = folder_paths.get_save_image_path(filename_prefix, self.output_dir, images[0].shape[1], images[0].shape[0])
results = list()
- for image in images:
+ for (batch_number, image) in enumerate(images):
i = 255. * image.cpu().numpy()
img = Image.fromarray(np.clip(i, 0, 255).astype(np.uint8))
metadata = None
@@ -1426,7 +1415,8 @@ class SaveImage:
for x in extra_pnginfo:
metadata.add_text(x, json.dumps(extra_pnginfo[x]))
- file = f"{filename}_{counter:05}_.png"
+ filename_with_batch_num = filename.replace("%batch_num%", str(batch_number))
+ file = f"{filename_with_batch_num}_{counter:05}_.png"
img.save(os.path.join(full_output_folder, file), pnginfo=metadata, compress_level=self.compress_level)
results.append({
"filename": file,
@@ -1754,6 +1744,7 @@ NODE_CLASS_MAPPINGS = {
"ConditioningConcat": ConditioningConcat,
"ConditioningSetArea": ConditioningSetArea,
"ConditioningSetAreaPercentage": ConditioningSetAreaPercentage,
+ "ConditioningSetAreaStrength": ConditioningSetAreaStrength,
"ConditioningSetMask": ConditioningSetMask,
"KSamplerAdvanced": KSamplerAdvanced,
"SetLatentNoiseMask": SetLatentNoiseMask,
@@ -1860,6 +1851,7 @@ def load_custom_node(module_path, ignore=set()):
sp = os.path.splitext(module_path)
module_name = sp[0]
try:
+ logging.debug("Trying to load custom node {}".format(module_path))
if os.path.isfile(module_path):
module_spec = importlib.util.spec_from_file_location(module_name, module_path)
module_dir = os.path.split(module_path)[0]
@@ -1884,11 +1876,11 @@ def load_custom_node(module_path, ignore=set()):
NODE_DISPLAY_NAME_MAPPINGS.update(module.NODE_DISPLAY_NAME_MAPPINGS)
return True
else:
- print(f"Skip {module_path} module for custom nodes due to the lack of NODE_CLASS_MAPPINGS.")
+ logging.warning(f"Skip {module_path} module for custom nodes due to the lack of NODE_CLASS_MAPPINGS.")
return False
except Exception as e:
- print(traceback.format_exc())
- print(f"Cannot import {module_path} module for custom nodes:", e)
+ logging.warning(traceback.format_exc())
+ logging.warning(f"Cannot import {module_path} module for custom nodes: {e}")
return False
def load_custom_nodes():
@@ -1909,14 +1901,14 @@ def load_custom_nodes():
node_import_times.append((time.perf_counter() - time_before, module_path, success))
if len(node_import_times) > 0:
- print("\nImport times for custom nodes:")
+ logging.info("\nImport times for custom nodes:")
for n in sorted(node_import_times):
if n[2]:
import_message = ""
else:
import_message = " (IMPORT FAILED)"
- print("{:6.1f} seconds{}:".format(n[0], import_message), n[1])
- print()
+ logging.info("{:6.1f} seconds{}: {}".format(n[0], import_message, n[1]))
+ logging.info("")
def init_custom_nodes():
extras_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), "comfy_extras")
@@ -1943,9 +1935,29 @@ def init_custom_nodes():
"nodes_perpneg.py",
"nodes_stable3d.py",
"nodes_sdupscale.py",
+ "nodes_photomaker.py",
+ "nodes_cond.py",
+ "nodes_morphology.py",
+ "nodes_stable_cascade.py",
+ "nodes_differential_diffusion.py",
+ "nodes_ip2p.py",
+ "nodes_model_merging_model_specific.py",
]
+ import_failed = []
for node_file in extras_files:
- load_custom_node(os.path.join(extras_dir, node_file))
+ if not load_custom_node(os.path.join(extras_dir, node_file)):
+ import_failed.append(node_file)
load_custom_nodes()
+
+ if len(import_failed) > 0:
+ logging.warning("WARNING: some comfy_extras/ nodes did not import correctly. This may be because they are missing some dependencies.\n")
+ for node in import_failed:
+ logging.warning("IMPORT FAILED: {}".format(node))
+ logging.warning("\nThis issue might be caused by new missing dependencies added the last time you updated ComfyUI.")
+ if args.windows_standalone_build:
+ logging.warning("Please run the update script: update/update_comfyui.bat")
+ else:
+ logging.warning("Please do a: pip install -r requirements.txt")
+ logging.warning("")
diff --git a/requirements.txt b/requirements.txt
index e804618e7..e7d8c0e9c 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -10,3 +10,4 @@ Pillow
scipy
tqdm
psutil
+kornia>=0.7.1
diff --git a/script_examples/websockets_api_example_ws_images.py b/script_examples/websockets_api_example_ws_images.py
new file mode 100644
index 000000000..737488621
--- /dev/null
+++ b/script_examples/websockets_api_example_ws_images.py
@@ -0,0 +1,159 @@
+#This is an example that uses the websockets api and the SaveImageWebsocket node to get images directly without
+#them being saved to disk
+
+import websocket #NOTE: websocket-client (https://github.com/websocket-client/websocket-client)
+import uuid
+import json
+import urllib.request
+import urllib.parse
+
+server_address = "127.0.0.1:8188"
+client_id = str(uuid.uuid4())
+
+def queue_prompt(prompt):
+ p = {"prompt": prompt, "client_id": client_id}
+ data = json.dumps(p).encode('utf-8')
+ req = urllib.request.Request("http://{}/prompt".format(server_address), data=data)
+ return json.loads(urllib.request.urlopen(req).read())
+
+def get_image(filename, subfolder, folder_type):
+ data = {"filename": filename, "subfolder": subfolder, "type": folder_type}
+ url_values = urllib.parse.urlencode(data)
+ with urllib.request.urlopen("http://{}/view?{}".format(server_address, url_values)) as response:
+ return response.read()
+
+def get_history(prompt_id):
+ with urllib.request.urlopen("http://{}/history/{}".format(server_address, prompt_id)) as response:
+ return json.loads(response.read())
+
+def get_images(ws, prompt):
+ prompt_id = queue_prompt(prompt)['prompt_id']
+ output_images = {}
+ current_node = ""
+ while True:
+ out = ws.recv()
+ if isinstance(out, str):
+ message = json.loads(out)
+ if message['type'] == 'executing':
+ data = message['data']
+ if data['prompt_id'] == prompt_id:
+ if data['node'] is None:
+ break #Execution is done
+ else:
+ current_node = data['node']
+ else:
+ if current_node == 'save_image_websocket_node':
+ images_output = output_images.get(current_node, [])
+ images_output.append(out[8:])
+ output_images[current_node] = images_output
+
+ return output_images
+
+prompt_text = """
+{
+ "3": {
+ "class_type": "KSampler",
+ "inputs": {
+ "cfg": 8,
+ "denoise": 1,
+ "latent_image": [
+ "5",
+ 0
+ ],
+ "model": [
+ "4",
+ 0
+ ],
+ "negative": [
+ "7",
+ 0
+ ],
+ "positive": [
+ "6",
+ 0
+ ],
+ "sampler_name": "euler",
+ "scheduler": "normal",
+ "seed": 8566257,
+ "steps": 20
+ }
+ },
+ "4": {
+ "class_type": "CheckpointLoaderSimple",
+ "inputs": {
+ "ckpt_name": "v1-5-pruned-emaonly.ckpt"
+ }
+ },
+ "5": {
+ "class_type": "EmptyLatentImage",
+ "inputs": {
+ "batch_size": 1,
+ "height": 512,
+ "width": 512
+ }
+ },
+ "6": {
+ "class_type": "CLIPTextEncode",
+ "inputs": {
+ "clip": [
+ "4",
+ 1
+ ],
+ "text": "masterpiece best quality girl"
+ }
+ },
+ "7": {
+ "class_type": "CLIPTextEncode",
+ "inputs": {
+ "clip": [
+ "4",
+ 1
+ ],
+ "text": "bad hands"
+ }
+ },
+ "8": {
+ "class_type": "VAEDecode",
+ "inputs": {
+ "samples": [
+ "3",
+ 0
+ ],
+ "vae": [
+ "4",
+ 2
+ ]
+ }
+ },
+ "save_image_websocket_node": {
+ "class_type": "SaveImageWebsocket",
+ "inputs": {
+ "images": [
+ "8",
+ 0
+ ]
+ }
+ }
+}
+"""
+
+prompt = json.loads(prompt_text)
+#set the text prompt for our positive CLIPTextEncode
+prompt["6"]["inputs"]["text"] = "masterpiece best quality man"
+
+#set the seed for our KSampler node
+prompt["3"]["inputs"]["seed"] = 5
+
+ws = websocket.WebSocket()
+ws.connect("ws://{}/ws?clientId={}".format(server_address, client_id))
+images = get_images(ws, prompt)
+
+#Commented out code to display the output images:
+
+# for node_id in images:
+# for image_data in images[node_id]:
+# from PIL import Image
+# import io
+# image = Image.open(io.BytesIO(image_data))
+# image.show()
+
diff --git a/server.py b/server.py
index 1135fff88..5642bd5e2 100644
--- a/server.py
+++ b/server.py
@@ -15,15 +15,9 @@ from PIL import Image, ImageOps
from PIL.PngImagePlugin import PngInfo
from io import BytesIO
-try:
- import aiohttp
- from aiohttp import web
-except ImportError:
- print("Module 'aiohttp' not installed. Please install it via:")
- print("pip install aiohttp")
- print("or")
- print("pip install -r requirements.txt")
- sys.exit()
+import aiohttp
+from aiohttp import web
+import logging
import mimetypes
from comfy.cli_args import args
@@ -40,7 +34,7 @@ async def send_socket_catch_exception(function, message):
try:
await function(message)
except (aiohttp.ClientError, aiohttp.ClientPayloadError, ConnectionResetError) as err:
- print("send error:", err)
+ logging.warning("send error: {}".format(err))
@web.middleware
async def cache_control(request: web.Request, handler):
@@ -118,7 +112,7 @@ class PromptServer():
async for msg in ws:
if msg.type == aiohttp.WSMsgType.ERROR:
- print('ws connection closed with exception %s' % ws.exception())
+ logging.warning('ws connection closed with exception %s' % ws.exception())
finally:
self.sockets.pop(sid, None)
return ws
@@ -419,8 +413,8 @@ class PromptServer():
try:
out[x] = node_info(x)
except Exception as e:
- print(f"[ERROR] An error occurred while retrieving information for the '{x}' node.", file=sys.stderr)
- traceback.print_exc()
+ logging.error(f"[ERROR] An error occurred while retrieving information for the '{x}' node.")
+ logging.error(traceback.format_exc())
return web.json_response(out)
@routes.get("/object_info/{node_class}")
@@ -453,7 +447,7 @@ class PromptServer():
@routes.post("/prompt")
async def post_prompt(request):
- print("got prompt")
+ logging.info("got prompt")
resp_code = 200
out_string = ""
json_data = await request.json()
@@ -485,7 +479,7 @@ class PromptServer():
response = {"prompt_id": prompt_id, "number": number, "node_errors": valid[3]}
return web.json_response(response)
else:
- print("invalid prompt:", valid[1])
+ logging.warning("invalid prompt: {}".format(valid[1]))
return web.json_response({"error": valid[1], "node_errors": valid[3]}, status=400)
else:
return web.json_response({"error": "no prompt", "node_errors": []}, status=400)
@@ -539,11 +533,11 @@ class PromptServer():
for name, dir in nodes.EXTENSION_WEB_DIRS.items():
self.app.add_routes([
- web.static('/extensions/' + urllib.parse.quote(name), dir, follow_symlinks=True),
+ web.static('/extensions/' + urllib.parse.quote(name), dir),
])
self.app.add_routes([
- web.static('/', self.web_root, follow_symlinks=True),
+ web.static('/', self.web_root),
])
def get_queue_info(self):
@@ -632,11 +626,9 @@ class PromptServer():
site = web.TCPSite(runner, address, port)
await site.start()
- if address == '':
- address = '0.0.0.0'
if verbose:
- print("Starting server\n")
- print("To see the GUI go to: http://{}:{}".format(address, port))
+ logging.info("Starting server\n")
+ logging.info("To see the GUI go to: http://{}:{}".format(address, port))
if call_on_start is not None:
call_on_start(address, port)
@@ -648,7 +640,7 @@ class PromptServer():
try:
json_data = handler(json_data)
except Exception as e:
- print(f"[ERROR] An error occurred during the on_prompt_handler processing")
- traceback.print_exc()
+ logging.warning(f"[ERROR] An error occurred during the on_prompt_handler processing")
+ logging.warning(traceback.format_exc())
return json_data
diff --git a/tests-ui/tests/groupNode.test.js b/tests-ui/tests/groupNode.test.js
index e6ebedd91..53a5828d1 100644
--- a/tests-ui/tests/groupNode.test.js
+++ b/tests-ui/tests/groupNode.test.js
@@ -947,7 +947,7 @@ describe("group node", () => {
expect(p1.widgets.value.widget.options?.step).toBe(80); // width/height step * 10
expect(p2.widgets.value.widget.options?.min).toBe(16); // width/height min
- expect(p2.widgets.value.widget.options?.max).toBe(8192); // width/height max
+ expect(p2.widgets.value.widget.options?.max).toBe(16384); // width/height max
expect(p2.widgets.value.widget.options?.step).toBe(80); // width/height step * 10
expect(p1.widgets.value.value).toBe(128);
diff --git a/web/extensions/core/groupNode.js b/web/extensions/core/groupNode.js
index 335bddbb7..0b0763d1d 100644
--- a/web/extensions/core/groupNode.js
+++ b/web/extensions/core/groupNode.js
@@ -910,6 +910,9 @@ export class GroupNodeHandler {
const self = this;
const onNodeCreated = this.node.onNodeCreated;
this.node.onNodeCreated = function () {
+ if (!this.widgets) {
+ return;
+ }
const config = self.groupData.nodeData.config;
if (config) {
for (const n in config) {
@@ -966,6 +969,26 @@ export class GroupNodeHandler {
api.removeEventListener("executing", executing);
api.removeEventListener("executed", executed);
};
+
+ this.node.refreshComboInNode = (defs) => {
+ // Update combo widget options
+ for (const widgetName in this.groupData.newToOldWidgetMap) {
+ const widget = this.node.widgets.find((w) => w.name === widgetName);
+ if (widget?.type === "combo") {
+ const old = this.groupData.newToOldWidgetMap[widgetName];
+ const def = defs[old.node.type];
+ const input = def?.input?.required?.[old.inputName] ?? def?.input?.optional?.[old.inputName];
+ if (!input) continue;
+
+ widget.options.values = input[0];
+
+ if (old.inputName !== "image" && !widget.options.values.includes(widget.value)) {
+ widget.value = widget.options.values[0];
+ widget.callback(widget.value);
+ }
+ }
+ }
+ };
}
updateInnerWidgets() {
@@ -1245,6 +1268,14 @@ const ext = {
node[GROUP] = new GroupNodeHandler(node);
}
},
+ async refreshComboInNodes(defs) {
+ // Re-register group nodes so new ones are created with the correct options
+ Object.assign(globalDefs, defs);
+ const nodes = app.graph.extra?.groupNodes;
+ if (nodes) {
+ await GroupNodeConfig.registerFromWorkflow(nodes, {});
+ }
+ }
};
app.registerExtension(ext);
diff --git a/web/extensions/core/groupNodeManage.css b/web/extensions/core/groupNodeManage.css
index 5ac89aee3..5470ecb5e 100644
--- a/web/extensions/core/groupNodeManage.css
+++ b/web/extensions/core/groupNodeManage.css
@@ -48,7 +48,7 @@
list-style: none;
}
.comfy-group-manage-list-items {
- max-height: 70vh;
+ max-height: calc(100% - 40px);
overflow-y: scroll;
overflow-x: hidden;
}
diff --git a/web/extensions/core/maskeditor.js b/web/extensions/core/maskeditor.js
index bb2f16d42..4f69ac760 100644
--- a/web/extensions/core/maskeditor.js
+++ b/web/extensions/core/maskeditor.js
@@ -62,7 +62,7 @@ async function uploadMask(filepath, formData) {
ClipspaceDialog.invalidatePreview();
}
-function prepare_mask(image, maskCanvas, maskCtx) {
+function prepare_mask(image, maskCanvas, maskCtx, maskColor) {
// paste mask data into alpha channel
maskCtx.drawImage(image, 0, 0, maskCanvas.width, maskCanvas.height);
const maskData = maskCtx.getImageData(0, 0, maskCanvas.width, maskCanvas.height);
@@ -74,9 +74,9 @@ function prepare_mask(image, maskCanvas, maskCtx) {
else
maskData.data[i+3] = 255;
- maskData.data[i] = 0;
- maskData.data[i+1] = 0;
- maskData.data[i+2] = 0;
+ maskData.data[i] = maskColor.r;
+ maskData.data[i+1] = maskColor.g;
+ maskData.data[i+2] = maskColor.b;
}
maskCtx.globalCompositeOperation = 'source-over';
@@ -110,6 +110,7 @@ class MaskEditorDialog extends ComfyDialog {
createButton(name, callback) {
var button = document.createElement("button");
+ button.style.pointerEvents = "auto";
button.innerText = name;
button.addEventListener("click", callback);
return button;
@@ -146,6 +147,7 @@ class MaskEditorDialog extends ComfyDialog {
divElement.style.display = "flex";
divElement.style.position = "relative";
divElement.style.top = "2px";
+ divElement.style.pointerEvents = "auto";
self.brush_slider_input = document.createElement('input');
self.brush_slider_input.setAttribute('type', 'range');
self.brush_slider_input.setAttribute('min', '1');
@@ -173,6 +175,7 @@ class MaskEditorDialog extends ComfyDialog {
bottom_panel.style.left = "20px";
bottom_panel.style.right = "20px";
bottom_panel.style.height = "50px";
+ bottom_panel.style.pointerEvents = "none";
var brush = document.createElement("div");
brush.id = "brush";
@@ -191,14 +194,29 @@ class MaskEditorDialog extends ComfyDialog {
this.element.appendChild(bottom_panel);
document.body.appendChild(brush);
+ var clearButton = this.createLeftButton("Clear", () => {
+ self.maskCtx.clearRect(0, 0, self.maskCanvas.width, self.maskCanvas.height);
+ });
+
this.brush_size_slider = this.createLeftSlider(self, "Thickness", (event) => {
self.brush_size = event.target.value;
self.updateBrushPreview(self, null, null);
});
- var clearButton = this.createLeftButton("Clear",
- () => {
- self.maskCtx.clearRect(0, 0, self.maskCanvas.width, self.maskCanvas.height);
- });
+
+ this.colorButton = this.createLeftButton(this.getColorButtonText(), () => {
+ if (self.brush_color_mode === "black") {
+ self.brush_color_mode = "white";
+ }
+ else if (self.brush_color_mode === "white") {
+ self.brush_color_mode = "negative";
+ }
+ else {
+ self.brush_color_mode = "black";
+ }
+
+ self.updateWhenBrushColorModeChanged();
+ });
+
var cancelButton = this.createRightButton("Cancel", () => {
document.removeEventListener("mouseup", MaskEditorDialog.handleMouseUp);
document.removeEventListener("keydown", MaskEditorDialog.handleKeyDown);
@@ -219,6 +237,7 @@ class MaskEditorDialog extends ComfyDialog {
bottom_panel.appendChild(this.saveButton);
bottom_panel.appendChild(cancelButton);
bottom_panel.appendChild(this.brush_size_slider);
+ bottom_panel.appendChild(this.colorButton);
imgCanvas.style.position = "absolute";
maskCanvas.style.position = "absolute";
@@ -228,6 +247,10 @@ class MaskEditorDialog extends ComfyDialog {
maskCanvas.style.top = imgCanvas.style.top;
maskCanvas.style.left = imgCanvas.style.left;
+
+ const maskCanvasStyle = this.getMaskCanvasStyle();
+ maskCanvas.style.mixBlendMode = maskCanvasStyle.mixBlendMode;
+ maskCanvas.style.opacity = maskCanvasStyle.opacity;
}
async show() {
@@ -313,7 +336,7 @@ class MaskEditorDialog extends ComfyDialog {
let maskCtx = this.maskCanvas.getContext('2d', {willReadFrequently: true });
imgCtx.drawImage(orig_image, 0, 0, orig_image.width, orig_image.height);
- prepare_mask(mask_image, this.maskCanvas, maskCtx);
+ prepare_mask(mask_image, this.maskCanvas, maskCtx, this.getMaskColor());
}
async setImages(imgCanvas) {
@@ -439,7 +462,84 @@ class MaskEditorDialog extends ComfyDialog {
}
}
+ getMaskCanvasStyle() {
+ if (this.brush_color_mode === "negative") {
+ return {
+ mixBlendMode: "difference",
+ opacity: "1",
+ };
+ }
+ else {
+ return {
+ mixBlendMode: "initial",
+ opacity: "0.7",
+ };
+ }
+ }
+
+ getMaskColor() {
+ if (this.brush_color_mode === "black") {
+ return { r: 0, g: 0, b: 0 };
+ }
+ if (this.brush_color_mode === "white") {
+ return { r: 255, g: 255, b: 255 };
+ }
+ if (this.brush_color_mode === "negative") {
+ // negative effect only works with white color
+ return { r: 255, g: 255, b: 255 };
+ }
+
+ return { r: 0, g: 0, b: 0 };
+ }
+
+ getMaskFillStyle() {
+ const maskColor = this.getMaskColor();
+
+ return "rgb(" + maskColor.r + "," + maskColor.g + "," + maskColor.b + ")";
+ }
+
+ getColorButtonText() {
+ let colorCaption = "unknown";
+
+ if (this.brush_color_mode === "black") {
+ colorCaption = "black";
+ }
+ else if (this.brush_color_mode === "white") {
+ colorCaption = "white";
+ }
+ else if (this.brush_color_mode === "negative") {
+ colorCaption = "negative";
+ }
+
+ return "Color: " + colorCaption;
+ }
+
+ updateWhenBrushColorModeChanged() {
+ this.colorButton.innerText = this.getColorButtonText();
+
+ // update mask canvas css styles
+
+ const maskCanvasStyle = this.getMaskCanvasStyle();
+ this.maskCanvas.style.mixBlendMode = maskCanvasStyle.mixBlendMode;
+ this.maskCanvas.style.opacity = maskCanvasStyle.opacity;
+
+ // update mask canvas rgb colors
+
+ const maskColor = this.getMaskColor();
+
+ const maskData = this.maskCtx.getImageData(0, 0, this.maskCanvas.width, this.maskCanvas.height);
+
+ for (let i = 0; i < maskData.data.length; i += 4) {
+ maskData.data[i] = maskColor.r;
+ maskData.data[i+1] = maskColor.g;
+ maskData.data[i+2] = maskColor.b;
+ }
+
+ this.maskCtx.putImageData(maskData, 0, 0);
+ }
+
brush_size = 10;
+ brush_color_mode = "black";
drawing_mode = false;
lastx = -1;
lasty = -1;
@@ -518,6 +618,19 @@ class MaskEditorDialog extends ComfyDialog {
event.preventDefault();
self.pan_move(self, event);
}
+
+ let left_button_down = window.TouchEvent && event instanceof TouchEvent || event.buttons == 1;
+
+ if(event.shiftKey && left_button_down) {
+ self.drawing_mode = false;
+
+ const y = event.clientY;
+ let delta = (self.zoom_lasty - y)*0.005;
+ self.zoom_ratio = Math.max(Math.min(10.0, self.last_zoom_ratio - delta), 0.2);
+
+ this.invalidatePanZoom();
+ return;
+ }
}
pan_move(self, event) {
@@ -535,7 +648,7 @@ class MaskEditorDialog extends ComfyDialog {
}
draw_move(self, event) {
- if(event.ctrlKey) {
+ if(event.ctrlKey || event.shiftKey) {
return;
}
@@ -546,7 +659,10 @@ class MaskEditorDialog extends ComfyDialog {
self.updateBrushPreview(self);
- if (window.TouchEvent && event instanceof TouchEvent || event.buttons == 1) {
+ let left_button_down = window.TouchEvent && event instanceof TouchEvent || event.buttons == 1;
+ let right_button_down = [2, 5, 32].includes(event.buttons);
+
+ if (!event.altKey && left_button_down) {
var diff = performance.now() - self.lasttime;
const maskRect = self.maskCanvas.getBoundingClientRect();
@@ -581,7 +697,7 @@ class MaskEditorDialog extends ComfyDialog {
if(diff > 20 && !this.drawing_mode)
requestAnimationFrame(() => {
self.maskCtx.beginPath();
- self.maskCtx.fillStyle = "rgb(0,0,0)";
+ self.maskCtx.fillStyle = this.getMaskFillStyle();
self.maskCtx.globalCompositeOperation = "source-over";
self.maskCtx.arc(x, y, brush_size, 0, Math.PI * 2, false);
self.maskCtx.fill();
@@ -591,7 +707,7 @@ class MaskEditorDialog extends ComfyDialog {
else
requestAnimationFrame(() => {
self.maskCtx.beginPath();
- self.maskCtx.fillStyle = "rgb(0,0,0)";
+ self.maskCtx.fillStyle = this.getMaskFillStyle();
self.maskCtx.globalCompositeOperation = "source-over";
var dx = x - self.lastx;
@@ -613,7 +729,7 @@ class MaskEditorDialog extends ComfyDialog {
self.lasttime = performance.now();
}
- else if(event.buttons == 2 || event.buttons == 5 || event.buttons == 32) {
+ else if((event.altKey && left_button_down) || right_button_down) {
const maskRect = self.maskCanvas.getBoundingClientRect();
const x = (event.offsetX || event.targetTouches[0].clientX - maskRect.left) / self.zoom_ratio;
const y = (event.offsetY || event.targetTouches[0].clientY - maskRect.top) / self.zoom_ratio;
@@ -687,13 +803,20 @@ class MaskEditorDialog extends ComfyDialog {
self.drawing_mode = true;
event.preventDefault();
+
+ if(event.shiftKey) {
+ self.zoom_lasty = event.clientY;
+ self.last_zoom_ratio = self.zoom_ratio;
+ return;
+ }
+
const maskRect = self.maskCanvas.getBoundingClientRect();
const x = (event.offsetX || event.targetTouches[0].clientX - maskRect.left) / self.zoom_ratio;
const y = (event.offsetY || event.targetTouches[0].clientY - maskRect.top) / self.zoom_ratio;
self.maskCtx.beginPath();
- if (event.button == 0) {
- self.maskCtx.fillStyle = "rgb(0,0,0)";
+ if (!event.altKey && event.button == 0) {
+ self.maskCtx.fillStyle = this.getMaskFillStyle();
self.maskCtx.globalCompositeOperation = "source-over";
} else {
self.maskCtx.globalCompositeOperation = "destination-out";
diff --git a/web/extensions/core/simpleTouchSupport.js b/web/extensions/core/simpleTouchSupport.js
new file mode 100644
index 000000000..041fc2c4c
--- /dev/null
+++ b/web/extensions/core/simpleTouchSupport.js
@@ -0,0 +1,102 @@
+import { app } from "../../scripts/app.js";
+
+let touchZooming;
+let touchCount = 0;
+
+app.registerExtension({
+ name: "Comfy.SimpleTouchSupport",
+ setup() {
+ let zoomPos;
+ let touchTime;
+ let lastTouch;
+
+ function getMultiTouchPos(e) {
+ return Math.hypot(e.touches[0].clientX - e.touches[1].clientX, e.touches[0].clientY - e.touches[1].clientY);
+ }
+
+ app.canvasEl.addEventListener(
+ "touchstart",
+ (e) => {
+ touchCount++;
+ lastTouch = null;
+ if (e.touches?.length === 1) {
+ // Store start time for press+hold for context menu
+ touchTime = new Date();
+ lastTouch = e.touches[0];
+ } else {
+ touchTime = null;
+ if (e.touches?.length === 2) {
+ // Store center pos for zoom
+ zoomPos = getMultiTouchPos(e);
+ app.canvas.pointer_is_down = false;
+ }
+ }
+ },
+ true
+ );
+
+ app.canvasEl.addEventListener("touchend", (e) => {
+ touchZooming = false;
+ touchCount = e.touches?.length ?? touchCount - 1;
+ if (touchTime && !e.touches?.length) {
+ if (new Date() - touchTime > 600) {
+ try {
+ // hack to get litegraph to use this event
+ e.constructor = CustomEvent;
+ } catch (error) {}
+ e.clientX = lastTouch.clientX;
+ e.clientY = lastTouch.clientY;
+
+ app.canvas.pointer_is_down = true;
+ app.canvas._mousedown_callback(e);
+ }
+ touchTime = null;
+ }
+ });
+
+ app.canvasEl.addEventListener(
+ "touchmove",
+ (e) => {
+ touchTime = null;
+ if (e.touches?.length === 2) {
+ app.canvas.pointer_is_down = false;
+ touchZooming = true;
+ LiteGraph.closeAllContextMenus();
+ app.canvas.search_box?.close();
+ const newZoomPos = getMultiTouchPos(e);
+
+ const midX = (e.touches[0].clientX + e.touches[1].clientX) / 2;
+ const midY = (e.touches[0].clientY + e.touches[1].clientY) / 2;
+
+ let scale = app.canvas.ds.scale;
+ const diff = zoomPos - newZoomPos;
+ if (diff > 0.5) {
+ scale *= 1 / 1.07;
+ } else if (diff < -0.5) {
+ scale *= 1.07;
+ }
+ app.canvas.ds.changeScale(scale, [midX, midY]);
+ app.canvas.setDirty(true, true);
+ zoomPos = newZoomPos;
+ }
+ },
+ true
+ );
+ },
+});
+
+const processMouseDown = LGraphCanvas.prototype.processMouseDown;
+LGraphCanvas.prototype.processMouseDown = function (e) {
+ if (touchZooming || touchCount) {
+ return;
+ }
+ return processMouseDown.apply(this, arguments);
+};
+
+const processMouseMove = LGraphCanvas.prototype.processMouseMove;
+LGraphCanvas.prototype.processMouseMove = function (e) {
+ if (touchZooming || touchCount > 1) {
+ return;
+ }
+ return processMouseMove.apply(this, arguments);
+};
diff --git a/web/extensions/core/widgetInputs.js b/web/extensions/core/widgetInputs.js
index 3f1c1f8c1..23f51d812 100644
--- a/web/extensions/core/widgetInputs.js
+++ b/web/extensions/core/widgetInputs.js
@@ -22,6 +22,7 @@ function isConvertableWidget(widget, config) {
}
function hideWidget(node, widget, suffix = "") {
+ if (widget.type?.startsWith(CONVERTED_TYPE)) return;
widget.origType = widget.type;
widget.origComputeSize = widget.computeSize;
widget.origSerializeValue = widget.serializeValue;
@@ -260,6 +261,12 @@ app.registerExtension({
async beforeRegisterNodeDef(nodeType, nodeData, app) {
// Add menu options to conver to/from widgets
const origGetExtraMenuOptions = nodeType.prototype.getExtraMenuOptions;
+ nodeType.prototype.convertWidgetToInput = function (widget) {
+ const config = getConfig.call(this, widget.name) ?? [widget.type, widget.options || {}];
+ if (!isConvertableWidget(widget, config)) return false;
+ convertToInput(this, widget, config);
+ return true;
+ };
nodeType.prototype.getExtraMenuOptions = function (_, options) {
const r = origGetExtraMenuOptions ? origGetExtraMenuOptions.apply(this, arguments) : undefined;
diff --git a/web/lib/litegraph.core.js b/web/lib/litegraph.core.js
index 434c4a83b..427a62b59 100644
--- a/web/lib/litegraph.core.js
+++ b/web/lib/litegraph.core.js
@@ -7247,7 +7247,7 @@ LGraphNode.prototype.executeAction = function(action)
//create links
for (var i = 0; i < clipboard_info.links.length; ++i) {
var link_info = clipboard_info.links[i];
- var origin_node;
+ var origin_node = undefined;
var origin_node_relative_id = link_info[0];
if (origin_node_relative_id != null) {
origin_node = nodes[origin_node_relative_id];
@@ -11496,7 +11496,7 @@ LGraphNode.prototype.executeAction = function(action)
}
timeout_close = setTimeout(function() {
dialog.close();
- }, 500);
+ }, typeof options.hide_on_mouse_leave === "number" ? options.hide_on_mouse_leave : 500);
});
// if filtering, check focus changed to comboboxes and prevent closing
if (options.do_type_filter){
@@ -11549,7 +11549,7 @@ LGraphNode.prototype.executeAction = function(action)
dialog.close();
} else if (e.keyCode == 13) {
if (selected) {
- select(selected.innerHTML);
+ select(unescape(selected.dataset["type"]));
} else if (first) {
select(first);
} else {
@@ -11910,7 +11910,7 @@ LGraphNode.prototype.executeAction = function(action)
var ctor = LiteGraph.registered_node_types[ type ];
if(filter && ctor.filter != filter )
return false;
- if ((!options.show_all_if_empty || str) && type.toLowerCase().indexOf(str) === -1)
+ if ((!options.show_all_if_empty || str) && type.toLowerCase().indexOf(str) === -1 && (!ctor.title || ctor.title.toLowerCase().indexOf(str) === -1))
return false;
// filter by slot IN, OUT types
@@ -11964,7 +11964,18 @@ LGraphNode.prototype.executeAction = function(action)
if (!first) {
first = type;
}
- help.innerText = type;
+
+ const nodeType = LiteGraph.registered_node_types[type];
+ if (nodeType?.title) {
+ help.innerText = nodeType?.title;
+ const typeEl = document.createElement("span");
+ typeEl.className = "litegraph lite-search-item-type";
+ typeEl.textContent = type;
+ help.append(typeEl);
+ } else {
+ help.innerText = type;
+ }
+
help.dataset["type"] = escape(type);
help.className = "litegraph lite-search-item";
if (className) {
diff --git a/web/lib/litegraph.css b/web/lib/litegraph.css
index 918858f41..5524e24ba 100644
--- a/web/lib/litegraph.css
+++ b/web/lib/litegraph.css
@@ -184,6 +184,7 @@
color: white;
padding-left: 10px;
margin-right: 5px;
+ max-width: 300px;
}
.litegraph.litesearchbox .name {
@@ -227,6 +228,18 @@
color: black;
}
+.litegraph.lite-search-item-type {
+ display: inline-block;
+ background: rgba(0,0,0,0.2);
+ margin-left: 5px;
+ font-size: 14px;
+ padding: 2px 5px;
+ position: relative;
+ top: -2px;
+ opacity: 0.8;
+ border-radius: 4px;
+ }
+
/* DIALOGs ******/
.litegraph .dialog {
diff --git a/web/scripts/api.js b/web/scripts/api.js
index 3a9bcc87a..8c8155be6 100644
--- a/web/scripts/api.js
+++ b/web/scripts/api.js
@@ -5,6 +5,7 @@ class ComfyApi extends EventTarget {
super();
this.api_host = location.host;
this.api_base = location.pathname.split('/').slice(0, -1).join('/');
+ this.initialClientId = sessionStorage.getItem("clientId");
}
apiURL(route) {
@@ -118,7 +119,8 @@ class ComfyApi extends EventTarget {
case "status":
if (msg.data.sid) {
this.clientId = msg.data.sid;
- window.name = this.clientId;
+ window.name = this.clientId; // use window name so it isnt reused when duplicating tabs
+ sessionStorage.setItem("clientId", this.clientId); // store in session storage so duplicate tab can load correct workflow
}
this.dispatchEvent(new CustomEvent("status", { detail: msg.data.status }));
break;
diff --git a/web/scripts/app.js b/web/scripts/app.js
index d4411bfef..c9c754148 100644
--- a/web/scripts/app.js
+++ b/web/scripts/app.js
@@ -1499,12 +1499,17 @@ export class ComfyApp {
// Load previous workflow
let restored = false;
try {
- const json = localStorage.getItem("workflow");
- if (json) {
- const workflow = JSON.parse(json);
- await this.loadGraphData(workflow);
- restored = true;
- }
+ const loadWorkflow = async (json) => {
+ if (json) {
+ const workflow = JSON.parse(json);
+ await this.loadGraphData(workflow);
+ return true;
+ }
+ };
+ const clientId = api.initialClientId ?? api.clientId;
+ restored =
+ (clientId && (await loadWorkflow(sessionStorage.getItem(`workflow:${clientId}`)))) ||
+ (await loadWorkflow(localStorage.getItem("workflow")));
} catch (err) {
console.error("Error loading previous workflow", err);
}
@@ -1515,7 +1520,13 @@ export class ComfyApp {
}
// Save current workflow automatically
- setInterval(() => localStorage.setItem("workflow", JSON.stringify(this.graph.serialize())), 1000);
+ setInterval(() => {
+ const workflow = JSON.stringify(this.graph.serialize());
+ localStorage.setItem("workflow", workflow);
+ if (api.clientId) {
+ sessionStorage.setItem(`workflow:${api.clientId}`, workflow);
+ }
+ }, 1000);
this.#addDrawNodeHandler();
this.#addDrawGroupsHandler();
@@ -2096,6 +2107,8 @@ export class ComfyApp {
this.loadGraphData(JSON.parse(pngInfo.Workflow)); // Support loading workflows from that webp custom node.
} else if (pngInfo.prompt) {
this.loadApiJson(JSON.parse(pngInfo.prompt));
+ } else if (pngInfo.Prompt) {
+ this.loadApiJson(JSON.parse(pngInfo.Prompt)); // Support loading prompts from that webp custom node.
}
}
} else if (file.type === "application/json" || file.name?.endsWith(".json")) {
@@ -2149,8 +2162,17 @@ export class ComfyApp {
if (value instanceof Array) {
const [fromId, fromSlot] = value;
const fromNode = app.graph.getNodeById(fromId);
- const toSlot = node.inputs?.findIndex((inp) => inp.name === input);
- if (toSlot !== -1) {
+ let toSlot = node.inputs?.findIndex((inp) => inp.name === input);
+ if (toSlot == null || toSlot === -1) {
+ try {
+ // Target has no matching input, most likely a converted widget
+ const widget = node.widgets?.find((w) => w.name === input);
+ if (widget && node.convertWidgetToInput?.(widget)) {
+ toSlot = node.inputs?.length - 1;
+ }
+ } catch (error) {}
+ }
+ if (toSlot != null || toSlot !== -1) {
fromNode.connect(fromSlot, node, toSlot);
}
} else {
@@ -2212,6 +2234,8 @@ export class ComfyApp {
}
}
}
+
+ await this.#invokeExtensionsAsync("refreshComboInNodes", defs);
}
/**
diff --git a/web/scripts/pnginfo.js b/web/scripts/pnginfo.js
index 83a4ebc86..7132fb60f 100644
--- a/web/scripts/pnginfo.js
+++ b/web/scripts/pnginfo.js
@@ -24,7 +24,7 @@ export function getPngMetadata(file) {
const length = dataView.getUint32(offset);
// Get the chunk type
const type = String.fromCharCode(...pngData.slice(offset + 4, offset + 8));
- if (type === "tEXt" || type == "comf") {
+ if (type === "tEXt" || type == "comf" || type === "iTXt") {
// Get the keyword
let keyword_end = offset + 8;
while (pngData[keyword_end] !== 0) {
@@ -33,7 +33,7 @@ export function getPngMetadata(file) {
const keyword = String.fromCharCode(...pngData.slice(offset + 8, keyword_end));
// Get the text
const contentArraySegment = pngData.slice(keyword_end + 1, offset + 8 + length);
- const contentJson = Array.from(contentArraySegment).map(s=>String.fromCharCode(s)).join('')
+ const contentJson = new TextDecoder("utf-8").decode(contentArraySegment);
txt_chunks[keyword] = contentJson;
}
@@ -170,9 +170,12 @@ export async function importA1111(graph, parameters) {
const opts = parameters
.substr(p)
.split("\n")[1]
- .split(",")
+ .match(new RegExp("\\s*([^:]+:\\s*([^\"\\{].*?|\".*?\"|\\{.*?\\}))\\s*(,|$)", "g"))
.reduce((p, n) => {
const s = n.split(":");
+ if (s[1].endsWith(',')) {
+ s[1] = s[1].substr(0, s[1].length -1);
+ }
p[s[0].trim().toLowerCase()] = s[1].trim();
return p;
}, {});
@@ -191,6 +194,7 @@ export async function importA1111(graph, parameters) {
const vaeLoaderNode = LiteGraph.createNode("VAELoader");
const saveNode = LiteGraph.createNode("SaveImage");
let hrSamplerNode = null;
+ let hrSteps = null;
const ceil64 = (v) => Math.ceil(v / 64) * 64;
@@ -290,6 +294,9 @@ export async function importA1111(graph, parameters) {
model(v) {
setWidgetValue(ckptNode, "ckpt_name", v, true);
},
+ "vae"(v) {
+ setWidgetValue(vaeLoaderNode, "vae_name", v, true);
+ },
"cfg scale"(v) {
setWidgetValue(samplerNode, "cfg", +v);
},
@@ -316,6 +323,7 @@ export async function importA1111(graph, parameters) {
const h = ceil64(+wxh[1]);
const hrUp = popOpt("hires upscale");
const hrSz = popOpt("hires resize");
+ hrSteps = popOpt("hires steps");
let hrMethod = popOpt("hires upscaler");
setWidgetValue(imageNode, "width", w);
@@ -398,7 +406,7 @@ export async function importA1111(graph, parameters) {
}
if (hrSamplerNode) {
- setWidgetValue(hrSamplerNode, "steps", getWidget(samplerNode, "steps").value);
+ setWidgetValue(hrSamplerNode, "steps", hrSteps? +hrSteps : getWidget(samplerNode, "steps").value);
setWidgetValue(hrSamplerNode, "cfg", getWidget(samplerNode, "cfg").value);
setWidgetValue(hrSamplerNode, "scheduler", getWidget(samplerNode, "scheduler").value);
setWidgetValue(hrSamplerNode, "sampler_name", getWidget(samplerNode, "sampler_name").value);
@@ -415,7 +423,7 @@ export async function importA1111(graph, parameters) {
graph.arrange();
- for (const opt of ["model hash", "ensd"]) {
+ for (const opt of ["model hash", "ensd", "version", "vae hash", "ti hashes", "lora hashes", "hashes"]) {
delete opts[opt];
}
diff --git a/web/scripts/ui.js b/web/scripts/ui.js
index 4437345a3..5ca6214eb 100644
--- a/web/scripts/ui.js
+++ b/web/scripts/ui.js
@@ -384,7 +384,7 @@ export class ComfyUI {
autoQueueModeEl.style.display = "none";
api.addEventListener("graphChanged", () => {
- if (this.autoQueueMode === "change") {
+ if (this.autoQueueMode === "change" && this.autoQueueEnabled === true) {
if (this.lastQueueSize === 0) {
this.graphHasChanged = false;
app.queuePrompt(0, this.batchCount);
@@ -394,18 +394,42 @@ export class ComfyUI {
}
});
- this.menuContainer = $el("div.comfy-menu", {parent: document.body}, [
- $el("div.drag-handle", {
+ this.menuHamburger = $el(
+ "div.comfy-menu-hamburger",
+ {
+ parent: document.body,
+ onclick: () => {
+ this.menuContainer.style.display = "block";
+ this.menuHamburger.style.display = "none";
+ },
+ },
+ [$el("div"), $el("div"), $el("div")]
+ );
+
+ this.menuContainer = $el("div.comfy-menu", { parent: document.body }, [
+ $el("div.drag-handle.comfy-menu-header", {
style: {
overflow: "hidden",
position: "relative",
width: "100%",
cursor: "default"
}
- }, [
+ }, [
$el("span.drag-handle"),
- $el("span", {$: (q) => (this.queueSize = q)}),
- $el("button.comfy-settings-btn", {textContent: "⚙️", onclick: () => this.settings.show()}),
+ $el("span.comfy-menu-queue-size", { $: (q) => (this.queueSize = q) }),
+ $el("div.comfy-menu-actions", [
+ $el("button.comfy-settings-btn", {
+ textContent: "⚙️",
+ onclick: () => this.settings.show(),
+ }),
+ $el("button.comfy-close-menu-btn", {
+ textContent: "\u00d7",
+ onclick: () => {
+ this.menuContainer.style.display = "none";
+ this.menuHamburger.style.display = "flex";
+ },
+ }),
+ ]),
]),
$el("button.comfy-queue-btn", {
id: "queue-button",
diff --git a/web/scripts/ui/settings.js b/web/scripts/ui/settings.js
index 1cdba5cfe..9e9d13af0 100644
--- a/web/scripts/ui/settings.js
+++ b/web/scripts/ui/settings.js
@@ -16,7 +16,17 @@ export class ComfySettingsDialog extends ComfyDialog {
},
[
$el("table.comfy-modal-content.comfy-table", [
- $el("caption", { textContent: "Settings" }),
+ $el(
+ "caption",
+ { textContent: "Settings" },
+ $el("button.comfy-btn", {
+ type: "button",
+ textContent: "\u00d7",
+ onclick: () => {
+ this.element.close();
+ },
+ })
+ ),
$el("tbody", { $: (tbody) => (this.textElement = tbody) }),
$el("button", {
type: "button",
diff --git a/web/scripts/widgets.js b/web/scripts/widgets.js
index 0529b1d80..678b1b8ec 100644
--- a/web/scripts/widgets.js
+++ b/web/scripts/widgets.js
@@ -81,6 +81,9 @@ export function addValueControlWidgets(node, targetWidget, defaultValue = "rando
const isCombo = targetWidget.type === "combo";
let comboFilter;
+ if (isCombo) {
+ valueControl.options.values.push("increment-wrap");
+ }
if (isCombo && options.addFilterList !== false) {
comboFilter = node.addWidget(
"string",
@@ -128,6 +131,12 @@ export function addValueControlWidgets(node, targetWidget, defaultValue = "rando
case "increment":
current_index += 1;
break;
+ case "increment-wrap":
+ current_index += 1;
+ if ( current_index >= current_length ) {
+ current_index = 0;
+ }
+ break;
case "decrement":
current_index -= 1;
break;
@@ -295,7 +304,7 @@ export const ComfyWidgets = {
let disable_rounding = app.ui.settings.getSettingValue("Comfy.DisableFloatRounding")
if (precision == 0) precision = undefined;
const { val, config } = getNumberDefaults(inputData, 0.5, precision, !disable_rounding);
- return { widget: node.addWidget(widgetType, inputName, val,
+ return { widget: node.addWidget(widgetType, inputName, val,
function (v) {
if (config.round) {
this.value = Math.round(v/config.round)*config.round;
diff --git a/web/style.css b/web/style.css
index ef838c822..d2a02f49a 100644
--- a/web/style.css
+++ b/web/style.css
@@ -82,6 +82,24 @@ body {
margin: 3px 3px 3px 4px;
}
+.comfy-menu-hamburger {
+ position: fixed;
+ top: 10px;
+ z-index: 9999;
+ right: 10px;
+ width: 30px;
+ display: none;
+ gap: 8px;
+ flex-direction: column;
+ cursor: pointer;
+}
+.comfy-menu-hamburger div {
+ height: 3px;
+ width: 100%;
+ border-radius: 20px;
+ background-color: white;
+}
+
.comfy-menu {
font-size: 15px;
position: absolute;
@@ -101,6 +119,44 @@ body {
box-shadow: 3px 3px 8px rgba(0, 0, 0, 0.4);
}
+.comfy-menu-header {
+ display: flex;
+}
+
+.comfy-menu-actions {
+ display: flex;
+ gap: 3px;
+ align-items: center;
+ height: 20px;
+ position: relative;
+ top: -1px;
+ font-size: 22px;
+}
+
+.comfy-menu .comfy-menu-actions button {
+ background-color: rgba(0, 0, 0, 0);
+ padding: 0;
+ border: none;
+ cursor: pointer;
+ font-size: inherit;
+}
+
+.comfy-menu .comfy-menu-actions .comfy-settings-btn {
+ font-size: 0.6em;
+}
+
+button.comfy-close-menu-btn {
+ font-size: 1em;
+ line-height: 12px;
+ color: #ccc;
+ position: relative;
+ top: -1px;
+}
+
+.comfy-menu-queue-size {
+ flex: auto;
+}
+
.comfy-menu button,
.comfy-modal button {
font-size: 12px;
@@ -121,7 +177,6 @@ body {
width: 100%;
}
-.comfy-toggle-switch,
.comfy-btn,
.comfy-menu > button,
.comfy-menu-btns button,
@@ -140,17 +195,12 @@ body {
.comfy-menu-btns button:hover,
.comfy-menu .comfy-list button:hover,
.comfy-modal button:hover,
-.comfy-settings-btn:hover {
+.comfy-menu-actions button:hover {
filter: brightness(1.2);
+ will-change: transform;
cursor: pointer;
}
-.comfy-menu span.drag-handle {
- position: absolute;
- top: 0;
- left: 0;
-}
-
span.drag-handle {
width: 10px;
height: 20px;
@@ -215,15 +265,6 @@ span.drag-handle::after {
font-size: 12px;
}
-button.comfy-settings-btn {
- background-color: rgba(0, 0, 0, 0);
- font-size: 12px;
- padding: 0;
- position: absolute;
- right: 0;
- border: none;
-}
-
button.comfy-queue-btn {
margin: 6px 0 !important;
}
@@ -269,7 +310,19 @@ button.comfy-queue-btn {
}
.comfy-menu span.drag-handle {
- visibility: hidden
+ display: none;
+ }
+
+ .comfy-menu-queue-size {
+ flex: unset;
+ }
+
+ .comfy-menu-header {
+ justify-content: space-between;
+ }
+ .comfy-menu-actions {
+ gap: 10px;
+ font-size: 28px;
}
}
@@ -320,7 +373,7 @@ dialog::backdrop {
text-align: right;
}
-#comfy-settings-dialog button {
+#comfy-settings-dialog tbody button, #comfy-settings-dialog table > button {
background-color: var(--bg-color);
border: 1px var(--border-color) solid;
border-radius: 0;
@@ -343,12 +396,33 @@ dialog::backdrop {
}
.comfy-table caption {
+ position: sticky;
+ top: 0;
background-color: var(--bg-color);
color: var(--input-text);
font-size: 1rem;
font-weight: bold;
padding: 8px;
text-align: center;
+ border-bottom: 1px solid var(--border-color);
+}
+
+.comfy-table caption .comfy-btn {
+ position: absolute;
+ top: -2px;
+ right: 0;
+ bottom: 0;
+ cursor: pointer;
+ border: none;
+ height: 100%;
+ border-radius: 0;
+ aspect-ratio: 1/1;
+ user-select: none;
+ font-size: 20px;
+}
+
+.comfy-table caption .comfy-btn:focus {
+ outline: none;
}
.comfy-table tr:nth-child(even) {
@@ -389,11 +463,13 @@ dialog::backdrop {
z-index: 9999 !important;
background-color: var(--comfy-menu-bg) !important;
filter: brightness(95%);
+ will-change: transform;
}
.litegraph.litecontextmenu .litemenu-entry:hover:not(.disabled):not(.separator) {
background-color: var(--comfy-menu-bg) !important;
filter: brightness(155%);
+ will-change: transform;
color: var(--input-text);
}
@@ -435,43 +511,6 @@ dialog::backdrop {
margin-left: 5px;
}
-.comfy-toggle-switch {
- border-width: 2px;
- display: flex;
- background-color: var(--comfy-input-bg);
- margin: 2px 0;
- white-space: nowrap;
-}
-
-.comfy-toggle-switch label {
- padding: 2px 0px 3px 6px;
- flex: auto;
- border-radius: 8px;
- align-items: center;
- display: flex;
- justify-content: center;
-}
-
-.comfy-toggle-switch label:first-child {
- border-top-left-radius: 8px;
- border-bottom-left-radius: 8px;
-}
-.comfy-toggle-switch label:last-child {
- border-top-right-radius: 8px;
- border-bottom-right-radius: 8px;
-}
-
-.comfy-toggle-switch .comfy-toggle-selected {
- background-color: var(--comfy-menu-bg);
-}
-
-#extraOptions {
- padding: 4px;
- background-color: var(--bg-color);
- margin-bottom: 4px;
- border-radius: 4px;
-}
-
/* Search box */
.litegraph.litesearchbox {
@@ -491,10 +530,30 @@ dialog::backdrop {
color: var(--input-text);
background-color: var(--comfy-input-bg);
filter: brightness(80%);
+ will-change: transform;
padding-left: 0.2em;
}
.litegraph.lite-search-item.generic_type {
color: var(--input-text);
filter: brightness(50%);
+ will-change: transform;
+}
+
+@media only screen and (max-width: 450px) {
+ #comfy-settings-dialog .comfy-table tbody {
+ display: grid;
+ }
+ #comfy-settings-dialog .comfy-table tr {
+ display: grid;
+ }
+ #comfy-settings-dialog tr > td:first-child {
+ text-align: center;
+ border-bottom: none;
+ padding-bottom: 0;
+ }
+ #comfy-settings-dialog tr > td:not(:first-child) {
+ text-align: center;
+ border-top: none;
+ }
}