mirror of
https://git.datalinker.icu/comfyanonymous/ComfyUI
synced 2026-11-23 03:56:59 +08:00
Merge branch 'master' into feature/pwa-inital-setup
This commit is contained in:
commit
739bf2964e
@ -2,6 +2,13 @@ name: "Windows Release cu118 dependencies 2"
|
|||||||
|
|
||||||
on:
|
on:
|
||||||
workflow_dispatch:
|
workflow_dispatch:
|
||||||
|
inputs:
|
||||||
|
xformers:
|
||||||
|
description: 'xformers version'
|
||||||
|
required: true
|
||||||
|
type: string
|
||||||
|
default: "xformers"
|
||||||
|
|
||||||
# push:
|
# push:
|
||||||
# branches:
|
# branches:
|
||||||
# - master
|
# - master
|
||||||
@ -17,7 +24,7 @@ jobs:
|
|||||||
|
|
||||||
- shell: bash
|
- shell: bash
|
||||||
run: |
|
run: |
|
||||||
python -m pip wheel --no-cache-dir torch torchvision torchaudio xformers --extra-index-url https://download.pytorch.org/whl/cu118 -r requirements.txt pygit2 -w ./temp_wheel_dir
|
python -m pip wheel --no-cache-dir torch torchvision torchaudio ${{ inputs.xformers }} --extra-index-url https://download.pytorch.org/whl/cu118 -r requirements.txt pygit2 -w ./temp_wheel_dir
|
||||||
python -m pip install --no-cache-dir ./temp_wheel_dir/*
|
python -m pip install --no-cache-dir ./temp_wheel_dir/*
|
||||||
echo installed basic
|
echo installed basic
|
||||||
ls -lah temp_wheel_dir
|
ls -lah temp_wheel_dir
|
||||||
|
|||||||
@ -31,7 +31,7 @@ jobs:
|
|||||||
echo 'import site' >> ./python311._pth
|
echo 'import site' >> ./python311._pth
|
||||||
curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py
|
curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py
|
||||||
./python.exe get-pip.py
|
./python.exe get-pip.py
|
||||||
python -m pip wheel torch torchvision torchaudio aiohttp==3.8.4 --pre --extra-index-url https://download.pytorch.org/whl/nightly/cu121 -r ../ComfyUI/requirements.txt pygit2 -w ../temp_wheel_dir
|
python -m pip wheel torch torchvision torchaudio aiohttp==3.8.5 --pre --extra-index-url https://download.pytorch.org/whl/nightly/cu121 -r ../ComfyUI/requirements.txt pygit2 -w ../temp_wheel_dir
|
||||||
ls ../temp_wheel_dir
|
ls ../temp_wheel_dir
|
||||||
./python.exe -s -m pip install --pre ../temp_wheel_dir/*
|
./python.exe -s -m pip install --pre ../temp_wheel_dir/*
|
||||||
sed -i '1i../ComfyUI' ./python311._pth
|
sed -i '1i../ComfyUI' ./python311._pth
|
||||||
|
|||||||
1
CODEOWNERS
Normal file
1
CODEOWNERS
Normal file
@ -0,0 +1 @@
|
|||||||
|
* @comfyanonymous
|
||||||
@ -47,6 +47,7 @@ Workflow examples can be found on the [Examples page](https://comfyanonymous.git
|
|||||||
| Ctrl + O | Load workflow |
|
| Ctrl + O | Load workflow |
|
||||||
| Ctrl + A | Select all nodes |
|
| Ctrl + A | Select all nodes |
|
||||||
| Ctrl + M | Mute/unmute selected nodes |
|
| Ctrl + M | Mute/unmute selected nodes |
|
||||||
|
| Ctrl + B | Bypass selected nodes (acts like the node was removed from the graph and the wires reconnected through) |
|
||||||
| Delete/Backspace | Delete selected nodes |
|
| Delete/Backspace | Delete selected nodes |
|
||||||
| Ctrl + Delete/Backspace | Delete the current graph |
|
| Ctrl + Delete/Backspace | Delete the current graph |
|
||||||
| Space | Move the canvas around when held and moving the cursor |
|
| Space | Move the canvas around when held and moving the cursor |
|
||||||
@ -126,10 +127,10 @@ After this you should have everything installed and can proceed to running Comfy
|
|||||||
|
|
||||||
You can install ComfyUI in Apple Mac silicon (M1 or M2) with any recent macOS version.
|
You can install ComfyUI in Apple Mac silicon (M1 or M2) with any recent macOS version.
|
||||||
|
|
||||||
1. Install pytorch. For instructions, read the [Accelerated PyTorch training on Mac](https://developer.apple.com/metal/pytorch/) Apple Developer guide.
|
1. Install pytorch nightly. For instructions, read the [Accelerated PyTorch training on Mac](https://developer.apple.com/metal/pytorch/) Apple Developer guide (make sure to install the latest pytorch nightly).
|
||||||
1. Follow the [ComfyUI manual installation](#manual-install-windows-linux) instructions for Windows and Linux.
|
1. Follow the [ComfyUI manual installation](#manual-install-windows-linux) instructions for Windows and Linux.
|
||||||
1. Install the ComfyUI [dependencies](#dependencies). If you have another Stable Diffusion UI [you might be able to reuse the dependencies](#i-already-have-another-ui-for-stable-diffusion-installed-do-i-really-have-to-install-all-of-these-dependencies).
|
1. Install the ComfyUI [dependencies](#dependencies). If you have another Stable Diffusion UI [you might be able to reuse the dependencies](#i-already-have-another-ui-for-stable-diffusion-installed-do-i-really-have-to-install-all-of-these-dependencies).
|
||||||
1. Launch ComfyUI by running `python main.py`.
|
1. Launch ComfyUI by running `python main.py --force-fp16`. Note that --force-fp16 will only work if you installed the latest pytorch nightly.
|
||||||
|
|
||||||
> **Note**: Remember to add your models, VAE, LoRAs etc. to the corresponding Comfy folders, as discussed in [ComfyUI manual installation](#manual-install-windows-linux).
|
> **Note**: Remember to add your models, VAE, LoRAs etc. to the corresponding Comfy folders, as discussed in [ComfyUI manual installation](#manual-install-windows-linux).
|
||||||
|
|
||||||
|
|||||||
@ -6,8 +6,6 @@ import torch as th
|
|||||||
import torch.nn as nn
|
import torch.nn as nn
|
||||||
|
|
||||||
from ..ldm.modules.diffusionmodules.util import (
|
from ..ldm.modules.diffusionmodules.util import (
|
||||||
conv_nd,
|
|
||||||
linear,
|
|
||||||
zero_module,
|
zero_module,
|
||||||
timestep_embedding,
|
timestep_embedding,
|
||||||
)
|
)
|
||||||
@ -15,7 +13,7 @@ from ..ldm.modules.diffusionmodules.util import (
|
|||||||
from ..ldm.modules.attention import SpatialTransformer
|
from ..ldm.modules.attention import SpatialTransformer
|
||||||
from ..ldm.modules.diffusionmodules.openaimodel import UNetModel, TimestepEmbedSequential, ResBlock, Downsample
|
from ..ldm.modules.diffusionmodules.openaimodel import UNetModel, TimestepEmbedSequential, ResBlock, Downsample
|
||||||
from ..ldm.util import exists
|
from ..ldm.util import exists
|
||||||
|
import comfy.ops
|
||||||
|
|
||||||
class ControlledUnetModel(UNetModel):
|
class ControlledUnetModel(UNetModel):
|
||||||
#implemented in the ldm unet
|
#implemented in the ldm unet
|
||||||
@ -55,6 +53,8 @@ class ControlNet(nn.Module):
|
|||||||
use_linear_in_transformer=False,
|
use_linear_in_transformer=False,
|
||||||
adm_in_channels=None,
|
adm_in_channels=None,
|
||||||
transformer_depth_middle=None,
|
transformer_depth_middle=None,
|
||||||
|
device=None,
|
||||||
|
operations=comfy.ops,
|
||||||
):
|
):
|
||||||
super().__init__()
|
super().__init__()
|
||||||
assert use_spatial_transformer == True, "use_spatial_transformer has to be true"
|
assert use_spatial_transformer == True, "use_spatial_transformer has to be true"
|
||||||
@ -117,9 +117,9 @@ class ControlNet(nn.Module):
|
|||||||
|
|
||||||
time_embed_dim = model_channels * 4
|
time_embed_dim = model_channels * 4
|
||||||
self.time_embed = nn.Sequential(
|
self.time_embed = nn.Sequential(
|
||||||
linear(model_channels, time_embed_dim),
|
operations.Linear(model_channels, time_embed_dim, dtype=self.dtype, device=device),
|
||||||
nn.SiLU(),
|
nn.SiLU(),
|
||||||
linear(time_embed_dim, time_embed_dim),
|
operations.Linear(time_embed_dim, time_embed_dim, dtype=self.dtype, device=device),
|
||||||
)
|
)
|
||||||
|
|
||||||
if self.num_classes is not None:
|
if self.num_classes is not None:
|
||||||
@ -132,9 +132,9 @@ class ControlNet(nn.Module):
|
|||||||
assert adm_in_channels is not None
|
assert adm_in_channels is not None
|
||||||
self.label_emb = nn.Sequential(
|
self.label_emb = nn.Sequential(
|
||||||
nn.Sequential(
|
nn.Sequential(
|
||||||
linear(adm_in_channels, time_embed_dim),
|
operations.Linear(adm_in_channels, time_embed_dim, dtype=self.dtype, device=device),
|
||||||
nn.SiLU(),
|
nn.SiLU(),
|
||||||
linear(time_embed_dim, time_embed_dim),
|
operations.Linear(time_embed_dim, time_embed_dim, dtype=self.dtype, device=device),
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
@ -143,28 +143,28 @@ class ControlNet(nn.Module):
|
|||||||
self.input_blocks = nn.ModuleList(
|
self.input_blocks = nn.ModuleList(
|
||||||
[
|
[
|
||||||
TimestepEmbedSequential(
|
TimestepEmbedSequential(
|
||||||
conv_nd(dims, in_channels, model_channels, 3, padding=1)
|
operations.conv_nd(dims, in_channels, model_channels, 3, padding=1, dtype=self.dtype, device=device)
|
||||||
)
|
)
|
||||||
]
|
]
|
||||||
)
|
)
|
||||||
self.zero_convs = nn.ModuleList([self.make_zero_conv(model_channels)])
|
self.zero_convs = nn.ModuleList([self.make_zero_conv(model_channels, operations=operations)])
|
||||||
|
|
||||||
self.input_hint_block = TimestepEmbedSequential(
|
self.input_hint_block = TimestepEmbedSequential(
|
||||||
conv_nd(dims, hint_channels, 16, 3, padding=1),
|
operations.conv_nd(dims, hint_channels, 16, 3, padding=1),
|
||||||
nn.SiLU(),
|
nn.SiLU(),
|
||||||
conv_nd(dims, 16, 16, 3, padding=1),
|
operations.conv_nd(dims, 16, 16, 3, padding=1),
|
||||||
nn.SiLU(),
|
nn.SiLU(),
|
||||||
conv_nd(dims, 16, 32, 3, padding=1, stride=2),
|
operations.conv_nd(dims, 16, 32, 3, padding=1, stride=2),
|
||||||
nn.SiLU(),
|
nn.SiLU(),
|
||||||
conv_nd(dims, 32, 32, 3, padding=1),
|
operations.conv_nd(dims, 32, 32, 3, padding=1),
|
||||||
nn.SiLU(),
|
nn.SiLU(),
|
||||||
conv_nd(dims, 32, 96, 3, padding=1, stride=2),
|
operations.conv_nd(dims, 32, 96, 3, padding=1, stride=2),
|
||||||
nn.SiLU(),
|
nn.SiLU(),
|
||||||
conv_nd(dims, 96, 96, 3, padding=1),
|
operations.conv_nd(dims, 96, 96, 3, padding=1),
|
||||||
nn.SiLU(),
|
nn.SiLU(),
|
||||||
conv_nd(dims, 96, 256, 3, padding=1, stride=2),
|
operations.conv_nd(dims, 96, 256, 3, padding=1, stride=2),
|
||||||
nn.SiLU(),
|
nn.SiLU(),
|
||||||
zero_module(conv_nd(dims, 256, model_channels, 3, padding=1))
|
zero_module(operations.conv_nd(dims, 256, model_channels, 3, padding=1))
|
||||||
)
|
)
|
||||||
|
|
||||||
self._feature_size = model_channels
|
self._feature_size = model_channels
|
||||||
@ -182,6 +182,7 @@ class ControlNet(nn.Module):
|
|||||||
dims=dims,
|
dims=dims,
|
||||||
use_checkpoint=use_checkpoint,
|
use_checkpoint=use_checkpoint,
|
||||||
use_scale_shift_norm=use_scale_shift_norm,
|
use_scale_shift_norm=use_scale_shift_norm,
|
||||||
|
operations=operations
|
||||||
)
|
)
|
||||||
]
|
]
|
||||||
ch = mult * model_channels
|
ch = mult * model_channels
|
||||||
@ -204,11 +205,11 @@ class ControlNet(nn.Module):
|
|||||||
SpatialTransformer(
|
SpatialTransformer(
|
||||||
ch, num_heads, dim_head, depth=transformer_depth[level], context_dim=context_dim,
|
ch, num_heads, dim_head, depth=transformer_depth[level], context_dim=context_dim,
|
||||||
disable_self_attn=disabled_sa, use_linear=use_linear_in_transformer,
|
disable_self_attn=disabled_sa, use_linear=use_linear_in_transformer,
|
||||||
use_checkpoint=use_checkpoint
|
use_checkpoint=use_checkpoint, operations=operations
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
self.input_blocks.append(TimestepEmbedSequential(*layers))
|
self.input_blocks.append(TimestepEmbedSequential(*layers))
|
||||||
self.zero_convs.append(self.make_zero_conv(ch))
|
self.zero_convs.append(self.make_zero_conv(ch, operations=operations))
|
||||||
self._feature_size += ch
|
self._feature_size += ch
|
||||||
input_block_chans.append(ch)
|
input_block_chans.append(ch)
|
||||||
if level != len(channel_mult) - 1:
|
if level != len(channel_mult) - 1:
|
||||||
@ -224,16 +225,17 @@ class ControlNet(nn.Module):
|
|||||||
use_checkpoint=use_checkpoint,
|
use_checkpoint=use_checkpoint,
|
||||||
use_scale_shift_norm=use_scale_shift_norm,
|
use_scale_shift_norm=use_scale_shift_norm,
|
||||||
down=True,
|
down=True,
|
||||||
|
operations=operations
|
||||||
)
|
)
|
||||||
if resblock_updown
|
if resblock_updown
|
||||||
else Downsample(
|
else Downsample(
|
||||||
ch, conv_resample, dims=dims, out_channels=out_ch
|
ch, conv_resample, dims=dims, out_channels=out_ch, operations=operations
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
ch = out_ch
|
ch = out_ch
|
||||||
input_block_chans.append(ch)
|
input_block_chans.append(ch)
|
||||||
self.zero_convs.append(self.make_zero_conv(ch))
|
self.zero_convs.append(self.make_zero_conv(ch, operations=operations))
|
||||||
ds *= 2
|
ds *= 2
|
||||||
self._feature_size += ch
|
self._feature_size += ch
|
||||||
|
|
||||||
@ -253,11 +255,12 @@ class ControlNet(nn.Module):
|
|||||||
dims=dims,
|
dims=dims,
|
||||||
use_checkpoint=use_checkpoint,
|
use_checkpoint=use_checkpoint,
|
||||||
use_scale_shift_norm=use_scale_shift_norm,
|
use_scale_shift_norm=use_scale_shift_norm,
|
||||||
|
operations=operations
|
||||||
),
|
),
|
||||||
SpatialTransformer( # always uses a self-attn
|
SpatialTransformer( # always uses a self-attn
|
||||||
ch, num_heads, dim_head, depth=transformer_depth_middle, context_dim=context_dim,
|
ch, num_heads, dim_head, depth=transformer_depth_middle, context_dim=context_dim,
|
||||||
disable_self_attn=disable_middle_self_attn, use_linear=use_linear_in_transformer,
|
disable_self_attn=disable_middle_self_attn, use_linear=use_linear_in_transformer,
|
||||||
use_checkpoint=use_checkpoint
|
use_checkpoint=use_checkpoint, operations=operations
|
||||||
),
|
),
|
||||||
ResBlock(
|
ResBlock(
|
||||||
ch,
|
ch,
|
||||||
@ -266,16 +269,17 @@ class ControlNet(nn.Module):
|
|||||||
dims=dims,
|
dims=dims,
|
||||||
use_checkpoint=use_checkpoint,
|
use_checkpoint=use_checkpoint,
|
||||||
use_scale_shift_norm=use_scale_shift_norm,
|
use_scale_shift_norm=use_scale_shift_norm,
|
||||||
|
operations=operations
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
self.middle_block_out = self.make_zero_conv(ch)
|
self.middle_block_out = self.make_zero_conv(ch, operations=operations)
|
||||||
self._feature_size += ch
|
self._feature_size += ch
|
||||||
|
|
||||||
def make_zero_conv(self, channels):
|
def make_zero_conv(self, channels, operations=None):
|
||||||
return TimestepEmbedSequential(zero_module(conv_nd(self.dims, channels, channels, 1, padding=0)))
|
return TimestepEmbedSequential(zero_module(operations.conv_nd(self.dims, channels, channels, 1, padding=0)))
|
||||||
|
|
||||||
def forward(self, x, hint, timesteps, context, y=None, **kwargs):
|
def forward(self, x, hint, timesteps, context, y=None, **kwargs):
|
||||||
t_emb = timestep_embedding(timesteps, self.model_channels, repeat_only=False)
|
t_emb = timestep_embedding(timesteps, self.model_channels, repeat_only=False).to(self.dtype)
|
||||||
emb = self.time_embed(t_emb)
|
emb = self.time_embed(t_emb)
|
||||||
|
|
||||||
guided_hint = self.input_hint_block(hint, emb, context)
|
guided_hint = self.input_hint_block(hint, emb, context)
|
||||||
@ -283,9 +287,6 @@ class ControlNet(nn.Module):
|
|||||||
outs = []
|
outs = []
|
||||||
|
|
||||||
hs = []
|
hs = []
|
||||||
t_emb = timestep_embedding(timesteps, self.model_channels, repeat_only=False)
|
|
||||||
emb = self.time_embed(t_emb)
|
|
||||||
|
|
||||||
if self.num_classes is not None:
|
if self.num_classes is not None:
|
||||||
assert y.shape[0] == x.shape[0]
|
assert y.shape[0] == x.shape[0]
|
||||||
emb = emb + self.label_emb(y)
|
emb = emb + self.label_emb(y)
|
||||||
|
|||||||
@ -38,7 +38,9 @@ parser.add_argument("--port", type=int, default=8188, help="Set the listen port.
|
|||||||
parser.add_argument("--enable-cors-header", type=str, default=None, metavar="ORIGIN", nargs="?", const="*", help="Enable CORS (Cross-Origin Resource Sharing) with optional origin or allow all with default '*'.")
|
parser.add_argument("--enable-cors-header", type=str, default=None, metavar="ORIGIN", nargs="?", const="*", help="Enable CORS (Cross-Origin Resource Sharing) with optional origin or allow all with default '*'.")
|
||||||
parser.add_argument("--extra-model-paths-config", type=str, default=None, metavar="PATH", nargs='+', action='append', help="Load one or more extra_model_paths.yaml files.")
|
parser.add_argument("--extra-model-paths-config", type=str, default=None, metavar="PATH", nargs='+', action='append', help="Load one or more extra_model_paths.yaml files.")
|
||||||
parser.add_argument("--output-directory", type=str, default=None, help="Set the ComfyUI output directory.")
|
parser.add_argument("--output-directory", type=str, default=None, help="Set the ComfyUI output directory.")
|
||||||
|
parser.add_argument("--temp-directory", type=str, default=None, help="Set the ComfyUI temp directory (default is in the ComfyUI directory).")
|
||||||
parser.add_argument("--auto-launch", action="store_true", help="Automatically launch ComfyUI in the default browser.")
|
parser.add_argument("--auto-launch", action="store_true", help="Automatically launch ComfyUI in the default browser.")
|
||||||
|
parser.add_argument("--disable-auto-launch", action="store_true", help="Disable auto launching the browser.")
|
||||||
parser.add_argument("--cuda-device", type=int, default=None, metavar="DEVICE_ID", help="Set the id of the cuda device this instance will use.")
|
parser.add_argument("--cuda-device", type=int, default=None, metavar="DEVICE_ID", help="Set the id of the cuda device this instance will use.")
|
||||||
cm_group = parser.add_mutually_exclusive_group()
|
cm_group = parser.add_mutually_exclusive_group()
|
||||||
cm_group.add_argument("--cuda-malloc", action="store_true", help="Enable cudaMallocAsync (enabled by default for torch 2.0 and up).")
|
cm_group.add_argument("--cuda-malloc", action="store_true", help="Enable cudaMallocAsync (enabled by default for torch 2.0 and up).")
|
||||||
@ -52,10 +54,13 @@ fp_group.add_argument("--force-fp16", action="store_true", help="Force fp16.")
|
|||||||
|
|
||||||
fpvae_group = parser.add_mutually_exclusive_group()
|
fpvae_group = parser.add_mutually_exclusive_group()
|
||||||
fpvae_group.add_argument("--fp16-vae", action="store_true", help="Run the VAE in fp16, might cause black images.")
|
fpvae_group.add_argument("--fp16-vae", action="store_true", help="Run the VAE in fp16, might cause black images.")
|
||||||
fpvae_group.add_argument("--bf16-vae", action="store_true", help="Run the VAE in bf16, might lower quality.")
|
fpvae_group.add_argument("--fp32-vae", action="store_true", help="Run the VAE in full precision fp32.")
|
||||||
|
fpvae_group.add_argument("--bf16-vae", action="store_true", help="Run the VAE in bf16.")
|
||||||
|
|
||||||
parser.add_argument("--directml", type=int, nargs="?", metavar="DIRECTML_DEVICE", const=-1, help="Use torch-directml.")
|
parser.add_argument("--directml", type=int, nargs="?", metavar="DIRECTML_DEVICE", const=-1, help="Use torch-directml.")
|
||||||
|
|
||||||
|
parser.add_argument("--disable-ipex-optimize", action="store_true", help="Disables ipex.optimize when loading models with Intel GPUs.")
|
||||||
|
|
||||||
class LatentPreviewMethod(enum.Enum):
|
class LatentPreviewMethod(enum.Enum):
|
||||||
NoPreviews = "none"
|
NoPreviews = "none"
|
||||||
Auto = "auto"
|
Auto = "auto"
|
||||||
@ -80,6 +85,9 @@ vram_group.add_argument("--novram", action="store_true", help="When lowvram isn'
|
|||||||
vram_group.add_argument("--cpu", action="store_true", help="To use the CPU for everything (slow).")
|
vram_group.add_argument("--cpu", action="store_true", help="To use the CPU for everything (slow).")
|
||||||
|
|
||||||
|
|
||||||
|
parser.add_argument("--disable-smart-memory", action="store_true", help="Force ComfyUI to agressively offload to regular ram instead of keeping models in vram when it can.")
|
||||||
|
|
||||||
|
|
||||||
parser.add_argument("--dont-print-server", action="store_true", help="Don't print server output.")
|
parser.add_argument("--dont-print-server", action="store_true", help="Don't print server output.")
|
||||||
parser.add_argument("--quick-test-for-ci", action="store_true", help="Quick test for CI.")
|
parser.add_argument("--quick-test-for-ci", action="store_true", help="Quick test for CI.")
|
||||||
parser.add_argument("--windows-standalone-build", action="store_true", help="Windows standalone build: Enable convenient things that most people using the standalone windows build will probably enjoy (like auto opening the page on startup).")
|
parser.add_argument("--windows-standalone-build", action="store_true", help="Windows standalone build: Enable convenient things that most people using the standalone windows build will probably enjoy (like auto opening the page on startup).")
|
||||||
@ -90,3 +98,6 @@ args = parser.parse_args()
|
|||||||
|
|
||||||
if args.windows_standalone_build:
|
if args.windows_standalone_build:
|
||||||
args.auto_launch = True
|
args.auto_launch = True
|
||||||
|
|
||||||
|
if args.disable_auto_launch:
|
||||||
|
args.auto_launch = False
|
||||||
|
|||||||
@ -2,14 +2,27 @@ from transformers import CLIPVisionModelWithProjection, CLIPVisionConfig, CLIPIm
|
|||||||
from .utils import load_torch_file, transformers_convert
|
from .utils import load_torch_file, transformers_convert
|
||||||
import os
|
import os
|
||||||
import torch
|
import torch
|
||||||
|
import contextlib
|
||||||
|
|
||||||
import comfy.ops
|
import comfy.ops
|
||||||
|
import comfy.model_patcher
|
||||||
|
import comfy.model_management
|
||||||
|
|
||||||
class ClipVisionModel():
|
class ClipVisionModel():
|
||||||
def __init__(self, json_config):
|
def __init__(self, json_config):
|
||||||
config = CLIPVisionConfig.from_json_file(json_config)
|
config = CLIPVisionConfig.from_json_file(json_config)
|
||||||
with comfy.ops.use_comfy_ops():
|
self.load_device = comfy.model_management.text_encoder_device()
|
||||||
|
offload_device = comfy.model_management.text_encoder_offload_device()
|
||||||
|
self.dtype = torch.float32
|
||||||
|
if comfy.model_management.should_use_fp16(self.load_device, prioritize_performance=False):
|
||||||
|
self.dtype = torch.float16
|
||||||
|
|
||||||
|
with comfy.ops.use_comfy_ops(offload_device, self.dtype):
|
||||||
with modeling_utils.no_init_weights():
|
with modeling_utils.no_init_weights():
|
||||||
self.model = CLIPVisionModelWithProjection(config)
|
self.model = CLIPVisionModelWithProjection(config)
|
||||||
|
self.model.to(self.dtype)
|
||||||
|
|
||||||
|
self.patcher = comfy.model_patcher.ModelPatcher(self.model, load_device=self.load_device, offload_device=offload_device)
|
||||||
self.processor = CLIPImageProcessor(crop_size=224,
|
self.processor = CLIPImageProcessor(crop_size=224,
|
||||||
do_center_crop=True,
|
do_center_crop=True,
|
||||||
do_convert_rgb=True,
|
do_convert_rgb=True,
|
||||||
@ -24,9 +37,24 @@ class ClipVisionModel():
|
|||||||
return self.model.load_state_dict(sd, strict=False)
|
return self.model.load_state_dict(sd, strict=False)
|
||||||
|
|
||||||
def encode_image(self, image):
|
def encode_image(self, image):
|
||||||
img = torch.clip((255. * image[0]), 0, 255).round().int()
|
img = torch.clip((255. * image), 0, 255).round().int()
|
||||||
inputs = self.processor(images=[img], return_tensors="pt")
|
img = list(map(lambda a: a, img))
|
||||||
outputs = self.model(**inputs)
|
inputs = self.processor(images=img, return_tensors="pt")
|
||||||
|
comfy.model_management.load_model_gpu(self.patcher)
|
||||||
|
pixel_values = inputs['pixel_values'].to(self.load_device)
|
||||||
|
|
||||||
|
if self.dtype != torch.float32:
|
||||||
|
precision_scope = torch.autocast
|
||||||
|
else:
|
||||||
|
precision_scope = lambda a, b: contextlib.nullcontext(a)
|
||||||
|
|
||||||
|
with precision_scope(comfy.model_management.get_autocast_device(self.load_device), torch.float32):
|
||||||
|
outputs = self.model(pixel_values=pixel_values)
|
||||||
|
|
||||||
|
for k in outputs:
|
||||||
|
t = outputs[k]
|
||||||
|
if t is not None:
|
||||||
|
outputs[k] = t.cpu()
|
||||||
return outputs
|
return outputs
|
||||||
|
|
||||||
def convert_to_transformers(sd, prefix):
|
def convert_to_transformers(sd, prefix):
|
||||||
@ -49,18 +77,22 @@ def convert_to_transformers(sd, prefix):
|
|||||||
if "{}proj".format(prefix) in sd_k:
|
if "{}proj".format(prefix) in sd_k:
|
||||||
sd['visual_projection.weight'] = sd.pop("{}proj".format(prefix)).transpose(0, 1)
|
sd['visual_projection.weight'] = sd.pop("{}proj".format(prefix)).transpose(0, 1)
|
||||||
|
|
||||||
sd = transformers_convert(sd, prefix, "vision_model.", 32)
|
sd = transformers_convert(sd, prefix, "vision_model.", 48)
|
||||||
return sd
|
return sd
|
||||||
|
|
||||||
def load_clipvision_from_sd(sd, prefix="", convert_keys=False):
|
def load_clipvision_from_sd(sd, prefix="", convert_keys=False):
|
||||||
if convert_keys:
|
if convert_keys:
|
||||||
sd = convert_to_transformers(sd, prefix)
|
sd = convert_to_transformers(sd, prefix)
|
||||||
if "vision_model.encoder.layers.30.layer_norm1.weight" in sd:
|
if "vision_model.encoder.layers.47.layer_norm1.weight" in sd:
|
||||||
|
json_config = os.path.join(os.path.dirname(os.path.realpath(__file__)), "clip_vision_config_g.json")
|
||||||
|
elif "vision_model.encoder.layers.30.layer_norm1.weight" in sd:
|
||||||
json_config = os.path.join(os.path.dirname(os.path.realpath(__file__)), "clip_vision_config_h.json")
|
json_config = os.path.join(os.path.dirname(os.path.realpath(__file__)), "clip_vision_config_h.json")
|
||||||
else:
|
else:
|
||||||
json_config = os.path.join(os.path.dirname(os.path.realpath(__file__)), "clip_vision_config_vitl.json")
|
json_config = os.path.join(os.path.dirname(os.path.realpath(__file__)), "clip_vision_config_vitl.json")
|
||||||
clip = ClipVisionModel(json_config)
|
clip = ClipVisionModel(json_config)
|
||||||
m, u = clip.load_sd(sd)
|
m, u = clip.load_sd(sd)
|
||||||
|
if len(m) > 0:
|
||||||
|
print("missing clip vision:", m)
|
||||||
u = set(u)
|
u = set(u)
|
||||||
keys = list(sd.keys())
|
keys = list(sd.keys())
|
||||||
for k in keys:
|
for k in keys:
|
||||||
@ -71,4 +103,7 @@ def load_clipvision_from_sd(sd, prefix="", convert_keys=False):
|
|||||||
|
|
||||||
def load(ckpt_path):
|
def load(ckpt_path):
|
||||||
sd = load_torch_file(ckpt_path)
|
sd = load_torch_file(ckpt_path)
|
||||||
return load_clipvision_from_sd(sd)
|
if "visual.transformer.resblocks.0.attn.in_proj_weight" in sd:
|
||||||
|
return load_clipvision_from_sd(sd, prefix="visual.", convert_keys=True)
|
||||||
|
else:
|
||||||
|
return load_clipvision_from_sd(sd)
|
||||||
|
|||||||
18
comfy/clip_vision_config_g.json
Normal file
18
comfy/clip_vision_config_g.json
Normal file
@ -0,0 +1,18 @@
|
|||||||
|
{
|
||||||
|
"attention_dropout": 0.0,
|
||||||
|
"dropout": 0.0,
|
||||||
|
"hidden_act": "gelu",
|
||||||
|
"hidden_size": 1664,
|
||||||
|
"image_size": 224,
|
||||||
|
"initializer_factor": 1.0,
|
||||||
|
"initializer_range": 0.02,
|
||||||
|
"intermediate_size": 8192,
|
||||||
|
"layer_norm_eps": 1e-05,
|
||||||
|
"model_type": "clip_vision_model",
|
||||||
|
"num_attention_heads": 16,
|
||||||
|
"num_channels": 3,
|
||||||
|
"num_hidden_layers": 48,
|
||||||
|
"patch_size": 14,
|
||||||
|
"projection_dim": 1280,
|
||||||
|
"torch_dtype": "float32"
|
||||||
|
}
|
||||||
480
comfy/controlnet.py
Normal file
480
comfy/controlnet.py
Normal file
@ -0,0 +1,480 @@
|
|||||||
|
import torch
|
||||||
|
import math
|
||||||
|
import os
|
||||||
|
import comfy.utils
|
||||||
|
import comfy.model_management
|
||||||
|
import comfy.model_detection
|
||||||
|
import comfy.model_patcher
|
||||||
|
|
||||||
|
import comfy.cldm.cldm
|
||||||
|
import comfy.t2i_adapter.adapter
|
||||||
|
|
||||||
|
|
||||||
|
def broadcast_image_to(tensor, target_batch_size, batched_number):
|
||||||
|
current_batch_size = tensor.shape[0]
|
||||||
|
#print(current_batch_size, target_batch_size)
|
||||||
|
if current_batch_size == 1:
|
||||||
|
return tensor
|
||||||
|
|
||||||
|
per_batch = target_batch_size // batched_number
|
||||||
|
tensor = tensor[:per_batch]
|
||||||
|
|
||||||
|
if per_batch > tensor.shape[0]:
|
||||||
|
tensor = torch.cat([tensor] * (per_batch // tensor.shape[0]) + [tensor[:(per_batch % tensor.shape[0])]], dim=0)
|
||||||
|
|
||||||
|
current_batch_size = tensor.shape[0]
|
||||||
|
if current_batch_size == target_batch_size:
|
||||||
|
return tensor
|
||||||
|
else:
|
||||||
|
return torch.cat([tensor] * batched_number, dim=0)
|
||||||
|
|
||||||
|
class ControlBase:
|
||||||
|
def __init__(self, device=None):
|
||||||
|
self.cond_hint_original = None
|
||||||
|
self.cond_hint = None
|
||||||
|
self.strength = 1.0
|
||||||
|
self.timestep_percent_range = (1.0, 0.0)
|
||||||
|
self.timestep_range = None
|
||||||
|
|
||||||
|
if device is None:
|
||||||
|
device = comfy.model_management.get_torch_device()
|
||||||
|
self.device = device
|
||||||
|
self.previous_controlnet = None
|
||||||
|
self.global_average_pooling = False
|
||||||
|
|
||||||
|
def set_cond_hint(self, cond_hint, strength=1.0, timestep_percent_range=(1.0, 0.0)):
|
||||||
|
self.cond_hint_original = cond_hint
|
||||||
|
self.strength = strength
|
||||||
|
self.timestep_percent_range = timestep_percent_range
|
||||||
|
return self
|
||||||
|
|
||||||
|
def pre_run(self, model, percent_to_timestep_function):
|
||||||
|
self.timestep_range = (percent_to_timestep_function(self.timestep_percent_range[0]), percent_to_timestep_function(self.timestep_percent_range[1]))
|
||||||
|
if self.previous_controlnet is not None:
|
||||||
|
self.previous_controlnet.pre_run(model, percent_to_timestep_function)
|
||||||
|
|
||||||
|
def set_previous_controlnet(self, controlnet):
|
||||||
|
self.previous_controlnet = controlnet
|
||||||
|
return self
|
||||||
|
|
||||||
|
def cleanup(self):
|
||||||
|
if self.previous_controlnet is not None:
|
||||||
|
self.previous_controlnet.cleanup()
|
||||||
|
if self.cond_hint is not None:
|
||||||
|
del self.cond_hint
|
||||||
|
self.cond_hint = None
|
||||||
|
self.timestep_range = None
|
||||||
|
|
||||||
|
def get_models(self):
|
||||||
|
out = []
|
||||||
|
if self.previous_controlnet is not None:
|
||||||
|
out += self.previous_controlnet.get_models()
|
||||||
|
return out
|
||||||
|
|
||||||
|
def copy_to(self, c):
|
||||||
|
c.cond_hint_original = self.cond_hint_original
|
||||||
|
c.strength = self.strength
|
||||||
|
c.timestep_percent_range = self.timestep_percent_range
|
||||||
|
|
||||||
|
def inference_memory_requirements(self, dtype):
|
||||||
|
if self.previous_controlnet is not None:
|
||||||
|
return self.previous_controlnet.inference_memory_requirements(dtype)
|
||||||
|
return 0
|
||||||
|
|
||||||
|
def control_merge(self, control_input, control_output, control_prev, output_dtype):
|
||||||
|
out = {'input':[], 'middle':[], 'output': []}
|
||||||
|
|
||||||
|
if control_input is not None:
|
||||||
|
for i in range(len(control_input)):
|
||||||
|
key = 'input'
|
||||||
|
x = control_input[i]
|
||||||
|
if x is not None:
|
||||||
|
x *= self.strength
|
||||||
|
if x.dtype != output_dtype:
|
||||||
|
x = x.to(output_dtype)
|
||||||
|
out[key].insert(0, x)
|
||||||
|
|
||||||
|
if control_output is not None:
|
||||||
|
for i in range(len(control_output)):
|
||||||
|
if i == (len(control_output) - 1):
|
||||||
|
key = 'middle'
|
||||||
|
index = 0
|
||||||
|
else:
|
||||||
|
key = 'output'
|
||||||
|
index = i
|
||||||
|
x = control_output[i]
|
||||||
|
if x is not None:
|
||||||
|
if self.global_average_pooling:
|
||||||
|
x = torch.mean(x, dim=(2, 3), keepdim=True).repeat(1, 1, x.shape[2], x.shape[3])
|
||||||
|
|
||||||
|
x *= self.strength
|
||||||
|
if x.dtype != output_dtype:
|
||||||
|
x = x.to(output_dtype)
|
||||||
|
|
||||||
|
out[key].append(x)
|
||||||
|
if control_prev is not None:
|
||||||
|
for x in ['input', 'middle', 'output']:
|
||||||
|
o = out[x]
|
||||||
|
for i in range(len(control_prev[x])):
|
||||||
|
prev_val = control_prev[x][i]
|
||||||
|
if i >= len(o):
|
||||||
|
o.append(prev_val)
|
||||||
|
elif prev_val is not None:
|
||||||
|
if o[i] is None:
|
||||||
|
o[i] = prev_val
|
||||||
|
else:
|
||||||
|
o[i] += prev_val
|
||||||
|
return out
|
||||||
|
|
||||||
|
class ControlNet(ControlBase):
|
||||||
|
def __init__(self, control_model, global_average_pooling=False, device=None):
|
||||||
|
super().__init__(device)
|
||||||
|
self.control_model = control_model
|
||||||
|
self.control_model_wrapped = comfy.model_patcher.ModelPatcher(self.control_model, load_device=comfy.model_management.get_torch_device(), offload_device=comfy.model_management.unet_offload_device())
|
||||||
|
self.global_average_pooling = global_average_pooling
|
||||||
|
|
||||||
|
def get_control(self, x_noisy, t, cond, batched_number):
|
||||||
|
control_prev = None
|
||||||
|
if self.previous_controlnet is not None:
|
||||||
|
control_prev = self.previous_controlnet.get_control(x_noisy, t, cond, batched_number)
|
||||||
|
|
||||||
|
if self.timestep_range is not None:
|
||||||
|
if t[0] > self.timestep_range[0] or t[0] < self.timestep_range[1]:
|
||||||
|
if control_prev is not None:
|
||||||
|
return control_prev
|
||||||
|
else:
|
||||||
|
return {}
|
||||||
|
|
||||||
|
output_dtype = x_noisy.dtype
|
||||||
|
if self.cond_hint is None or x_noisy.shape[2] * 8 != self.cond_hint.shape[2] or x_noisy.shape[3] * 8 != self.cond_hint.shape[3]:
|
||||||
|
if self.cond_hint is not None:
|
||||||
|
del self.cond_hint
|
||||||
|
self.cond_hint = None
|
||||||
|
self.cond_hint = comfy.utils.common_upscale(self.cond_hint_original, x_noisy.shape[3] * 8, x_noisy.shape[2] * 8, 'nearest-exact', "center").to(self.control_model.dtype).to(self.device)
|
||||||
|
if x_noisy.shape[0] != self.cond_hint.shape[0]:
|
||||||
|
self.cond_hint = broadcast_image_to(self.cond_hint, x_noisy.shape[0], batched_number)
|
||||||
|
|
||||||
|
|
||||||
|
context = torch.cat(cond['c_crossattn'], 1)
|
||||||
|
y = cond.get('c_adm', None)
|
||||||
|
if y is not None:
|
||||||
|
y = y.to(self.control_model.dtype)
|
||||||
|
control = self.control_model(x=x_noisy.to(self.control_model.dtype), hint=self.cond_hint, timesteps=t, context=context.to(self.control_model.dtype), y=y)
|
||||||
|
return self.control_merge(None, control, control_prev, output_dtype)
|
||||||
|
|
||||||
|
def copy(self):
|
||||||
|
c = ControlNet(self.control_model, global_average_pooling=self.global_average_pooling)
|
||||||
|
self.copy_to(c)
|
||||||
|
return c
|
||||||
|
|
||||||
|
def get_models(self):
|
||||||
|
out = super().get_models()
|
||||||
|
out.append(self.control_model_wrapped)
|
||||||
|
return out
|
||||||
|
|
||||||
|
class ControlLoraOps:
|
||||||
|
class Linear(torch.nn.Module):
|
||||||
|
def __init__(self, in_features: int, out_features: int, bias: bool = True,
|
||||||
|
device=None, dtype=None) -> None:
|
||||||
|
factory_kwargs = {'device': device, 'dtype': dtype}
|
||||||
|
super().__init__()
|
||||||
|
self.in_features = in_features
|
||||||
|
self.out_features = out_features
|
||||||
|
self.weight = None
|
||||||
|
self.up = None
|
||||||
|
self.down = None
|
||||||
|
self.bias = None
|
||||||
|
|
||||||
|
def forward(self, input):
|
||||||
|
if self.up is not None:
|
||||||
|
return torch.nn.functional.linear(input, self.weight.to(input.device) + (torch.mm(self.up.flatten(start_dim=1), self.down.flatten(start_dim=1))).reshape(self.weight.shape).type(input.dtype), self.bias)
|
||||||
|
else:
|
||||||
|
return torch.nn.functional.linear(input, self.weight.to(input.device), self.bias)
|
||||||
|
|
||||||
|
class Conv2d(torch.nn.Module):
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
in_channels,
|
||||||
|
out_channels,
|
||||||
|
kernel_size,
|
||||||
|
stride=1,
|
||||||
|
padding=0,
|
||||||
|
dilation=1,
|
||||||
|
groups=1,
|
||||||
|
bias=True,
|
||||||
|
padding_mode='zeros',
|
||||||
|
device=None,
|
||||||
|
dtype=None
|
||||||
|
):
|
||||||
|
super().__init__()
|
||||||
|
self.in_channels = in_channels
|
||||||
|
self.out_channels = out_channels
|
||||||
|
self.kernel_size = kernel_size
|
||||||
|
self.stride = stride
|
||||||
|
self.padding = padding
|
||||||
|
self.dilation = dilation
|
||||||
|
self.transposed = False
|
||||||
|
self.output_padding = 0
|
||||||
|
self.groups = groups
|
||||||
|
self.padding_mode = padding_mode
|
||||||
|
|
||||||
|
self.weight = None
|
||||||
|
self.bias = None
|
||||||
|
self.up = None
|
||||||
|
self.down = None
|
||||||
|
|
||||||
|
|
||||||
|
def forward(self, input):
|
||||||
|
if self.up is not None:
|
||||||
|
return torch.nn.functional.conv2d(input, self.weight.to(input.device) + (torch.mm(self.up.flatten(start_dim=1), self.down.flatten(start_dim=1))).reshape(self.weight.shape).type(input.dtype), self.bias, self.stride, self.padding, self.dilation, self.groups)
|
||||||
|
else:
|
||||||
|
return torch.nn.functional.conv2d(input, self.weight.to(input.device), self.bias, self.stride, self.padding, self.dilation, self.groups)
|
||||||
|
|
||||||
|
def conv_nd(self, dims, *args, **kwargs):
|
||||||
|
if dims == 2:
|
||||||
|
return self.Conv2d(*args, **kwargs)
|
||||||
|
else:
|
||||||
|
raise ValueError(f"unsupported dimensions: {dims}")
|
||||||
|
|
||||||
|
|
||||||
|
class ControlLora(ControlNet):
|
||||||
|
def __init__(self, control_weights, global_average_pooling=False, device=None):
|
||||||
|
ControlBase.__init__(self, device)
|
||||||
|
self.control_weights = control_weights
|
||||||
|
self.global_average_pooling = global_average_pooling
|
||||||
|
|
||||||
|
def pre_run(self, model, percent_to_timestep_function):
|
||||||
|
super().pre_run(model, percent_to_timestep_function)
|
||||||
|
controlnet_config = model.model_config.unet_config.copy()
|
||||||
|
controlnet_config.pop("out_channels")
|
||||||
|
controlnet_config["hint_channels"] = self.control_weights["input_hint_block.0.weight"].shape[1]
|
||||||
|
controlnet_config["operations"] = ControlLoraOps()
|
||||||
|
self.control_model = comfy.cldm.cldm.ControlNet(**controlnet_config)
|
||||||
|
dtype = model.get_dtype()
|
||||||
|
self.control_model.to(dtype)
|
||||||
|
self.control_model.to(comfy.model_management.get_torch_device())
|
||||||
|
diffusion_model = model.diffusion_model
|
||||||
|
sd = diffusion_model.state_dict()
|
||||||
|
cm = self.control_model.state_dict()
|
||||||
|
|
||||||
|
for k in sd:
|
||||||
|
weight = comfy.model_management.resolve_lowvram_weight(sd[k], diffusion_model, k)
|
||||||
|
try:
|
||||||
|
comfy.utils.set_attr(self.control_model, k, weight)
|
||||||
|
except:
|
||||||
|
pass
|
||||||
|
|
||||||
|
for k in self.control_weights:
|
||||||
|
if k not in {"lora_controlnet"}:
|
||||||
|
comfy.utils.set_attr(self.control_model, k, self.control_weights[k].to(dtype).to(comfy.model_management.get_torch_device()))
|
||||||
|
|
||||||
|
def copy(self):
|
||||||
|
c = ControlLora(self.control_weights, global_average_pooling=self.global_average_pooling)
|
||||||
|
self.copy_to(c)
|
||||||
|
return c
|
||||||
|
|
||||||
|
def cleanup(self):
|
||||||
|
del self.control_model
|
||||||
|
self.control_model = None
|
||||||
|
super().cleanup()
|
||||||
|
|
||||||
|
def get_models(self):
|
||||||
|
out = ControlBase.get_models(self)
|
||||||
|
return out
|
||||||
|
|
||||||
|
def inference_memory_requirements(self, dtype):
|
||||||
|
return comfy.utils.calculate_parameters(self.control_weights) * comfy.model_management.dtype_size(dtype) + ControlBase.inference_memory_requirements(self, dtype)
|
||||||
|
|
||||||
|
def load_controlnet(ckpt_path, model=None):
|
||||||
|
controlnet_data = comfy.utils.load_torch_file(ckpt_path, safe_load=True)
|
||||||
|
if "lora_controlnet" in controlnet_data:
|
||||||
|
return ControlLora(controlnet_data)
|
||||||
|
|
||||||
|
controlnet_config = None
|
||||||
|
if "controlnet_cond_embedding.conv_in.weight" in controlnet_data: #diffusers format
|
||||||
|
use_fp16 = comfy.model_management.should_use_fp16()
|
||||||
|
controlnet_config = comfy.model_detection.unet_config_from_diffusers_unet(controlnet_data, use_fp16)
|
||||||
|
diffusers_keys = comfy.utils.unet_to_diffusers(controlnet_config)
|
||||||
|
diffusers_keys["controlnet_mid_block.weight"] = "middle_block_out.0.weight"
|
||||||
|
diffusers_keys["controlnet_mid_block.bias"] = "middle_block_out.0.bias"
|
||||||
|
|
||||||
|
count = 0
|
||||||
|
loop = True
|
||||||
|
while loop:
|
||||||
|
suffix = [".weight", ".bias"]
|
||||||
|
for s in suffix:
|
||||||
|
k_in = "controlnet_down_blocks.{}{}".format(count, s)
|
||||||
|
k_out = "zero_convs.{}.0{}".format(count, s)
|
||||||
|
if k_in not in controlnet_data:
|
||||||
|
loop = False
|
||||||
|
break
|
||||||
|
diffusers_keys[k_in] = k_out
|
||||||
|
count += 1
|
||||||
|
|
||||||
|
count = 0
|
||||||
|
loop = True
|
||||||
|
while loop:
|
||||||
|
suffix = [".weight", ".bias"]
|
||||||
|
for s in suffix:
|
||||||
|
if count == 0:
|
||||||
|
k_in = "controlnet_cond_embedding.conv_in{}".format(s)
|
||||||
|
else:
|
||||||
|
k_in = "controlnet_cond_embedding.blocks.{}{}".format(count - 1, s)
|
||||||
|
k_out = "input_hint_block.{}{}".format(count * 2, s)
|
||||||
|
if k_in not in controlnet_data:
|
||||||
|
k_in = "controlnet_cond_embedding.conv_out{}".format(s)
|
||||||
|
loop = False
|
||||||
|
diffusers_keys[k_in] = k_out
|
||||||
|
count += 1
|
||||||
|
|
||||||
|
new_sd = {}
|
||||||
|
for k in diffusers_keys:
|
||||||
|
if k in controlnet_data:
|
||||||
|
new_sd[diffusers_keys[k]] = controlnet_data.pop(k)
|
||||||
|
|
||||||
|
leftover_keys = controlnet_data.keys()
|
||||||
|
if len(leftover_keys) > 0:
|
||||||
|
print("leftover keys:", leftover_keys)
|
||||||
|
controlnet_data = new_sd
|
||||||
|
|
||||||
|
pth_key = 'control_model.zero_convs.0.0.weight'
|
||||||
|
pth = False
|
||||||
|
key = 'zero_convs.0.0.weight'
|
||||||
|
if pth_key in controlnet_data:
|
||||||
|
pth = True
|
||||||
|
key = pth_key
|
||||||
|
prefix = "control_model."
|
||||||
|
elif key in controlnet_data:
|
||||||
|
prefix = ""
|
||||||
|
else:
|
||||||
|
net = load_t2i_adapter(controlnet_data)
|
||||||
|
if net is None:
|
||||||
|
print("error checkpoint does not contain controlnet or t2i adapter data", ckpt_path)
|
||||||
|
return net
|
||||||
|
|
||||||
|
if controlnet_config is None:
|
||||||
|
use_fp16 = comfy.model_management.should_use_fp16()
|
||||||
|
controlnet_config = comfy.model_detection.model_config_from_unet(controlnet_data, prefix, use_fp16).unet_config
|
||||||
|
controlnet_config.pop("out_channels")
|
||||||
|
controlnet_config["hint_channels"] = controlnet_data["{}input_hint_block.0.weight".format(prefix)].shape[1]
|
||||||
|
control_model = comfy.cldm.cldm.ControlNet(**controlnet_config)
|
||||||
|
|
||||||
|
if pth:
|
||||||
|
if 'difference' in controlnet_data:
|
||||||
|
if model is not None:
|
||||||
|
comfy.model_management.load_models_gpu([model])
|
||||||
|
model_sd = model.model_state_dict()
|
||||||
|
for x in controlnet_data:
|
||||||
|
c_m = "control_model."
|
||||||
|
if x.startswith(c_m):
|
||||||
|
sd_key = "diffusion_model.{}".format(x[len(c_m):])
|
||||||
|
if sd_key in model_sd:
|
||||||
|
cd = controlnet_data[x]
|
||||||
|
cd += model_sd[sd_key].type(cd.dtype).to(cd.device)
|
||||||
|
else:
|
||||||
|
print("WARNING: Loaded a diff controlnet without a model. It will very likely not work.")
|
||||||
|
|
||||||
|
class WeightsLoader(torch.nn.Module):
|
||||||
|
pass
|
||||||
|
w = WeightsLoader()
|
||||||
|
w.control_model = control_model
|
||||||
|
missing, unexpected = w.load_state_dict(controlnet_data, strict=False)
|
||||||
|
else:
|
||||||
|
missing, unexpected = control_model.load_state_dict(controlnet_data, strict=False)
|
||||||
|
print(missing, unexpected)
|
||||||
|
|
||||||
|
if use_fp16:
|
||||||
|
control_model = control_model.half()
|
||||||
|
|
||||||
|
global_average_pooling = False
|
||||||
|
filename = os.path.splitext(ckpt_path)[0]
|
||||||
|
if filename.endswith("_shuffle") or filename.endswith("_shuffle_fp16"): #TODO: smarter way of enabling global_average_pooling
|
||||||
|
global_average_pooling = True
|
||||||
|
|
||||||
|
control = ControlNet(control_model, global_average_pooling=global_average_pooling)
|
||||||
|
return control
|
||||||
|
|
||||||
|
class T2IAdapter(ControlBase):
|
||||||
|
def __init__(self, t2i_model, channels_in, device=None):
|
||||||
|
super().__init__(device)
|
||||||
|
self.t2i_model = t2i_model
|
||||||
|
self.channels_in = channels_in
|
||||||
|
self.control_input = None
|
||||||
|
|
||||||
|
def scale_image_to(self, width, height):
|
||||||
|
unshuffle_amount = self.t2i_model.unshuffle_amount
|
||||||
|
width = math.ceil(width / unshuffle_amount) * unshuffle_amount
|
||||||
|
height = math.ceil(height / unshuffle_amount) * unshuffle_amount
|
||||||
|
return width, height
|
||||||
|
|
||||||
|
def get_control(self, x_noisy, t, cond, batched_number):
|
||||||
|
control_prev = None
|
||||||
|
if self.previous_controlnet is not None:
|
||||||
|
control_prev = self.previous_controlnet.get_control(x_noisy, t, cond, batched_number)
|
||||||
|
|
||||||
|
if self.timestep_range is not None:
|
||||||
|
if t[0] > self.timestep_range[0] or t[0] < self.timestep_range[1]:
|
||||||
|
if control_prev is not None:
|
||||||
|
return control_prev
|
||||||
|
else:
|
||||||
|
return {}
|
||||||
|
|
||||||
|
if self.cond_hint is None or x_noisy.shape[2] * 8 != self.cond_hint.shape[2] or x_noisy.shape[3] * 8 != self.cond_hint.shape[3]:
|
||||||
|
if self.cond_hint is not None:
|
||||||
|
del self.cond_hint
|
||||||
|
self.control_input = None
|
||||||
|
self.cond_hint = None
|
||||||
|
width, height = self.scale_image_to(x_noisy.shape[3] * 8, x_noisy.shape[2] * 8)
|
||||||
|
self.cond_hint = comfy.utils.common_upscale(self.cond_hint_original, width, height, 'nearest-exact', "center").float().to(self.device)
|
||||||
|
if self.channels_in == 1 and self.cond_hint.shape[1] > 1:
|
||||||
|
self.cond_hint = torch.mean(self.cond_hint, 1, keepdim=True)
|
||||||
|
if x_noisy.shape[0] != self.cond_hint.shape[0]:
|
||||||
|
self.cond_hint = broadcast_image_to(self.cond_hint, x_noisy.shape[0], batched_number)
|
||||||
|
if self.control_input is None:
|
||||||
|
self.t2i_model.to(x_noisy.dtype)
|
||||||
|
self.t2i_model.to(self.device)
|
||||||
|
self.control_input = self.t2i_model(self.cond_hint.to(x_noisy.dtype))
|
||||||
|
self.t2i_model.cpu()
|
||||||
|
|
||||||
|
control_input = list(map(lambda a: None if a is None else a.clone(), self.control_input))
|
||||||
|
mid = None
|
||||||
|
if self.t2i_model.xl == True:
|
||||||
|
mid = control_input[-1:]
|
||||||
|
control_input = control_input[:-1]
|
||||||
|
return self.control_merge(control_input, mid, control_prev, x_noisy.dtype)
|
||||||
|
|
||||||
|
def copy(self):
|
||||||
|
c = T2IAdapter(self.t2i_model, self.channels_in)
|
||||||
|
self.copy_to(c)
|
||||||
|
return c
|
||||||
|
|
||||||
|
def load_t2i_adapter(t2i_data):
|
||||||
|
keys = t2i_data.keys()
|
||||||
|
if 'adapter' in keys:
|
||||||
|
t2i_data = t2i_data['adapter']
|
||||||
|
keys = t2i_data.keys()
|
||||||
|
if "body.0.in_conv.weight" in keys:
|
||||||
|
cin = t2i_data['body.0.in_conv.weight'].shape[1]
|
||||||
|
model_ad = comfy.t2i_adapter.adapter.Adapter_light(cin=cin, channels=[320, 640, 1280, 1280], nums_rb=4)
|
||||||
|
elif 'conv_in.weight' in keys:
|
||||||
|
cin = t2i_data['conv_in.weight'].shape[1]
|
||||||
|
channel = t2i_data['conv_in.weight'].shape[0]
|
||||||
|
ksize = t2i_data['body.0.block2.weight'].shape[2]
|
||||||
|
use_conv = False
|
||||||
|
down_opts = list(filter(lambda a: a.endswith("down_opt.op.weight"), keys))
|
||||||
|
if len(down_opts) > 0:
|
||||||
|
use_conv = True
|
||||||
|
xl = False
|
||||||
|
if cin == 256 or cin == 768:
|
||||||
|
xl = True
|
||||||
|
model_ad = comfy.t2i_adapter.adapter.Adapter(cin=cin, channels=[channel, channel*2, channel*4, channel*4][:4], nums_rb=2, ksize=ksize, sk=True, use_conv=use_conv, xl=xl)
|
||||||
|
else:
|
||||||
|
return None
|
||||||
|
missing, unexpected = model_ad.load_state_dict(t2i_data)
|
||||||
|
if len(missing) > 0:
|
||||||
|
print("t2i missing", missing)
|
||||||
|
|
||||||
|
if len(unexpected) > 0:
|
||||||
|
print("t2i unexpected", unexpected)
|
||||||
|
|
||||||
|
return T2IAdapter(model_ad, model_ad.input_channels)
|
||||||
@ -1,87 +1,36 @@
|
|||||||
import json
|
import json
|
||||||
import os
|
import os
|
||||||
import yaml
|
|
||||||
|
|
||||||
import folder_paths
|
import comfy.sd
|
||||||
from comfy.sd import load_checkpoint
|
|
||||||
import os.path as osp
|
|
||||||
import re
|
|
||||||
import torch
|
|
||||||
from safetensors.torch import load_file, save_file
|
|
||||||
from . import diffusers_convert
|
|
||||||
|
|
||||||
|
def first_file(path, filenames):
|
||||||
|
for f in filenames:
|
||||||
|
p = os.path.join(path, f)
|
||||||
|
if os.path.exists(p):
|
||||||
|
return p
|
||||||
|
return None
|
||||||
|
|
||||||
def load_diffusers(model_path, fp16=True, output_vae=True, output_clip=True, embedding_directory=None):
|
def load_diffusers(model_path, output_vae=True, output_clip=True, embedding_directory=None):
|
||||||
diffusers_unet_conf = json.load(open(osp.join(model_path, "unet/config.json")))
|
diffusion_model_names = ["diffusion_pytorch_model.fp16.safetensors", "diffusion_pytorch_model.safetensors", "diffusion_pytorch_model.fp16.bin", "diffusion_pytorch_model.bin"]
|
||||||
diffusers_scheduler_conf = json.load(open(osp.join(model_path, "scheduler/scheduler_config.json")))
|
unet_path = first_file(os.path.join(model_path, "unet"), diffusion_model_names)
|
||||||
|
vae_path = first_file(os.path.join(model_path, "vae"), diffusion_model_names)
|
||||||
|
|
||||||
# magic
|
text_encoder_model_names = ["model.fp16.safetensors", "model.safetensors", "pytorch_model.fp16.bin", "pytorch_model.bin"]
|
||||||
v2 = diffusers_unet_conf["sample_size"] == 96
|
text_encoder1_path = first_file(os.path.join(model_path, "text_encoder"), text_encoder_model_names)
|
||||||
if 'prediction_type' in diffusers_scheduler_conf:
|
text_encoder2_path = first_file(os.path.join(model_path, "text_encoder_2"), text_encoder_model_names)
|
||||||
v_pred = diffusers_scheduler_conf['prediction_type'] == 'v_prediction'
|
|
||||||
|
|
||||||
if v2:
|
text_encoder_paths = [text_encoder1_path]
|
||||||
if v_pred:
|
if text_encoder2_path is not None:
|
||||||
config_path = folder_paths.get_full_path("configs", 'v2-inference-v.yaml')
|
text_encoder_paths.append(text_encoder2_path)
|
||||||
else:
|
|
||||||
config_path = folder_paths.get_full_path("configs", 'v2-inference.yaml')
|
|
||||||
else:
|
|
||||||
config_path = folder_paths.get_full_path("configs", 'v1-inference.yaml')
|
|
||||||
|
|
||||||
with open(config_path, 'r') as stream:
|
unet = comfy.sd.load_unet(unet_path)
|
||||||
config = yaml.safe_load(stream)
|
|
||||||
|
|
||||||
model_config_params = config['model']['params']
|
clip = None
|
||||||
clip_config = model_config_params['cond_stage_config']
|
if output_clip:
|
||||||
scale_factor = model_config_params['scale_factor']
|
clip = comfy.sd.load_clip(text_encoder_paths, embedding_directory=embedding_directory)
|
||||||
vae_config = model_config_params['first_stage_config']
|
|
||||||
vae_config['scale_factor'] = scale_factor
|
|
||||||
model_config_params["unet_config"]["params"]["use_fp16"] = fp16
|
|
||||||
|
|
||||||
unet_path = osp.join(model_path, "unet", "diffusion_pytorch_model.safetensors")
|
vae = None
|
||||||
vae_path = osp.join(model_path, "vae", "diffusion_pytorch_model.safetensors")
|
if output_vae:
|
||||||
text_enc_path = osp.join(model_path, "text_encoder", "model.safetensors")
|
vae = comfy.sd.VAE(ckpt_path=vae_path)
|
||||||
|
|
||||||
# Load models from safetensors if it exists, if it doesn't pytorch
|
return (unet, clip, vae)
|
||||||
if osp.exists(unet_path):
|
|
||||||
unet_state_dict = load_file(unet_path, device="cpu")
|
|
||||||
else:
|
|
||||||
unet_path = osp.join(model_path, "unet", "diffusion_pytorch_model.bin")
|
|
||||||
unet_state_dict = torch.load(unet_path, map_location="cpu")
|
|
||||||
|
|
||||||
if osp.exists(vae_path):
|
|
||||||
vae_state_dict = load_file(vae_path, device="cpu")
|
|
||||||
else:
|
|
||||||
vae_path = osp.join(model_path, "vae", "diffusion_pytorch_model.bin")
|
|
||||||
vae_state_dict = torch.load(vae_path, map_location="cpu")
|
|
||||||
|
|
||||||
if osp.exists(text_enc_path):
|
|
||||||
text_enc_dict = load_file(text_enc_path, device="cpu")
|
|
||||||
else:
|
|
||||||
text_enc_path = osp.join(model_path, "text_encoder", "pytorch_model.bin")
|
|
||||||
text_enc_dict = torch.load(text_enc_path, map_location="cpu")
|
|
||||||
|
|
||||||
# Convert the UNet model
|
|
||||||
unet_state_dict = diffusers_convert.convert_unet_state_dict(unet_state_dict)
|
|
||||||
unet_state_dict = {"model.diffusion_model." + k: v for k, v in unet_state_dict.items()}
|
|
||||||
|
|
||||||
# Convert the VAE model
|
|
||||||
vae_state_dict = diffusers_convert.convert_vae_state_dict(vae_state_dict)
|
|
||||||
vae_state_dict = {"first_stage_model." + k: v for k, v in vae_state_dict.items()}
|
|
||||||
|
|
||||||
# Easiest way to identify v2.0 model seems to be that the text encoder (OpenCLIP) is deeper
|
|
||||||
is_v20_model = "text_model.encoder.layers.22.layer_norm2.bias" in text_enc_dict
|
|
||||||
|
|
||||||
if is_v20_model:
|
|
||||||
# Need to add the tag 'transformer' in advance so we can knock it out from the final layer-norm
|
|
||||||
text_enc_dict = {"transformer." + k: v for k, v in text_enc_dict.items()}
|
|
||||||
text_enc_dict = diffusers_convert.convert_text_enc_state_dict_v20(text_enc_dict)
|
|
||||||
text_enc_dict = {"cond_stage_model.model." + k: v for k, v in text_enc_dict.items()}
|
|
||||||
else:
|
|
||||||
text_enc_dict = diffusers_convert.convert_text_enc_state_dict(text_enc_dict)
|
|
||||||
text_enc_dict = {"cond_stage_model.transformer." + k: v for k, v in text_enc_dict.items()}
|
|
||||||
|
|
||||||
# Put together new checkpoint
|
|
||||||
sd = {**unet_state_dict, **vae_state_dict, **text_enc_dict}
|
|
||||||
|
|
||||||
return load_checkpoint(embedding_directory=embedding_directory, state_dict=sd, config=config)
|
|
||||||
|
|||||||
@ -244,30 +244,15 @@ class Gligen(nn.Module):
|
|||||||
self.position_net = position_net
|
self.position_net = position_net
|
||||||
self.key_dim = key_dim
|
self.key_dim = key_dim
|
||||||
self.max_objs = 30
|
self.max_objs = 30
|
||||||
self.lowvram = False
|
self.current_device = torch.device("cpu")
|
||||||
|
|
||||||
def _set_position(self, boxes, masks, positive_embeddings):
|
def _set_position(self, boxes, masks, positive_embeddings):
|
||||||
if self.lowvram == True:
|
|
||||||
self.position_net.to(boxes.device)
|
|
||||||
|
|
||||||
objs = self.position_net(boxes, masks, positive_embeddings)
|
objs = self.position_net(boxes, masks, positive_embeddings)
|
||||||
|
def func(x, extra_options):
|
||||||
if self.lowvram == True:
|
key = extra_options["transformer_index"]
|
||||||
self.position_net.cpu()
|
module = self.module_list[key]
|
||||||
def func_lowvram(x, extra_options):
|
return module(x, objs)
|
||||||
key = extra_options["transformer_index"]
|
return func
|
||||||
module = self.module_list[key]
|
|
||||||
module.to(x.device)
|
|
||||||
r = module(x, objs)
|
|
||||||
module.cpu()
|
|
||||||
return r
|
|
||||||
return func_lowvram
|
|
||||||
else:
|
|
||||||
def func(x, extra_options):
|
|
||||||
key = extra_options["transformer_index"]
|
|
||||||
module = self.module_list[key]
|
|
||||||
return module(x, objs)
|
|
||||||
return func
|
|
||||||
|
|
||||||
def set_position(self, latent_image_shape, position_params, device):
|
def set_position(self, latent_image_shape, position_params, device):
|
||||||
batch, c, h, w = latent_image_shape
|
batch, c, h, w = latent_image_shape
|
||||||
@ -312,14 +297,6 @@ class Gligen(nn.Module):
|
|||||||
masks.to(device),
|
masks.to(device),
|
||||||
conds.to(device))
|
conds.to(device))
|
||||||
|
|
||||||
def set_lowvram(self, value=True):
|
|
||||||
self.lowvram = value
|
|
||||||
|
|
||||||
def cleanup(self):
|
|
||||||
self.lowvram = False
|
|
||||||
|
|
||||||
def get_models(self):
|
|
||||||
return [self]
|
|
||||||
|
|
||||||
def load_gligen(sd):
|
def load_gligen(sd):
|
||||||
sd_k = sd.keys()
|
sd_k = sd.keys()
|
||||||
|
|||||||
@ -631,23 +631,78 @@ def sample_dpmpp_2m_sde(model, x, sigmas, extra_args=None, callback=None, disabl
|
|||||||
elif solver_type == 'midpoint':
|
elif solver_type == 'midpoint':
|
||||||
x = x + 0.5 * (-h - eta_h).expm1().neg() * (1 / r) * (denoised - old_denoised)
|
x = x + 0.5 * (-h - eta_h).expm1().neg() * (1 / r) * (denoised - old_denoised)
|
||||||
|
|
||||||
x = x + noise_sampler(sigmas[i], sigmas[i + 1]) * sigmas[i + 1] * (-2 * eta_h).expm1().neg().sqrt() * s_noise
|
if eta:
|
||||||
|
x = x + noise_sampler(sigmas[i], sigmas[i + 1]) * sigmas[i + 1] * (-2 * eta_h).expm1().neg().sqrt() * s_noise
|
||||||
|
|
||||||
old_denoised = denoised
|
old_denoised = denoised
|
||||||
h_last = h
|
h_last = h
|
||||||
return x
|
return x
|
||||||
|
|
||||||
|
@torch.no_grad()
|
||||||
|
def sample_dpmpp_3m_sde(model, x, sigmas, extra_args=None, callback=None, disable=None, eta=1., s_noise=1., noise_sampler=None):
|
||||||
|
"""DPM-Solver++(3M) SDE."""
|
||||||
|
|
||||||
|
seed = extra_args.get("seed", None)
|
||||||
|
sigma_min, sigma_max = sigmas[sigmas > 0].min(), sigmas.max()
|
||||||
|
noise_sampler = BrownianTreeNoiseSampler(x, sigma_min, sigma_max, seed=seed, cpu=True) if noise_sampler is None else noise_sampler
|
||||||
|
extra_args = {} if extra_args is None else extra_args
|
||||||
|
s_in = x.new_ones([x.shape[0]])
|
||||||
|
|
||||||
|
denoised_1, denoised_2 = None, None
|
||||||
|
h, h_1, h_2 = None, None, None
|
||||||
|
|
||||||
|
for i in trange(len(sigmas) - 1, disable=disable):
|
||||||
|
denoised = model(x, sigmas[i] * s_in, **extra_args)
|
||||||
|
if callback is not None:
|
||||||
|
callback({'x': x, 'i': i, 'sigma': sigmas[i], 'sigma_hat': sigmas[i], 'denoised': denoised})
|
||||||
|
if sigmas[i + 1] == 0:
|
||||||
|
# Denoising step
|
||||||
|
x = denoised
|
||||||
|
else:
|
||||||
|
t, s = -sigmas[i].log(), -sigmas[i + 1].log()
|
||||||
|
h = s - t
|
||||||
|
h_eta = h * (eta + 1)
|
||||||
|
|
||||||
|
x = torch.exp(-h_eta) * x + (-h_eta).expm1().neg() * denoised
|
||||||
|
|
||||||
|
if h_2 is not None:
|
||||||
|
r0 = h_1 / h
|
||||||
|
r1 = h_2 / h
|
||||||
|
d1_0 = (denoised - denoised_1) / r0
|
||||||
|
d1_1 = (denoised_1 - denoised_2) / r1
|
||||||
|
d1 = d1_0 + (d1_0 - d1_1) * r0 / (r0 + r1)
|
||||||
|
d2 = (d1_0 - d1_1) / (r0 + r1)
|
||||||
|
phi_2 = h_eta.neg().expm1() / h_eta + 1
|
||||||
|
phi_3 = phi_2 / h_eta - 0.5
|
||||||
|
x = x + phi_2 * d1 - phi_3 * d2
|
||||||
|
elif h_1 is not None:
|
||||||
|
r = h_1 / h
|
||||||
|
d = (denoised - denoised_1) / r
|
||||||
|
phi_2 = h_eta.neg().expm1() / h_eta + 1
|
||||||
|
x = x + phi_2 * d
|
||||||
|
|
||||||
|
if eta:
|
||||||
|
x = x + noise_sampler(sigmas[i], sigmas[i + 1]) * sigmas[i + 1] * (-2 * h * eta).expm1().neg().sqrt() * s_noise
|
||||||
|
|
||||||
|
denoised_1, denoised_2 = denoised, denoised_1
|
||||||
|
h_1, h_2 = h, h_1
|
||||||
|
return x
|
||||||
|
|
||||||
|
@torch.no_grad()
|
||||||
|
def sample_dpmpp_3m_sde_gpu(model, x, sigmas, extra_args=None, callback=None, disable=None, eta=1., s_noise=1., noise_sampler=None):
|
||||||
|
sigma_min, sigma_max = sigmas[sigmas > 0].min(), sigmas.max()
|
||||||
|
noise_sampler = BrownianTreeNoiseSampler(x, sigma_min, sigma_max, seed=extra_args.get("seed", None), cpu=False) if noise_sampler is None else noise_sampler
|
||||||
|
return sample_dpmpp_3m_sde(model, x, sigmas, extra_args=extra_args, callback=callback, disable=disable, eta=eta, s_noise=s_noise, noise_sampler=noise_sampler)
|
||||||
|
|
||||||
@torch.no_grad()
|
@torch.no_grad()
|
||||||
def sample_dpmpp_2m_sde_gpu(model, x, sigmas, extra_args=None, callback=None, disable=None, eta=1., s_noise=1., noise_sampler=None, solver_type='midpoint'):
|
def sample_dpmpp_2m_sde_gpu(model, x, sigmas, extra_args=None, callback=None, disable=None, eta=1., s_noise=1., noise_sampler=None, solver_type='midpoint'):
|
||||||
sigma_min, sigma_max = sigmas[sigmas > 0].min(), sigmas.max()
|
sigma_min, sigma_max = sigmas[sigmas > 0].min(), sigmas.max()
|
||||||
noise_sampler = BrownianTreeNoiseSampler(x, sigma_min, sigma_max, seed=extra_args.get("seed", None), cpu=False) if noise_sampler is None else noise_sampler
|
noise_sampler = BrownianTreeNoiseSampler(x, sigma_min, sigma_max, seed=extra_args.get("seed", None), cpu=False) if noise_sampler is None else noise_sampler
|
||||||
return sample_dpmpp_2m_sde(model, x, sigmas, extra_args=extra_args, callback=callback, disable=disable, eta=eta, s_noise=s_noise, noise_sampler=noise_sampler, solver_type=solver_type)
|
return sample_dpmpp_2m_sde(model, x, sigmas, extra_args=extra_args, callback=callback, disable=disable, eta=eta, s_noise=s_noise, noise_sampler=noise_sampler, solver_type=solver_type)
|
||||||
|
|
||||||
|
|
||||||
@torch.no_grad()
|
@torch.no_grad()
|
||||||
def sample_dpmpp_sde_gpu(model, x, sigmas, extra_args=None, callback=None, disable=None, eta=1., s_noise=1., noise_sampler=None, r=1 / 2):
|
def sample_dpmpp_sde_gpu(model, x, sigmas, extra_args=None, callback=None, disable=None, eta=1., s_noise=1., noise_sampler=None, r=1 / 2):
|
||||||
sigma_min, sigma_max = sigmas[sigmas > 0].min(), sigmas.max()
|
sigma_min, sigma_max = sigmas[sigmas > 0].min(), sigmas.max()
|
||||||
noise_sampler = BrownianTreeNoiseSampler(x, sigma_min, sigma_max, seed=extra_args.get("seed", None), cpu=False) if noise_sampler is None else noise_sampler
|
noise_sampler = BrownianTreeNoiseSampler(x, sigma_min, sigma_max, seed=extra_args.get("seed", None), cpu=False) if noise_sampler is None else noise_sampler
|
||||||
return sample_dpmpp_sde(model, x, sigmas, extra_args=extra_args, callback=callback, disable=disable, eta=eta, s_noise=s_noise, noise_sampler=noise_sampler, r=r)
|
return sample_dpmpp_sde(model, x, sigmas, extra_args=extra_args, callback=callback, disable=disable, eta=eta, s_noise=s_noise, noise_sampler=noise_sampler, r=r)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@ -10,13 +10,14 @@ from .diffusionmodules.util import checkpoint
|
|||||||
from .sub_quadratic_attention import efficient_dot_product_attention
|
from .sub_quadratic_attention import efficient_dot_product_attention
|
||||||
|
|
||||||
from comfy import model_management
|
from comfy import model_management
|
||||||
import comfy.ops
|
|
||||||
|
|
||||||
if model_management.xformers_enabled():
|
if model_management.xformers_enabled():
|
||||||
import xformers
|
import xformers
|
||||||
import xformers.ops
|
import xformers.ops
|
||||||
|
|
||||||
from comfy.cli_args import args
|
from comfy.cli_args import args
|
||||||
|
import comfy.ops
|
||||||
|
|
||||||
# CrossAttn precision handling
|
# CrossAttn precision handling
|
||||||
if args.dont_upcast_attention:
|
if args.dont_upcast_attention:
|
||||||
print("disabling upcasting of attention")
|
print("disabling upcasting of attention")
|
||||||
@ -52,9 +53,9 @@ def init_(tensor):
|
|||||||
|
|
||||||
# feedforward
|
# feedforward
|
||||||
class GEGLU(nn.Module):
|
class GEGLU(nn.Module):
|
||||||
def __init__(self, dim_in, dim_out, dtype=None, device=None):
|
def __init__(self, dim_in, dim_out, dtype=None, device=None, operations=comfy.ops):
|
||||||
super().__init__()
|
super().__init__()
|
||||||
self.proj = comfy.ops.Linear(dim_in, dim_out * 2, dtype=dtype, device=device)
|
self.proj = operations.Linear(dim_in, dim_out * 2, dtype=dtype, device=device)
|
||||||
|
|
||||||
def forward(self, x):
|
def forward(self, x):
|
||||||
x, gate = self.proj(x).chunk(2, dim=-1)
|
x, gate = self.proj(x).chunk(2, dim=-1)
|
||||||
@ -62,19 +63,19 @@ class GEGLU(nn.Module):
|
|||||||
|
|
||||||
|
|
||||||
class FeedForward(nn.Module):
|
class FeedForward(nn.Module):
|
||||||
def __init__(self, dim, dim_out=None, mult=4, glu=False, dropout=0., dtype=None, device=None):
|
def __init__(self, dim, dim_out=None, mult=4, glu=False, dropout=0., dtype=None, device=None, operations=comfy.ops):
|
||||||
super().__init__()
|
super().__init__()
|
||||||
inner_dim = int(dim * mult)
|
inner_dim = int(dim * mult)
|
||||||
dim_out = default(dim_out, dim)
|
dim_out = default(dim_out, dim)
|
||||||
project_in = nn.Sequential(
|
project_in = nn.Sequential(
|
||||||
comfy.ops.Linear(dim, inner_dim, dtype=dtype, device=device),
|
operations.Linear(dim, inner_dim, dtype=dtype, device=device),
|
||||||
nn.GELU()
|
nn.GELU()
|
||||||
) if not glu else GEGLU(dim, inner_dim, dtype=dtype, device=device)
|
) if not glu else GEGLU(dim, inner_dim, dtype=dtype, device=device, operations=operations)
|
||||||
|
|
||||||
self.net = nn.Sequential(
|
self.net = nn.Sequential(
|
||||||
project_in,
|
project_in,
|
||||||
nn.Dropout(dropout),
|
nn.Dropout(dropout),
|
||||||
comfy.ops.Linear(inner_dim, dim_out, dtype=dtype, device=device)
|
operations.Linear(inner_dim, dim_out, dtype=dtype, device=device)
|
||||||
)
|
)
|
||||||
|
|
||||||
def forward(self, x):
|
def forward(self, x):
|
||||||
@ -148,7 +149,7 @@ class SpatialSelfAttention(nn.Module):
|
|||||||
|
|
||||||
|
|
||||||
class CrossAttentionBirchSan(nn.Module):
|
class CrossAttentionBirchSan(nn.Module):
|
||||||
def __init__(self, query_dim, context_dim=None, heads=8, dim_head=64, dropout=0., dtype=None, device=None):
|
def __init__(self, query_dim, context_dim=None, heads=8, dim_head=64, dropout=0., dtype=None, device=None, operations=comfy.ops):
|
||||||
super().__init__()
|
super().__init__()
|
||||||
inner_dim = dim_head * heads
|
inner_dim = dim_head * heads
|
||||||
context_dim = default(context_dim, query_dim)
|
context_dim = default(context_dim, query_dim)
|
||||||
@ -156,12 +157,12 @@ class CrossAttentionBirchSan(nn.Module):
|
|||||||
self.scale = dim_head ** -0.5
|
self.scale = dim_head ** -0.5
|
||||||
self.heads = heads
|
self.heads = heads
|
||||||
|
|
||||||
self.to_q = comfy.ops.Linear(query_dim, inner_dim, bias=False, dtype=dtype, device=device)
|
self.to_q = operations.Linear(query_dim, inner_dim, bias=False, dtype=dtype, device=device)
|
||||||
self.to_k = comfy.ops.Linear(context_dim, inner_dim, bias=False, dtype=dtype, device=device)
|
self.to_k = operations.Linear(context_dim, inner_dim, bias=False, dtype=dtype, device=device)
|
||||||
self.to_v = comfy.ops.Linear(context_dim, inner_dim, bias=False, dtype=dtype, device=device)
|
self.to_v = operations.Linear(context_dim, inner_dim, bias=False, dtype=dtype, device=device)
|
||||||
|
|
||||||
self.to_out = nn.Sequential(
|
self.to_out = nn.Sequential(
|
||||||
comfy.ops.Linear(inner_dim, query_dim, dtype=dtype, device=device),
|
operations.Linear(inner_dim, query_dim, dtype=dtype, device=device),
|
||||||
nn.Dropout(dropout)
|
nn.Dropout(dropout)
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -245,7 +246,7 @@ class CrossAttentionBirchSan(nn.Module):
|
|||||||
|
|
||||||
|
|
||||||
class CrossAttentionDoggettx(nn.Module):
|
class CrossAttentionDoggettx(nn.Module):
|
||||||
def __init__(self, query_dim, context_dim=None, heads=8, dim_head=64, dropout=0., dtype=None, device=None):
|
def __init__(self, query_dim, context_dim=None, heads=8, dim_head=64, dropout=0., dtype=None, device=None, operations=comfy.ops):
|
||||||
super().__init__()
|
super().__init__()
|
||||||
inner_dim = dim_head * heads
|
inner_dim = dim_head * heads
|
||||||
context_dim = default(context_dim, query_dim)
|
context_dim = default(context_dim, query_dim)
|
||||||
@ -253,12 +254,12 @@ class CrossAttentionDoggettx(nn.Module):
|
|||||||
self.scale = dim_head ** -0.5
|
self.scale = dim_head ** -0.5
|
||||||
self.heads = heads
|
self.heads = heads
|
||||||
|
|
||||||
self.to_q = comfy.ops.Linear(query_dim, inner_dim, bias=False, dtype=dtype, device=device)
|
self.to_q = operations.Linear(query_dim, inner_dim, bias=False, dtype=dtype, device=device)
|
||||||
self.to_k = comfy.ops.Linear(context_dim, inner_dim, bias=False, dtype=dtype, device=device)
|
self.to_k = operations.Linear(context_dim, inner_dim, bias=False, dtype=dtype, device=device)
|
||||||
self.to_v = comfy.ops.Linear(context_dim, inner_dim, bias=False, dtype=dtype, device=device)
|
self.to_v = operations.Linear(context_dim, inner_dim, bias=False, dtype=dtype, device=device)
|
||||||
|
|
||||||
self.to_out = nn.Sequential(
|
self.to_out = nn.Sequential(
|
||||||
comfy.ops.Linear(inner_dim, query_dim, dtype=dtype, device=device),
|
operations.Linear(inner_dim, query_dim, dtype=dtype, device=device),
|
||||||
nn.Dropout(dropout)
|
nn.Dropout(dropout)
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -343,7 +344,7 @@ class CrossAttentionDoggettx(nn.Module):
|
|||||||
return self.to_out(r2)
|
return self.to_out(r2)
|
||||||
|
|
||||||
class CrossAttention(nn.Module):
|
class CrossAttention(nn.Module):
|
||||||
def __init__(self, query_dim, context_dim=None, heads=8, dim_head=64, dropout=0., dtype=None, device=None):
|
def __init__(self, query_dim, context_dim=None, heads=8, dim_head=64, dropout=0., dtype=None, device=None, operations=comfy.ops):
|
||||||
super().__init__()
|
super().__init__()
|
||||||
inner_dim = dim_head * heads
|
inner_dim = dim_head * heads
|
||||||
context_dim = default(context_dim, query_dim)
|
context_dim = default(context_dim, query_dim)
|
||||||
@ -351,12 +352,12 @@ class CrossAttention(nn.Module):
|
|||||||
self.scale = dim_head ** -0.5
|
self.scale = dim_head ** -0.5
|
||||||
self.heads = heads
|
self.heads = heads
|
||||||
|
|
||||||
self.to_q = comfy.ops.Linear(query_dim, inner_dim, bias=False, dtype=dtype, device=device)
|
self.to_q = operations.Linear(query_dim, inner_dim, bias=False, dtype=dtype, device=device)
|
||||||
self.to_k = comfy.ops.Linear(context_dim, inner_dim, bias=False, dtype=dtype, device=device)
|
self.to_k = operations.Linear(context_dim, inner_dim, bias=False, dtype=dtype, device=device)
|
||||||
self.to_v = comfy.ops.Linear(context_dim, inner_dim, bias=False, dtype=dtype, device=device)
|
self.to_v = operations.Linear(context_dim, inner_dim, bias=False, dtype=dtype, device=device)
|
||||||
|
|
||||||
self.to_out = nn.Sequential(
|
self.to_out = nn.Sequential(
|
||||||
comfy.ops.Linear(inner_dim, query_dim, dtype=dtype, device=device),
|
operations.Linear(inner_dim, query_dim, dtype=dtype, device=device),
|
||||||
nn.Dropout(dropout)
|
nn.Dropout(dropout)
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -399,7 +400,7 @@ class CrossAttention(nn.Module):
|
|||||||
|
|
||||||
class MemoryEfficientCrossAttention(nn.Module):
|
class MemoryEfficientCrossAttention(nn.Module):
|
||||||
# https://github.com/MatthieuTPHR/diffusers/blob/d80b531ff8060ec1ea982b65a1b8df70f73aa67c/src/diffusers/models/attention.py#L223
|
# https://github.com/MatthieuTPHR/diffusers/blob/d80b531ff8060ec1ea982b65a1b8df70f73aa67c/src/diffusers/models/attention.py#L223
|
||||||
def __init__(self, query_dim, context_dim=None, heads=8, dim_head=64, dropout=0.0, dtype=None, device=None):
|
def __init__(self, query_dim, context_dim=None, heads=8, dim_head=64, dropout=0.0, dtype=None, device=None, operations=comfy.ops):
|
||||||
super().__init__()
|
super().__init__()
|
||||||
print(f"Setting up {self.__class__.__name__}. Query dim is {query_dim}, context_dim is {context_dim} and using "
|
print(f"Setting up {self.__class__.__name__}. Query dim is {query_dim}, context_dim is {context_dim} and using "
|
||||||
f"{heads} heads.")
|
f"{heads} heads.")
|
||||||
@ -409,11 +410,11 @@ class MemoryEfficientCrossAttention(nn.Module):
|
|||||||
self.heads = heads
|
self.heads = heads
|
||||||
self.dim_head = dim_head
|
self.dim_head = dim_head
|
||||||
|
|
||||||
self.to_q = comfy.ops.Linear(query_dim, inner_dim, bias=False, dtype=dtype, device=device)
|
self.to_q = operations.Linear(query_dim, inner_dim, bias=False, dtype=dtype, device=device)
|
||||||
self.to_k = comfy.ops.Linear(context_dim, inner_dim, bias=False, dtype=dtype, device=device)
|
self.to_k = operations.Linear(context_dim, inner_dim, bias=False, dtype=dtype, device=device)
|
||||||
self.to_v = comfy.ops.Linear(context_dim, inner_dim, bias=False, dtype=dtype, device=device)
|
self.to_v = operations.Linear(context_dim, inner_dim, bias=False, dtype=dtype, device=device)
|
||||||
|
|
||||||
self.to_out = nn.Sequential(comfy.ops.Linear(inner_dim, query_dim, dtype=dtype, device=device), nn.Dropout(dropout))
|
self.to_out = nn.Sequential(operations.Linear(inner_dim, query_dim, dtype=dtype, device=device), nn.Dropout(dropout))
|
||||||
self.attention_op: Optional[Any] = None
|
self.attention_op: Optional[Any] = None
|
||||||
|
|
||||||
def forward(self, x, context=None, value=None, mask=None):
|
def forward(self, x, context=None, value=None, mask=None):
|
||||||
@ -450,7 +451,7 @@ class MemoryEfficientCrossAttention(nn.Module):
|
|||||||
return self.to_out(out)
|
return self.to_out(out)
|
||||||
|
|
||||||
class CrossAttentionPytorch(nn.Module):
|
class CrossAttentionPytorch(nn.Module):
|
||||||
def __init__(self, query_dim, context_dim=None, heads=8, dim_head=64, dropout=0., dtype=None, device=None):
|
def __init__(self, query_dim, context_dim=None, heads=8, dim_head=64, dropout=0., dtype=None, device=None, operations=comfy.ops):
|
||||||
super().__init__()
|
super().__init__()
|
||||||
inner_dim = dim_head * heads
|
inner_dim = dim_head * heads
|
||||||
context_dim = default(context_dim, query_dim)
|
context_dim = default(context_dim, query_dim)
|
||||||
@ -458,11 +459,11 @@ class CrossAttentionPytorch(nn.Module):
|
|||||||
self.heads = heads
|
self.heads = heads
|
||||||
self.dim_head = dim_head
|
self.dim_head = dim_head
|
||||||
|
|
||||||
self.to_q = comfy.ops.Linear(query_dim, inner_dim, bias=False, dtype=dtype, device=device)
|
self.to_q = operations.Linear(query_dim, inner_dim, bias=False, dtype=dtype, device=device)
|
||||||
self.to_k = comfy.ops.Linear(context_dim, inner_dim, bias=False, dtype=dtype, device=device)
|
self.to_k = operations.Linear(context_dim, inner_dim, bias=False, dtype=dtype, device=device)
|
||||||
self.to_v = comfy.ops.Linear(context_dim, inner_dim, bias=False, dtype=dtype, device=device)
|
self.to_v = operations.Linear(context_dim, inner_dim, bias=False, dtype=dtype, device=device)
|
||||||
|
|
||||||
self.to_out = nn.Sequential(comfy.ops.Linear(inner_dim, query_dim, dtype=dtype, device=device), nn.Dropout(dropout))
|
self.to_out = nn.Sequential(operations.Linear(inner_dim, query_dim, dtype=dtype, device=device), nn.Dropout(dropout))
|
||||||
self.attention_op: Optional[Any] = None
|
self.attention_op: Optional[Any] = None
|
||||||
|
|
||||||
def forward(self, x, context=None, value=None, mask=None):
|
def forward(self, x, context=None, value=None, mask=None):
|
||||||
@ -508,14 +509,14 @@ else:
|
|||||||
|
|
||||||
class BasicTransformerBlock(nn.Module):
|
class BasicTransformerBlock(nn.Module):
|
||||||
def __init__(self, dim, n_heads, d_head, dropout=0., context_dim=None, gated_ff=True, checkpoint=True,
|
def __init__(self, dim, n_heads, d_head, dropout=0., context_dim=None, gated_ff=True, checkpoint=True,
|
||||||
disable_self_attn=False, dtype=None, device=None):
|
disable_self_attn=False, dtype=None, device=None, operations=comfy.ops):
|
||||||
super().__init__()
|
super().__init__()
|
||||||
self.disable_self_attn = disable_self_attn
|
self.disable_self_attn = disable_self_attn
|
||||||
self.attn1 = CrossAttention(query_dim=dim, heads=n_heads, dim_head=d_head, dropout=dropout,
|
self.attn1 = CrossAttention(query_dim=dim, heads=n_heads, dim_head=d_head, dropout=dropout,
|
||||||
context_dim=context_dim if self.disable_self_attn else None, dtype=dtype, device=device) # is a self-attention if not self.disable_self_attn
|
context_dim=context_dim if self.disable_self_attn else None, dtype=dtype, device=device, operations=operations) # is a self-attention if not self.disable_self_attn
|
||||||
self.ff = FeedForward(dim, dropout=dropout, glu=gated_ff, dtype=dtype, device=device)
|
self.ff = FeedForward(dim, dropout=dropout, glu=gated_ff, dtype=dtype, device=device, operations=operations)
|
||||||
self.attn2 = CrossAttention(query_dim=dim, context_dim=context_dim,
|
self.attn2 = CrossAttention(query_dim=dim, context_dim=context_dim,
|
||||||
heads=n_heads, dim_head=d_head, dropout=dropout, dtype=dtype, device=device) # is self-attn if context is none
|
heads=n_heads, dim_head=d_head, dropout=dropout, dtype=dtype, device=device, operations=operations) # is self-attn if context is none
|
||||||
self.norm1 = nn.LayerNorm(dim, dtype=dtype, device=device)
|
self.norm1 = nn.LayerNorm(dim, dtype=dtype, device=device)
|
||||||
self.norm2 = nn.LayerNorm(dim, dtype=dtype, device=device)
|
self.norm2 = nn.LayerNorm(dim, dtype=dtype, device=device)
|
||||||
self.norm3 = nn.LayerNorm(dim, dtype=dtype, device=device)
|
self.norm3 = nn.LayerNorm(dim, dtype=dtype, device=device)
|
||||||
@ -648,7 +649,7 @@ class SpatialTransformer(nn.Module):
|
|||||||
def __init__(self, in_channels, n_heads, d_head,
|
def __init__(self, in_channels, n_heads, d_head,
|
||||||
depth=1, dropout=0., context_dim=None,
|
depth=1, dropout=0., context_dim=None,
|
||||||
disable_self_attn=False, use_linear=False,
|
disable_self_attn=False, use_linear=False,
|
||||||
use_checkpoint=True, dtype=None, device=None):
|
use_checkpoint=True, dtype=None, device=None, operations=comfy.ops):
|
||||||
super().__init__()
|
super().__init__()
|
||||||
if exists(context_dim) and not isinstance(context_dim, list):
|
if exists(context_dim) and not isinstance(context_dim, list):
|
||||||
context_dim = [context_dim] * depth
|
context_dim = [context_dim] * depth
|
||||||
@ -656,26 +657,26 @@ class SpatialTransformer(nn.Module):
|
|||||||
inner_dim = n_heads * d_head
|
inner_dim = n_heads * d_head
|
||||||
self.norm = Normalize(in_channels, dtype=dtype, device=device)
|
self.norm = Normalize(in_channels, dtype=dtype, device=device)
|
||||||
if not use_linear:
|
if not use_linear:
|
||||||
self.proj_in = nn.Conv2d(in_channels,
|
self.proj_in = operations.Conv2d(in_channels,
|
||||||
inner_dim,
|
inner_dim,
|
||||||
kernel_size=1,
|
kernel_size=1,
|
||||||
stride=1,
|
stride=1,
|
||||||
padding=0, dtype=dtype, device=device)
|
padding=0, dtype=dtype, device=device)
|
||||||
else:
|
else:
|
||||||
self.proj_in = comfy.ops.Linear(in_channels, inner_dim, dtype=dtype, device=device)
|
self.proj_in = operations.Linear(in_channels, inner_dim, dtype=dtype, device=device)
|
||||||
|
|
||||||
self.transformer_blocks = nn.ModuleList(
|
self.transformer_blocks = nn.ModuleList(
|
||||||
[BasicTransformerBlock(inner_dim, n_heads, d_head, dropout=dropout, context_dim=context_dim[d],
|
[BasicTransformerBlock(inner_dim, n_heads, d_head, dropout=dropout, context_dim=context_dim[d],
|
||||||
disable_self_attn=disable_self_attn, checkpoint=use_checkpoint, dtype=dtype, device=device)
|
disable_self_attn=disable_self_attn, checkpoint=use_checkpoint, dtype=dtype, device=device, operations=operations)
|
||||||
for d in range(depth)]
|
for d in range(depth)]
|
||||||
)
|
)
|
||||||
if not use_linear:
|
if not use_linear:
|
||||||
self.proj_out = nn.Conv2d(inner_dim,in_channels,
|
self.proj_out = operations.Conv2d(inner_dim,in_channels,
|
||||||
kernel_size=1,
|
kernel_size=1,
|
||||||
stride=1,
|
stride=1,
|
||||||
padding=0, dtype=dtype, device=device)
|
padding=0, dtype=dtype, device=device)
|
||||||
else:
|
else:
|
||||||
self.proj_out = comfy.ops.Linear(in_channels, inner_dim, dtype=dtype, device=device)
|
self.proj_out = operations.Linear(in_channels, inner_dim, dtype=dtype, device=device)
|
||||||
self.use_linear = use_linear
|
self.use_linear = use_linear
|
||||||
|
|
||||||
def forward(self, x, context=None, transformer_options={}):
|
def forward(self, x, context=None, transformer_options={}):
|
||||||
|
|||||||
@ -56,7 +56,18 @@ class Upsample(nn.Module):
|
|||||||
padding=1)
|
padding=1)
|
||||||
|
|
||||||
def forward(self, x):
|
def forward(self, x):
|
||||||
x = torch.nn.functional.interpolate(x, scale_factor=2.0, mode="nearest")
|
try:
|
||||||
|
x = torch.nn.functional.interpolate(x, scale_factor=2.0, mode="nearest")
|
||||||
|
except: #operation not implemented for bf16
|
||||||
|
b, c, h, w = x.shape
|
||||||
|
out = torch.empty((b, c, h*2, w*2), dtype=x.dtype, layout=x.layout, device=x.device)
|
||||||
|
split = 8
|
||||||
|
l = out.shape[1] // split
|
||||||
|
for i in range(0, out.shape[1], l):
|
||||||
|
out[:,i:i+l] = torch.nn.functional.interpolate(x[:,i:i+l].to(torch.float32), scale_factor=2.0, mode="nearest").to(x.dtype)
|
||||||
|
del x
|
||||||
|
x = out
|
||||||
|
|
||||||
if self.with_conv:
|
if self.with_conv:
|
||||||
x = self.conv(x)
|
x = self.conv(x)
|
||||||
return x
|
return x
|
||||||
@ -74,11 +85,10 @@ class Downsample(nn.Module):
|
|||||||
stride=2,
|
stride=2,
|
||||||
padding=0)
|
padding=0)
|
||||||
|
|
||||||
def forward(self, x, already_padded=False):
|
def forward(self, x):
|
||||||
if self.with_conv:
|
if self.with_conv:
|
||||||
if not already_padded:
|
pad = (0,1,0,1)
|
||||||
pad = (0,1,0,1)
|
x = torch.nn.functional.pad(x, pad, mode="constant", value=0)
|
||||||
x = torch.nn.functional.pad(x, pad, mode="constant", value=0)
|
|
||||||
x = self.conv(x)
|
x = self.conv(x)
|
||||||
else:
|
else:
|
||||||
x = torch.nn.functional.avg_pool2d(x, kernel_size=2, stride=2)
|
x = torch.nn.functional.avg_pool2d(x, kernel_size=2, stride=2)
|
||||||
@ -275,25 +285,17 @@ class MemoryEfficientAttnBlock(nn.Module):
|
|||||||
|
|
||||||
# compute attention
|
# compute attention
|
||||||
B, C, H, W = q.shape
|
B, C, H, W = q.shape
|
||||||
q, k, v = map(lambda x: rearrange(x, 'b c h w -> b (h w) c'), (q, k, v))
|
|
||||||
|
|
||||||
q, k, v = map(
|
q, k, v = map(
|
||||||
lambda t: t.unsqueeze(3)
|
lambda t: t.view(B, C, -1).transpose(1, 2).contiguous(),
|
||||||
.reshape(B, t.shape[1], 1, C)
|
|
||||||
.permute(0, 2, 1, 3)
|
|
||||||
.reshape(B * 1, t.shape[1], C)
|
|
||||||
.contiguous(),
|
|
||||||
(q, k, v),
|
(q, k, v),
|
||||||
)
|
)
|
||||||
out = xformers.ops.memory_efficient_attention(q, k, v, attn_bias=None, op=self.attention_op)
|
|
||||||
|
|
||||||
out = (
|
try:
|
||||||
out.unsqueeze(0)
|
out = xformers.ops.memory_efficient_attention(q, k, v, attn_bias=None, op=self.attention_op)
|
||||||
.reshape(B, 1, out.shape[1], C)
|
out = out.transpose(1, 2).reshape(B, C, H, W)
|
||||||
.permute(0, 2, 1, 3)
|
except NotImplementedError as e:
|
||||||
.reshape(B, out.shape[1], C)
|
out = slice_attention(q.view(B, -1, C), k.view(B, -1, C).transpose(1, 2), v.view(B, -1, C).transpose(1, 2)).reshape(B, C, H, W)
|
||||||
)
|
|
||||||
out = rearrange(out, 'b (h w) c -> b c h w', b=B, h=H, w=W, c=C)
|
|
||||||
out = self.proj_out(out)
|
out = self.proj_out(out)
|
||||||
return x+out
|
return x+out
|
||||||
|
|
||||||
@ -603,9 +605,6 @@ class Encoder(nn.Module):
|
|||||||
def forward(self, x):
|
def forward(self, x):
|
||||||
# timestep embedding
|
# timestep embedding
|
||||||
temb = None
|
temb = None
|
||||||
pad = (0,1,0,1)
|
|
||||||
x = torch.nn.functional.pad(x, pad, mode="constant", value=0)
|
|
||||||
already_padded = True
|
|
||||||
# downsampling
|
# downsampling
|
||||||
h = self.conv_in(x)
|
h = self.conv_in(x)
|
||||||
for i_level in range(self.num_resolutions):
|
for i_level in range(self.num_resolutions):
|
||||||
@ -614,8 +613,7 @@ class Encoder(nn.Module):
|
|||||||
if len(self.down[i_level].attn) > 0:
|
if len(self.down[i_level].attn) > 0:
|
||||||
h = self.down[i_level].attn[i_block](h)
|
h = self.down[i_level].attn[i_block](h)
|
||||||
if i_level != self.num_resolutions-1:
|
if i_level != self.num_resolutions-1:
|
||||||
h = self.down[i_level].downsample(h, already_padded)
|
h = self.down[i_level].downsample(h)
|
||||||
already_padded = False
|
|
||||||
|
|
||||||
# middle
|
# middle
|
||||||
h = self.mid.block_1(h, temb)
|
h = self.mid.block_1(h, temb)
|
||||||
|
|||||||
@ -8,8 +8,6 @@ import torch.nn.functional as F
|
|||||||
|
|
||||||
from .util import (
|
from .util import (
|
||||||
checkpoint,
|
checkpoint,
|
||||||
conv_nd,
|
|
||||||
linear,
|
|
||||||
avg_pool_nd,
|
avg_pool_nd,
|
||||||
zero_module,
|
zero_module,
|
||||||
normalization,
|
normalization,
|
||||||
@ -17,7 +15,7 @@ from .util import (
|
|||||||
)
|
)
|
||||||
from ..attention import SpatialTransformer
|
from ..attention import SpatialTransformer
|
||||||
from comfy.ldm.util import exists
|
from comfy.ldm.util import exists
|
||||||
|
import comfy.ops
|
||||||
|
|
||||||
class TimestepBlock(nn.Module):
|
class TimestepBlock(nn.Module):
|
||||||
"""
|
"""
|
||||||
@ -72,14 +70,14 @@ class Upsample(nn.Module):
|
|||||||
upsampling occurs in the inner-two dimensions.
|
upsampling occurs in the inner-two dimensions.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
def __init__(self, channels, use_conv, dims=2, out_channels=None, padding=1, dtype=None, device=None):
|
def __init__(self, channels, use_conv, dims=2, out_channels=None, padding=1, dtype=None, device=None, operations=comfy.ops):
|
||||||
super().__init__()
|
super().__init__()
|
||||||
self.channels = channels
|
self.channels = channels
|
||||||
self.out_channels = out_channels or channels
|
self.out_channels = out_channels or channels
|
||||||
self.use_conv = use_conv
|
self.use_conv = use_conv
|
||||||
self.dims = dims
|
self.dims = dims
|
||||||
if use_conv:
|
if use_conv:
|
||||||
self.conv = conv_nd(dims, self.channels, self.out_channels, 3, padding=padding, dtype=dtype, device=device)
|
self.conv = operations.conv_nd(dims, self.channels, self.out_channels, 3, padding=padding, dtype=dtype, device=device)
|
||||||
|
|
||||||
def forward(self, x, output_shape=None):
|
def forward(self, x, output_shape=None):
|
||||||
assert x.shape[1] == self.channels
|
assert x.shape[1] == self.channels
|
||||||
@ -108,7 +106,7 @@ class Downsample(nn.Module):
|
|||||||
downsampling occurs in the inner-two dimensions.
|
downsampling occurs in the inner-two dimensions.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
def __init__(self, channels, use_conv, dims=2, out_channels=None, padding=1, dtype=None, device=None):
|
def __init__(self, channels, use_conv, dims=2, out_channels=None, padding=1, dtype=None, device=None, operations=comfy.ops):
|
||||||
super().__init__()
|
super().__init__()
|
||||||
self.channels = channels
|
self.channels = channels
|
||||||
self.out_channels = out_channels or channels
|
self.out_channels = out_channels or channels
|
||||||
@ -116,7 +114,7 @@ class Downsample(nn.Module):
|
|||||||
self.dims = dims
|
self.dims = dims
|
||||||
stride = 2 if dims != 3 else (1, 2, 2)
|
stride = 2 if dims != 3 else (1, 2, 2)
|
||||||
if use_conv:
|
if use_conv:
|
||||||
self.op = conv_nd(
|
self.op = operations.conv_nd(
|
||||||
dims, self.channels, self.out_channels, 3, stride=stride, padding=padding, dtype=dtype, device=device
|
dims, self.channels, self.out_channels, 3, stride=stride, padding=padding, dtype=dtype, device=device
|
||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
@ -158,6 +156,7 @@ class ResBlock(TimestepBlock):
|
|||||||
down=False,
|
down=False,
|
||||||
dtype=None,
|
dtype=None,
|
||||||
device=None,
|
device=None,
|
||||||
|
operations=comfy.ops
|
||||||
):
|
):
|
||||||
super().__init__()
|
super().__init__()
|
||||||
self.channels = channels
|
self.channels = channels
|
||||||
@ -171,7 +170,7 @@ class ResBlock(TimestepBlock):
|
|||||||
self.in_layers = nn.Sequential(
|
self.in_layers = nn.Sequential(
|
||||||
nn.GroupNorm(32, channels, dtype=dtype, device=device),
|
nn.GroupNorm(32, channels, dtype=dtype, device=device),
|
||||||
nn.SiLU(),
|
nn.SiLU(),
|
||||||
conv_nd(dims, channels, self.out_channels, 3, padding=1, dtype=dtype, device=device),
|
operations.conv_nd(dims, channels, self.out_channels, 3, padding=1, dtype=dtype, device=device),
|
||||||
)
|
)
|
||||||
|
|
||||||
self.updown = up or down
|
self.updown = up or down
|
||||||
@ -187,7 +186,7 @@ class ResBlock(TimestepBlock):
|
|||||||
|
|
||||||
self.emb_layers = nn.Sequential(
|
self.emb_layers = nn.Sequential(
|
||||||
nn.SiLU(),
|
nn.SiLU(),
|
||||||
linear(
|
operations.Linear(
|
||||||
emb_channels,
|
emb_channels,
|
||||||
2 * self.out_channels if use_scale_shift_norm else self.out_channels, dtype=dtype, device=device
|
2 * self.out_channels if use_scale_shift_norm else self.out_channels, dtype=dtype, device=device
|
||||||
),
|
),
|
||||||
@ -197,18 +196,18 @@ class ResBlock(TimestepBlock):
|
|||||||
nn.SiLU(),
|
nn.SiLU(),
|
||||||
nn.Dropout(p=dropout),
|
nn.Dropout(p=dropout),
|
||||||
zero_module(
|
zero_module(
|
||||||
conv_nd(dims, self.out_channels, self.out_channels, 3, padding=1, dtype=dtype, device=device)
|
operations.conv_nd(dims, self.out_channels, self.out_channels, 3, padding=1, dtype=dtype, device=device)
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
if self.out_channels == channels:
|
if self.out_channels == channels:
|
||||||
self.skip_connection = nn.Identity()
|
self.skip_connection = nn.Identity()
|
||||||
elif use_conv:
|
elif use_conv:
|
||||||
self.skip_connection = conv_nd(
|
self.skip_connection = operations.conv_nd(
|
||||||
dims, channels, self.out_channels, 3, padding=1, dtype=dtype, device=device
|
dims, channels, self.out_channels, 3, padding=1, dtype=dtype, device=device
|
||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
self.skip_connection = conv_nd(dims, channels, self.out_channels, 1, dtype=dtype, device=device)
|
self.skip_connection = operations.conv_nd(dims, channels, self.out_channels, 1, dtype=dtype, device=device)
|
||||||
|
|
||||||
def forward(self, x, emb):
|
def forward(self, x, emb):
|
||||||
"""
|
"""
|
||||||
@ -317,6 +316,7 @@ class UNetModel(nn.Module):
|
|||||||
adm_in_channels=None,
|
adm_in_channels=None,
|
||||||
transformer_depth_middle=None,
|
transformer_depth_middle=None,
|
||||||
device=None,
|
device=None,
|
||||||
|
operations=comfy.ops,
|
||||||
):
|
):
|
||||||
super().__init__()
|
super().__init__()
|
||||||
assert use_spatial_transformer == True, "use_spatial_transformer has to be true"
|
assert use_spatial_transformer == True, "use_spatial_transformer has to be true"
|
||||||
@ -379,9 +379,9 @@ class UNetModel(nn.Module):
|
|||||||
|
|
||||||
time_embed_dim = model_channels * 4
|
time_embed_dim = model_channels * 4
|
||||||
self.time_embed = nn.Sequential(
|
self.time_embed = nn.Sequential(
|
||||||
linear(model_channels, time_embed_dim, dtype=self.dtype, device=device),
|
operations.Linear(model_channels, time_embed_dim, dtype=self.dtype, device=device),
|
||||||
nn.SiLU(),
|
nn.SiLU(),
|
||||||
linear(time_embed_dim, time_embed_dim, dtype=self.dtype, device=device),
|
operations.Linear(time_embed_dim, time_embed_dim, dtype=self.dtype, device=device),
|
||||||
)
|
)
|
||||||
|
|
||||||
if self.num_classes is not None:
|
if self.num_classes is not None:
|
||||||
@ -394,9 +394,9 @@ class UNetModel(nn.Module):
|
|||||||
assert adm_in_channels is not None
|
assert adm_in_channels is not None
|
||||||
self.label_emb = nn.Sequential(
|
self.label_emb = nn.Sequential(
|
||||||
nn.Sequential(
|
nn.Sequential(
|
||||||
linear(adm_in_channels, time_embed_dim, dtype=self.dtype, device=device),
|
operations.Linear(adm_in_channels, time_embed_dim, dtype=self.dtype, device=device),
|
||||||
nn.SiLU(),
|
nn.SiLU(),
|
||||||
linear(time_embed_dim, time_embed_dim, dtype=self.dtype, device=device),
|
operations.Linear(time_embed_dim, time_embed_dim, dtype=self.dtype, device=device),
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
@ -405,7 +405,7 @@ class UNetModel(nn.Module):
|
|||||||
self.input_blocks = nn.ModuleList(
|
self.input_blocks = nn.ModuleList(
|
||||||
[
|
[
|
||||||
TimestepEmbedSequential(
|
TimestepEmbedSequential(
|
||||||
conv_nd(dims, in_channels, model_channels, 3, padding=1, dtype=self.dtype, device=device)
|
operations.conv_nd(dims, in_channels, model_channels, 3, padding=1, dtype=self.dtype, device=device)
|
||||||
)
|
)
|
||||||
]
|
]
|
||||||
)
|
)
|
||||||
@ -426,6 +426,7 @@ class UNetModel(nn.Module):
|
|||||||
use_scale_shift_norm=use_scale_shift_norm,
|
use_scale_shift_norm=use_scale_shift_norm,
|
||||||
dtype=self.dtype,
|
dtype=self.dtype,
|
||||||
device=device,
|
device=device,
|
||||||
|
operations=operations,
|
||||||
)
|
)
|
||||||
]
|
]
|
||||||
ch = mult * model_channels
|
ch = mult * model_channels
|
||||||
@ -447,7 +448,7 @@ class UNetModel(nn.Module):
|
|||||||
layers.append(SpatialTransformer(
|
layers.append(SpatialTransformer(
|
||||||
ch, num_heads, dim_head, depth=transformer_depth[level], context_dim=context_dim,
|
ch, num_heads, dim_head, depth=transformer_depth[level], context_dim=context_dim,
|
||||||
disable_self_attn=disabled_sa, use_linear=use_linear_in_transformer,
|
disable_self_attn=disabled_sa, use_linear=use_linear_in_transformer,
|
||||||
use_checkpoint=use_checkpoint, dtype=self.dtype, device=device
|
use_checkpoint=use_checkpoint, dtype=self.dtype, device=device, operations=operations
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
self.input_blocks.append(TimestepEmbedSequential(*layers))
|
self.input_blocks.append(TimestepEmbedSequential(*layers))
|
||||||
@ -468,10 +469,11 @@ class UNetModel(nn.Module):
|
|||||||
down=True,
|
down=True,
|
||||||
dtype=self.dtype,
|
dtype=self.dtype,
|
||||||
device=device,
|
device=device,
|
||||||
|
operations=operations
|
||||||
)
|
)
|
||||||
if resblock_updown
|
if resblock_updown
|
||||||
else Downsample(
|
else Downsample(
|
||||||
ch, conv_resample, dims=dims, out_channels=out_ch, dtype=self.dtype, device=device
|
ch, conv_resample, dims=dims, out_channels=out_ch, dtype=self.dtype, device=device, operations=operations
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
@ -498,11 +500,12 @@ class UNetModel(nn.Module):
|
|||||||
use_scale_shift_norm=use_scale_shift_norm,
|
use_scale_shift_norm=use_scale_shift_norm,
|
||||||
dtype=self.dtype,
|
dtype=self.dtype,
|
||||||
device=device,
|
device=device,
|
||||||
|
operations=operations
|
||||||
),
|
),
|
||||||
SpatialTransformer( # always uses a self-attn
|
SpatialTransformer( # always uses a self-attn
|
||||||
ch, num_heads, dim_head, depth=transformer_depth_middle, context_dim=context_dim,
|
ch, num_heads, dim_head, depth=transformer_depth_middle, context_dim=context_dim,
|
||||||
disable_self_attn=disable_middle_self_attn, use_linear=use_linear_in_transformer,
|
disable_self_attn=disable_middle_self_attn, use_linear=use_linear_in_transformer,
|
||||||
use_checkpoint=use_checkpoint, dtype=self.dtype, device=device
|
use_checkpoint=use_checkpoint, dtype=self.dtype, device=device, operations=operations
|
||||||
),
|
),
|
||||||
ResBlock(
|
ResBlock(
|
||||||
ch,
|
ch,
|
||||||
@ -513,6 +516,7 @@ class UNetModel(nn.Module):
|
|||||||
use_scale_shift_norm=use_scale_shift_norm,
|
use_scale_shift_norm=use_scale_shift_norm,
|
||||||
dtype=self.dtype,
|
dtype=self.dtype,
|
||||||
device=device,
|
device=device,
|
||||||
|
operations=operations
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
self._feature_size += ch
|
self._feature_size += ch
|
||||||
@ -532,6 +536,7 @@ class UNetModel(nn.Module):
|
|||||||
use_scale_shift_norm=use_scale_shift_norm,
|
use_scale_shift_norm=use_scale_shift_norm,
|
||||||
dtype=self.dtype,
|
dtype=self.dtype,
|
||||||
device=device,
|
device=device,
|
||||||
|
operations=operations
|
||||||
)
|
)
|
||||||
]
|
]
|
||||||
ch = model_channels * mult
|
ch = model_channels * mult
|
||||||
@ -554,7 +559,7 @@ class UNetModel(nn.Module):
|
|||||||
SpatialTransformer(
|
SpatialTransformer(
|
||||||
ch, num_heads, dim_head, depth=transformer_depth[level], context_dim=context_dim,
|
ch, num_heads, dim_head, depth=transformer_depth[level], context_dim=context_dim,
|
||||||
disable_self_attn=disabled_sa, use_linear=use_linear_in_transformer,
|
disable_self_attn=disabled_sa, use_linear=use_linear_in_transformer,
|
||||||
use_checkpoint=use_checkpoint, dtype=self.dtype, device=device
|
use_checkpoint=use_checkpoint, dtype=self.dtype, device=device, operations=operations
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
if level and i == self.num_res_blocks[level]:
|
if level and i == self.num_res_blocks[level]:
|
||||||
@ -571,9 +576,10 @@ class UNetModel(nn.Module):
|
|||||||
up=True,
|
up=True,
|
||||||
dtype=self.dtype,
|
dtype=self.dtype,
|
||||||
device=device,
|
device=device,
|
||||||
|
operations=operations
|
||||||
)
|
)
|
||||||
if resblock_updown
|
if resblock_updown
|
||||||
else Upsample(ch, conv_resample, dims=dims, out_channels=out_ch, dtype=self.dtype, device=device)
|
else Upsample(ch, conv_resample, dims=dims, out_channels=out_ch, dtype=self.dtype, device=device, operations=operations)
|
||||||
)
|
)
|
||||||
ds //= 2
|
ds //= 2
|
||||||
self.output_blocks.append(TimestepEmbedSequential(*layers))
|
self.output_blocks.append(TimestepEmbedSequential(*layers))
|
||||||
@ -582,12 +588,12 @@ class UNetModel(nn.Module):
|
|||||||
self.out = nn.Sequential(
|
self.out = nn.Sequential(
|
||||||
nn.GroupNorm(32, ch, dtype=self.dtype, device=device),
|
nn.GroupNorm(32, ch, dtype=self.dtype, device=device),
|
||||||
nn.SiLU(),
|
nn.SiLU(),
|
||||||
zero_module(conv_nd(dims, model_channels, out_channels, 3, padding=1, dtype=self.dtype, device=device)),
|
zero_module(operations.conv_nd(dims, model_channels, out_channels, 3, padding=1, dtype=self.dtype, device=device)),
|
||||||
)
|
)
|
||||||
if self.predict_codebook_ids:
|
if self.predict_codebook_ids:
|
||||||
self.id_predictor = nn.Sequential(
|
self.id_predictor = nn.Sequential(
|
||||||
nn.GroupNorm(32, ch, dtype=self.dtype, device=device),
|
nn.GroupNorm(32, ch, dtype=self.dtype, device=device),
|
||||||
conv_nd(dims, model_channels, n_embed, 1, dtype=self.dtype, device=device),
|
operations.conv_nd(dims, model_channels, n_embed, 1, dtype=self.dtype, device=device),
|
||||||
#nn.LogSoftmax(dim=1) # change to cross_entropy and produce non-normalized logits
|
#nn.LogSoftmax(dim=1) # change to cross_entropy and produce non-normalized logits
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -626,7 +632,9 @@ class UNetModel(nn.Module):
|
|||||||
transformer_options["block"] = ("middle", 0)
|
transformer_options["block"] = ("middle", 0)
|
||||||
h = forward_timestep_embed(self.middle_block, h, emb, context, transformer_options)
|
h = forward_timestep_embed(self.middle_block, h, emb, context, transformer_options)
|
||||||
if control is not None and 'middle' in control and len(control['middle']) > 0:
|
if control is not None and 'middle' in control and len(control['middle']) > 0:
|
||||||
h += control['middle'].pop()
|
ctrl = control['middle'].pop()
|
||||||
|
if ctrl is not None:
|
||||||
|
h += ctrl
|
||||||
|
|
||||||
for id, module in enumerate(self.output_blocks):
|
for id, module in enumerate(self.output_blocks):
|
||||||
transformer_options["block"] = ("output", id)
|
transformer_options["block"] = ("output", id)
|
||||||
|
|||||||
199
comfy/lora.py
Normal file
199
comfy/lora.py
Normal file
@ -0,0 +1,199 @@
|
|||||||
|
import comfy.utils
|
||||||
|
|
||||||
|
LORA_CLIP_MAP = {
|
||||||
|
"mlp.fc1": "mlp_fc1",
|
||||||
|
"mlp.fc2": "mlp_fc2",
|
||||||
|
"self_attn.k_proj": "self_attn_k_proj",
|
||||||
|
"self_attn.q_proj": "self_attn_q_proj",
|
||||||
|
"self_attn.v_proj": "self_attn_v_proj",
|
||||||
|
"self_attn.out_proj": "self_attn_out_proj",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def load_lora(lora, to_load):
|
||||||
|
patch_dict = {}
|
||||||
|
loaded_keys = set()
|
||||||
|
for x in to_load:
|
||||||
|
alpha_name = "{}.alpha".format(x)
|
||||||
|
alpha = None
|
||||||
|
if alpha_name in lora.keys():
|
||||||
|
alpha = lora[alpha_name].item()
|
||||||
|
loaded_keys.add(alpha_name)
|
||||||
|
|
||||||
|
regular_lora = "{}.lora_up.weight".format(x)
|
||||||
|
diffusers_lora = "{}_lora.up.weight".format(x)
|
||||||
|
transformers_lora = "{}.lora_linear_layer.up.weight".format(x)
|
||||||
|
A_name = None
|
||||||
|
|
||||||
|
if regular_lora in lora.keys():
|
||||||
|
A_name = regular_lora
|
||||||
|
B_name = "{}.lora_down.weight".format(x)
|
||||||
|
mid_name = "{}.lora_mid.weight".format(x)
|
||||||
|
elif diffusers_lora in lora.keys():
|
||||||
|
A_name = diffusers_lora
|
||||||
|
B_name = "{}_lora.down.weight".format(x)
|
||||||
|
mid_name = None
|
||||||
|
elif transformers_lora in lora.keys():
|
||||||
|
A_name = transformers_lora
|
||||||
|
B_name ="{}.lora_linear_layer.down.weight".format(x)
|
||||||
|
mid_name = None
|
||||||
|
|
||||||
|
if A_name is not None:
|
||||||
|
mid = None
|
||||||
|
if mid_name is not None and mid_name in lora.keys():
|
||||||
|
mid = lora[mid_name]
|
||||||
|
loaded_keys.add(mid_name)
|
||||||
|
patch_dict[to_load[x]] = (lora[A_name], lora[B_name], alpha, mid)
|
||||||
|
loaded_keys.add(A_name)
|
||||||
|
loaded_keys.add(B_name)
|
||||||
|
|
||||||
|
|
||||||
|
######## loha
|
||||||
|
hada_w1_a_name = "{}.hada_w1_a".format(x)
|
||||||
|
hada_w1_b_name = "{}.hada_w1_b".format(x)
|
||||||
|
hada_w2_a_name = "{}.hada_w2_a".format(x)
|
||||||
|
hada_w2_b_name = "{}.hada_w2_b".format(x)
|
||||||
|
hada_t1_name = "{}.hada_t1".format(x)
|
||||||
|
hada_t2_name = "{}.hada_t2".format(x)
|
||||||
|
if hada_w1_a_name in lora.keys():
|
||||||
|
hada_t1 = None
|
||||||
|
hada_t2 = None
|
||||||
|
if hada_t1_name in lora.keys():
|
||||||
|
hada_t1 = lora[hada_t1_name]
|
||||||
|
hada_t2 = lora[hada_t2_name]
|
||||||
|
loaded_keys.add(hada_t1_name)
|
||||||
|
loaded_keys.add(hada_t2_name)
|
||||||
|
|
||||||
|
patch_dict[to_load[x]] = (lora[hada_w1_a_name], lora[hada_w1_b_name], alpha, lora[hada_w2_a_name], lora[hada_w2_b_name], hada_t1, hada_t2)
|
||||||
|
loaded_keys.add(hada_w1_a_name)
|
||||||
|
loaded_keys.add(hada_w1_b_name)
|
||||||
|
loaded_keys.add(hada_w2_a_name)
|
||||||
|
loaded_keys.add(hada_w2_b_name)
|
||||||
|
|
||||||
|
|
||||||
|
######## lokr
|
||||||
|
lokr_w1_name = "{}.lokr_w1".format(x)
|
||||||
|
lokr_w2_name = "{}.lokr_w2".format(x)
|
||||||
|
lokr_w1_a_name = "{}.lokr_w1_a".format(x)
|
||||||
|
lokr_w1_b_name = "{}.lokr_w1_b".format(x)
|
||||||
|
lokr_t2_name = "{}.lokr_t2".format(x)
|
||||||
|
lokr_w2_a_name = "{}.lokr_w2_a".format(x)
|
||||||
|
lokr_w2_b_name = "{}.lokr_w2_b".format(x)
|
||||||
|
|
||||||
|
lokr_w1 = None
|
||||||
|
if lokr_w1_name in lora.keys():
|
||||||
|
lokr_w1 = lora[lokr_w1_name]
|
||||||
|
loaded_keys.add(lokr_w1_name)
|
||||||
|
|
||||||
|
lokr_w2 = None
|
||||||
|
if lokr_w2_name in lora.keys():
|
||||||
|
lokr_w2 = lora[lokr_w2_name]
|
||||||
|
loaded_keys.add(lokr_w2_name)
|
||||||
|
|
||||||
|
lokr_w1_a = None
|
||||||
|
if lokr_w1_a_name in lora.keys():
|
||||||
|
lokr_w1_a = lora[lokr_w1_a_name]
|
||||||
|
loaded_keys.add(lokr_w1_a_name)
|
||||||
|
|
||||||
|
lokr_w1_b = None
|
||||||
|
if lokr_w1_b_name in lora.keys():
|
||||||
|
lokr_w1_b = lora[lokr_w1_b_name]
|
||||||
|
loaded_keys.add(lokr_w1_b_name)
|
||||||
|
|
||||||
|
lokr_w2_a = None
|
||||||
|
if lokr_w2_a_name in lora.keys():
|
||||||
|
lokr_w2_a = lora[lokr_w2_a_name]
|
||||||
|
loaded_keys.add(lokr_w2_a_name)
|
||||||
|
|
||||||
|
lokr_w2_b = None
|
||||||
|
if lokr_w2_b_name in lora.keys():
|
||||||
|
lokr_w2_b = lora[lokr_w2_b_name]
|
||||||
|
loaded_keys.add(lokr_w2_b_name)
|
||||||
|
|
||||||
|
lokr_t2 = None
|
||||||
|
if lokr_t2_name in lora.keys():
|
||||||
|
lokr_t2 = lora[lokr_t2_name]
|
||||||
|
loaded_keys.add(lokr_t2_name)
|
||||||
|
|
||||||
|
if (lokr_w1 is not None) or (lokr_w2 is not None) or (lokr_w1_a is not None) or (lokr_w2_a is not None):
|
||||||
|
patch_dict[to_load[x]] = (lokr_w1, lokr_w2, alpha, lokr_w1_a, lokr_w1_b, lokr_w2_a, lokr_w2_b, lokr_t2)
|
||||||
|
|
||||||
|
|
||||||
|
w_norm_name = "{}.w_norm".format(x)
|
||||||
|
b_norm_name = "{}.b_norm".format(x)
|
||||||
|
w_norm = lora.get(w_norm_name, None)
|
||||||
|
b_norm = lora.get(b_norm_name, None)
|
||||||
|
|
||||||
|
if w_norm is not None:
|
||||||
|
loaded_keys.add(w_norm_name)
|
||||||
|
patch_dict[to_load[x]] = (w_norm,)
|
||||||
|
if b_norm is not None:
|
||||||
|
loaded_keys.add(b_norm_name)
|
||||||
|
patch_dict["{}.bias".format(to_load[x][:-len(".weight")])] = (b_norm,)
|
||||||
|
|
||||||
|
for x in lora.keys():
|
||||||
|
if x not in loaded_keys:
|
||||||
|
print("lora key not loaded", x)
|
||||||
|
return patch_dict
|
||||||
|
|
||||||
|
def model_lora_keys_clip(model, key_map={}):
|
||||||
|
sdk = model.state_dict().keys()
|
||||||
|
|
||||||
|
text_model_lora_key = "lora_te_text_model_encoder_layers_{}_{}"
|
||||||
|
clip_l_present = False
|
||||||
|
for b in range(32):
|
||||||
|
for c in LORA_CLIP_MAP:
|
||||||
|
k = "transformer.text_model.encoder.layers.{}.{}.weight".format(b, c)
|
||||||
|
if k in sdk:
|
||||||
|
lora_key = text_model_lora_key.format(b, LORA_CLIP_MAP[c])
|
||||||
|
key_map[lora_key] = k
|
||||||
|
lora_key = "lora_te1_text_model_encoder_layers_{}_{}".format(b, LORA_CLIP_MAP[c])
|
||||||
|
key_map[lora_key] = k
|
||||||
|
lora_key = "text_encoder.text_model.encoder.layers.{}.{}".format(b, c) #diffusers lora
|
||||||
|
key_map[lora_key] = k
|
||||||
|
|
||||||
|
k = "clip_l.transformer.text_model.encoder.layers.{}.{}.weight".format(b, c)
|
||||||
|
if k in sdk:
|
||||||
|
lora_key = "lora_te1_text_model_encoder_layers_{}_{}".format(b, LORA_CLIP_MAP[c]) #SDXL base
|
||||||
|
key_map[lora_key] = k
|
||||||
|
clip_l_present = True
|
||||||
|
lora_key = "text_encoder.text_model.encoder.layers.{}.{}".format(b, c) #diffusers lora
|
||||||
|
key_map[lora_key] = k
|
||||||
|
|
||||||
|
k = "clip_g.transformer.text_model.encoder.layers.{}.{}.weight".format(b, c)
|
||||||
|
if k in sdk:
|
||||||
|
if clip_l_present:
|
||||||
|
lora_key = "lora_te2_text_model_encoder_layers_{}_{}".format(b, LORA_CLIP_MAP[c]) #SDXL base
|
||||||
|
key_map[lora_key] = k
|
||||||
|
lora_key = "text_encoder_2.text_model.encoder.layers.{}.{}".format(b, c) #diffusers lora
|
||||||
|
key_map[lora_key] = k
|
||||||
|
else:
|
||||||
|
lora_key = "lora_te_text_model_encoder_layers_{}_{}".format(b, LORA_CLIP_MAP[c]) #TODO: test if this is correct for SDXL-Refiner
|
||||||
|
key_map[lora_key] = k
|
||||||
|
lora_key = "text_encoder.text_model.encoder.layers.{}.{}".format(b, c) #diffusers lora
|
||||||
|
key_map[lora_key] = k
|
||||||
|
|
||||||
|
return key_map
|
||||||
|
|
||||||
|
def model_lora_keys_unet(model, key_map={}):
|
||||||
|
sdk = model.state_dict().keys()
|
||||||
|
|
||||||
|
for k in sdk:
|
||||||
|
if k.startswith("diffusion_model.") and k.endswith(".weight"):
|
||||||
|
key_lora = k[len("diffusion_model."):-len(".weight")].replace(".", "_")
|
||||||
|
key_map["lora_unet_{}".format(key_lora)] = k
|
||||||
|
|
||||||
|
diffusers_keys = comfy.utils.unet_to_diffusers(model.model_config.unet_config)
|
||||||
|
for k in diffusers_keys:
|
||||||
|
if k.endswith(".weight"):
|
||||||
|
unet_key = "diffusion_model.{}".format(diffusers_keys[k])
|
||||||
|
key_lora = k[:-len(".weight")].replace(".", "_")
|
||||||
|
key_map["lora_unet_{}".format(key_lora)] = unet_key
|
||||||
|
|
||||||
|
diffusers_lora_prefix = ["", "unet."]
|
||||||
|
for p in diffusers_lora_prefix:
|
||||||
|
diffusers_lora_key = "{}{}".format(p, k[:-len(".weight")].replace(".to_", ".processor.to_"))
|
||||||
|
if diffusers_lora_key.endswith(".to_out.0"):
|
||||||
|
diffusers_lora_key = diffusers_lora_key[:-2]
|
||||||
|
key_map[diffusers_lora_key] = unet_key
|
||||||
|
return key_map
|
||||||
@ -3,6 +3,7 @@ from comfy.ldm.modules.diffusionmodules.openaimodel import UNetModel
|
|||||||
from comfy.ldm.modules.encoders.noise_aug_modules import CLIPEmbeddingNoiseAugmentation
|
from comfy.ldm.modules.encoders.noise_aug_modules import CLIPEmbeddingNoiseAugmentation
|
||||||
from comfy.ldm.modules.diffusionmodules.util import make_beta_schedule
|
from comfy.ldm.modules.diffusionmodules.util import make_beta_schedule
|
||||||
from comfy.ldm.modules.diffusionmodules.openaimodel import Timestep
|
from comfy.ldm.modules.diffusionmodules.openaimodel import Timestep
|
||||||
|
import comfy.model_management
|
||||||
import numpy as np
|
import numpy as np
|
||||||
from enum import Enum
|
from enum import Enum
|
||||||
from . import utils
|
from . import utils
|
||||||
@ -18,8 +19,9 @@ class BaseModel(torch.nn.Module):
|
|||||||
unet_config = model_config.unet_config
|
unet_config = model_config.unet_config
|
||||||
self.latent_format = model_config.latent_format
|
self.latent_format = model_config.latent_format
|
||||||
self.model_config = model_config
|
self.model_config = model_config
|
||||||
self.register_schedule(given_betas=None, beta_schedule="linear", timesteps=1000, linear_start=0.00085, linear_end=0.012, cosine_s=8e-3)
|
self.register_schedule(given_betas=None, beta_schedule=model_config.beta_schedule, timesteps=1000, linear_start=0.00085, linear_end=0.012, cosine_s=8e-3)
|
||||||
self.diffusion_model = UNetModel(**unet_config, device=device)
|
if not unet_config.get("disable_unet_model_creation", False):
|
||||||
|
self.diffusion_model = UNetModel(**unet_config, device=device)
|
||||||
self.model_type = model_type
|
self.model_type = model_type
|
||||||
self.adm_channels = unet_config.get("adm_in_channels", None)
|
self.adm_channels = unet_config.get("adm_in_channels", None)
|
||||||
if self.adm_channels is None:
|
if self.adm_channels is None:
|
||||||
@ -93,7 +95,11 @@ class BaseModel(torch.nn.Module):
|
|||||||
|
|
||||||
def state_dict_for_saving(self, clip_state_dict, vae_state_dict):
|
def state_dict_for_saving(self, clip_state_dict, vae_state_dict):
|
||||||
clip_state_dict = self.model_config.process_clip_state_dict_for_saving(clip_state_dict)
|
clip_state_dict = self.model_config.process_clip_state_dict_for_saving(clip_state_dict)
|
||||||
unet_state_dict = self.diffusion_model.state_dict()
|
unet_sd = self.diffusion_model.state_dict()
|
||||||
|
unet_state_dict = {}
|
||||||
|
for k in unet_sd:
|
||||||
|
unet_state_dict[k] = comfy.model_management.resolve_lowvram_weight(unet_sd[k], self.diffusion_model, k)
|
||||||
|
|
||||||
unet_state_dict = self.model_config.process_unet_state_dict_for_saving(unet_state_dict)
|
unet_state_dict = self.model_config.process_unet_state_dict_for_saving(unet_state_dict)
|
||||||
vae_state_dict = self.model_config.process_vae_state_dict_for_saving(vae_state_dict)
|
vae_state_dict = self.model_config.process_vae_state_dict_for_saving(vae_state_dict)
|
||||||
if self.get_dtype() == torch.float16:
|
if self.get_dtype() == torch.float16:
|
||||||
@ -105,6 +111,29 @@ class BaseModel(torch.nn.Module):
|
|||||||
|
|
||||||
return {**unet_state_dict, **vae_state_dict, **clip_state_dict}
|
return {**unet_state_dict, **vae_state_dict, **clip_state_dict}
|
||||||
|
|
||||||
|
def unclip_adm(unclip_conditioning, device, noise_augmentor, noise_augment_merge=0.0):
|
||||||
|
adm_inputs = []
|
||||||
|
weights = []
|
||||||
|
noise_aug = []
|
||||||
|
for unclip_cond in unclip_conditioning:
|
||||||
|
for adm_cond in unclip_cond["clip_vision_output"].image_embeds:
|
||||||
|
weight = unclip_cond["strength"]
|
||||||
|
noise_augment = unclip_cond["noise_augmentation"]
|
||||||
|
noise_level = round((noise_augmentor.max_noise_level - 1) * noise_augment)
|
||||||
|
c_adm, noise_level_emb = noise_augmentor(adm_cond.to(device), noise_level=torch.tensor([noise_level], device=device))
|
||||||
|
adm_out = torch.cat((c_adm, noise_level_emb), 1) * weight
|
||||||
|
weights.append(weight)
|
||||||
|
noise_aug.append(noise_augment)
|
||||||
|
adm_inputs.append(adm_out)
|
||||||
|
|
||||||
|
if len(noise_aug) > 1:
|
||||||
|
adm_out = torch.stack(adm_inputs).sum(0)
|
||||||
|
noise_augment = noise_augment_merge
|
||||||
|
noise_level = round((noise_augmentor.max_noise_level - 1) * noise_augment)
|
||||||
|
c_adm, noise_level_emb = noise_augmentor(adm_out[:, :noise_augmentor.time_embed.dim], noise_level=torch.tensor([noise_level], device=device))
|
||||||
|
adm_out = torch.cat((c_adm, noise_level_emb), 1)
|
||||||
|
|
||||||
|
return adm_out
|
||||||
|
|
||||||
class SD21UNCLIP(BaseModel):
|
class SD21UNCLIP(BaseModel):
|
||||||
def __init__(self, model_config, noise_aug_config, model_type=ModelType.V_PREDICTION, device=None):
|
def __init__(self, model_config, noise_aug_config, model_type=ModelType.V_PREDICTION, device=None):
|
||||||
@ -114,46 +143,31 @@ class SD21UNCLIP(BaseModel):
|
|||||||
def encode_adm(self, **kwargs):
|
def encode_adm(self, **kwargs):
|
||||||
unclip_conditioning = kwargs.get("unclip_conditioning", None)
|
unclip_conditioning = kwargs.get("unclip_conditioning", None)
|
||||||
device = kwargs["device"]
|
device = kwargs["device"]
|
||||||
|
if unclip_conditioning is None:
|
||||||
if unclip_conditioning is not None:
|
return torch.zeros((1, self.adm_channels))
|
||||||
adm_inputs = []
|
|
||||||
weights = []
|
|
||||||
noise_aug = []
|
|
||||||
for unclip_cond in unclip_conditioning:
|
|
||||||
adm_cond = unclip_cond["clip_vision_output"].image_embeds
|
|
||||||
weight = unclip_cond["strength"]
|
|
||||||
noise_augment = unclip_cond["noise_augmentation"]
|
|
||||||
noise_level = round((self.noise_augmentor.max_noise_level - 1) * noise_augment)
|
|
||||||
c_adm, noise_level_emb = self.noise_augmentor(adm_cond.to(device), noise_level=torch.tensor([noise_level], device=device))
|
|
||||||
adm_out = torch.cat((c_adm, noise_level_emb), 1) * weight
|
|
||||||
weights.append(weight)
|
|
||||||
noise_aug.append(noise_augment)
|
|
||||||
adm_inputs.append(adm_out)
|
|
||||||
|
|
||||||
if len(noise_aug) > 1:
|
|
||||||
adm_out = torch.stack(adm_inputs).sum(0)
|
|
||||||
#TODO: add a way to control this
|
|
||||||
noise_augment = 0.05
|
|
||||||
noise_level = round((self.noise_augmentor.max_noise_level - 1) * noise_augment)
|
|
||||||
c_adm, noise_level_emb = self.noise_augmentor(adm_out[:, :self.noise_augmentor.time_embed.dim], noise_level=torch.tensor([noise_level], device=device))
|
|
||||||
adm_out = torch.cat((c_adm, noise_level_emb), 1)
|
|
||||||
else:
|
else:
|
||||||
adm_out = torch.zeros((1, self.adm_channels))
|
return unclip_adm(unclip_conditioning, device, self.noise_augmentor, kwargs.get("unclip_noise_augment_merge", 0.05))
|
||||||
|
|
||||||
return adm_out
|
|
||||||
|
|
||||||
class SDInpaint(BaseModel):
|
class SDInpaint(BaseModel):
|
||||||
def __init__(self, model_config, model_type=ModelType.EPS, device=None):
|
def __init__(self, model_config, model_type=ModelType.EPS, device=None):
|
||||||
super().__init__(model_config, model_type, device=device)
|
super().__init__(model_config, model_type, device=device)
|
||||||
self.concat_keys = ("mask", "masked_image")
|
self.concat_keys = ("mask", "masked_image")
|
||||||
|
|
||||||
|
def sdxl_pooled(args, noise_augmentor):
|
||||||
|
if "unclip_conditioning" in args:
|
||||||
|
return unclip_adm(args.get("unclip_conditioning", None), args["device"], noise_augmentor)[:,:1280]
|
||||||
|
else:
|
||||||
|
return args["pooled_output"]
|
||||||
|
|
||||||
class SDXLRefiner(BaseModel):
|
class SDXLRefiner(BaseModel):
|
||||||
def __init__(self, model_config, model_type=ModelType.EPS, device=None):
|
def __init__(self, model_config, model_type=ModelType.EPS, device=None):
|
||||||
super().__init__(model_config, model_type, device=device)
|
super().__init__(model_config, model_type, device=device)
|
||||||
self.embedder = Timestep(256)
|
self.embedder = Timestep(256)
|
||||||
|
self.noise_augmentor = CLIPEmbeddingNoiseAugmentation(**{"noise_schedule_config": {"timesteps": 1000, "beta_schedule": "squaredcos_cap_v2"}, "timestep_dim": 1280})
|
||||||
|
|
||||||
def encode_adm(self, **kwargs):
|
def encode_adm(self, **kwargs):
|
||||||
clip_pooled = kwargs["pooled_output"]
|
clip_pooled = sdxl_pooled(kwargs, self.noise_augmentor)
|
||||||
width = kwargs.get("width", 768)
|
width = kwargs.get("width", 768)
|
||||||
height = kwargs.get("height", 768)
|
height = kwargs.get("height", 768)
|
||||||
crop_w = kwargs.get("crop_w", 0)
|
crop_w = kwargs.get("crop_w", 0)
|
||||||
@ -177,9 +191,10 @@ class SDXL(BaseModel):
|
|||||||
def __init__(self, model_config, model_type=ModelType.EPS, device=None):
|
def __init__(self, model_config, model_type=ModelType.EPS, device=None):
|
||||||
super().__init__(model_config, model_type, device=device)
|
super().__init__(model_config, model_type, device=device)
|
||||||
self.embedder = Timestep(256)
|
self.embedder = Timestep(256)
|
||||||
|
self.noise_augmentor = CLIPEmbeddingNoiseAugmentation(**{"noise_schedule_config": {"timesteps": 1000, "beta_schedule": "squaredcos_cap_v2"}, "timestep_dim": 1280})
|
||||||
|
|
||||||
def encode_adm(self, **kwargs):
|
def encode_adm(self, **kwargs):
|
||||||
clip_pooled = kwargs["pooled_output"]
|
clip_pooled = sdxl_pooled(kwargs, self.noise_augmentor)
|
||||||
width = kwargs.get("width", 768)
|
width = kwargs.get("width", 768)
|
||||||
height = kwargs.get("height", 768)
|
height = kwargs.get("height", 768)
|
||||||
crop_w = kwargs.get("crop_w", 0)
|
crop_w = kwargs.get("crop_w", 0)
|
||||||
|
|||||||
@ -113,6 +113,7 @@ def model_config_from_unet_config(unet_config):
|
|||||||
if model_config.matches(unet_config):
|
if model_config.matches(unet_config):
|
||||||
return model_config(unet_config)
|
return model_config(unet_config)
|
||||||
|
|
||||||
|
print("no match", unet_config)
|
||||||
return None
|
return None
|
||||||
|
|
||||||
def model_config_from_unet(state_dict, unet_key_prefix, use_fp16):
|
def model_config_from_unet(state_dict, unet_key_prefix, use_fp16):
|
||||||
@ -120,9 +121,20 @@ def model_config_from_unet(state_dict, unet_key_prefix, use_fp16):
|
|||||||
return model_config_from_unet_config(unet_config)
|
return model_config_from_unet_config(unet_config)
|
||||||
|
|
||||||
|
|
||||||
def model_config_from_diffusers_unet(state_dict, use_fp16):
|
def unet_config_from_diffusers_unet(state_dict, use_fp16):
|
||||||
match = {}
|
match = {}
|
||||||
match["context_dim"] = state_dict["down_blocks.1.attentions.1.transformer_blocks.0.attn2.to_k.weight"].shape[1]
|
attention_resolutions = []
|
||||||
|
|
||||||
|
attn_res = 1
|
||||||
|
for i in range(5):
|
||||||
|
k = "down_blocks.{}.attentions.1.transformer_blocks.0.attn2.to_k.weight".format(i)
|
||||||
|
if k in state_dict:
|
||||||
|
match["context_dim"] = state_dict[k].shape[1]
|
||||||
|
attention_resolutions.append(attn_res)
|
||||||
|
attn_res *= 2
|
||||||
|
|
||||||
|
match["attention_resolutions"] = attention_resolutions
|
||||||
|
|
||||||
match["model_channels"] = state_dict["conv_in.weight"].shape[0]
|
match["model_channels"] = state_dict["conv_in.weight"].shape[0]
|
||||||
match["in_channels"] = state_dict["conv_in.weight"].shape[1]
|
match["in_channels"] = state_dict["conv_in.weight"].shape[1]
|
||||||
match["adm_in_channels"] = None
|
match["adm_in_channels"] = None
|
||||||
@ -134,22 +146,22 @@ def model_config_from_diffusers_unet(state_dict, use_fp16):
|
|||||||
SDXL = {'use_checkpoint': False, 'image_size': 32, 'out_channels': 4, 'use_spatial_transformer': True, 'legacy': False,
|
SDXL = {'use_checkpoint': False, 'image_size': 32, 'out_channels': 4, 'use_spatial_transformer': True, 'legacy': False,
|
||||||
'num_classes': 'sequential', 'adm_in_channels': 2816, 'use_fp16': use_fp16, 'in_channels': 4, 'model_channels': 320,
|
'num_classes': 'sequential', 'adm_in_channels': 2816, 'use_fp16': use_fp16, 'in_channels': 4, 'model_channels': 320,
|
||||||
'num_res_blocks': 2, 'attention_resolutions': [2, 4], 'transformer_depth': [0, 2, 10], 'channel_mult': [1, 2, 4],
|
'num_res_blocks': 2, 'attention_resolutions': [2, 4], 'transformer_depth': [0, 2, 10], 'channel_mult': [1, 2, 4],
|
||||||
'transformer_depth_middle': 10, 'use_linear_in_transformer': True, 'context_dim': 2048}
|
'transformer_depth_middle': 10, 'use_linear_in_transformer': True, 'context_dim': 2048, "num_head_channels": 64}
|
||||||
|
|
||||||
SDXL_refiner = {'use_checkpoint': False, 'image_size': 32, 'out_channels': 4, 'use_spatial_transformer': True, 'legacy': False,
|
SDXL_refiner = {'use_checkpoint': False, 'image_size': 32, 'out_channels': 4, 'use_spatial_transformer': True, 'legacy': False,
|
||||||
'num_classes': 'sequential', 'adm_in_channels': 2560, 'use_fp16': use_fp16, 'in_channels': 4, 'model_channels': 384,
|
'num_classes': 'sequential', 'adm_in_channels': 2560, 'use_fp16': use_fp16, 'in_channels': 4, 'model_channels': 384,
|
||||||
'num_res_blocks': 2, 'attention_resolutions': [2, 4], 'transformer_depth': [0, 4, 4, 0], 'channel_mult': [1, 2, 4, 4],
|
'num_res_blocks': 2, 'attention_resolutions': [2, 4], 'transformer_depth': [0, 4, 4, 0], 'channel_mult': [1, 2, 4, 4],
|
||||||
'transformer_depth_middle': 4, 'use_linear_in_transformer': True, 'context_dim': 1280}
|
'transformer_depth_middle': 4, 'use_linear_in_transformer': True, 'context_dim': 1280, "num_head_channels": 64}
|
||||||
|
|
||||||
SD21 = {'use_checkpoint': False, 'image_size': 32, 'out_channels': 4, 'use_spatial_transformer': True, 'legacy': False,
|
SD21 = {'use_checkpoint': False, 'image_size': 32, 'out_channels': 4, 'use_spatial_transformer': True, 'legacy': False,
|
||||||
'adm_in_channels': None, 'use_fp16': use_fp16, 'in_channels': 4, 'model_channels': 320, 'num_res_blocks': 2,
|
'adm_in_channels': None, 'use_fp16': use_fp16, 'in_channels': 4, 'model_channels': 320, 'num_res_blocks': 2,
|
||||||
'attention_resolutions': [1, 2, 4], 'transformer_depth': [1, 1, 1, 0], 'channel_mult': [1, 2, 4, 4],
|
'attention_resolutions': [1, 2, 4], 'transformer_depth': [1, 1, 1, 0], 'channel_mult': [1, 2, 4, 4],
|
||||||
'transformer_depth_middle': 1, 'use_linear_in_transformer': True, 'context_dim': 1024}
|
'transformer_depth_middle': 1, 'use_linear_in_transformer': True, 'context_dim': 1024, "num_head_channels": 64}
|
||||||
|
|
||||||
SD21_uncliph = {'use_checkpoint': False, 'image_size': 32, 'out_channels': 4, 'use_spatial_transformer': True, 'legacy': False,
|
SD21_uncliph = {'use_checkpoint': False, 'image_size': 32, 'out_channels': 4, 'use_spatial_transformer': True, 'legacy': False,
|
||||||
'num_classes': 'sequential', 'adm_in_channels': 2048, 'use_fp16': use_fp16, 'in_channels': 4, 'model_channels': 320,
|
'num_classes': 'sequential', 'adm_in_channels': 2048, 'use_fp16': use_fp16, 'in_channels': 4, 'model_channels': 320,
|
||||||
'num_res_blocks': 2, 'attention_resolutions': [1, 2, 4], 'transformer_depth': [1, 1, 1, 0], 'channel_mult': [1, 2, 4, 4],
|
'num_res_blocks': 2, 'attention_resolutions': [1, 2, 4], 'transformer_depth': [1, 1, 1, 0], 'channel_mult': [1, 2, 4, 4],
|
||||||
'transformer_depth_middle': 1, 'use_linear_in_transformer': True, 'context_dim': 1024}
|
'transformer_depth_middle': 1, 'use_linear_in_transformer': True, 'context_dim': 1024, "num_head_channels": 64}
|
||||||
|
|
||||||
SD21_unclipl = {'use_checkpoint': False, 'image_size': 32, 'out_channels': 4, 'use_spatial_transformer': True, 'legacy': False,
|
SD21_unclipl = {'use_checkpoint': False, 'image_size': 32, 'out_channels': 4, 'use_spatial_transformer': True, 'legacy': False,
|
||||||
'num_classes': 'sequential', 'adm_in_channels': 1536, 'use_fp16': use_fp16, 'in_channels': 4, 'model_channels': 320,
|
'num_classes': 'sequential', 'adm_in_channels': 1536, 'use_fp16': use_fp16, 'in_channels': 4, 'model_channels': 320,
|
||||||
@ -159,9 +171,20 @@ def model_config_from_diffusers_unet(state_dict, use_fp16):
|
|||||||
SD15 = {'use_checkpoint': False, 'image_size': 32, 'out_channels': 4, 'use_spatial_transformer': True, 'legacy': False,
|
SD15 = {'use_checkpoint': False, 'image_size': 32, 'out_channels': 4, 'use_spatial_transformer': True, 'legacy': False,
|
||||||
'adm_in_channels': None, 'use_fp16': use_fp16, 'in_channels': 4, 'model_channels': 320, 'num_res_blocks': 2,
|
'adm_in_channels': None, 'use_fp16': use_fp16, 'in_channels': 4, 'model_channels': 320, 'num_res_blocks': 2,
|
||||||
'attention_resolutions': [1, 2, 4], 'transformer_depth': [1, 1, 1, 0], 'channel_mult': [1, 2, 4, 4],
|
'attention_resolutions': [1, 2, 4], 'transformer_depth': [1, 1, 1, 0], 'channel_mult': [1, 2, 4, 4],
|
||||||
'transformer_depth_middle': 1, 'use_linear_in_transformer': False, 'context_dim': 768}
|
'transformer_depth_middle': 1, 'use_linear_in_transformer': False, 'context_dim': 768, "num_heads": 8}
|
||||||
|
|
||||||
supported_models = [SDXL, SDXL_refiner, SD21, SD15, SD21_uncliph, SD21_unclipl]
|
SDXL_mid_cnet = {'use_checkpoint': False, 'image_size': 32, 'out_channels': 4, 'use_spatial_transformer': True, 'legacy': False,
|
||||||
|
'num_classes': 'sequential', 'adm_in_channels': 2816, 'use_fp16': use_fp16, 'in_channels': 4, 'model_channels': 320,
|
||||||
|
'num_res_blocks': 2, 'attention_resolutions': [4], 'transformer_depth': [0, 0, 1], 'channel_mult': [1, 2, 4],
|
||||||
|
'transformer_depth_middle': 1, 'use_linear_in_transformer': True, 'context_dim': 2048, "num_head_channels": 64}
|
||||||
|
|
||||||
|
SDXL_small_cnet = {'use_checkpoint': False, 'image_size': 32, 'out_channels': 4, 'use_spatial_transformer': True, 'legacy': False,
|
||||||
|
'num_classes': 'sequential', 'adm_in_channels': 2816, 'use_fp16': use_fp16, 'in_channels': 4, 'model_channels': 320,
|
||||||
|
'num_res_blocks': 2, 'attention_resolutions': [], 'transformer_depth': [0, 0, 0], 'channel_mult': [1, 2, 4],
|
||||||
|
'transformer_depth_middle': 0, 'use_linear_in_transformer': True, "num_head_channels": 64, 'context_dim': 1}
|
||||||
|
|
||||||
|
|
||||||
|
supported_models = [SDXL, SDXL_refiner, SD21, SD15, SD21_uncliph, SD21_unclipl, SDXL_mid_cnet, SDXL_small_cnet]
|
||||||
|
|
||||||
for unet_config in supported_models:
|
for unet_config in supported_models:
|
||||||
matches = True
|
matches = True
|
||||||
@ -170,5 +193,11 @@ def model_config_from_diffusers_unet(state_dict, use_fp16):
|
|||||||
matches = False
|
matches = False
|
||||||
break
|
break
|
||||||
if matches:
|
if matches:
|
||||||
return model_config_from_unet_config(unet_config)
|
return unet_config
|
||||||
|
return None
|
||||||
|
|
||||||
|
def model_config_from_diffusers_unet(state_dict, use_fp16):
|
||||||
|
unet_config = unet_config_from_diffusers_unet(state_dict, use_fp16)
|
||||||
|
if unet_config is not None:
|
||||||
|
return model_config_from_unet_config(unet_config)
|
||||||
return None
|
return None
|
||||||
|
|||||||
@ -1,7 +1,9 @@
|
|||||||
import psutil
|
import psutil
|
||||||
from enum import Enum
|
from enum import Enum
|
||||||
from comfy.cli_args import args
|
from comfy.cli_args import args
|
||||||
|
import comfy.utils
|
||||||
import torch
|
import torch
|
||||||
|
import sys
|
||||||
|
|
||||||
class VRAMState(Enum):
|
class VRAMState(Enum):
|
||||||
DISABLED = 0 #No vram present: no need to move models to vram
|
DISABLED = 0 #No vram present: no need to move models to vram
|
||||||
@ -87,8 +89,10 @@ def get_total_memory(dev=None, torch_total_too=False):
|
|||||||
mem_total = 1024 * 1024 * 1024 #TODO
|
mem_total = 1024 * 1024 * 1024 #TODO
|
||||||
mem_total_torch = mem_total
|
mem_total_torch = mem_total
|
||||||
elif xpu_available:
|
elif xpu_available:
|
||||||
|
stats = torch.xpu.memory_stats(dev)
|
||||||
|
mem_reserved = stats['reserved_bytes.all.current']
|
||||||
mem_total = torch.xpu.get_device_properties(dev).total_memory
|
mem_total = torch.xpu.get_device_properties(dev).total_memory
|
||||||
mem_total_torch = mem_total
|
mem_total_torch = mem_reserved
|
||||||
else:
|
else:
|
||||||
stats = torch.cuda.memory_stats(dev)
|
stats = torch.cuda.memory_stats(dev)
|
||||||
mem_reserved = stats['reserved_bytes.all.current']
|
mem_reserved = stats['reserved_bytes.all.current']
|
||||||
@ -108,9 +112,6 @@ if not args.normalvram and not args.cpu:
|
|||||||
if lowvram_available and total_vram <= 4096:
|
if lowvram_available and total_vram <= 4096:
|
||||||
print("Trying to enable lowvram mode because your GPU seems to have 4GB or less. If you don't want this use: --normalvram")
|
print("Trying to enable lowvram mode because your GPU seems to have 4GB or less. If you don't want this use: --normalvram")
|
||||||
set_vram_to = VRAMState.LOW_VRAM
|
set_vram_to = VRAMState.LOW_VRAM
|
||||||
elif total_vram > total_ram * 1.1 and total_vram > 14336:
|
|
||||||
print("Enabling highvram mode because your GPU has more vram than your computer has ram. If you don't want this use: --normalvram")
|
|
||||||
vram_state = VRAMState.HIGH_VRAM
|
|
||||||
|
|
||||||
try:
|
try:
|
||||||
OOM_EXCEPTION = torch.cuda.OutOfMemoryError
|
OOM_EXCEPTION = torch.cuda.OutOfMemoryError
|
||||||
@ -147,15 +148,27 @@ def is_nvidia():
|
|||||||
return True
|
return True
|
||||||
|
|
||||||
ENABLE_PYTORCH_ATTENTION = args.use_pytorch_cross_attention
|
ENABLE_PYTORCH_ATTENTION = args.use_pytorch_cross_attention
|
||||||
|
VAE_DTYPE = torch.float32
|
||||||
|
|
||||||
if ENABLE_PYTORCH_ATTENTION == False and XFORMERS_IS_AVAILABLE == False and args.use_split_cross_attention == False and args.use_quad_cross_attention == False:
|
|
||||||
try:
|
try:
|
||||||
if is_nvidia():
|
if is_nvidia():
|
||||||
torch_version = torch.version.__version__
|
torch_version = torch.version.__version__
|
||||||
if int(torch_version[0]) >= 2:
|
if int(torch_version[0]) >= 2:
|
||||||
|
if ENABLE_PYTORCH_ATTENTION == False and XFORMERS_IS_AVAILABLE == False and args.use_split_cross_attention == False and args.use_quad_cross_attention == False:
|
||||||
ENABLE_PYTORCH_ATTENTION = True
|
ENABLE_PYTORCH_ATTENTION = True
|
||||||
except:
|
if torch.cuda.is_bf16_supported():
|
||||||
pass
|
VAE_DTYPE = torch.bfloat16
|
||||||
|
except:
|
||||||
|
pass
|
||||||
|
|
||||||
|
if args.fp16_vae:
|
||||||
|
VAE_DTYPE = torch.float16
|
||||||
|
elif args.bf16_vae:
|
||||||
|
VAE_DTYPE = torch.bfloat16
|
||||||
|
elif args.fp32_vae:
|
||||||
|
VAE_DTYPE = torch.float32
|
||||||
|
|
||||||
|
|
||||||
if ENABLE_PYTORCH_ATTENTION:
|
if ENABLE_PYTORCH_ATTENTION:
|
||||||
torch.backends.cuda.enable_math_sdp(True)
|
torch.backends.cuda.enable_math_sdp(True)
|
||||||
@ -201,8 +214,13 @@ if cpu_state == CPUState.MPS:
|
|||||||
|
|
||||||
print(f"Set vram state to: {vram_state.name}")
|
print(f"Set vram state to: {vram_state.name}")
|
||||||
|
|
||||||
|
DISABLE_SMART_MEMORY = args.disable_smart_memory
|
||||||
|
|
||||||
|
if DISABLE_SMART_MEMORY:
|
||||||
|
print("Disabling smart memory management")
|
||||||
|
|
||||||
def get_torch_device_name(device):
|
def get_torch_device_name(device):
|
||||||
|
global xpu_available
|
||||||
if hasattr(device, 'type'):
|
if hasattr(device, 'type'):
|
||||||
if device.type == "cuda":
|
if device.type == "cuda":
|
||||||
try:
|
try:
|
||||||
@ -212,6 +230,8 @@ def get_torch_device_name(device):
|
|||||||
return "{} {} : {}".format(device, torch.cuda.get_device_name(device), allocator_backend)
|
return "{} {} : {}".format(device, torch.cuda.get_device_name(device), allocator_backend)
|
||||||
else:
|
else:
|
||||||
return "{}".format(device.type)
|
return "{}".format(device.type)
|
||||||
|
elif xpu_available:
|
||||||
|
return "{} {}".format(device, torch.xpu.get_device_name(device))
|
||||||
else:
|
else:
|
||||||
return "CUDA {}: {}".format(device, torch.cuda.get_device_name(device))
|
return "CUDA {}: {}".format(device, torch.cuda.get_device_name(device))
|
||||||
|
|
||||||
@ -220,133 +240,175 @@ try:
|
|||||||
except:
|
except:
|
||||||
print("Could not pick default device.")
|
print("Could not pick default device.")
|
||||||
|
|
||||||
|
print("VAE dtype:", VAE_DTYPE)
|
||||||
|
|
||||||
current_loaded_model = None
|
current_loaded_models = []
|
||||||
current_gpu_controlnets = []
|
|
||||||
|
|
||||||
model_accelerated = False
|
class LoadedModel:
|
||||||
|
def __init__(self, model):
|
||||||
|
self.model = model
|
||||||
|
self.model_accelerated = False
|
||||||
|
self.device = model.load_device
|
||||||
|
|
||||||
|
def model_memory(self):
|
||||||
|
return self.model.model_size()
|
||||||
|
|
||||||
def unload_model():
|
def model_memory_required(self, device):
|
||||||
global current_loaded_model
|
if device == self.model.current_device:
|
||||||
global model_accelerated
|
return 0
|
||||||
global current_gpu_controlnets
|
else:
|
||||||
global vram_state
|
return self.model_memory()
|
||||||
|
|
||||||
if current_loaded_model is not None:
|
def model_load(self, lowvram_model_memory=0):
|
||||||
if model_accelerated:
|
global xpu_available
|
||||||
accelerate.hooks.remove_hook_from_submodules(current_loaded_model.model)
|
patch_model_to = None
|
||||||
model_accelerated = False
|
if lowvram_model_memory == 0:
|
||||||
|
patch_model_to = self.device
|
||||||
|
|
||||||
current_loaded_model.unpatch_model()
|
self.model.model_patches_to(self.device)
|
||||||
current_loaded_model.model.to(current_loaded_model.offload_device)
|
self.model.model_patches_to(self.model.model_dtype())
|
||||||
current_loaded_model.model_patches_to(current_loaded_model.offload_device)
|
|
||||||
current_loaded_model = None
|
|
||||||
if vram_state != VRAMState.HIGH_VRAM:
|
|
||||||
soft_empty_cache()
|
|
||||||
|
|
||||||
if vram_state != VRAMState.HIGH_VRAM:
|
try:
|
||||||
if len(current_gpu_controlnets) > 0:
|
self.real_model = self.model.patch_model(device_to=patch_model_to) #TODO: do something with loras and offloading to CPU
|
||||||
for n in current_gpu_controlnets:
|
except Exception as e:
|
||||||
n.cpu()
|
self.model.unpatch_model(self.model.offload_device)
|
||||||
current_gpu_controlnets = []
|
self.model_unload()
|
||||||
|
raise e
|
||||||
|
|
||||||
|
if lowvram_model_memory > 0:
|
||||||
|
print("loading in lowvram mode", lowvram_model_memory/(1024 * 1024))
|
||||||
|
device_map = accelerate.infer_auto_device_map(self.real_model, max_memory={0: "{}MiB".format(lowvram_model_memory // (1024 * 1024)), "cpu": "16GiB"})
|
||||||
|
accelerate.dispatch_model(self.real_model, device_map=device_map, main_device=self.device)
|
||||||
|
self.model_accelerated = True
|
||||||
|
|
||||||
|
if xpu_available and not args.disable_ipex_optimize:
|
||||||
|
self.real_model = torch.xpu.optimize(self.real_model.eval(), inplace=True, auto_kernel_selection=True, graph_mode=True)
|
||||||
|
|
||||||
|
return self.real_model
|
||||||
|
|
||||||
|
def model_unload(self):
|
||||||
|
if self.model_accelerated:
|
||||||
|
accelerate.hooks.remove_hook_from_submodules(self.real_model)
|
||||||
|
self.model_accelerated = False
|
||||||
|
|
||||||
|
self.model.unpatch_model(self.model.offload_device)
|
||||||
|
self.model.model_patches_to(self.model.offload_device)
|
||||||
|
|
||||||
|
def __eq__(self, other):
|
||||||
|
return self.model is other.model
|
||||||
|
|
||||||
def minimum_inference_memory():
|
def minimum_inference_memory():
|
||||||
return (768 * 1024 * 1024)
|
return (1024 * 1024 * 1024)
|
||||||
|
|
||||||
|
def unload_model_clones(model):
|
||||||
|
to_unload = []
|
||||||
|
for i in range(len(current_loaded_models)):
|
||||||
|
if model.is_clone(current_loaded_models[i].model):
|
||||||
|
to_unload = [i] + to_unload
|
||||||
|
|
||||||
|
for i in to_unload:
|
||||||
|
print("unload clone", i)
|
||||||
|
current_loaded_models.pop(i).model_unload()
|
||||||
|
|
||||||
|
def free_memory(memory_required, device, keep_loaded=[]):
|
||||||
|
unloaded_model = False
|
||||||
|
for i in range(len(current_loaded_models) -1, -1, -1):
|
||||||
|
if not DISABLE_SMART_MEMORY:
|
||||||
|
if get_free_memory(device) > memory_required:
|
||||||
|
break
|
||||||
|
shift_model = current_loaded_models[i]
|
||||||
|
if shift_model.device == device:
|
||||||
|
if shift_model not in keep_loaded:
|
||||||
|
m = current_loaded_models.pop(i)
|
||||||
|
m.model_unload()
|
||||||
|
del m
|
||||||
|
unloaded_model = True
|
||||||
|
|
||||||
|
if unloaded_model:
|
||||||
|
soft_empty_cache()
|
||||||
|
|
||||||
|
|
||||||
|
def load_models_gpu(models, memory_required=0):
|
||||||
|
global vram_state
|
||||||
|
|
||||||
|
inference_memory = minimum_inference_memory()
|
||||||
|
extra_mem = max(inference_memory, memory_required)
|
||||||
|
|
||||||
|
models_to_load = []
|
||||||
|
models_already_loaded = []
|
||||||
|
for x in models:
|
||||||
|
loaded_model = LoadedModel(x)
|
||||||
|
|
||||||
|
if loaded_model in current_loaded_models:
|
||||||
|
index = current_loaded_models.index(loaded_model)
|
||||||
|
current_loaded_models.insert(0, current_loaded_models.pop(index))
|
||||||
|
models_already_loaded.append(loaded_model)
|
||||||
|
else:
|
||||||
|
models_to_load.append(loaded_model)
|
||||||
|
|
||||||
|
if len(models_to_load) == 0:
|
||||||
|
devs = set(map(lambda a: a.device, models_already_loaded))
|
||||||
|
for d in devs:
|
||||||
|
if d != torch.device("cpu"):
|
||||||
|
free_memory(extra_mem, d, models_already_loaded)
|
||||||
|
return
|
||||||
|
|
||||||
|
print("loading new")
|
||||||
|
|
||||||
|
total_memory_required = {}
|
||||||
|
for loaded_model in models_to_load:
|
||||||
|
unload_model_clones(loaded_model.model)
|
||||||
|
total_memory_required[loaded_model.device] = total_memory_required.get(loaded_model.device, 0) + loaded_model.model_memory_required(loaded_model.device)
|
||||||
|
|
||||||
|
for device in total_memory_required:
|
||||||
|
if device != torch.device("cpu"):
|
||||||
|
free_memory(total_memory_required[device] * 1.3 + extra_mem, device, models_already_loaded)
|
||||||
|
|
||||||
|
for loaded_model in models_to_load:
|
||||||
|
model = loaded_model.model
|
||||||
|
torch_dev = model.load_device
|
||||||
|
if is_device_cpu(torch_dev):
|
||||||
|
vram_set_state = VRAMState.DISABLED
|
||||||
|
else:
|
||||||
|
vram_set_state = vram_state
|
||||||
|
lowvram_model_memory = 0
|
||||||
|
if lowvram_available and (vram_set_state == VRAMState.LOW_VRAM or vram_set_state == VRAMState.NORMAL_VRAM):
|
||||||
|
model_size = loaded_model.model_memory_required(torch_dev)
|
||||||
|
current_free_mem = get_free_memory(torch_dev)
|
||||||
|
lowvram_model_memory = int(max(256 * (1024 * 1024), (current_free_mem - 1024 * (1024 * 1024)) / 1.3 ))
|
||||||
|
if model_size > (current_free_mem - inference_memory): #only switch to lowvram if really necessary
|
||||||
|
vram_set_state = VRAMState.LOW_VRAM
|
||||||
|
else:
|
||||||
|
lowvram_model_memory = 0
|
||||||
|
|
||||||
|
if vram_set_state == VRAMState.NO_VRAM:
|
||||||
|
lowvram_model_memory = 256 * 1024 * 1024
|
||||||
|
|
||||||
|
cur_loaded_model = loaded_model.model_load(lowvram_model_memory)
|
||||||
|
current_loaded_models.insert(0, loaded_model)
|
||||||
|
return
|
||||||
|
|
||||||
|
|
||||||
def load_model_gpu(model):
|
def load_model_gpu(model):
|
||||||
global current_loaded_model
|
return load_models_gpu([model])
|
||||||
global vram_state
|
|
||||||
global model_accelerated
|
|
||||||
|
|
||||||
if model is current_loaded_model:
|
def cleanup_models():
|
||||||
return
|
to_delete = []
|
||||||
unload_model()
|
for i in range(len(current_loaded_models)):
|
||||||
|
print(sys.getrefcount(current_loaded_models[i].model))
|
||||||
|
if sys.getrefcount(current_loaded_models[i].model) <= 2:
|
||||||
|
to_delete = [i] + to_delete
|
||||||
|
|
||||||
torch_dev = model.load_device
|
for i in to_delete:
|
||||||
model.model_patches_to(torch_dev)
|
x = current_loaded_models.pop(i)
|
||||||
model.model_patches_to(model.model_dtype())
|
x.model_unload()
|
||||||
current_loaded_model = model
|
del x
|
||||||
|
|
||||||
if is_device_cpu(torch_dev):
|
def dtype_size(dtype):
|
||||||
vram_set_state = VRAMState.DISABLED
|
dtype_size = 4
|
||||||
else:
|
if dtype == torch.float16 or dtype == torch.bfloat16:
|
||||||
vram_set_state = vram_state
|
dtype_size = 2
|
||||||
|
return dtype_size
|
||||||
if lowvram_available and (vram_set_state == VRAMState.LOW_VRAM or vram_set_state == VRAMState.NORMAL_VRAM):
|
|
||||||
model_size = model.model_size()
|
|
||||||
current_free_mem = get_free_memory(torch_dev)
|
|
||||||
lowvram_model_memory = int(max(256 * (1024 * 1024), (current_free_mem - 1024 * (1024 * 1024)) / 1.3 ))
|
|
||||||
if model_size > (current_free_mem - minimum_inference_memory()): #only switch to lowvram if really necessary
|
|
||||||
vram_set_state = VRAMState.LOW_VRAM
|
|
||||||
|
|
||||||
real_model = model.model
|
|
||||||
patch_model_to = None
|
|
||||||
if vram_set_state == VRAMState.DISABLED:
|
|
||||||
pass
|
|
||||||
elif vram_set_state == VRAMState.NORMAL_VRAM or vram_set_state == VRAMState.HIGH_VRAM or vram_set_state == VRAMState.SHARED:
|
|
||||||
model_accelerated = False
|
|
||||||
patch_model_to = torch_dev
|
|
||||||
|
|
||||||
try:
|
|
||||||
real_model = model.patch_model(device_to=patch_model_to)
|
|
||||||
except Exception as e:
|
|
||||||
model.unpatch_model()
|
|
||||||
unload_model()
|
|
||||||
raise e
|
|
||||||
|
|
||||||
if patch_model_to is not None:
|
|
||||||
real_model.to(torch_dev)
|
|
||||||
|
|
||||||
if vram_set_state == VRAMState.NO_VRAM:
|
|
||||||
device_map = accelerate.infer_auto_device_map(real_model, max_memory={0: "256MiB", "cpu": "16GiB"})
|
|
||||||
accelerate.dispatch_model(real_model, device_map=device_map, main_device=torch_dev)
|
|
||||||
model_accelerated = True
|
|
||||||
elif vram_set_state == VRAMState.LOW_VRAM:
|
|
||||||
device_map = accelerate.infer_auto_device_map(real_model, max_memory={0: "{}MiB".format(lowvram_model_memory // (1024 * 1024)), "cpu": "16GiB"})
|
|
||||||
accelerate.dispatch_model(real_model, device_map=device_map, main_device=torch_dev)
|
|
||||||
model_accelerated = True
|
|
||||||
|
|
||||||
return current_loaded_model
|
|
||||||
|
|
||||||
def load_controlnet_gpu(control_models):
|
|
||||||
global current_gpu_controlnets
|
|
||||||
global vram_state
|
|
||||||
if vram_state == VRAMState.DISABLED:
|
|
||||||
return
|
|
||||||
|
|
||||||
if vram_state == VRAMState.LOW_VRAM or vram_state == VRAMState.NO_VRAM:
|
|
||||||
for m in control_models:
|
|
||||||
if hasattr(m, 'set_lowvram'):
|
|
||||||
m.set_lowvram(True)
|
|
||||||
#don't load controlnets like this if low vram because they will be loaded right before running and unloaded right after
|
|
||||||
return
|
|
||||||
|
|
||||||
models = []
|
|
||||||
for m in control_models:
|
|
||||||
models += m.get_models()
|
|
||||||
|
|
||||||
for m in current_gpu_controlnets:
|
|
||||||
if m not in models:
|
|
||||||
m.cpu()
|
|
||||||
|
|
||||||
device = get_torch_device()
|
|
||||||
current_gpu_controlnets = []
|
|
||||||
for m in models:
|
|
||||||
current_gpu_controlnets.append(m.to(device))
|
|
||||||
|
|
||||||
|
|
||||||
def load_if_low_vram(model):
|
|
||||||
global vram_state
|
|
||||||
if vram_state == VRAMState.LOW_VRAM or vram_state == VRAMState.NO_VRAM:
|
|
||||||
return model.to(get_torch_device())
|
|
||||||
return model
|
|
||||||
|
|
||||||
def unload_if_low_vram(model):
|
|
||||||
global vram_state
|
|
||||||
if vram_state == VRAMState.LOW_VRAM or vram_state == VRAMState.NO_VRAM:
|
|
||||||
return model.cpu()
|
|
||||||
return model
|
|
||||||
|
|
||||||
def unet_offload_device():
|
def unet_offload_device():
|
||||||
if vram_state == VRAMState.HIGH_VRAM:
|
if vram_state == VRAMState.HIGH_VRAM:
|
||||||
@ -354,6 +416,24 @@ def unet_offload_device():
|
|||||||
else:
|
else:
|
||||||
return torch.device("cpu")
|
return torch.device("cpu")
|
||||||
|
|
||||||
|
def unet_inital_load_device(parameters, dtype):
|
||||||
|
torch_dev = get_torch_device()
|
||||||
|
if vram_state == VRAMState.HIGH_VRAM:
|
||||||
|
return torch_dev
|
||||||
|
|
||||||
|
cpu_dev = torch.device("cpu")
|
||||||
|
if DISABLE_SMART_MEMORY:
|
||||||
|
return cpu_dev
|
||||||
|
|
||||||
|
model_size = dtype_size(dtype) * parameters
|
||||||
|
|
||||||
|
mem_dev = get_free_memory(torch_dev)
|
||||||
|
mem_cpu = get_free_memory(cpu_dev)
|
||||||
|
if mem_dev > mem_cpu and model_size < mem_dev:
|
||||||
|
return torch_dev
|
||||||
|
else:
|
||||||
|
return cpu_dev
|
||||||
|
|
||||||
def text_encoder_offload_device():
|
def text_encoder_offload_device():
|
||||||
if args.gpu_only:
|
if args.gpu_only:
|
||||||
return get_torch_device()
|
return get_torch_device()
|
||||||
@ -364,7 +444,7 @@ def text_encoder_device():
|
|||||||
if args.gpu_only:
|
if args.gpu_only:
|
||||||
return get_torch_device()
|
return get_torch_device()
|
||||||
elif vram_state == VRAMState.HIGH_VRAM or vram_state == VRAMState.NORMAL_VRAM:
|
elif vram_state == VRAMState.HIGH_VRAM or vram_state == VRAMState.NORMAL_VRAM:
|
||||||
if torch.get_num_threads() < 4: #leaving the text encoder on the CPU is faster than shifting it if the CPU is fast enough.
|
if should_use_fp16(prioritize_performance=False):
|
||||||
return get_torch_device()
|
return get_torch_device()
|
||||||
else:
|
else:
|
||||||
return torch.device("cpu")
|
return torch.device("cpu")
|
||||||
@ -381,12 +461,8 @@ def vae_offload_device():
|
|||||||
return torch.device("cpu")
|
return torch.device("cpu")
|
||||||
|
|
||||||
def vae_dtype():
|
def vae_dtype():
|
||||||
if args.fp16_vae:
|
global VAE_DTYPE
|
||||||
return torch.float16
|
return VAE_DTYPE
|
||||||
elif args.bf16_vae:
|
|
||||||
return torch.bfloat16
|
|
||||||
else:
|
|
||||||
return torch.float32
|
|
||||||
|
|
||||||
def get_autocast_device(dev):
|
def get_autocast_device(dev):
|
||||||
if hasattr(dev, 'type'):
|
if hasattr(dev, 'type'):
|
||||||
@ -440,8 +516,12 @@ def get_free_memory(dev=None, torch_free_too=False):
|
|||||||
mem_free_total = 1024 * 1024 * 1024 #TODO
|
mem_free_total = 1024 * 1024 * 1024 #TODO
|
||||||
mem_free_torch = mem_free_total
|
mem_free_torch = mem_free_total
|
||||||
elif xpu_available:
|
elif xpu_available:
|
||||||
mem_free_total = torch.xpu.get_device_properties(dev).total_memory - torch.xpu.memory_allocated(dev)
|
stats = torch.xpu.memory_stats(dev)
|
||||||
mem_free_torch = mem_free_total
|
mem_active = stats['active_bytes.all.current']
|
||||||
|
mem_allocated = stats['allocated_bytes.all.current']
|
||||||
|
mem_reserved = stats['reserved_bytes.all.current']
|
||||||
|
mem_free_torch = mem_reserved - mem_active
|
||||||
|
mem_free_total = torch.xpu.get_device_properties(dev).total_memory - mem_allocated
|
||||||
else:
|
else:
|
||||||
stats = torch.cuda.memory_stats(dev)
|
stats = torch.cuda.memory_stats(dev)
|
||||||
mem_active = stats['active_bytes.all.current']
|
mem_active = stats['active_bytes.all.current']
|
||||||
@ -455,6 +535,13 @@ def get_free_memory(dev=None, torch_free_too=False):
|
|||||||
else:
|
else:
|
||||||
return mem_free_total
|
return mem_free_total
|
||||||
|
|
||||||
|
def batch_area_memory(area):
|
||||||
|
if xformers_enabled() or pytorch_attention_flash_attention():
|
||||||
|
#TODO: these formulas are copied from maximum_batch_area below
|
||||||
|
return (area / 20) * (1024 * 1024)
|
||||||
|
else:
|
||||||
|
return (((area * 0.6) / 0.9) + 1024) * (1024 * 1024)
|
||||||
|
|
||||||
def maximum_batch_area():
|
def maximum_batch_area():
|
||||||
global vram_state
|
global vram_state
|
||||||
if vram_state == VRAMState.NO_VRAM:
|
if vram_state == VRAMState.NO_VRAM:
|
||||||
@ -489,15 +576,19 @@ def is_device_mps(device):
|
|||||||
return True
|
return True
|
||||||
return False
|
return False
|
||||||
|
|
||||||
def should_use_fp16(device=None, model_params=0):
|
def should_use_fp16(device=None, model_params=0, prioritize_performance=True):
|
||||||
global xpu_available
|
global xpu_available
|
||||||
global directml_enabled
|
global directml_enabled
|
||||||
|
|
||||||
|
if device is not None:
|
||||||
|
if is_device_cpu(device):
|
||||||
|
return False
|
||||||
|
|
||||||
if FORCE_FP16:
|
if FORCE_FP16:
|
||||||
return True
|
return True
|
||||||
|
|
||||||
if device is not None: #TODO
|
if device is not None: #TODO
|
||||||
if is_device_cpu(device) or is_device_mps(device):
|
if is_device_mps(device):
|
||||||
return False
|
return False
|
||||||
|
|
||||||
if FORCE_FP32:
|
if FORCE_FP32:
|
||||||
@ -506,9 +597,12 @@ def should_use_fp16(device=None, model_params=0):
|
|||||||
if directml_enabled:
|
if directml_enabled:
|
||||||
return False
|
return False
|
||||||
|
|
||||||
if cpu_mode() or mps_mode() or xpu_available:
|
if cpu_mode() or mps_mode():
|
||||||
return False #TODO ?
|
return False #TODO ?
|
||||||
|
|
||||||
|
if xpu_available:
|
||||||
|
return True
|
||||||
|
|
||||||
if torch.cuda.is_bf16_supported():
|
if torch.cuda.is_bf16_supported():
|
||||||
return True
|
return True
|
||||||
|
|
||||||
@ -527,14 +621,14 @@ def should_use_fp16(device=None, model_params=0):
|
|||||||
|
|
||||||
if fp16_works:
|
if fp16_works:
|
||||||
free_model_memory = (get_free_memory() * 0.9 - minimum_inference_memory())
|
free_model_memory = (get_free_memory() * 0.9 - minimum_inference_memory())
|
||||||
if model_params * 4 > free_model_memory:
|
if (not prioritize_performance) or model_params * 4 > free_model_memory:
|
||||||
return True
|
return True
|
||||||
|
|
||||||
if props.major < 7:
|
if props.major < 7:
|
||||||
return False
|
return False
|
||||||
|
|
||||||
#FP16 is just broken on these cards
|
#FP16 is just broken on these cards
|
||||||
nvidia_16_series = ["1660", "1650", "1630", "T500", "T550", "T600", "MX550", "MX450"]
|
nvidia_16_series = ["1660", "1650", "1630", "T500", "T550", "T600", "MX550", "MX450", "CMP 30HX"]
|
||||||
for x in nvidia_16_series:
|
for x in nvidia_16_series:
|
||||||
if x in props.name:
|
if x in props.name:
|
||||||
return False
|
return False
|
||||||
@ -553,6 +647,13 @@ def soft_empty_cache():
|
|||||||
torch.cuda.empty_cache()
|
torch.cuda.empty_cache()
|
||||||
torch.cuda.ipc_collect()
|
torch.cuda.ipc_collect()
|
||||||
|
|
||||||
|
def resolve_lowvram_weight(weight, model, key):
|
||||||
|
if weight.device == torch.device("meta"): #lowvram NOTE: this depends on the inner working of the accelerate library so it might break.
|
||||||
|
key_split = key.split('.') # I have no idea why they don't just leave the weight there instead of using the meta device.
|
||||||
|
op = comfy.utils.get_attr(model, '.'.join(key_split[:-1]))
|
||||||
|
weight = op._hf_hook.weights_map[key_split[-1]]
|
||||||
|
return weight
|
||||||
|
|
||||||
#TODO: might be cleaner to put this somewhere else
|
#TODO: might be cleaner to put this somewhere else
|
||||||
import threading
|
import threading
|
||||||
|
|
||||||
|
|||||||
270
comfy/model_patcher.py
Normal file
270
comfy/model_patcher.py
Normal file
@ -0,0 +1,270 @@
|
|||||||
|
import torch
|
||||||
|
import copy
|
||||||
|
import inspect
|
||||||
|
|
||||||
|
import comfy.utils
|
||||||
|
|
||||||
|
class ModelPatcher:
|
||||||
|
def __init__(self, model, load_device, offload_device, size=0, current_device=None):
|
||||||
|
self.size = size
|
||||||
|
self.model = model
|
||||||
|
self.patches = {}
|
||||||
|
self.backup = {}
|
||||||
|
self.model_options = {"transformer_options":{}}
|
||||||
|
self.model_size()
|
||||||
|
self.load_device = load_device
|
||||||
|
self.offload_device = offload_device
|
||||||
|
if current_device is None:
|
||||||
|
self.current_device = self.offload_device
|
||||||
|
else:
|
||||||
|
self.current_device = current_device
|
||||||
|
|
||||||
|
def model_size(self):
|
||||||
|
if self.size > 0:
|
||||||
|
return self.size
|
||||||
|
model_sd = self.model.state_dict()
|
||||||
|
size = 0
|
||||||
|
for k in model_sd:
|
||||||
|
t = model_sd[k]
|
||||||
|
size += t.nelement() * t.element_size()
|
||||||
|
self.size = size
|
||||||
|
self.model_keys = set(model_sd.keys())
|
||||||
|
return size
|
||||||
|
|
||||||
|
def clone(self):
|
||||||
|
n = ModelPatcher(self.model, self.load_device, self.offload_device, self.size, self.current_device)
|
||||||
|
n.patches = {}
|
||||||
|
for k in self.patches:
|
||||||
|
n.patches[k] = self.patches[k][:]
|
||||||
|
|
||||||
|
n.model_options = copy.deepcopy(self.model_options)
|
||||||
|
n.model_keys = self.model_keys
|
||||||
|
return n
|
||||||
|
|
||||||
|
def is_clone(self, other):
|
||||||
|
if hasattr(other, 'model') and self.model is other.model:
|
||||||
|
return True
|
||||||
|
return False
|
||||||
|
|
||||||
|
def set_model_sampler_cfg_function(self, sampler_cfg_function):
|
||||||
|
if len(inspect.signature(sampler_cfg_function).parameters) == 3:
|
||||||
|
self.model_options["sampler_cfg_function"] = lambda args: sampler_cfg_function(args["cond"], args["uncond"], args["cond_scale"]) #Old way
|
||||||
|
else:
|
||||||
|
self.model_options["sampler_cfg_function"] = sampler_cfg_function
|
||||||
|
|
||||||
|
def set_model_unet_function_wrapper(self, unet_wrapper_function):
|
||||||
|
self.model_options["model_function_wrapper"] = unet_wrapper_function
|
||||||
|
|
||||||
|
def set_model_patch(self, patch, name):
|
||||||
|
to = self.model_options["transformer_options"]
|
||||||
|
if "patches" not in to:
|
||||||
|
to["patches"] = {}
|
||||||
|
to["patches"][name] = to["patches"].get(name, []) + [patch]
|
||||||
|
|
||||||
|
def set_model_patch_replace(self, patch, name, block_name, number):
|
||||||
|
to = self.model_options["transformer_options"]
|
||||||
|
if "patches_replace" not in to:
|
||||||
|
to["patches_replace"] = {}
|
||||||
|
if name not in to["patches_replace"]:
|
||||||
|
to["patches_replace"][name] = {}
|
||||||
|
to["patches_replace"][name][(block_name, number)] = patch
|
||||||
|
|
||||||
|
def set_model_attn1_patch(self, patch):
|
||||||
|
self.set_model_patch(patch, "attn1_patch")
|
||||||
|
|
||||||
|
def set_model_attn2_patch(self, patch):
|
||||||
|
self.set_model_patch(patch, "attn2_patch")
|
||||||
|
|
||||||
|
def set_model_attn1_replace(self, patch, block_name, number):
|
||||||
|
self.set_model_patch_replace(patch, "attn1", block_name, number)
|
||||||
|
|
||||||
|
def set_model_attn2_replace(self, patch, block_name, number):
|
||||||
|
self.set_model_patch_replace(patch, "attn2", block_name, number)
|
||||||
|
|
||||||
|
def set_model_attn1_output_patch(self, patch):
|
||||||
|
self.set_model_patch(patch, "attn1_output_patch")
|
||||||
|
|
||||||
|
def set_model_attn2_output_patch(self, patch):
|
||||||
|
self.set_model_patch(patch, "attn2_output_patch")
|
||||||
|
|
||||||
|
def model_patches_to(self, device):
|
||||||
|
to = self.model_options["transformer_options"]
|
||||||
|
if "patches" in to:
|
||||||
|
patches = to["patches"]
|
||||||
|
for name in patches:
|
||||||
|
patch_list = patches[name]
|
||||||
|
for i in range(len(patch_list)):
|
||||||
|
if hasattr(patch_list[i], "to"):
|
||||||
|
patch_list[i] = patch_list[i].to(device)
|
||||||
|
if "patches_replace" in to:
|
||||||
|
patches = to["patches_replace"]
|
||||||
|
for name in patches:
|
||||||
|
patch_list = patches[name]
|
||||||
|
for k in patch_list:
|
||||||
|
if hasattr(patch_list[k], "to"):
|
||||||
|
patch_list[k] = patch_list[k].to(device)
|
||||||
|
|
||||||
|
def model_dtype(self):
|
||||||
|
if hasattr(self.model, "get_dtype"):
|
||||||
|
return self.model.get_dtype()
|
||||||
|
|
||||||
|
def add_patches(self, patches, strength_patch=1.0, strength_model=1.0):
|
||||||
|
p = set()
|
||||||
|
for k in patches:
|
||||||
|
if k in self.model_keys:
|
||||||
|
p.add(k)
|
||||||
|
current_patches = self.patches.get(k, [])
|
||||||
|
current_patches.append((strength_patch, patches[k], strength_model))
|
||||||
|
self.patches[k] = current_patches
|
||||||
|
|
||||||
|
return list(p)
|
||||||
|
|
||||||
|
def get_key_patches(self, filter_prefix=None):
|
||||||
|
model_sd = self.model_state_dict()
|
||||||
|
p = {}
|
||||||
|
for k in model_sd:
|
||||||
|
if filter_prefix is not None:
|
||||||
|
if not k.startswith(filter_prefix):
|
||||||
|
continue
|
||||||
|
if k in self.patches:
|
||||||
|
p[k] = [model_sd[k]] + self.patches[k]
|
||||||
|
else:
|
||||||
|
p[k] = (model_sd[k],)
|
||||||
|
return p
|
||||||
|
|
||||||
|
def model_state_dict(self, filter_prefix=None):
|
||||||
|
sd = self.model.state_dict()
|
||||||
|
keys = list(sd.keys())
|
||||||
|
if filter_prefix is not None:
|
||||||
|
for k in keys:
|
||||||
|
if not k.startswith(filter_prefix):
|
||||||
|
sd.pop(k)
|
||||||
|
return sd
|
||||||
|
|
||||||
|
def patch_model(self, device_to=None):
|
||||||
|
model_sd = self.model_state_dict()
|
||||||
|
for key in self.patches:
|
||||||
|
if key not in model_sd:
|
||||||
|
print("could not patch. key doesn't exist in model:", key)
|
||||||
|
continue
|
||||||
|
|
||||||
|
weight = model_sd[key]
|
||||||
|
|
||||||
|
if key not in self.backup:
|
||||||
|
self.backup[key] = weight.to(self.offload_device)
|
||||||
|
|
||||||
|
if device_to is not None:
|
||||||
|
temp_weight = weight.float().to(device_to, copy=True)
|
||||||
|
else:
|
||||||
|
temp_weight = weight.to(torch.float32, copy=True)
|
||||||
|
out_weight = self.calculate_weight(self.patches[key], temp_weight, key).to(weight.dtype)
|
||||||
|
comfy.utils.set_attr(self.model, key, out_weight)
|
||||||
|
del temp_weight
|
||||||
|
|
||||||
|
if device_to is not None:
|
||||||
|
self.model.to(device_to)
|
||||||
|
self.current_device = device_to
|
||||||
|
|
||||||
|
return self.model
|
||||||
|
|
||||||
|
def calculate_weight(self, patches, weight, key):
|
||||||
|
for p in patches:
|
||||||
|
alpha = p[0]
|
||||||
|
v = p[1]
|
||||||
|
strength_model = p[2]
|
||||||
|
|
||||||
|
if strength_model != 1.0:
|
||||||
|
weight *= strength_model
|
||||||
|
|
||||||
|
if isinstance(v, list):
|
||||||
|
v = (self.calculate_weight(v[1:], v[0].clone(), key), )
|
||||||
|
|
||||||
|
if len(v) == 1:
|
||||||
|
w1 = v[0]
|
||||||
|
if alpha != 0.0:
|
||||||
|
if w1.shape != weight.shape:
|
||||||
|
print("WARNING SHAPE MISMATCH {} WEIGHT NOT MERGED {} != {}".format(key, w1.shape, weight.shape))
|
||||||
|
else:
|
||||||
|
weight += alpha * w1.type(weight.dtype).to(weight.device)
|
||||||
|
elif len(v) == 4: #lora/locon
|
||||||
|
mat1 = v[0].float().to(weight.device)
|
||||||
|
mat2 = v[1].float().to(weight.device)
|
||||||
|
if v[2] is not None:
|
||||||
|
alpha *= v[2] / mat2.shape[0]
|
||||||
|
if v[3] is not None:
|
||||||
|
#locon mid weights, hopefully the math is fine because I didn't properly test it
|
||||||
|
mat3 = v[3].float().to(weight.device)
|
||||||
|
final_shape = [mat2.shape[1], mat2.shape[0], mat3.shape[2], mat3.shape[3]]
|
||||||
|
mat2 = torch.mm(mat2.transpose(0, 1).flatten(start_dim=1), mat3.transpose(0, 1).flatten(start_dim=1)).reshape(final_shape).transpose(0, 1)
|
||||||
|
try:
|
||||||
|
weight += (alpha * torch.mm(mat1.flatten(start_dim=1), mat2.flatten(start_dim=1))).reshape(weight.shape).type(weight.dtype)
|
||||||
|
except Exception as e:
|
||||||
|
print("ERROR", key, e)
|
||||||
|
elif len(v) == 8: #lokr
|
||||||
|
w1 = v[0]
|
||||||
|
w2 = v[1]
|
||||||
|
w1_a = v[3]
|
||||||
|
w1_b = v[4]
|
||||||
|
w2_a = v[5]
|
||||||
|
w2_b = v[6]
|
||||||
|
t2 = v[7]
|
||||||
|
dim = None
|
||||||
|
|
||||||
|
if w1 is None:
|
||||||
|
dim = w1_b.shape[0]
|
||||||
|
w1 = torch.mm(w1_a.float(), w1_b.float())
|
||||||
|
else:
|
||||||
|
w1 = w1.float().to(weight.device)
|
||||||
|
|
||||||
|
if w2 is None:
|
||||||
|
dim = w2_b.shape[0]
|
||||||
|
if t2 is None:
|
||||||
|
w2 = torch.mm(w2_a.float().to(weight.device), w2_b.float().to(weight.device))
|
||||||
|
else:
|
||||||
|
w2 = torch.einsum('i j k l, j r, i p -> p r k l', t2.float().to(weight.device), w2_b.float().to(weight.device), w2_a.float().to(weight.device))
|
||||||
|
else:
|
||||||
|
w2 = w2.float().to(weight.device)
|
||||||
|
|
||||||
|
if len(w2.shape) == 4:
|
||||||
|
w1 = w1.unsqueeze(2).unsqueeze(2)
|
||||||
|
if v[2] is not None and dim is not None:
|
||||||
|
alpha *= v[2] / dim
|
||||||
|
|
||||||
|
try:
|
||||||
|
weight += alpha * torch.kron(w1, w2).reshape(weight.shape).type(weight.dtype)
|
||||||
|
except Exception as e:
|
||||||
|
print("ERROR", key, e)
|
||||||
|
else: #loha
|
||||||
|
w1a = v[0]
|
||||||
|
w1b = v[1]
|
||||||
|
if v[2] is not None:
|
||||||
|
alpha *= v[2] / w1b.shape[0]
|
||||||
|
w2a = v[3]
|
||||||
|
w2b = v[4]
|
||||||
|
if v[5] is not None: #cp decomposition
|
||||||
|
t1 = v[5]
|
||||||
|
t2 = v[6]
|
||||||
|
m1 = torch.einsum('i j k l, j r, i p -> p r k l', t1.float().to(weight.device), w1b.float().to(weight.device), w1a.float().to(weight.device))
|
||||||
|
m2 = torch.einsum('i j k l, j r, i p -> p r k l', t2.float().to(weight.device), w2b.float().to(weight.device), w2a.float().to(weight.device))
|
||||||
|
else:
|
||||||
|
m1 = torch.mm(w1a.float().to(weight.device), w1b.float().to(weight.device))
|
||||||
|
m2 = torch.mm(w2a.float().to(weight.device), w2b.float().to(weight.device))
|
||||||
|
|
||||||
|
try:
|
||||||
|
weight += (alpha * m1 * m2).reshape(weight.shape).type(weight.dtype)
|
||||||
|
except Exception as e:
|
||||||
|
print("ERROR", key, e)
|
||||||
|
|
||||||
|
return weight
|
||||||
|
|
||||||
|
def unpatch_model(self, device_to=None):
|
||||||
|
keys = list(self.backup.keys())
|
||||||
|
|
||||||
|
for k in keys:
|
||||||
|
comfy.utils.set_attr(self.model, k, self.backup[k])
|
||||||
|
|
||||||
|
self.backup = {}
|
||||||
|
|
||||||
|
if device_to is not None:
|
||||||
|
self.model.to(device_to)
|
||||||
|
self.current_device = device_to
|
||||||
18
comfy/ops.py
18
comfy/ops.py
@ -21,11 +21,25 @@ class Conv2d(torch.nn.Conv2d):
|
|||||||
def reset_parameters(self):
|
def reset_parameters(self):
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
def conv_nd(dims, *args, **kwargs):
|
||||||
|
if dims == 2:
|
||||||
|
return Conv2d(*args, **kwargs)
|
||||||
|
else:
|
||||||
|
raise ValueError(f"unsupported dimensions: {dims}")
|
||||||
|
|
||||||
@contextmanager
|
@contextmanager
|
||||||
def use_comfy_ops(): # Kind of an ugly hack but I can't think of a better way
|
def use_comfy_ops(device=None, dtype=None): # Kind of an ugly hack but I can't think of a better way
|
||||||
old_torch_nn_linear = torch.nn.Linear
|
old_torch_nn_linear = torch.nn.Linear
|
||||||
torch.nn.Linear = Linear
|
force_device = device
|
||||||
|
force_dtype = dtype
|
||||||
|
def linear_with_dtype(in_features: int, out_features: int, bias: bool = True, device=None, dtype=None):
|
||||||
|
if force_device is not None:
|
||||||
|
device = force_device
|
||||||
|
if force_dtype is not None:
|
||||||
|
dtype = force_dtype
|
||||||
|
return Linear(in_features, out_features, bias=bias, device=device, dtype=dtype)
|
||||||
|
|
||||||
|
torch.nn.Linear = linear_with_dtype
|
||||||
try:
|
try:
|
||||||
yield
|
yield
|
||||||
finally:
|
finally:
|
||||||
|
|||||||
@ -51,19 +51,26 @@ def get_models_from_cond(cond, model_type):
|
|||||||
models += [c[1][model_type]]
|
models += [c[1][model_type]]
|
||||||
return models
|
return models
|
||||||
|
|
||||||
def load_additional_models(positive, negative, dtype):
|
def get_additional_models(positive, negative, dtype):
|
||||||
"""loads additional models in positive and negative conditioning"""
|
"""loads additional models in positive and negative conditioning"""
|
||||||
control_nets = get_models_from_cond(positive, "control") + get_models_from_cond(negative, "control")
|
control_nets = set(get_models_from_cond(positive, "control") + get_models_from_cond(negative, "control"))
|
||||||
|
|
||||||
|
inference_memory = 0
|
||||||
|
control_models = []
|
||||||
|
for m in control_nets:
|
||||||
|
control_models += m.get_models()
|
||||||
|
inference_memory += m.inference_memory_requirements(dtype)
|
||||||
|
|
||||||
gligen = get_models_from_cond(positive, "gligen") + get_models_from_cond(negative, "gligen")
|
gligen = get_models_from_cond(positive, "gligen") + get_models_from_cond(negative, "gligen")
|
||||||
gligen = [x[1].to(dtype) for x in gligen]
|
gligen = [x[1] for x in gligen]
|
||||||
models = control_nets + gligen
|
models = control_models + gligen
|
||||||
comfy.model_management.load_controlnet_gpu(models)
|
return models, inference_memory
|
||||||
return models
|
|
||||||
|
|
||||||
def cleanup_additional_models(models):
|
def cleanup_additional_models(models):
|
||||||
"""cleanup additional models that were loaded"""
|
"""cleanup additional models that were loaded"""
|
||||||
for m in models:
|
for m in models:
|
||||||
m.cleanup()
|
if hasattr(m, 'cleanup'):
|
||||||
|
m.cleanup()
|
||||||
|
|
||||||
def sample(model, noise, steps, cfg, sampler_name, scheduler, positive, negative, latent_image, denoise=1.0, disable_noise=False, start_step=None, last_step=None, force_full_denoise=False, noise_mask=None, sigmas=None, callback=None, disable_pbar=False, seed=None):
|
def sample(model, noise, steps, cfg, sampler_name, scheduler, positive, negative, latent_image, denoise=1.0, disable_noise=False, start_step=None, last_step=None, force_full_denoise=False, noise_mask=None, sigmas=None, callback=None, disable_pbar=False, seed=None):
|
||||||
device = comfy.model_management.get_torch_device()
|
device = comfy.model_management.get_torch_device()
|
||||||
@ -72,7 +79,8 @@ def sample(model, noise, steps, cfg, sampler_name, scheduler, positive, negative
|
|||||||
noise_mask = prepare_mask(noise_mask, noise.shape, device)
|
noise_mask = prepare_mask(noise_mask, noise.shape, device)
|
||||||
|
|
||||||
real_model = None
|
real_model = None
|
||||||
comfy.model_management.load_model_gpu(model)
|
models, inference_memory = get_additional_models(positive, negative, model.model_dtype())
|
||||||
|
comfy.model_management.load_models_gpu([model] + models, comfy.model_management.batch_area_memory(noise.shape[0] * noise.shape[2] * noise.shape[3]) + inference_memory)
|
||||||
real_model = model.model
|
real_model = model.model
|
||||||
|
|
||||||
noise = noise.to(device)
|
noise = noise.to(device)
|
||||||
@ -81,7 +89,6 @@ def sample(model, noise, steps, cfg, sampler_name, scheduler, positive, negative
|
|||||||
positive_copy = broadcast_cond(positive, noise.shape[0], device)
|
positive_copy = broadcast_cond(positive, noise.shape[0], device)
|
||||||
negative_copy = broadcast_cond(negative, noise.shape[0], device)
|
negative_copy = broadcast_cond(negative, noise.shape[0], device)
|
||||||
|
|
||||||
models = load_additional_models(positive, negative, model.model_dtype())
|
|
||||||
|
|
||||||
sampler = comfy.samplers.KSampler(real_model, steps=steps, device=device, sampler=sampler_name, scheduler=scheduler, denoise=denoise, model_options=model.model_options)
|
sampler = comfy.samplers.KSampler(real_model, steps=steps, device=device, sampler=sampler_name, scheduler=scheduler, denoise=denoise, model_options=model.model_options)
|
||||||
|
|
||||||
|
|||||||
@ -88,9 +88,9 @@ def sampling_function(model_function, x, timestep, uncond, cond, cond_scale, con
|
|||||||
gligen_type = gligen[0]
|
gligen_type = gligen[0]
|
||||||
gligen_model = gligen[1]
|
gligen_model = gligen[1]
|
||||||
if gligen_type == "position":
|
if gligen_type == "position":
|
||||||
gligen_patch = gligen_model.set_position(input_x.shape, gligen[2], input_x.device)
|
gligen_patch = gligen_model.model.set_position(input_x.shape, gligen[2], input_x.device)
|
||||||
else:
|
else:
|
||||||
gligen_patch = gligen_model.set_empty(input_x.shape, input_x.device)
|
gligen_patch = gligen_model.model.set_empty(input_x.shape, input_x.device)
|
||||||
|
|
||||||
patches['middle_patch'] = [gligen_patch]
|
patches['middle_patch'] = [gligen_patch]
|
||||||
|
|
||||||
@ -189,12 +189,13 @@ def sampling_function(model_function, x, timestep, uncond, cond, cond_scale, con
|
|||||||
continue
|
continue
|
||||||
|
|
||||||
to_run += [(p, COND)]
|
to_run += [(p, COND)]
|
||||||
for x in uncond:
|
if uncond is not None:
|
||||||
p = get_area_and_mult(x, x_in, cond_concat_in, timestep)
|
for x in uncond:
|
||||||
if p is None:
|
p = get_area_and_mult(x, x_in, cond_concat_in, timestep)
|
||||||
continue
|
if p is None:
|
||||||
|
continue
|
||||||
|
|
||||||
to_run += [(p, UNCOND)]
|
to_run += [(p, UNCOND)]
|
||||||
|
|
||||||
while len(to_run) > 0:
|
while len(to_run) > 0:
|
||||||
first = to_run[0]
|
first = to_run[0]
|
||||||
@ -282,6 +283,9 @@ def sampling_function(model_function, x, timestep, uncond, cond, cond_scale, con
|
|||||||
|
|
||||||
|
|
||||||
max_total_area = model_management.maximum_batch_area()
|
max_total_area = model_management.maximum_batch_area()
|
||||||
|
if math.isclose(cond_scale, 1.0):
|
||||||
|
uncond = None
|
||||||
|
|
||||||
cond, uncond = calc_cond_uncond_batch(model_function, cond, uncond, x, timestep, max_total_area, cond_concat, model_options)
|
cond, uncond = calc_cond_uncond_batch(model_function, cond, uncond, x, timestep, max_total_area, cond_concat, model_options)
|
||||||
if "sampler_cfg_function" in model_options:
|
if "sampler_cfg_function" in model_options:
|
||||||
args = {"cond": cond, "uncond": uncond, "cond_scale": cond_scale, "timestep": timestep}
|
args = {"cond": cond, "uncond": uncond, "cond_scale": cond_scale, "timestep": timestep}
|
||||||
@ -343,6 +347,17 @@ def ddim_scheduler(model, steps):
|
|||||||
sigs += [0.0]
|
sigs += [0.0]
|
||||||
return torch.FloatTensor(sigs)
|
return torch.FloatTensor(sigs)
|
||||||
|
|
||||||
|
def sgm_scheduler(model, steps):
|
||||||
|
sigs = []
|
||||||
|
timesteps = torch.linspace(model.inner_model.inner_model.num_timesteps - 1, 0, steps + 1)[:-1].type(torch.int)
|
||||||
|
for x in range(len(timesteps)):
|
||||||
|
ts = timesteps[x]
|
||||||
|
if ts > 999:
|
||||||
|
ts = 999
|
||||||
|
sigs.append(model.t_to_sigma(torch.tensor(ts)))
|
||||||
|
sigs += [0.0]
|
||||||
|
return torch.FloatTensor(sigs)
|
||||||
|
|
||||||
def blank_inpaint_image_like(latent_image):
|
def blank_inpaint_image_like(latent_image):
|
||||||
blank_image = torch.ones_like(latent_image)
|
blank_image = torch.ones_like(latent_image)
|
||||||
# these are the values for "zero" in pixel space translated to latent space
|
# these are the values for "zero" in pixel space translated to latent space
|
||||||
@ -463,7 +478,7 @@ def pre_run_control(model, conds):
|
|||||||
timestep_end = None
|
timestep_end = None
|
||||||
percent_to_timestep_function = lambda a: model.sigma_to_t(model.t_to_sigma(torch.tensor(a) * 999.0))
|
percent_to_timestep_function = lambda a: model.sigma_to_t(model.t_to_sigma(torch.tensor(a) * 999.0))
|
||||||
if 'control' in x[1]:
|
if 'control' in x[1]:
|
||||||
x[1]['control'].pre_run(model.inner_model, percent_to_timestep_function)
|
x[1]['control'].pre_run(model.inner_model.inner_model, percent_to_timestep_function)
|
||||||
|
|
||||||
def apply_empty_x_to_equal_area(conds, uncond, name, uncond_fill_func):
|
def apply_empty_x_to_equal_area(conds, uncond, name, uncond_fill_func):
|
||||||
cond_cnets = []
|
cond_cnets = []
|
||||||
@ -521,10 +536,10 @@ def encode_adm(model, conds, batch_size, width, height, device, prompt_type):
|
|||||||
|
|
||||||
|
|
||||||
class KSampler:
|
class KSampler:
|
||||||
SCHEDULERS = ["normal", "karras", "exponential", "simple", "ddim_uniform"]
|
SCHEDULERS = ["normal", "karras", "exponential", "sgm_uniform", "simple", "ddim_uniform"]
|
||||||
SAMPLERS = ["euler", "euler_ancestral", "heun", "dpm_2", "dpm_2_ancestral",
|
SAMPLERS = ["euler", "euler_ancestral", "heun", "dpm_2", "dpm_2_ancestral",
|
||||||
"lms", "dpm_fast", "dpm_adaptive", "dpmpp_2s_ancestral", "dpmpp_sde", "dpmpp_sde_gpu",
|
"lms", "dpm_fast", "dpm_adaptive", "dpmpp_2s_ancestral", "dpmpp_sde", "dpmpp_sde_gpu",
|
||||||
"dpmpp_2m", "dpmpp_2m_sde", "dpmpp_2m_sde_gpu", "ddim", "uni_pc", "uni_pc_bh2"]
|
"dpmpp_2m", "dpmpp_2m_sde", "dpmpp_2m_sde_gpu", "dpmpp_3m_sde", "dpmpp_3m_sde_gpu", "ddim", "uni_pc", "uni_pc_bh2"]
|
||||||
|
|
||||||
def __init__(self, model, steps, device, sampler=None, scheduler=None, denoise=None, model_options={}):
|
def __init__(self, model, steps, device, sampler=None, scheduler=None, denoise=None, model_options={}):
|
||||||
self.model = model
|
self.model = model
|
||||||
@ -566,6 +581,8 @@ class KSampler:
|
|||||||
sigmas = simple_scheduler(self.model_wrap, steps)
|
sigmas = simple_scheduler(self.model_wrap, steps)
|
||||||
elif self.scheduler == "ddim_uniform":
|
elif self.scheduler == "ddim_uniform":
|
||||||
sigmas = ddim_scheduler(self.model_wrap, steps)
|
sigmas = ddim_scheduler(self.model_wrap, steps)
|
||||||
|
elif self.scheduler == "sgm_uniform":
|
||||||
|
sigmas = sgm_scheduler(self.model_wrap, steps)
|
||||||
else:
|
else:
|
||||||
print("error invalid scheduler", self.scheduler)
|
print("error invalid scheduler", self.scheduler)
|
||||||
|
|
||||||
|
|||||||
904
comfy/sd.py
904
comfy/sd.py
File diff suppressed because it is too large
Load Diff
@ -43,7 +43,7 @@ class SD1ClipModel(torch.nn.Module, ClipTokenWeightEncoder):
|
|||||||
"hidden"
|
"hidden"
|
||||||
]
|
]
|
||||||
def __init__(self, version="openai/clip-vit-large-patch14", device="cpu", max_length=77,
|
def __init__(self, version="openai/clip-vit-large-patch14", device="cpu", max_length=77,
|
||||||
freeze=True, layer="last", layer_idx=None, textmodel_json_config=None, textmodel_path=None): # clip-vit-base-patch32
|
freeze=True, layer="last", layer_idx=None, textmodel_json_config=None, textmodel_path=None, dtype=None): # clip-vit-base-patch32
|
||||||
super().__init__()
|
super().__init__()
|
||||||
assert layer in self.LAYERS
|
assert layer in self.LAYERS
|
||||||
self.num_layers = 12
|
self.num_layers = 12
|
||||||
@ -54,17 +54,21 @@ class SD1ClipModel(torch.nn.Module, ClipTokenWeightEncoder):
|
|||||||
textmodel_json_config = os.path.join(os.path.dirname(os.path.realpath(__file__)), "sd1_clip_config.json")
|
textmodel_json_config = os.path.join(os.path.dirname(os.path.realpath(__file__)), "sd1_clip_config.json")
|
||||||
config = CLIPTextConfig.from_json_file(textmodel_json_config)
|
config = CLIPTextConfig.from_json_file(textmodel_json_config)
|
||||||
self.num_layers = config.num_hidden_layers
|
self.num_layers = config.num_hidden_layers
|
||||||
with comfy.ops.use_comfy_ops():
|
with comfy.ops.use_comfy_ops(device, dtype):
|
||||||
with modeling_utils.no_init_weights():
|
with modeling_utils.no_init_weights():
|
||||||
self.transformer = CLIPTextModel(config)
|
self.transformer = CLIPTextModel(config)
|
||||||
|
|
||||||
|
if dtype is not None:
|
||||||
|
self.transformer.to(dtype)
|
||||||
self.max_length = max_length
|
self.max_length = max_length
|
||||||
if freeze:
|
if freeze:
|
||||||
self.freeze()
|
self.freeze()
|
||||||
self.layer = layer
|
self.layer = layer
|
||||||
self.layer_idx = None
|
self.layer_idx = None
|
||||||
self.empty_tokens = [[49406] + [49407] * 76]
|
self.empty_tokens = [[49406] + [49407] * 76]
|
||||||
self.text_projection = None
|
self.text_projection = torch.nn.Parameter(torch.eye(self.transformer.get_input_embeddings().weight.shape[1]))
|
||||||
|
self.logit_scale = torch.nn.Parameter(torch.tensor(4.6055))
|
||||||
|
|
||||||
self.layer_norm_hidden_state = True
|
self.layer_norm_hidden_state = True
|
||||||
if layer == "hidden":
|
if layer == "hidden":
|
||||||
assert layer_idx is not None
|
assert layer_idx is not None
|
||||||
@ -91,13 +95,15 @@ class SD1ClipModel(torch.nn.Module, ClipTokenWeightEncoder):
|
|||||||
|
|
||||||
def set_up_textual_embeddings(self, tokens, current_embeds):
|
def set_up_textual_embeddings(self, tokens, current_embeds):
|
||||||
out_tokens = []
|
out_tokens = []
|
||||||
next_new_token = token_dict_size = current_embeds.weight.shape[0]
|
next_new_token = token_dict_size = current_embeds.weight.shape[0] - 1
|
||||||
embedding_weights = []
|
embedding_weights = []
|
||||||
|
|
||||||
for x in tokens:
|
for x in tokens:
|
||||||
tokens_temp = []
|
tokens_temp = []
|
||||||
for y in x:
|
for y in x:
|
||||||
if isinstance(y, int):
|
if isinstance(y, int):
|
||||||
|
if y == token_dict_size: #EOS token
|
||||||
|
y = -1
|
||||||
tokens_temp += [y]
|
tokens_temp += [y]
|
||||||
else:
|
else:
|
||||||
if y.shape[0] == current_embeds.weight.shape[1]:
|
if y.shape[0] == current_embeds.weight.shape[1]:
|
||||||
@ -110,15 +116,21 @@ class SD1ClipModel(torch.nn.Module, ClipTokenWeightEncoder):
|
|||||||
tokens_temp += [self.empty_tokens[0][-1]]
|
tokens_temp += [self.empty_tokens[0][-1]]
|
||||||
out_tokens += [tokens_temp]
|
out_tokens += [tokens_temp]
|
||||||
|
|
||||||
|
n = token_dict_size
|
||||||
if len(embedding_weights) > 0:
|
if len(embedding_weights) > 0:
|
||||||
new_embedding = torch.nn.Embedding(next_new_token, current_embeds.weight.shape[1], device=current_embeds.weight.device, dtype=current_embeds.weight.dtype)
|
new_embedding = torch.nn.Embedding(next_new_token + 1, current_embeds.weight.shape[1], device=current_embeds.weight.device, dtype=current_embeds.weight.dtype)
|
||||||
new_embedding.weight[:token_dict_size] = current_embeds.weight[:]
|
new_embedding.weight[:token_dict_size] = current_embeds.weight[:-1]
|
||||||
n = token_dict_size
|
|
||||||
for x in embedding_weights:
|
for x in embedding_weights:
|
||||||
new_embedding.weight[n] = x
|
new_embedding.weight[n] = x
|
||||||
n += 1
|
n += 1
|
||||||
|
new_embedding.weight[n] = current_embeds.weight[-1] #EOS embedding
|
||||||
self.transformer.set_input_embeddings(new_embedding)
|
self.transformer.set_input_embeddings(new_embedding)
|
||||||
return out_tokens
|
|
||||||
|
processed_tokens = []
|
||||||
|
for x in out_tokens:
|
||||||
|
processed_tokens += [list(map(lambda a: n if a == -1 else a, x))] #The EOS token should always be the largest one
|
||||||
|
|
||||||
|
return processed_tokens
|
||||||
|
|
||||||
def forward(self, tokens):
|
def forward(self, tokens):
|
||||||
backup_embeds = self.transformer.get_input_embeddings()
|
backup_embeds = self.transformer.get_input_embeddings()
|
||||||
@ -129,9 +141,9 @@ class SD1ClipModel(torch.nn.Module, ClipTokenWeightEncoder):
|
|||||||
if backup_embeds.weight.dtype != torch.float32:
|
if backup_embeds.weight.dtype != torch.float32:
|
||||||
precision_scope = torch.autocast
|
precision_scope = torch.autocast
|
||||||
else:
|
else:
|
||||||
precision_scope = contextlib.nullcontext
|
precision_scope = lambda a, b: contextlib.nullcontext(a)
|
||||||
|
|
||||||
with precision_scope(model_management.get_autocast_device(device)):
|
with precision_scope(model_management.get_autocast_device(device), torch.float32):
|
||||||
outputs = self.transformer(input_ids=tokens, output_hidden_states=self.layer=="hidden")
|
outputs = self.transformer(input_ids=tokens, output_hidden_states=self.layer=="hidden")
|
||||||
self.transformer.set_input_embeddings(backup_embeds)
|
self.transformer.set_input_embeddings(backup_embeds)
|
||||||
|
|
||||||
@ -146,13 +158,17 @@ class SD1ClipModel(torch.nn.Module, ClipTokenWeightEncoder):
|
|||||||
|
|
||||||
pooled_output = outputs.pooler_output
|
pooled_output = outputs.pooler_output
|
||||||
if self.text_projection is not None:
|
if self.text_projection is not None:
|
||||||
pooled_output = pooled_output.to(self.text_projection.device) @ self.text_projection
|
pooled_output = pooled_output.float().to(self.text_projection.device) @ self.text_projection.float()
|
||||||
return z.float(), pooled_output.float()
|
return z.float(), pooled_output.float()
|
||||||
|
|
||||||
def encode(self, tokens):
|
def encode(self, tokens):
|
||||||
return self(tokens)
|
return self(tokens)
|
||||||
|
|
||||||
def load_sd(self, sd):
|
def load_sd(self, sd):
|
||||||
|
if "text_projection" in sd:
|
||||||
|
self.text_projection[:] = sd.pop("text_projection")
|
||||||
|
if "text_projection.weight" in sd:
|
||||||
|
self.text_projection[:] = sd.pop("text_projection.weight").transpose(0, 1)
|
||||||
return self.transformer.load_state_dict(sd, strict=False)
|
return self.transformer.load_state_dict(sd, strict=False)
|
||||||
|
|
||||||
def parse_parentheses(string):
|
def parse_parentheses(string):
|
||||||
|
|||||||
@ -3,13 +3,13 @@ import torch
|
|||||||
import os
|
import os
|
||||||
|
|
||||||
class SD2ClipModel(sd1_clip.SD1ClipModel):
|
class SD2ClipModel(sd1_clip.SD1ClipModel):
|
||||||
def __init__(self, arch="ViT-H-14", device="cpu", max_length=77, freeze=True, layer="penultimate", layer_idx=None, textmodel_path=None):
|
def __init__(self, arch="ViT-H-14", device="cpu", max_length=77, freeze=True, layer="penultimate", layer_idx=None, textmodel_path=None, dtype=None):
|
||||||
if layer == "penultimate":
|
if layer == "penultimate":
|
||||||
layer="hidden"
|
layer="hidden"
|
||||||
layer_idx=23
|
layer_idx=23
|
||||||
|
|
||||||
textmodel_json_config = os.path.join(os.path.dirname(os.path.realpath(__file__)), "sd2_clip_config.json")
|
textmodel_json_config = os.path.join(os.path.dirname(os.path.realpath(__file__)), "sd2_clip_config.json")
|
||||||
super().__init__(device=device, freeze=freeze, layer=layer, layer_idx=layer_idx, textmodel_json_config=textmodel_json_config, textmodel_path=textmodel_path)
|
super().__init__(device=device, freeze=freeze, layer=layer, layer_idx=layer_idx, textmodel_json_config=textmodel_json_config, textmodel_path=textmodel_path, dtype=dtype)
|
||||||
self.empty_tokens = [[49406] + [49407] + [0] * 75]
|
self.empty_tokens = [[49406] + [49407] + [0] * 75]
|
||||||
|
|
||||||
def clip_layer(self, layer_idx):
|
def clip_layer(self, layer_idx):
|
||||||
|
|||||||
@ -17,7 +17,7 @@
|
|||||||
"num_attention_heads": 16,
|
"num_attention_heads": 16,
|
||||||
"num_hidden_layers": 24,
|
"num_hidden_layers": 24,
|
||||||
"pad_token_id": 1,
|
"pad_token_id": 1,
|
||||||
"projection_dim": 512,
|
"projection_dim": 1024,
|
||||||
"torch_dtype": "float32",
|
"torch_dtype": "float32",
|
||||||
"vocab_size": 49408
|
"vocab_size": 49408
|
||||||
}
|
}
|
||||||
|
|||||||
@ -3,23 +3,17 @@ import torch
|
|||||||
import os
|
import os
|
||||||
|
|
||||||
class SDXLClipG(sd1_clip.SD1ClipModel):
|
class SDXLClipG(sd1_clip.SD1ClipModel):
|
||||||
def __init__(self, device="cpu", max_length=77, freeze=True, layer="penultimate", layer_idx=None, textmodel_path=None):
|
def __init__(self, device="cpu", max_length=77, freeze=True, layer="penultimate", layer_idx=None, textmodel_path=None, dtype=None):
|
||||||
if layer == "penultimate":
|
if layer == "penultimate":
|
||||||
layer="hidden"
|
layer="hidden"
|
||||||
layer_idx=-2
|
layer_idx=-2
|
||||||
|
|
||||||
textmodel_json_config = os.path.join(os.path.dirname(os.path.realpath(__file__)), "clip_config_bigg.json")
|
textmodel_json_config = os.path.join(os.path.dirname(os.path.realpath(__file__)), "clip_config_bigg.json")
|
||||||
super().__init__(device=device, freeze=freeze, layer=layer, layer_idx=layer_idx, textmodel_json_config=textmodel_json_config, textmodel_path=textmodel_path)
|
super().__init__(device=device, freeze=freeze, layer=layer, layer_idx=layer_idx, textmodel_json_config=textmodel_json_config, textmodel_path=textmodel_path, dtype=dtype)
|
||||||
self.empty_tokens = [[49406] + [49407] + [0] * 75]
|
self.empty_tokens = [[49406] + [49407] + [0] * 75]
|
||||||
self.text_projection = torch.nn.Parameter(torch.empty(1280, 1280))
|
|
||||||
self.logit_scale = torch.nn.Parameter(torch.tensor(4.6055))
|
|
||||||
self.layer_norm_hidden_state = False
|
self.layer_norm_hidden_state = False
|
||||||
|
|
||||||
def load_sd(self, sd):
|
def load_sd(self, sd):
|
||||||
if "text_projection" in sd:
|
|
||||||
self.text_projection[:] = sd.pop("text_projection")
|
|
||||||
if "text_projection.weight" in sd:
|
|
||||||
self.text_projection[:] = sd.pop("text_projection.weight").transpose(0, 1)
|
|
||||||
return super().load_sd(sd)
|
return super().load_sd(sd)
|
||||||
|
|
||||||
class SDXLClipGTokenizer(sd1_clip.SD1Tokenizer):
|
class SDXLClipGTokenizer(sd1_clip.SD1Tokenizer):
|
||||||
@ -42,11 +36,11 @@ class SDXLTokenizer(sd1_clip.SD1Tokenizer):
|
|||||||
return self.clip_g.untokenize(token_weight_pair)
|
return self.clip_g.untokenize(token_weight_pair)
|
||||||
|
|
||||||
class SDXLClipModel(torch.nn.Module):
|
class SDXLClipModel(torch.nn.Module):
|
||||||
def __init__(self, device="cpu"):
|
def __init__(self, device="cpu", dtype=None):
|
||||||
super().__init__()
|
super().__init__()
|
||||||
self.clip_l = sd1_clip.SD1ClipModel(layer="hidden", layer_idx=11, device=device)
|
self.clip_l = sd1_clip.SD1ClipModel(layer="hidden", layer_idx=11, device=device, dtype=dtype)
|
||||||
self.clip_l.layer_norm_hidden_state = False
|
self.clip_l.layer_norm_hidden_state = False
|
||||||
self.clip_g = SDXLClipG(device=device)
|
self.clip_g = SDXLClipG(device=device, dtype=dtype)
|
||||||
|
|
||||||
def clip_layer(self, layer_idx):
|
def clip_layer(self, layer_idx):
|
||||||
self.clip_l.clip_layer(layer_idx)
|
self.clip_l.clip_layer(layer_idx)
|
||||||
@ -70,9 +64,9 @@ class SDXLClipModel(torch.nn.Module):
|
|||||||
return self.clip_l.load_sd(sd)
|
return self.clip_l.load_sd(sd)
|
||||||
|
|
||||||
class SDXLRefinerClipModel(torch.nn.Module):
|
class SDXLRefinerClipModel(torch.nn.Module):
|
||||||
def __init__(self, device="cpu"):
|
def __init__(self, device="cpu", dtype=None):
|
||||||
super().__init__()
|
super().__init__()
|
||||||
self.clip_g = SDXLClipG(device=device)
|
self.clip_g = SDXLClipG(device=device, dtype=dtype)
|
||||||
|
|
||||||
def clip_layer(self, layer_idx):
|
def clip_layer(self, layer_idx):
|
||||||
self.clip_g.clip_layer(layer_idx)
|
self.clip_g.clip_layer(layer_idx)
|
||||||
|
|||||||
@ -1,6 +1,7 @@
|
|||||||
import torch
|
import torch
|
||||||
from . import model_base
|
from . import model_base
|
||||||
from . import utils
|
from . import utils
|
||||||
|
from . import latent_formats
|
||||||
|
|
||||||
|
|
||||||
def state_dict_key_replace(state_dict, keys_to_replace):
|
def state_dict_key_replace(state_dict, keys_to_replace):
|
||||||
@ -33,6 +34,8 @@ class BASE:
|
|||||||
clip_prefix = []
|
clip_prefix = []
|
||||||
clip_vision_prefix = None
|
clip_vision_prefix = None
|
||||||
noise_aug_config = None
|
noise_aug_config = None
|
||||||
|
beta_schedule = "linear"
|
||||||
|
latent_format = latent_formats.LatentFormat
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def matches(s, unet_config):
|
def matches(s, unet_config):
|
||||||
|
|||||||
@ -101,17 +101,30 @@ class ResnetBlock(nn.Module):
|
|||||||
|
|
||||||
|
|
||||||
class Adapter(nn.Module):
|
class Adapter(nn.Module):
|
||||||
def __init__(self, channels=[320, 640, 1280, 1280], nums_rb=3, cin=64, ksize=3, sk=False, use_conv=True):
|
def __init__(self, channels=[320, 640, 1280, 1280], nums_rb=3, cin=64, ksize=3, sk=False, use_conv=True, xl=True):
|
||||||
super(Adapter, self).__init__()
|
super(Adapter, self).__init__()
|
||||||
self.unshuffle = nn.PixelUnshuffle(8)
|
self.unshuffle_amount = 8
|
||||||
|
resblock_no_downsample = []
|
||||||
|
resblock_downsample = [3, 2, 1]
|
||||||
|
self.xl = xl
|
||||||
|
if self.xl:
|
||||||
|
self.unshuffle_amount = 16
|
||||||
|
resblock_no_downsample = [1]
|
||||||
|
resblock_downsample = [2]
|
||||||
|
|
||||||
|
self.input_channels = cin // (self.unshuffle_amount * self.unshuffle_amount)
|
||||||
|
self.unshuffle = nn.PixelUnshuffle(self.unshuffle_amount)
|
||||||
self.channels = channels
|
self.channels = channels
|
||||||
self.nums_rb = nums_rb
|
self.nums_rb = nums_rb
|
||||||
self.body = []
|
self.body = []
|
||||||
for i in range(len(channels)):
|
for i in range(len(channels)):
|
||||||
for j in range(nums_rb):
|
for j in range(nums_rb):
|
||||||
if (i != 0) and (j == 0):
|
if (i in resblock_downsample) and (j == 0):
|
||||||
self.body.append(
|
self.body.append(
|
||||||
ResnetBlock(channels[i - 1], channels[i], down=True, ksize=ksize, sk=sk, use_conv=use_conv))
|
ResnetBlock(channels[i - 1], channels[i], down=True, ksize=ksize, sk=sk, use_conv=use_conv))
|
||||||
|
elif (i in resblock_no_downsample) and (j == 0):
|
||||||
|
self.body.append(
|
||||||
|
ResnetBlock(channels[i - 1], channels[i], down=False, ksize=ksize, sk=sk, use_conv=use_conv))
|
||||||
else:
|
else:
|
||||||
self.body.append(
|
self.body.append(
|
||||||
ResnetBlock(channels[i], channels[i], down=False, ksize=ksize, sk=sk, use_conv=use_conv))
|
ResnetBlock(channels[i], channels[i], down=False, ksize=ksize, sk=sk, use_conv=use_conv))
|
||||||
@ -128,6 +141,16 @@ class Adapter(nn.Module):
|
|||||||
for j in range(self.nums_rb):
|
for j in range(self.nums_rb):
|
||||||
idx = i * self.nums_rb + j
|
idx = i * self.nums_rb + j
|
||||||
x = self.body[idx](x)
|
x = self.body[idx](x)
|
||||||
|
if self.xl:
|
||||||
|
features.append(None)
|
||||||
|
if i == 0:
|
||||||
|
features.append(None)
|
||||||
|
features.append(None)
|
||||||
|
if i == 2:
|
||||||
|
features.append(None)
|
||||||
|
else:
|
||||||
|
features.append(None)
|
||||||
|
features.append(None)
|
||||||
features.append(x)
|
features.append(x)
|
||||||
|
|
||||||
return features
|
return features
|
||||||
@ -241,10 +264,14 @@ class extractor(nn.Module):
|
|||||||
class Adapter_light(nn.Module):
|
class Adapter_light(nn.Module):
|
||||||
def __init__(self, channels=[320, 640, 1280, 1280], nums_rb=3, cin=64):
|
def __init__(self, channels=[320, 640, 1280, 1280], nums_rb=3, cin=64):
|
||||||
super(Adapter_light, self).__init__()
|
super(Adapter_light, self).__init__()
|
||||||
self.unshuffle = nn.PixelUnshuffle(8)
|
self.unshuffle_amount = 8
|
||||||
|
self.unshuffle = nn.PixelUnshuffle(self.unshuffle_amount)
|
||||||
|
self.input_channels = cin // (self.unshuffle_amount * self.unshuffle_amount)
|
||||||
self.channels = channels
|
self.channels = channels
|
||||||
self.nums_rb = nums_rb
|
self.nums_rb = nums_rb
|
||||||
self.body = []
|
self.body = []
|
||||||
|
self.xl = False
|
||||||
|
|
||||||
for i in range(len(channels)):
|
for i in range(len(channels)):
|
||||||
if i == 0:
|
if i == 0:
|
||||||
self.body.append(extractor(in_c=cin, inter_c=channels[i]//4, out_c=channels[i], nums_rb=nums_rb, down=False))
|
self.body.append(extractor(in_c=cin, inter_c=channels[i]//4, out_c=channels[i], nums_rb=nums_rb, down=False))
|
||||||
@ -259,6 +286,8 @@ class Adapter_light(nn.Module):
|
|||||||
features = []
|
features = []
|
||||||
for i in range(len(self.channels)):
|
for i in range(len(self.channels)):
|
||||||
x = self.body[i](x)
|
x = self.body[i](x)
|
||||||
|
features.append(None)
|
||||||
|
features.append(None)
|
||||||
features.append(x)
|
features.append(x)
|
||||||
|
|
||||||
return features
|
return features
|
||||||
|
|||||||
@ -32,6 +32,13 @@ def save_torch_file(sd, ckpt, metadata=None):
|
|||||||
else:
|
else:
|
||||||
safetensors.torch.save_file(sd, ckpt)
|
safetensors.torch.save_file(sd, ckpt)
|
||||||
|
|
||||||
|
def calculate_parameters(sd, prefix=""):
|
||||||
|
params = 0
|
||||||
|
for k in sd.keys():
|
||||||
|
if k.startswith(prefix):
|
||||||
|
params += sd[k].nelement()
|
||||||
|
return params
|
||||||
|
|
||||||
def transformers_convert(sd, prefix_from, prefix_to, number):
|
def transformers_convert(sd, prefix_from, prefix_to, number):
|
||||||
keys_to_replace = {
|
keys_to_replace = {
|
||||||
"{}positional_embedding": "{}embeddings.position_embedding.weight",
|
"{}positional_embedding": "{}embeddings.position_embedding.weight",
|
||||||
@ -230,6 +237,20 @@ def safetensors_header(safetensors_path, max_size=100*1024*1024):
|
|||||||
return None
|
return None
|
||||||
return f.read(length_of_header)
|
return f.read(length_of_header)
|
||||||
|
|
||||||
|
def set_attr(obj, attr, value):
|
||||||
|
attrs = attr.split(".")
|
||||||
|
for name in attrs[:-1]:
|
||||||
|
obj = getattr(obj, name)
|
||||||
|
prev = getattr(obj, attrs[-1])
|
||||||
|
setattr(obj, attrs[-1], torch.nn.Parameter(value))
|
||||||
|
del prev
|
||||||
|
|
||||||
|
def get_attr(obj, attr):
|
||||||
|
attrs = attr.split(".")
|
||||||
|
for name in attrs:
|
||||||
|
obj = getattr(obj, name)
|
||||||
|
return obj
|
||||||
|
|
||||||
def bislerp(samples, width, height):
|
def bislerp(samples, width, height):
|
||||||
def slerp(b1, b2, r):
|
def slerp(b1, b2, r):
|
||||||
'''slerps batches b1, b2 according to ratio r, batches should be flat e.g. NxC'''
|
'''slerps batches b1, b2 according to ratio r, batches should be flat e.g. NxC'''
|
||||||
|
|||||||
@ -3,7 +3,7 @@ import math
|
|||||||
|
|
||||||
import torch
|
import torch
|
||||||
import torch.nn.functional as F
|
import torch.nn.functional as F
|
||||||
|
import comfy.model_management
|
||||||
|
|
||||||
def get_canny_nms_kernel(device=None, dtype=None):
|
def get_canny_nms_kernel(device=None, dtype=None):
|
||||||
"""Utility function that returns 3x3 kernels for the Canny Non-maximal suppression."""
|
"""Utility function that returns 3x3 kernels for the Canny Non-maximal suppression."""
|
||||||
@ -290,8 +290,8 @@ class Canny:
|
|||||||
CATEGORY = "image/preprocessors"
|
CATEGORY = "image/preprocessors"
|
||||||
|
|
||||||
def detect_edge(self, image, low_threshold, high_threshold):
|
def detect_edge(self, image, low_threshold, high_threshold):
|
||||||
output = canny(image.movedim(-1, 1), low_threshold, high_threshold)
|
output = canny(image.to(comfy.model_management.get_torch_device()).movedim(-1, 1), low_threshold, high_threshold)
|
||||||
img_out = output[1].repeat(1, 3, 1, 1).movedim(1, -1)
|
img_out = output[1].cpu().repeat(1, 3, 1, 1).movedim(1, -1)
|
||||||
return (img_out,)
|
return (img_out,)
|
||||||
|
|
||||||
NODE_CLASS_MAPPINGS = {
|
NODE_CLASS_MAPPINGS = {
|
||||||
|
|||||||
@ -1,7 +1,40 @@
|
|||||||
|
import numpy as np
|
||||||
|
from scipy.ndimage import grey_dilation
|
||||||
import torch
|
import torch
|
||||||
|
|
||||||
from nodes import MAX_RESOLUTION
|
from nodes import MAX_RESOLUTION
|
||||||
|
|
||||||
|
def composite(destination, source, x, y, mask = None, multiplier = 8, resize_source = False):
|
||||||
|
if resize_source:
|
||||||
|
source = torch.nn.functional.interpolate(source, size=(destination.shape[2], destination.shape[3]), mode="bilinear")
|
||||||
|
|
||||||
|
x = max(-source.shape[3] * multiplier, min(x, destination.shape[3] * multiplier))
|
||||||
|
y = max(-source.shape[2] * multiplier, min(y, destination.shape[2] * multiplier))
|
||||||
|
|
||||||
|
left, top = (x // multiplier, y // multiplier)
|
||||||
|
right, bottom = (left + source.shape[3], top + source.shape[2],)
|
||||||
|
|
||||||
|
if mask is None:
|
||||||
|
mask = torch.ones_like(source)
|
||||||
|
else:
|
||||||
|
mask = mask.clone()
|
||||||
|
mask = torch.nn.functional.interpolate(mask[None, None], size=(source.shape[2], source.shape[3]), mode="bilinear")
|
||||||
|
mask = mask.repeat((source.shape[0], source.shape[1], 1, 1))
|
||||||
|
|
||||||
|
# calculate the bounds of the source that will be overlapping the destination
|
||||||
|
# this prevents the source trying to overwrite latent pixels that are out of bounds
|
||||||
|
# of the destination
|
||||||
|
visible_width, visible_height = (destination.shape[3] - left + min(0, x), destination.shape[2] - top + min(0, y),)
|
||||||
|
|
||||||
|
mask = mask[:, :, :visible_height, :visible_width]
|
||||||
|
inverse_mask = torch.ones_like(mask) - mask
|
||||||
|
|
||||||
|
source_portion = mask * source[:, :, :visible_height, :visible_width]
|
||||||
|
destination_portion = inverse_mask * destination[:, :, top:bottom, left:right]
|
||||||
|
|
||||||
|
destination[:, :, top:bottom, left:right] = source_portion + destination_portion
|
||||||
|
return destination
|
||||||
|
|
||||||
class LatentCompositeMasked:
|
class LatentCompositeMasked:
|
||||||
@classmethod
|
@classmethod
|
||||||
def INPUT_TYPES(s):
|
def INPUT_TYPES(s):
|
||||||
@ -11,6 +44,7 @@ class LatentCompositeMasked:
|
|||||||
"source": ("LATENT",),
|
"source": ("LATENT",),
|
||||||
"x": ("INT", {"default": 0, "min": 0, "max": MAX_RESOLUTION, "step": 8}),
|
"x": ("INT", {"default": 0, "min": 0, "max": MAX_RESOLUTION, "step": 8}),
|
||||||
"y": ("INT", {"default": 0, "min": 0, "max": MAX_RESOLUTION, "step": 8}),
|
"y": ("INT", {"default": 0, "min": 0, "max": MAX_RESOLUTION, "step": 8}),
|
||||||
|
"resize_source": ("BOOLEAN", {"default": False}),
|
||||||
},
|
},
|
||||||
"optional": {
|
"optional": {
|
||||||
"mask": ("MASK",),
|
"mask": ("MASK",),
|
||||||
@ -21,40 +55,36 @@ class LatentCompositeMasked:
|
|||||||
|
|
||||||
CATEGORY = "latent"
|
CATEGORY = "latent"
|
||||||
|
|
||||||
def composite(self, destination, source, x, y, mask = None):
|
def composite(self, destination, source, x, y, resize_source, mask = None):
|
||||||
output = destination.copy()
|
output = destination.copy()
|
||||||
destination = destination["samples"].clone()
|
destination = destination["samples"].clone()
|
||||||
source = source["samples"]
|
source = source["samples"]
|
||||||
|
output["samples"] = composite(destination, source, x, y, mask, 8, resize_source)
|
||||||
|
return (output,)
|
||||||
|
|
||||||
x = max(-source.shape[3] * 8, min(x, destination.shape[3] * 8))
|
class ImageCompositeMasked:
|
||||||
y = max(-source.shape[2] * 8, min(y, destination.shape[2] * 8))
|
@classmethod
|
||||||
|
def INPUT_TYPES(s):
|
||||||
|
return {
|
||||||
|
"required": {
|
||||||
|
"destination": ("IMAGE",),
|
||||||
|
"source": ("IMAGE",),
|
||||||
|
"x": ("INT", {"default": 0, "min": 0, "max": MAX_RESOLUTION, "step": 1}),
|
||||||
|
"y": ("INT", {"default": 0, "min": 0, "max": MAX_RESOLUTION, "step": 1}),
|
||||||
|
"resize_source": ("BOOLEAN", {"default": False}),
|
||||||
|
},
|
||||||
|
"optional": {
|
||||||
|
"mask": ("MASK",),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
RETURN_TYPES = ("IMAGE",)
|
||||||
|
FUNCTION = "composite"
|
||||||
|
|
||||||
left, top = (x // 8, y // 8)
|
CATEGORY = "image"
|
||||||
right, bottom = (left + source.shape[3], top + source.shape[2],)
|
|
||||||
|
|
||||||
|
|
||||||
if mask is None:
|
|
||||||
mask = torch.ones_like(source)
|
|
||||||
else:
|
|
||||||
mask = mask.clone()
|
|
||||||
mask = torch.nn.functional.interpolate(mask[None, None], size=(source.shape[2], source.shape[3]), mode="bilinear")
|
|
||||||
mask = mask.repeat((source.shape[0], source.shape[1], 1, 1))
|
|
||||||
|
|
||||||
# calculate the bounds of the source that will be overlapping the destination
|
|
||||||
# this prevents the source trying to overwrite latent pixels that are out of bounds
|
|
||||||
# of the destination
|
|
||||||
visible_width, visible_height = (destination.shape[3] - left + min(0, x), destination.shape[2] - top + min(0, y),)
|
|
||||||
|
|
||||||
mask = mask[:, :, :visible_height, :visible_width]
|
|
||||||
inverse_mask = torch.ones_like(mask) - mask
|
|
||||||
|
|
||||||
source_portion = mask * source[:, :, :visible_height, :visible_width]
|
|
||||||
destination_portion = inverse_mask * destination[:, :, top:bottom, left:right]
|
|
||||||
|
|
||||||
destination[:, :, top:bottom, left:right] = source_portion + destination_portion
|
|
||||||
|
|
||||||
output["samples"] = destination
|
|
||||||
|
|
||||||
|
def composite(self, destination, source, x, y, resize_source, mask = None):
|
||||||
|
destination = destination.clone().movedim(-1, 1)
|
||||||
|
output = composite(destination, source.movedim(-1, 1), x, y, mask, 1, resize_source).movedim(1, -1)
|
||||||
return (output,)
|
return (output,)
|
||||||
|
|
||||||
class MaskToImage:
|
class MaskToImage:
|
||||||
@ -95,6 +125,27 @@ class ImageToMask:
|
|||||||
mask = image[0, :, :, channels.index(channel)]
|
mask = image[0, :, :, channels.index(channel)]
|
||||||
return (mask,)
|
return (mask,)
|
||||||
|
|
||||||
|
class ImageColorToMask:
|
||||||
|
@classmethod
|
||||||
|
def INPUT_TYPES(s):
|
||||||
|
return {
|
||||||
|
"required": {
|
||||||
|
"image": ("IMAGE",),
|
||||||
|
"color": ("INT", {"default": 0, "min": 0, "max": 0xFFFFFF, "step": 1, "display": "color"}),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
CATEGORY = "mask"
|
||||||
|
|
||||||
|
RETURN_TYPES = ("MASK",)
|
||||||
|
FUNCTION = "image_to_mask"
|
||||||
|
|
||||||
|
def image_to_mask(self, image, color):
|
||||||
|
temp = (torch.clamp(image[0], 0, 1.0) * 255.0).round().to(torch.int)
|
||||||
|
temp = torch.bitwise_left_shift(temp[:,:,0], 16) + torch.bitwise_left_shift(temp[:,:,1], 8) + temp[:,:,2]
|
||||||
|
mask = torch.where(temp == color, 255, 0).float()
|
||||||
|
return (mask,)
|
||||||
|
|
||||||
class SolidMask:
|
class SolidMask:
|
||||||
@classmethod
|
@classmethod
|
||||||
def INPUT_TYPES(cls):
|
def INPUT_TYPES(cls):
|
||||||
@ -248,18 +299,50 @@ class FeatherMask:
|
|||||||
output[-y, :] *= feather_rate
|
output[-y, :] *= feather_rate
|
||||||
|
|
||||||
return (output,)
|
return (output,)
|
||||||
|
|
||||||
|
class GrowMask:
|
||||||
|
@classmethod
|
||||||
|
def INPUT_TYPES(cls):
|
||||||
|
return {
|
||||||
|
"required": {
|
||||||
|
"mask": ("MASK",),
|
||||||
|
"expand": ("INT", {"default": 0, "min": 0, "max": MAX_RESOLUTION, "step": 1}),
|
||||||
|
"tapered_corners": ("BOOLEAN", {"default": True}),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
CATEGORY = "mask"
|
||||||
|
|
||||||
|
RETURN_TYPES = ("MASK",)
|
||||||
|
|
||||||
|
FUNCTION = "expand_mask"
|
||||||
|
|
||||||
|
def expand_mask(self, mask, expand, tapered_corners):
|
||||||
|
c = 0 if tapered_corners else 1
|
||||||
|
kernel = np.array([[c, 1, c],
|
||||||
|
[1, 1, 1],
|
||||||
|
[c, 1, c]])
|
||||||
|
output = mask.numpy().copy()
|
||||||
|
while expand > 0:
|
||||||
|
output = grey_dilation(output, footprint=kernel)
|
||||||
|
expand -= 1
|
||||||
|
output = torch.from_numpy(output)
|
||||||
|
return (output,)
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
NODE_CLASS_MAPPINGS = {
|
NODE_CLASS_MAPPINGS = {
|
||||||
"LatentCompositeMasked": LatentCompositeMasked,
|
"LatentCompositeMasked": LatentCompositeMasked,
|
||||||
|
"ImageCompositeMasked": ImageCompositeMasked,
|
||||||
"MaskToImage": MaskToImage,
|
"MaskToImage": MaskToImage,
|
||||||
"ImageToMask": ImageToMask,
|
"ImageToMask": ImageToMask,
|
||||||
|
"ImageColorToMask": ImageColorToMask,
|
||||||
"SolidMask": SolidMask,
|
"SolidMask": SolidMask,
|
||||||
"InvertMask": InvertMask,
|
"InvertMask": InvertMask,
|
||||||
"CropMask": CropMask,
|
"CropMask": CropMask,
|
||||||
"MaskComposite": MaskComposite,
|
"MaskComposite": MaskComposite,
|
||||||
"FeatherMask": FeatherMask,
|
"FeatherMask": FeatherMask,
|
||||||
|
"GrowMask": GrowMask,
|
||||||
}
|
}
|
||||||
|
|
||||||
NODE_DISPLAY_NAME_MAPPINGS = {
|
NODE_DISPLAY_NAME_MAPPINGS = {
|
||||||
|
|||||||
@ -2,6 +2,7 @@ import numpy as np
|
|||||||
import torch
|
import torch
|
||||||
import torch.nn.functional as F
|
import torch.nn.functional as F
|
||||||
from PIL import Image
|
from PIL import Image
|
||||||
|
import math
|
||||||
|
|
||||||
import comfy.utils
|
import comfy.utils
|
||||||
|
|
||||||
@ -59,8 +60,8 @@ class Blend:
|
|||||||
def g(self, x):
|
def g(self, x):
|
||||||
return torch.where(x <= 0.25, ((16 * x - 12) * x + 4) * x, torch.sqrt(x))
|
return torch.where(x <= 0.25, ((16 * x - 12) * x + 4) * x, torch.sqrt(x))
|
||||||
|
|
||||||
def gaussian_kernel(kernel_size: int, sigma: float):
|
def gaussian_kernel(kernel_size: int, sigma: float, device=None):
|
||||||
x, y = torch.meshgrid(torch.linspace(-1, 1, kernel_size), torch.linspace(-1, 1, kernel_size), indexing="ij")
|
x, y = torch.meshgrid(torch.linspace(-1, 1, kernel_size, device=device), torch.linspace(-1, 1, kernel_size, device=device), indexing="ij")
|
||||||
d = torch.sqrt(x * x + y * y)
|
d = torch.sqrt(x * x + y * y)
|
||||||
g = torch.exp(-(d * d) / (2.0 * sigma * sigma))
|
g = torch.exp(-(d * d) / (2.0 * sigma * sigma))
|
||||||
return g / g.sum()
|
return g / g.sum()
|
||||||
@ -101,7 +102,7 @@ class Blur:
|
|||||||
batch_size, height, width, channels = image.shape
|
batch_size, height, width, channels = image.shape
|
||||||
|
|
||||||
kernel_size = blur_radius * 2 + 1
|
kernel_size = blur_radius * 2 + 1
|
||||||
kernel = gaussian_kernel(kernel_size, sigma).repeat(channels, 1, 1).unsqueeze(1)
|
kernel = gaussian_kernel(kernel_size, sigma, device=image.device).repeat(channels, 1, 1).unsqueeze(1)
|
||||||
|
|
||||||
image = image.permute(0, 3, 1, 2) # Torch wants (B, C, H, W) we use (B, H, W, C)
|
image = image.permute(0, 3, 1, 2) # Torch wants (B, C, H, W) we use (B, H, W, C)
|
||||||
padded_image = F.pad(image, (blur_radius,blur_radius,blur_radius,blur_radius), 'reflect')
|
padded_image = F.pad(image, (blur_radius,blur_radius,blur_radius,blur_radius), 'reflect')
|
||||||
@ -209,9 +210,36 @@ class Sharpen:
|
|||||||
|
|
||||||
return (result,)
|
return (result,)
|
||||||
|
|
||||||
|
class ImageScaleToTotalPixels:
|
||||||
|
upscale_methods = ["nearest-exact", "bilinear", "area", "bicubic"]
|
||||||
|
crop_methods = ["disabled", "center"]
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def INPUT_TYPES(s):
|
||||||
|
return {"required": { "image": ("IMAGE",), "upscale_method": (s.upscale_methods,),
|
||||||
|
"megapixels": ("FLOAT", {"default": 1.0, "min": 0.01, "max": 16.0, "step": 0.01}),
|
||||||
|
}}
|
||||||
|
RETURN_TYPES = ("IMAGE",)
|
||||||
|
FUNCTION = "upscale"
|
||||||
|
|
||||||
|
CATEGORY = "image/upscaling"
|
||||||
|
|
||||||
|
def upscale(self, image, upscale_method, megapixels):
|
||||||
|
samples = image.movedim(-1,1)
|
||||||
|
total = int(megapixels * 1024 * 1024)
|
||||||
|
|
||||||
|
scale_by = math.sqrt(total / (samples.shape[3] * samples.shape[2]))
|
||||||
|
width = round(samples.shape[3] * scale_by)
|
||||||
|
height = round(samples.shape[2] * scale_by)
|
||||||
|
|
||||||
|
s = comfy.utils.common_upscale(samples, width, height, upscale_method, "disabled")
|
||||||
|
s = s.movedim(1,-1)
|
||||||
|
return (s,)
|
||||||
|
|
||||||
NODE_CLASS_MAPPINGS = {
|
NODE_CLASS_MAPPINGS = {
|
||||||
"ImageBlend": Blend,
|
"ImageBlend": Blend,
|
||||||
"ImageBlur": Blur,
|
"ImageBlur": Blur,
|
||||||
"ImageQuantize": Quantize,
|
"ImageQuantize": Quantize,
|
||||||
"ImageSharpen": Sharpen,
|
"ImageSharpen": Sharpen,
|
||||||
|
"ImageScaleToTotalPixels": ImageScaleToTotalPixels,
|
||||||
}
|
}
|
||||||
|
|||||||
@ -36,12 +36,15 @@ def get_gpu_names():
|
|||||||
else:
|
else:
|
||||||
return set()
|
return set()
|
||||||
|
|
||||||
def cuda_malloc_supported():
|
blacklist = {"GeForce GTX TITAN X", "GeForce GTX 980", "GeForce GTX 970", "GeForce GTX 960", "GeForce GTX 950", "GeForce 945M",
|
||||||
blacklist = {"GeForce GTX TITAN X", "GeForce GTX 980", "GeForce GTX 970", "GeForce GTX 960", "GeForce GTX 950", "GeForce 945M",
|
"GeForce 940M", "GeForce 930M", "GeForce 920M", "GeForce 910M", "GeForce GTX 750", "GeForce GTX 745", "Quadro K620",
|
||||||
"GeForce 940M", "GeForce 930M", "GeForce 920M", "GeForce 910M", "GeForce GTX 750", "GeForce GTX 745", "Quadro K620",
|
"Quadro K1200", "Quadro K2200", "Quadro M500", "Quadro M520", "Quadro M600", "Quadro M620", "Quadro M1000",
|
||||||
"Quadro K1200", "Quadro K2200", "Quadro M500", "Quadro M520", "Quadro M600", "Quadro M620", "Quadro M1000",
|
"Quadro M1200", "Quadro M2000", "Quadro M2200", "Quadro M3000", "Quadro M4000", "Quadro M5000", "Quadro M5500", "Quadro M6000",
|
||||||
"Quadro M1200", "Quadro M2000", "Quadro M2200", "Quadro M3000", "Quadro M4000", "Quadro M5000", "Quadro M5500", "Quadro M6000"}
|
"GeForce MX110", "GeForce MX130", "GeForce 830M", "GeForce 840M", "GeForce GTX 850M", "GeForce GTX 860M",
|
||||||
|
"GeForce GTX 1650", "GeForce GTX 1630"
|
||||||
|
}
|
||||||
|
|
||||||
|
def cuda_malloc_supported():
|
||||||
try:
|
try:
|
||||||
names = get_gpu_names()
|
names = get_gpu_names()
|
||||||
except:
|
except:
|
||||||
|
|||||||
@ -51,9 +51,10 @@ class Example:
|
|||||||
"default": 0,
|
"default": 0,
|
||||||
"min": 0, #Minimum value
|
"min": 0, #Minimum value
|
||||||
"max": 4096, #Maximum value
|
"max": 4096, #Maximum value
|
||||||
"step": 64 #Slider's step
|
"step": 64, #Slider's step
|
||||||
|
"display": "number" # Cosmetic only: display as "number" or "slider"
|
||||||
}),
|
}),
|
||||||
"float_field": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 10.0, "step": 0.01}),
|
"float_field": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 10.0, "step": 0.01, "display": "number"}),
|
||||||
"print_to_screen": (["enable", "disable"],),
|
"print_to_screen": (["enable", "disable"],),
|
||||||
"string_field": ("STRING", {
|
"string_field": ("STRING", {
|
||||||
"multiline": False, #True if you want the field to look like the one on the ClipTextEncode node
|
"multiline": False, #True if you want the field to look like the one on the ClipTextEncode node
|
||||||
|
|||||||
18
execution.py
18
execution.py
@ -42,11 +42,14 @@ def get_input_data(inputs, class_def, unique_id, outputs={}, prompt={}, extra_da
|
|||||||
|
|
||||||
def map_node_over_list(obj, input_data_all, func, allow_interrupt=False):
|
def map_node_over_list(obj, input_data_all, func, allow_interrupt=False):
|
||||||
# check if node wants the lists
|
# check if node wants the lists
|
||||||
intput_is_list = False
|
input_is_list = False
|
||||||
if hasattr(obj, "INPUT_IS_LIST"):
|
if hasattr(obj, "INPUT_IS_LIST"):
|
||||||
intput_is_list = obj.INPUT_IS_LIST
|
input_is_list = obj.INPUT_IS_LIST
|
||||||
|
|
||||||
max_len_input = max([len(x) for x in input_data_all.values()])
|
if len(input_data_all) == 0:
|
||||||
|
max_len_input = 0
|
||||||
|
else:
|
||||||
|
max_len_input = max([len(x) for x in input_data_all.values()])
|
||||||
|
|
||||||
# get a slice of inputs, repeat last input when list isn't long enough
|
# get a slice of inputs, repeat last input when list isn't long enough
|
||||||
def slice_dict(d, i):
|
def slice_dict(d, i):
|
||||||
@ -56,11 +59,15 @@ def map_node_over_list(obj, input_data_all, func, allow_interrupt=False):
|
|||||||
return d_new
|
return d_new
|
||||||
|
|
||||||
results = []
|
results = []
|
||||||
if intput_is_list:
|
if input_is_list:
|
||||||
if allow_interrupt:
|
if allow_interrupt:
|
||||||
nodes.before_node_execution()
|
nodes.before_node_execution()
|
||||||
results.append(getattr(obj, func)(**input_data_all))
|
results.append(getattr(obj, func)(**input_data_all))
|
||||||
else:
|
elif max_len_input == 0:
|
||||||
|
if allow_interrupt:
|
||||||
|
nodes.before_node_execution()
|
||||||
|
results.append(getattr(obj, func)())
|
||||||
|
else:
|
||||||
for i in range(max_len_input):
|
for i in range(max_len_input):
|
||||||
if allow_interrupt:
|
if allow_interrupt:
|
||||||
nodes.before_node_execution()
|
nodes.before_node_execution()
|
||||||
@ -347,6 +354,7 @@ class PromptExecutor:
|
|||||||
d = self.outputs_ui.pop(x)
|
d = self.outputs_ui.pop(x)
|
||||||
del d
|
del d
|
||||||
|
|
||||||
|
comfy.model_management.cleanup_models()
|
||||||
if self.server.client_id is not None:
|
if self.server.client_id is not None:
|
||||||
self.server.send_sync("execution_cached", { "nodes": list(current_outputs) , "prompt_id": prompt_id}, self.server.client_id)
|
self.server.send_sync("execution_cached", { "nodes": list(current_outputs) , "prompt_id": prompt_id}, self.server.client_id)
|
||||||
executed = set()
|
executed = set()
|
||||||
|
|||||||
@ -43,6 +43,10 @@ def set_output_directory(output_dir):
|
|||||||
global output_directory
|
global output_directory
|
||||||
output_directory = output_dir
|
output_directory = output_dir
|
||||||
|
|
||||||
|
def set_temp_directory(temp_dir):
|
||||||
|
global temp_directory
|
||||||
|
temp_directory = temp_dir
|
||||||
|
|
||||||
def get_output_directory():
|
def get_output_directory():
|
||||||
global output_directory
|
global output_directory
|
||||||
return output_directory
|
return output_directory
|
||||||
@ -111,6 +115,8 @@ def add_model_folder_path(folder_name, full_folder_path):
|
|||||||
global folder_names_and_paths
|
global folder_names_and_paths
|
||||||
if folder_name in folder_names_and_paths:
|
if folder_name in folder_names_and_paths:
|
||||||
folder_names_and_paths[folder_name][0].append(full_folder_path)
|
folder_names_and_paths[folder_name][0].append(full_folder_path)
|
||||||
|
else:
|
||||||
|
folder_names_and_paths[folder_name] = ([full_folder_path], set())
|
||||||
|
|
||||||
def get_folder_paths(folder_name):
|
def get_folder_paths(folder_name):
|
||||||
return folder_names_and_paths[folder_name][0][:]
|
return folder_names_and_paths[folder_name][0][:]
|
||||||
|
|||||||
22
main.py
22
main.py
@ -72,6 +72,17 @@ from server import BinaryEventTypes
|
|||||||
from nodes import init_custom_nodes
|
from nodes import init_custom_nodes
|
||||||
import comfy.model_management
|
import comfy.model_management
|
||||||
|
|
||||||
|
def cuda_malloc_warning():
|
||||||
|
device = comfy.model_management.get_torch_device()
|
||||||
|
device_name = comfy.model_management.get_torch_device_name(device)
|
||||||
|
cuda_malloc_warning = False
|
||||||
|
if "cudaMallocAsync" in device_name:
|
||||||
|
for b in cuda_malloc.blacklist:
|
||||||
|
if b in device_name:
|
||||||
|
cuda_malloc_warning = True
|
||||||
|
if cuda_malloc_warning:
|
||||||
|
print("\nWARNING: this card most likely does not support cuda-malloc, if you get \"CUDA error\" please run ComfyUI with: --disable-cuda-malloc\n")
|
||||||
|
|
||||||
def prompt_worker(q, server):
|
def prompt_worker(q, server):
|
||||||
e = execution.PromptExecutor(server)
|
e = execution.PromptExecutor(server)
|
||||||
while True:
|
while True:
|
||||||
@ -100,7 +111,7 @@ def hijack_progress(server):
|
|||||||
|
|
||||||
|
|
||||||
def cleanup_temp():
|
def cleanup_temp():
|
||||||
temp_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), "temp")
|
temp_dir = folder_paths.get_temp_directory()
|
||||||
if os.path.exists(temp_dir):
|
if os.path.exists(temp_dir):
|
||||||
shutil.rmtree(temp_dir, ignore_errors=True)
|
shutil.rmtree(temp_dir, ignore_errors=True)
|
||||||
|
|
||||||
@ -127,6 +138,10 @@ def load_extra_path_config(yaml_path):
|
|||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
|
if args.temp_directory:
|
||||||
|
temp_dir = os.path.join(os.path.abspath(args.temp_directory), "temp")
|
||||||
|
print(f"Setting temp directory to: {temp_dir}")
|
||||||
|
folder_paths.set_temp_directory(temp_dir)
|
||||||
cleanup_temp()
|
cleanup_temp()
|
||||||
|
|
||||||
loop = asyncio.new_event_loop()
|
loop = asyncio.new_event_loop()
|
||||||
@ -143,6 +158,9 @@ if __name__ == "__main__":
|
|||||||
load_extra_path_config(config_path)
|
load_extra_path_config(config_path)
|
||||||
|
|
||||||
init_custom_nodes()
|
init_custom_nodes()
|
||||||
|
|
||||||
|
cuda_malloc_warning()
|
||||||
|
|
||||||
server.add_routes()
|
server.add_routes()
|
||||||
hijack_progress(server)
|
hijack_progress(server)
|
||||||
|
|
||||||
@ -160,6 +178,8 @@ if __name__ == "__main__":
|
|||||||
if args.auto_launch:
|
if args.auto_launch:
|
||||||
def startup_server(address, port):
|
def startup_server(address, port):
|
||||||
import webbrowser
|
import webbrowser
|
||||||
|
if os.name == 'nt' and address == '0.0.0.0':
|
||||||
|
address = '127.0.0.1'
|
||||||
webbrowser.open(f"http://{address}:{port}")
|
webbrowser.open(f"http://{address}:{port}")
|
||||||
call_on_start = startup_server
|
call_on_start = startup_server
|
||||||
|
|
||||||
|
|||||||
134
nodes.py
134
nodes.py
@ -22,6 +22,7 @@ import comfy.samplers
|
|||||||
import comfy.sample
|
import comfy.sample
|
||||||
import comfy.sd
|
import comfy.sd
|
||||||
import comfy.utils
|
import comfy.utils
|
||||||
|
import comfy.controlnet
|
||||||
|
|
||||||
import comfy.clip_vision
|
import comfy.clip_vision
|
||||||
|
|
||||||
@ -243,14 +244,16 @@ class VAEDecode:
|
|||||||
class VAEDecodeTiled:
|
class VAEDecodeTiled:
|
||||||
@classmethod
|
@classmethod
|
||||||
def INPUT_TYPES(s):
|
def INPUT_TYPES(s):
|
||||||
return {"required": { "samples": ("LATENT", ), "vae": ("VAE", )}}
|
return {"required": {"samples": ("LATENT", ), "vae": ("VAE", ),
|
||||||
|
"tile_size": ("INT", {"default": 512, "min": 192, "max": 4096, "step": 64})
|
||||||
|
}}
|
||||||
RETURN_TYPES = ("IMAGE",)
|
RETURN_TYPES = ("IMAGE",)
|
||||||
FUNCTION = "decode"
|
FUNCTION = "decode"
|
||||||
|
|
||||||
CATEGORY = "_for_testing"
|
CATEGORY = "_for_testing"
|
||||||
|
|
||||||
def decode(self, vae, samples):
|
def decode(self, vae, samples, tile_size):
|
||||||
return (vae.decode_tiled(samples["samples"]), )
|
return (vae.decode_tiled(samples["samples"], tile_x=tile_size // 8, tile_y=tile_size // 8, ), )
|
||||||
|
|
||||||
class VAEEncode:
|
class VAEEncode:
|
||||||
@classmethod
|
@classmethod
|
||||||
@ -279,15 +282,17 @@ class VAEEncode:
|
|||||||
class VAEEncodeTiled:
|
class VAEEncodeTiled:
|
||||||
@classmethod
|
@classmethod
|
||||||
def INPUT_TYPES(s):
|
def INPUT_TYPES(s):
|
||||||
return {"required": { "pixels": ("IMAGE", ), "vae": ("VAE", )}}
|
return {"required": {"pixels": ("IMAGE", ), "vae": ("VAE", ),
|
||||||
|
"tile_size": ("INT", {"default": 512, "min": 320, "max": 4096, "step": 64})
|
||||||
|
}}
|
||||||
RETURN_TYPES = ("LATENT",)
|
RETURN_TYPES = ("LATENT",)
|
||||||
FUNCTION = "encode"
|
FUNCTION = "encode"
|
||||||
|
|
||||||
CATEGORY = "_for_testing"
|
CATEGORY = "_for_testing"
|
||||||
|
|
||||||
def encode(self, vae, pixels):
|
def encode(self, vae, pixels, tile_size):
|
||||||
pixels = VAEEncode.vae_encode_crop_pixels(pixels)
|
pixels = VAEEncode.vae_encode_crop_pixels(pixels)
|
||||||
t = vae.encode_tiled(pixels[:,:,:,:3])
|
t = vae.encode_tiled(pixels[:,:,:,:3], tile_x=tile_size, tile_y=tile_size, )
|
||||||
return ({"samples":t}, )
|
return ({"samples":t}, )
|
||||||
|
|
||||||
class VAEEncodeForInpaint:
|
class VAEEncodeForInpaint:
|
||||||
@ -362,6 +367,14 @@ class SaveLatent:
|
|||||||
metadata[x] = json.dumps(extra_pnginfo[x])
|
metadata[x] = json.dumps(extra_pnginfo[x])
|
||||||
|
|
||||||
file = f"{filename}_{counter:05}_.latent"
|
file = f"{filename}_{counter:05}_.latent"
|
||||||
|
|
||||||
|
results = list()
|
||||||
|
results.append({
|
||||||
|
"filename": file,
|
||||||
|
"subfolder": subfolder,
|
||||||
|
"type": "output"
|
||||||
|
})
|
||||||
|
|
||||||
file = os.path.join(full_output_folder, file)
|
file = os.path.join(full_output_folder, file)
|
||||||
|
|
||||||
output = {}
|
output = {}
|
||||||
@ -369,7 +382,7 @@ class SaveLatent:
|
|||||||
output["latent_format_version_0"] = torch.tensor([])
|
output["latent_format_version_0"] = torch.tensor([])
|
||||||
|
|
||||||
comfy.utils.save_torch_file(output, file, metadata=metadata)
|
comfy.utils.save_torch_file(output, file, metadata=metadata)
|
||||||
return {}
|
return { "ui": { "latents": results } }
|
||||||
|
|
||||||
|
|
||||||
class LoadLatent:
|
class LoadLatent:
|
||||||
@ -462,7 +475,7 @@ class DiffusersLoader:
|
|||||||
model_path = path
|
model_path = path
|
||||||
break
|
break
|
||||||
|
|
||||||
return comfy.diffusers_load.load_diffusers(model_path, fp16=comfy.model_management.should_use_fp16(), output_vae=output_vae, output_clip=output_clip, embedding_directory=folder_paths.get_folder_paths("embeddings"))
|
return comfy.diffusers_load.load_diffusers(model_path, output_vae=output_vae, output_clip=output_clip, embedding_directory=folder_paths.get_folder_paths("embeddings"))
|
||||||
|
|
||||||
|
|
||||||
class unCLIPCheckpointLoader:
|
class unCLIPCheckpointLoader:
|
||||||
@ -561,7 +574,7 @@ class ControlNetLoader:
|
|||||||
|
|
||||||
def load_controlnet(self, control_net_name):
|
def load_controlnet(self, control_net_name):
|
||||||
controlnet_path = folder_paths.get_full_path("controlnet", control_net_name)
|
controlnet_path = folder_paths.get_full_path("controlnet", control_net_name)
|
||||||
controlnet = comfy.sd.load_controlnet(controlnet_path)
|
controlnet = comfy.controlnet.load_controlnet(controlnet_path)
|
||||||
return (controlnet,)
|
return (controlnet,)
|
||||||
|
|
||||||
class DiffControlNetLoader:
|
class DiffControlNetLoader:
|
||||||
@ -577,7 +590,7 @@ class DiffControlNetLoader:
|
|||||||
|
|
||||||
def load_controlnet(self, model, control_net_name):
|
def load_controlnet(self, model, control_net_name):
|
||||||
controlnet_path = folder_paths.get_full_path("controlnet", control_net_name)
|
controlnet_path = folder_paths.get_full_path("controlnet", control_net_name)
|
||||||
controlnet = comfy.sd.load_controlnet(controlnet_path, model)
|
controlnet = comfy.controlnet.load_controlnet(controlnet_path, model)
|
||||||
return (controlnet,)
|
return (controlnet,)
|
||||||
|
|
||||||
|
|
||||||
@ -763,7 +776,7 @@ class StyleModelApply:
|
|||||||
CATEGORY = "conditioning/style_model"
|
CATEGORY = "conditioning/style_model"
|
||||||
|
|
||||||
def apply_stylemodel(self, clip_vision_output, style_model, conditioning):
|
def apply_stylemodel(self, clip_vision_output, style_model, conditioning):
|
||||||
cond = style_model.get_cond(clip_vision_output)
|
cond = style_model.get_cond(clip_vision_output).flatten(start_dim=0, end_dim=1).unsqueeze(dim=0)
|
||||||
c = []
|
c = []
|
||||||
for t in conditioning:
|
for t in conditioning:
|
||||||
n = [torch.cat((t[0], cond), dim=1), t[1].copy()]
|
n = [torch.cat((t[0], cond), dim=1), t[1].copy()]
|
||||||
@ -1047,6 +1060,47 @@ class LatentComposite:
|
|||||||
samples_out["samples"] = s
|
samples_out["samples"] = s
|
||||||
return (samples_out,)
|
return (samples_out,)
|
||||||
|
|
||||||
|
class LatentBlend:
|
||||||
|
@classmethod
|
||||||
|
def INPUT_TYPES(s):
|
||||||
|
return {"required": {
|
||||||
|
"samples1": ("LATENT",),
|
||||||
|
"samples2": ("LATENT",),
|
||||||
|
"blend_factor": ("FLOAT", {
|
||||||
|
"default": 0.5,
|
||||||
|
"min": 0,
|
||||||
|
"max": 1,
|
||||||
|
"step": 0.01
|
||||||
|
}),
|
||||||
|
}}
|
||||||
|
|
||||||
|
RETURN_TYPES = ("LATENT",)
|
||||||
|
FUNCTION = "blend"
|
||||||
|
|
||||||
|
CATEGORY = "_for_testing"
|
||||||
|
|
||||||
|
def blend(self, samples1, samples2, blend_factor:float, blend_mode: str="normal"):
|
||||||
|
|
||||||
|
samples_out = samples1.copy()
|
||||||
|
samples1 = samples1["samples"]
|
||||||
|
samples2 = samples2["samples"]
|
||||||
|
|
||||||
|
if samples1.shape != samples2.shape:
|
||||||
|
samples2.permute(0, 3, 1, 2)
|
||||||
|
samples2 = comfy.utils.common_upscale(samples2, samples1.shape[3], samples1.shape[2], 'bicubic', crop='center')
|
||||||
|
samples2.permute(0, 2, 3, 1)
|
||||||
|
|
||||||
|
samples_blended = self.blend_mode(samples1, samples2, blend_mode)
|
||||||
|
samples_blended = samples1 * blend_factor + samples_blended * (1 - blend_factor)
|
||||||
|
samples_out["samples"] = samples_blended
|
||||||
|
return (samples_out,)
|
||||||
|
|
||||||
|
def blend_mode(self, img1, img2, mode):
|
||||||
|
if mode == "normal":
|
||||||
|
return img2
|
||||||
|
else:
|
||||||
|
raise ValueError(f"Unsupported blend mode: {mode}")
|
||||||
|
|
||||||
class LatentCrop:
|
class LatentCrop:
|
||||||
@classmethod
|
@classmethod
|
||||||
def INPUT_TYPES(s):
|
def INPUT_TYPES(s):
|
||||||
@ -1257,7 +1311,7 @@ class LoadImage:
|
|||||||
input_dir = folder_paths.get_input_directory()
|
input_dir = folder_paths.get_input_directory()
|
||||||
files = [f for f in os.listdir(input_dir) if os.path.isfile(os.path.join(input_dir, f))]
|
files = [f for f in os.listdir(input_dir) if os.path.isfile(os.path.join(input_dir, f))]
|
||||||
return {"required":
|
return {"required":
|
||||||
{"image": (sorted(files), )},
|
{"image": (sorted(files), {"image_upload": True})},
|
||||||
}
|
}
|
||||||
|
|
||||||
CATEGORY = "image"
|
CATEGORY = "image"
|
||||||
@ -1300,7 +1354,7 @@ class LoadImageMask:
|
|||||||
input_dir = folder_paths.get_input_directory()
|
input_dir = folder_paths.get_input_directory()
|
||||||
files = [f for f in os.listdir(input_dir) if os.path.isfile(os.path.join(input_dir, f))]
|
files = [f for f in os.listdir(input_dir) if os.path.isfile(os.path.join(input_dir, f))]
|
||||||
return {"required":
|
return {"required":
|
||||||
{"image": (sorted(files), ),
|
{"image": (sorted(files), {"image_upload": True}),
|
||||||
"channel": (s._color_channels, ), }
|
"channel": (s._color_channels, ), }
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -1399,6 +1453,44 @@ class ImageInvert:
|
|||||||
s = 1.0 - image
|
s = 1.0 - image
|
||||||
return (s,)
|
return (s,)
|
||||||
|
|
||||||
|
class ImageBatch:
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def INPUT_TYPES(s):
|
||||||
|
return {"required": { "image1": ("IMAGE",), "image2": ("IMAGE",)}}
|
||||||
|
|
||||||
|
RETURN_TYPES = ("IMAGE",)
|
||||||
|
FUNCTION = "batch"
|
||||||
|
|
||||||
|
CATEGORY = "image"
|
||||||
|
|
||||||
|
def batch(self, image1, image2):
|
||||||
|
if image1.shape[1:] != image2.shape[1:]:
|
||||||
|
image2 = comfy.utils.common_upscale(image2.movedim(-1,1), image1.shape[2], image1.shape[1], "bilinear", "center").movedim(1,-1)
|
||||||
|
s = torch.cat((image1, image2), dim=0)
|
||||||
|
return (s,)
|
||||||
|
|
||||||
|
class EmptyImage:
|
||||||
|
def __init__(self, device="cpu"):
|
||||||
|
self.device = device
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def INPUT_TYPES(s):
|
||||||
|
return {"required": { "width": ("INT", {"default": 512, "min": 1, "max": MAX_RESOLUTION, "step": 1}),
|
||||||
|
"height": ("INT", {"default": 512, "min": 1, "max": MAX_RESOLUTION, "step": 1}),
|
||||||
|
"batch_size": ("INT", {"default": 1, "min": 1, "max": 64}),
|
||||||
|
"color": ("INT", {"default": 0, "min": 0, "max": 0xFFFFFF, "step": 1, "display": "color"}),
|
||||||
|
}}
|
||||||
|
RETURN_TYPES = ("IMAGE",)
|
||||||
|
FUNCTION = "generate"
|
||||||
|
|
||||||
|
CATEGORY = "image"
|
||||||
|
|
||||||
|
def generate(self, width, height, batch_size=1, color=0):
|
||||||
|
r = torch.full([batch_size, height, width, 1], ((color >> 16) & 0xFF) / 0xFF)
|
||||||
|
g = torch.full([batch_size, height, width, 1], ((color >> 8) & 0xFF) / 0xFF)
|
||||||
|
b = torch.full([batch_size, height, width, 1], ((color) & 0xFF) / 0xFF)
|
||||||
|
return (torch.cat((r, g, b), dim=-1), )
|
||||||
|
|
||||||
class ImagePadForOutpaint:
|
class ImagePadForOutpaint:
|
||||||
|
|
||||||
@ -1484,7 +1576,9 @@ NODE_CLASS_MAPPINGS = {
|
|||||||
"ImageScale": ImageScale,
|
"ImageScale": ImageScale,
|
||||||
"ImageScaleBy": ImageScaleBy,
|
"ImageScaleBy": ImageScaleBy,
|
||||||
"ImageInvert": ImageInvert,
|
"ImageInvert": ImageInvert,
|
||||||
|
"ImageBatch": ImageBatch,
|
||||||
"ImagePadForOutpaint": ImagePadForOutpaint,
|
"ImagePadForOutpaint": ImagePadForOutpaint,
|
||||||
|
"EmptyImage": EmptyImage,
|
||||||
"ConditioningAverage ": ConditioningAverage ,
|
"ConditioningAverage ": ConditioningAverage ,
|
||||||
"ConditioningCombine": ConditioningCombine,
|
"ConditioningCombine": ConditioningCombine,
|
||||||
"ConditioningConcat": ConditioningConcat,
|
"ConditioningConcat": ConditioningConcat,
|
||||||
@ -1493,6 +1587,7 @@ NODE_CLASS_MAPPINGS = {
|
|||||||
"KSamplerAdvanced": KSamplerAdvanced,
|
"KSamplerAdvanced": KSamplerAdvanced,
|
||||||
"SetLatentNoiseMask": SetLatentNoiseMask,
|
"SetLatentNoiseMask": SetLatentNoiseMask,
|
||||||
"LatentComposite": LatentComposite,
|
"LatentComposite": LatentComposite,
|
||||||
|
"LatentBlend": LatentBlend,
|
||||||
"LatentRotate": LatentRotate,
|
"LatentRotate": LatentRotate,
|
||||||
"LatentFlip": LatentFlip,
|
"LatentFlip": LatentFlip,
|
||||||
"LatentCrop": LatentCrop,
|
"LatentCrop": LatentCrop,
|
||||||
@ -1564,6 +1659,7 @@ NODE_DISPLAY_NAME_MAPPINGS = {
|
|||||||
"LatentUpscale": "Upscale Latent",
|
"LatentUpscale": "Upscale Latent",
|
||||||
"LatentUpscaleBy": "Upscale Latent By",
|
"LatentUpscaleBy": "Upscale Latent By",
|
||||||
"LatentComposite": "Latent Composite",
|
"LatentComposite": "Latent Composite",
|
||||||
|
"LatentBlend": "Latent Blend",
|
||||||
"LatentFromBatch" : "Latent From Batch",
|
"LatentFromBatch" : "Latent From Batch",
|
||||||
"RepeatLatentBatch": "Repeat Latent Batch",
|
"RepeatLatentBatch": "Repeat Latent Batch",
|
||||||
# Image
|
# Image
|
||||||
@ -1576,11 +1672,14 @@ NODE_DISPLAY_NAME_MAPPINGS = {
|
|||||||
"ImageUpscaleWithModel": "Upscale Image (using Model)",
|
"ImageUpscaleWithModel": "Upscale Image (using Model)",
|
||||||
"ImageInvert": "Invert Image",
|
"ImageInvert": "Invert Image",
|
||||||
"ImagePadForOutpaint": "Pad Image for Outpainting",
|
"ImagePadForOutpaint": "Pad Image for Outpainting",
|
||||||
|
"ImageBatch": "Batch Images",
|
||||||
# _for_testing
|
# _for_testing
|
||||||
"VAEDecodeTiled": "VAE Decode (Tiled)",
|
"VAEDecodeTiled": "VAE Decode (Tiled)",
|
||||||
"VAEEncodeTiled": "VAE Encode (Tiled)",
|
"VAEEncodeTiled": "VAE Encode (Tiled)",
|
||||||
}
|
}
|
||||||
|
|
||||||
|
EXTENSION_WEB_DIRS = {}
|
||||||
|
|
||||||
def load_custom_node(module_path, ignore=set()):
|
def load_custom_node(module_path, ignore=set()):
|
||||||
module_name = os.path.basename(module_path)
|
module_name = os.path.basename(module_path)
|
||||||
if os.path.isfile(module_path):
|
if os.path.isfile(module_path):
|
||||||
@ -1589,11 +1688,20 @@ def load_custom_node(module_path, ignore=set()):
|
|||||||
try:
|
try:
|
||||||
if os.path.isfile(module_path):
|
if os.path.isfile(module_path):
|
||||||
module_spec = importlib.util.spec_from_file_location(module_name, module_path)
|
module_spec = importlib.util.spec_from_file_location(module_name, module_path)
|
||||||
|
module_dir = os.path.split(module_path)[0]
|
||||||
else:
|
else:
|
||||||
module_spec = importlib.util.spec_from_file_location(module_name, os.path.join(module_path, "__init__.py"))
|
module_spec = importlib.util.spec_from_file_location(module_name, os.path.join(module_path, "__init__.py"))
|
||||||
|
module_dir = module_path
|
||||||
|
|
||||||
module = importlib.util.module_from_spec(module_spec)
|
module = importlib.util.module_from_spec(module_spec)
|
||||||
sys.modules[module_name] = module
|
sys.modules[module_name] = module
|
||||||
module_spec.loader.exec_module(module)
|
module_spec.loader.exec_module(module)
|
||||||
|
|
||||||
|
if hasattr(module, "WEB_DIRECTORY") and getattr(module, "WEB_DIRECTORY") is not None:
|
||||||
|
web_dir = os.path.abspath(os.path.join(module_dir, getattr(module, "WEB_DIRECTORY")))
|
||||||
|
if os.path.isdir(web_dir):
|
||||||
|
EXTENSION_WEB_DIRS[module_name] = web_dir
|
||||||
|
|
||||||
if hasattr(module, "NODE_CLASS_MAPPINGS") and getattr(module, "NODE_CLASS_MAPPINGS") is not None:
|
if hasattr(module, "NODE_CLASS_MAPPINGS") and getattr(module, "NODE_CLASS_MAPPINGS") is not None:
|
||||||
for name in module.NODE_CLASS_MAPPINGS:
|
for name in module.NODE_CLASS_MAPPINGS:
|
||||||
if name not in ignore:
|
if name not in ignore:
|
||||||
|
|||||||
@ -75,6 +75,8 @@
|
|||||||
"#!wget -c https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0/resolve/main/sd_xl_base_1.0.safetensors -P ./models/checkpoints/\n",
|
"#!wget -c https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0/resolve/main/sd_xl_base_1.0.safetensors -P ./models/checkpoints/\n",
|
||||||
"#!wget -c https://huggingface.co/stabilityai/stable-diffusion-xl-refiner-1.0/resolve/main/sd_xl_refiner_1.0.safetensors -P ./models/checkpoints/\n",
|
"#!wget -c https://huggingface.co/stabilityai/stable-diffusion-xl-refiner-1.0/resolve/main/sd_xl_refiner_1.0.safetensors -P ./models/checkpoints/\n",
|
||||||
"\n",
|
"\n",
|
||||||
|
"# SDXL ReVision\n",
|
||||||
|
"#!wget -c https://huggingface.co/comfyanonymous/clip_vision_g/resolve/main/clip_vision_g.safetensors -P ./models/clip_vision/\n",
|
||||||
"\n",
|
"\n",
|
||||||
"# SD1.5\n",
|
"# SD1.5\n",
|
||||||
"!wget -c https://huggingface.co/runwayml/stable-diffusion-v1-5/resolve/main/v1-5-pruned-emaonly.ckpt -P ./models/checkpoints/\n",
|
"!wget -c https://huggingface.co/runwayml/stable-diffusion-v1-5/resolve/main/v1-5-pruned-emaonly.ckpt -P ./models/checkpoints/\n",
|
||||||
@ -142,6 +144,11 @@
|
|||||||
"#!wget -c https://huggingface.co/comfyanonymous/ControlNet-v1-1_fp16_safetensors/resolve/main/control_v11p_sd15s2_lineart_anime_fp16.safetensors -P ./models/controlnet/\n",
|
"#!wget -c https://huggingface.co/comfyanonymous/ControlNet-v1-1_fp16_safetensors/resolve/main/control_v11p_sd15s2_lineart_anime_fp16.safetensors -P ./models/controlnet/\n",
|
||||||
"#!wget -c https://huggingface.co/comfyanonymous/ControlNet-v1-1_fp16_safetensors/resolve/main/control_v11u_sd15_tile_fp16.safetensors -P ./models/controlnet/\n",
|
"#!wget -c https://huggingface.co/comfyanonymous/ControlNet-v1-1_fp16_safetensors/resolve/main/control_v11u_sd15_tile_fp16.safetensors -P ./models/controlnet/\n",
|
||||||
"\n",
|
"\n",
|
||||||
|
"# ControlNet SDXL\n",
|
||||||
|
"#!wget -c https://huggingface.co/stabilityai/control-lora/resolve/main/control-LoRAs-rank256/control-lora-canny-rank256.safetensors -P ./models/controlnet/\n",
|
||||||
|
"#!wget -c https://huggingface.co/stabilityai/control-lora/resolve/main/control-LoRAs-rank256/control-lora-depth-rank256.safetensors -P ./models/controlnet/\n",
|
||||||
|
"#!wget -c https://huggingface.co/stabilityai/control-lora/resolve/main/control-LoRAs-rank256/control-lora-recolor-rank256.safetensors -P ./models/controlnet/\n",
|
||||||
|
"#!wget -c https://huggingface.co/stabilityai/control-lora/resolve/main/control-LoRAs-rank256/control-lora-sketch-rank256.safetensors -P ./models/controlnet/\n",
|
||||||
"\n",
|
"\n",
|
||||||
"# Controlnet Preprocessor nodes by Fannovel16\n",
|
"# Controlnet Preprocessor nodes by Fannovel16\n",
|
||||||
"#!cd custom_nodes && git clone https://github.com/Fannovel16/comfy_controlnet_preprocessors; cd comfy_controlnet_preprocessors && python install.py\n",
|
"#!cd custom_nodes && git clone https://github.com/Fannovel16/comfy_controlnet_preprocessors; cd comfy_controlnet_preprocessors && python install.py\n",
|
||||||
@ -159,13 +166,64 @@
|
|||||||
"\n"
|
"\n"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {
|
||||||
|
"id": "kkkkkkkkkkkkkkk"
|
||||||
|
},
|
||||||
|
"source": [
|
||||||
|
"### Run ComfyUI with cloudflared (Recommended Way)\n",
|
||||||
|
"\n",
|
||||||
|
"\n"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {
|
||||||
|
"id": "jjjjjjjjjjjjjj"
|
||||||
|
},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"!wget https://github.com/cloudflare/cloudflared/releases/latest/download/cloudflared-linux-amd64.deb\n",
|
||||||
|
"!dpkg -i cloudflared-linux-amd64.deb\n",
|
||||||
|
"\n",
|
||||||
|
"import subprocess\n",
|
||||||
|
"import threading\n",
|
||||||
|
"import time\n",
|
||||||
|
"import socket\n",
|
||||||
|
"import urllib.request\n",
|
||||||
|
"\n",
|
||||||
|
"def iframe_thread(port):\n",
|
||||||
|
" while True:\n",
|
||||||
|
" time.sleep(0.5)\n",
|
||||||
|
" sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)\n",
|
||||||
|
" result = sock.connect_ex(('127.0.0.1', port))\n",
|
||||||
|
" if result == 0:\n",
|
||||||
|
" break\n",
|
||||||
|
" sock.close()\n",
|
||||||
|
" print(\"\\nComfyUI finished loading, trying to launch cloudflared (if it gets stuck here cloudflared is having issues)\\n\")\n",
|
||||||
|
"\n",
|
||||||
|
" p = subprocess.Popen([\"cloudflared\", \"tunnel\", \"--url\", \"http://127.0.0.1:{}\".format(port)], stdout=subprocess.PIPE, stderr=subprocess.PIPE)\n",
|
||||||
|
" for line in p.stderr:\n",
|
||||||
|
" l = line.decode()\n",
|
||||||
|
" if \"trycloudflare.com \" in l:\n",
|
||||||
|
" print(\"This is the URL to access ComfyUI:\", l[l.find(\"http\"):], end='')\n",
|
||||||
|
" #print(l, end='')\n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
"threading.Thread(target=iframe_thread, daemon=True, args=(8188,)).start()\n",
|
||||||
|
"\n",
|
||||||
|
"!python main.py --dont-print-server"
|
||||||
|
]
|
||||||
|
},
|
||||||
{
|
{
|
||||||
"cell_type": "markdown",
|
"cell_type": "markdown",
|
||||||
"metadata": {
|
"metadata": {
|
||||||
"id": "kkkkkkkkkkkkkk"
|
"id": "kkkkkkkkkkkkkk"
|
||||||
},
|
},
|
||||||
"source": [
|
"source": [
|
||||||
"### Run ComfyUI with localtunnel (Recommended Way)\n",
|
"### Run ComfyUI with localtunnel\n",
|
||||||
"\n",
|
"\n",
|
||||||
"\n"
|
"\n"
|
||||||
]
|
]
|
||||||
|
|||||||
@ -9,3 +9,4 @@ pyyaml
|
|||||||
Pillow
|
Pillow
|
||||||
scipy
|
scipy
|
||||||
tqdm
|
tqdm
|
||||||
|
psutil
|
||||||
|
|||||||
55
server.py
55
server.py
@ -1,14 +1,18 @@
|
|||||||
import os
|
import os
|
||||||
import sys
|
import sys
|
||||||
import asyncio
|
import asyncio
|
||||||
|
import traceback
|
||||||
|
|
||||||
import nodes
|
import nodes
|
||||||
import folder_paths
|
import folder_paths
|
||||||
import execution
|
import execution
|
||||||
import uuid
|
import uuid
|
||||||
|
import urllib
|
||||||
import json
|
import json
|
||||||
import glob
|
import glob
|
||||||
import struct
|
import struct
|
||||||
from PIL import Image, ImageOps
|
from PIL import Image, ImageOps
|
||||||
|
from PIL.PngImagePlugin import PngInfo
|
||||||
from io import BytesIO
|
from io import BytesIO
|
||||||
|
|
||||||
try:
|
try:
|
||||||
@ -67,6 +71,8 @@ class PromptServer():
|
|||||||
|
|
||||||
mimetypes.init()
|
mimetypes.init()
|
||||||
mimetypes.types_map['.js'] = 'application/javascript; charset=utf-8'
|
mimetypes.types_map['.js'] = 'application/javascript; charset=utf-8'
|
||||||
|
|
||||||
|
self.supports = ["custom_nodes_from_web"]
|
||||||
self.prompt_queue = None
|
self.prompt_queue = None
|
||||||
self.loop = loop
|
self.loop = loop
|
||||||
self.messages = asyncio.Queue()
|
self.messages = asyncio.Queue()
|
||||||
@ -76,7 +82,7 @@ class PromptServer():
|
|||||||
if args.enable_cors_header:
|
if args.enable_cors_header:
|
||||||
middlewares.append(create_cors_middleware(args.enable_cors_header))
|
middlewares.append(create_cors_middleware(args.enable_cors_header))
|
||||||
|
|
||||||
self.app = web.Application(client_max_size=20971520, middlewares=middlewares)
|
self.app = web.Application(client_max_size=104857600, middlewares=middlewares)
|
||||||
self.sockets = dict()
|
self.sockets = dict()
|
||||||
self.web_root = os.path.join(os.path.dirname(
|
self.web_root = os.path.join(os.path.dirname(
|
||||||
os.path.realpath(__file__)), "web")
|
os.path.realpath(__file__)), "web")
|
||||||
@ -85,6 +91,8 @@ class PromptServer():
|
|||||||
self.last_node_id = None
|
self.last_node_id = None
|
||||||
self.client_id = None
|
self.client_id = None
|
||||||
|
|
||||||
|
self.on_prompt_handlers = []
|
||||||
|
|
||||||
@routes.get('/ws')
|
@routes.get('/ws')
|
||||||
async def websocket_handler(request):
|
async def websocket_handler(request):
|
||||||
ws = web.WebSocketResponse()
|
ws = web.WebSocketResponse()
|
||||||
@ -119,12 +127,21 @@ class PromptServer():
|
|||||||
@routes.get("/embeddings")
|
@routes.get("/embeddings")
|
||||||
def get_embeddings(self):
|
def get_embeddings(self):
|
||||||
embeddings = folder_paths.get_filename_list("embeddings")
|
embeddings = folder_paths.get_filename_list("embeddings")
|
||||||
return web.json_response(list(map(lambda a: os.path.splitext(a)[0].lower(), embeddings)))
|
return web.json_response(list(map(lambda a: os.path.splitext(a)[0], embeddings)))
|
||||||
|
|
||||||
@routes.get("/extensions")
|
@routes.get("/extensions")
|
||||||
async def get_extensions(request):
|
async def get_extensions(request):
|
||||||
files = glob.glob(os.path.join(self.web_root, 'extensions/**/*.js'), recursive=True)
|
files = glob.glob(os.path.join(
|
||||||
return web.json_response(list(map(lambda f: "/" + os.path.relpath(f, self.web_root).replace("\\", "/"), files)))
|
self.web_root, 'extensions/**/*.js'), recursive=True)
|
||||||
|
|
||||||
|
extensions = list(map(lambda f: "/" + os.path.relpath(f, self.web_root).replace("\\", "/"), files))
|
||||||
|
|
||||||
|
for name, dir in nodes.EXTENSION_WEB_DIRS.items():
|
||||||
|
files = glob.glob(os.path.join(dir, '**/*.js'), recursive=True)
|
||||||
|
extensions.extend(list(map(lambda f: "/extensions/" + urllib.parse.quote(
|
||||||
|
name) + "/" + os.path.relpath(f, dir).replace("\\", "/"), files)))
|
||||||
|
|
||||||
|
return web.json_response(extensions)
|
||||||
|
|
||||||
def get_dir_by_type(dir_type):
|
def get_dir_by_type(dir_type):
|
||||||
if dir_type is None:
|
if dir_type is None:
|
||||||
@ -217,13 +234,17 @@ class PromptServer():
|
|||||||
|
|
||||||
if os.path.isfile(file):
|
if os.path.isfile(file):
|
||||||
with Image.open(file) as original_pil:
|
with Image.open(file) as original_pil:
|
||||||
|
metadata = PngInfo()
|
||||||
|
if hasattr(original_pil,'text'):
|
||||||
|
for key in original_pil.text:
|
||||||
|
metadata.add_text(key, original_pil.text[key])
|
||||||
original_pil = original_pil.convert('RGBA')
|
original_pil = original_pil.convert('RGBA')
|
||||||
mask_pil = Image.open(image.file).convert('RGBA')
|
mask_pil = Image.open(image.file).convert('RGBA')
|
||||||
|
|
||||||
# alpha copy
|
# alpha copy
|
||||||
new_alpha = mask_pil.getchannel('A')
|
new_alpha = mask_pil.getchannel('A')
|
||||||
original_pil.putalpha(new_alpha)
|
original_pil.putalpha(new_alpha)
|
||||||
original_pil.save(filepath, compress_level=4)
|
original_pil.save(filepath, compress_level=4, pnginfo=metadata)
|
||||||
|
|
||||||
return image_upload(post, image_save_function)
|
return image_upload(post, image_save_function)
|
||||||
|
|
||||||
@ -345,6 +366,11 @@ class PromptServer():
|
|||||||
vram_total, torch_vram_total = comfy.model_management.get_total_memory(device, torch_total_too=True)
|
vram_total, torch_vram_total = comfy.model_management.get_total_memory(device, torch_total_too=True)
|
||||||
vram_free, torch_vram_free = comfy.model_management.get_free_memory(device, torch_free_too=True)
|
vram_free, torch_vram_free = comfy.model_management.get_free_memory(device, torch_free_too=True)
|
||||||
system_stats = {
|
system_stats = {
|
||||||
|
"system": {
|
||||||
|
"os": os.name,
|
||||||
|
"python_version": sys.version,
|
||||||
|
"embedded_python": os.path.split(os.path.split(sys.executable)[0])[1] == "python_embeded"
|
||||||
|
},
|
||||||
"devices": [
|
"devices": [
|
||||||
{
|
{
|
||||||
"name": device_name,
|
"name": device_name,
|
||||||
@ -421,6 +447,7 @@ class PromptServer():
|
|||||||
resp_code = 200
|
resp_code = 200
|
||||||
out_string = ""
|
out_string = ""
|
||||||
json_data = await request.json()
|
json_data = await request.json()
|
||||||
|
json_data = self.trigger_on_prompt(json_data)
|
||||||
|
|
||||||
if "number" in json_data:
|
if "number" in json_data:
|
||||||
number = float(json_data['number'])
|
number = float(json_data['number'])
|
||||||
@ -487,6 +514,12 @@ class PromptServer():
|
|||||||
|
|
||||||
def add_routes(self):
|
def add_routes(self):
|
||||||
self.app.add_routes(self.routes)
|
self.app.add_routes(self.routes)
|
||||||
|
|
||||||
|
for name, dir in nodes.EXTENSION_WEB_DIRS.items():
|
||||||
|
self.app.add_routes([
|
||||||
|
web.static('/extensions/' + urllib.parse.quote(name), dir, follow_symlinks=True),
|
||||||
|
])
|
||||||
|
|
||||||
self.app.add_routes([
|
self.app.add_routes([
|
||||||
web.static('/', self.web_root, follow_symlinks=True),
|
web.static('/', self.web_root, follow_symlinks=True),
|
||||||
])
|
])
|
||||||
@ -583,3 +616,15 @@ class PromptServer():
|
|||||||
if call_on_start is not None:
|
if call_on_start is not None:
|
||||||
call_on_start(address, port)
|
call_on_start(address, port)
|
||||||
|
|
||||||
|
def add_on_prompt_handler(self, handler):
|
||||||
|
self.on_prompt_handlers.append(handler)
|
||||||
|
|
||||||
|
def trigger_on_prompt(self, json_data):
|
||||||
|
for handler in self.on_prompt_handlers:
|
||||||
|
try:
|
||||||
|
json_data = handler(json_data)
|
||||||
|
except Exception as e:
|
||||||
|
print(f"[ERROR] An error occurred during the on_prompt_handler processing")
|
||||||
|
traceback.print_exc()
|
||||||
|
|
||||||
|
return json_data
|
||||||
|
|||||||
@ -1,4 +1,4 @@
|
|||||||
import {app} from "/scripts/app.js";
|
import {app} from "../../scripts/app.js";
|
||||||
|
|
||||||
// Adds filtering to combo context menus
|
// Adds filtering to combo context menus
|
||||||
|
|
||||||
@ -27,10 +27,13 @@ const ext = {
|
|||||||
const clickedComboValue = currentNode.widgets
|
const clickedComboValue = currentNode.widgets
|
||||||
.filter(w => w.type === "combo" && w.options.values.length === values.length)
|
.filter(w => w.type === "combo" && w.options.values.length === values.length)
|
||||||
.find(w => w.options.values.every((v, i) => v === values[i]))
|
.find(w => w.options.values.every((v, i) => v === values[i]))
|
||||||
.value;
|
?.value;
|
||||||
|
|
||||||
let selectedIndex = values.findIndex(v => v === clickedComboValue);
|
let selectedIndex = clickedComboValue ? values.findIndex(v => v === clickedComboValue) : 0;
|
||||||
let selectedItem = displayedItems?.[selectedIndex];
|
if (selectedIndex < 0) {
|
||||||
|
selectedIndex = 0;
|
||||||
|
}
|
||||||
|
let selectedItem = displayedItems[selectedIndex];
|
||||||
updateSelected();
|
updateSelected();
|
||||||
|
|
||||||
// Apply highlighting to the selected item
|
// Apply highlighting to the selected item
|
||||||
|
|||||||
167
web/extensions/core/groupOptions.js
Normal file
167
web/extensions/core/groupOptions.js
Normal file
@ -0,0 +1,167 @@
|
|||||||
|
import {app} from "../../scripts/app.js";
|
||||||
|
|
||||||
|
function setNodeMode(node, mode) {
|
||||||
|
node.mode = mode;
|
||||||
|
node.graph.change();
|
||||||
|
}
|
||||||
|
|
||||||
|
app.registerExtension({
|
||||||
|
name: "Comfy.GroupOptions",
|
||||||
|
setup() {
|
||||||
|
const orig = LGraphCanvas.prototype.getCanvasMenuOptions;
|
||||||
|
// graph_mouse
|
||||||
|
LGraphCanvas.prototype.getCanvasMenuOptions = function () {
|
||||||
|
const options = orig.apply(this, arguments);
|
||||||
|
const group = this.graph.getGroupOnPos(this.graph_mouse[0], this.graph_mouse[1]);
|
||||||
|
if (!group) {
|
||||||
|
return options;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Group nodes aren't recomputed until the group is moved, this ensures the nodes are up-to-date
|
||||||
|
group.recomputeInsideNodes();
|
||||||
|
const nodesInGroup = group._nodes;
|
||||||
|
|
||||||
|
// No nodes in group, return default options
|
||||||
|
if (nodesInGroup.length === 0) {
|
||||||
|
return options;
|
||||||
|
} else {
|
||||||
|
// Add a separator between the default options and the group options
|
||||||
|
options.push(null);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check if all nodes are the same mode
|
||||||
|
let allNodesAreSameMode = true;
|
||||||
|
for (let i = 1; i < nodesInGroup.length; i++) {
|
||||||
|
if (nodesInGroup[i].mode !== nodesInGroup[0].mode) {
|
||||||
|
allNodesAreSameMode = false;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Modes
|
||||||
|
// 0: Always
|
||||||
|
// 1: On Event
|
||||||
|
// 2: Never
|
||||||
|
// 3: On Trigger
|
||||||
|
// 4: Bypass
|
||||||
|
// If all nodes are the same mode, add a menu option to change the mode
|
||||||
|
if (allNodesAreSameMode) {
|
||||||
|
const mode = nodesInGroup[0].mode;
|
||||||
|
switch (mode) {
|
||||||
|
case 0:
|
||||||
|
// All nodes are always, option to disable, and bypass
|
||||||
|
options.push({
|
||||||
|
content: "Set Group Nodes to Never",
|
||||||
|
callback: () => {
|
||||||
|
for (const node of nodesInGroup) {
|
||||||
|
setNodeMode(node, 2);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
options.push({
|
||||||
|
content: "Bypass Group Nodes",
|
||||||
|
callback: () => {
|
||||||
|
for (const node of nodesInGroup) {
|
||||||
|
setNodeMode(node, 4);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
break;
|
||||||
|
case 2:
|
||||||
|
// All nodes are never, option to enable, and bypass
|
||||||
|
options.push({
|
||||||
|
content: "Set Group Nodes to Always",
|
||||||
|
callback: () => {
|
||||||
|
for (const node of nodesInGroup) {
|
||||||
|
setNodeMode(node, 0);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
options.push({
|
||||||
|
content: "Bypass Group Nodes",
|
||||||
|
callback: () => {
|
||||||
|
for (const node of nodesInGroup) {
|
||||||
|
setNodeMode(node, 4);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
break;
|
||||||
|
case 4:
|
||||||
|
// All nodes are bypass, option to enable, and disable
|
||||||
|
options.push({
|
||||||
|
content: "Set Group Nodes to Always",
|
||||||
|
callback: () => {
|
||||||
|
for (const node of nodesInGroup) {
|
||||||
|
setNodeMode(node, 0);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
options.push({
|
||||||
|
content: "Set Group Nodes to Never",
|
||||||
|
callback: () => {
|
||||||
|
for (const node of nodesInGroup) {
|
||||||
|
setNodeMode(node, 2);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
break;
|
||||||
|
default:
|
||||||
|
// All nodes are On Trigger or On Event(Or other?), option to disable, set to always, or bypass
|
||||||
|
options.push({
|
||||||
|
content: "Set Group Nodes to Always",
|
||||||
|
callback: () => {
|
||||||
|
for (const node of nodesInGroup) {
|
||||||
|
setNodeMode(node, 0);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
options.push({
|
||||||
|
content: "Set Group Nodes to Never",
|
||||||
|
callback: () => {
|
||||||
|
for (const node of nodesInGroup) {
|
||||||
|
setNodeMode(node, 2);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
options.push({
|
||||||
|
content: "Bypass Group Nodes",
|
||||||
|
callback: () => {
|
||||||
|
for (const node of nodesInGroup) {
|
||||||
|
setNodeMode(node, 4);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
// Nodes are not all the same mode, add a menu option to change the mode to always, never, or bypass
|
||||||
|
options.push({
|
||||||
|
content: "Set Group Nodes to Always",
|
||||||
|
callback: () => {
|
||||||
|
for (const node of nodesInGroup) {
|
||||||
|
setNodeMode(node, 0);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
options.push({
|
||||||
|
content: "Set Group Nodes to Never",
|
||||||
|
callback: () => {
|
||||||
|
for (const node of nodesInGroup) {
|
||||||
|
setNodeMode(node, 2);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
options.push({
|
||||||
|
content: "Bypass Group Nodes",
|
||||||
|
callback: () => {
|
||||||
|
for (const node of nodesInGroup) {
|
||||||
|
setNodeMode(node, 4);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
return options
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
25
web/extensions/core/linkRenderMode.js
Normal file
25
web/extensions/core/linkRenderMode.js
Normal file
@ -0,0 +1,25 @@
|
|||||||
|
import { app } from "../../scripts/app.js";
|
||||||
|
|
||||||
|
const id = "Comfy.LinkRenderMode";
|
||||||
|
const ext = {
|
||||||
|
name: id,
|
||||||
|
async setup(app) {
|
||||||
|
app.ui.settings.addSetting({
|
||||||
|
id,
|
||||||
|
name: "Link Render Mode",
|
||||||
|
defaultValue: 2,
|
||||||
|
type: "combo",
|
||||||
|
options: LiteGraph.LINK_RENDER_MODES.map((m, i) => ({
|
||||||
|
value: i,
|
||||||
|
text: m,
|
||||||
|
selected: i == app.canvas.links_render_mode,
|
||||||
|
})),
|
||||||
|
onChange(value) {
|
||||||
|
app.canvas.links_render_mode = +value;
|
||||||
|
app.graph.setDirtyCanvas(true);
|
||||||
|
},
|
||||||
|
});
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
app.registerExtension(ext);
|
||||||
@ -5,7 +5,7 @@ import { app } from "../../scripts/app.js";
|
|||||||
app.registerExtension({
|
app.registerExtension({
|
||||||
name: "Comfy.UploadImage",
|
name: "Comfy.UploadImage",
|
||||||
async beforeRegisterNodeDef(nodeType, nodeData, app) {
|
async beforeRegisterNodeDef(nodeType, nodeData, app) {
|
||||||
if (nodeData.name === "LoadImage" || nodeData.name === "LoadImageMask") {
|
if (nodeData?.input?.required?.image?.[1]?.image_upload === true) {
|
||||||
nodeData.input.required.upload = ["IMAGEUPLOAD"];
|
nodeData.input.required.upload = ["IMAGEUPLOAD"];
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
|||||||
@ -2,7 +2,7 @@ import { ComfyWidgets, addValueControlWidget } from "../../scripts/widgets.js";
|
|||||||
import { app } from "../../scripts/app.js";
|
import { app } from "../../scripts/app.js";
|
||||||
|
|
||||||
const CONVERTED_TYPE = "converted-widget";
|
const CONVERTED_TYPE = "converted-widget";
|
||||||
const VALID_TYPES = ["STRING", "combo", "number"];
|
const VALID_TYPES = ["STRING", "combo", "number", "BOOLEAN"];
|
||||||
|
|
||||||
function isConvertableWidget(widget, config) {
|
function isConvertableWidget(widget, config) {
|
||||||
return VALID_TYPES.includes(widget.type) || VALID_TYPES.includes(config[0]);
|
return VALID_TYPES.includes(widget.type) || VALID_TYPES.includes(config[0]);
|
||||||
|
|||||||
@ -8,6 +8,7 @@
|
|||||||
<link rel="stylesheet" type="text/css" href="./style.css" />
|
<link rel="stylesheet" type="text/css" href="./style.css" />
|
||||||
<link rel="manifest" href="./manifest.json">
|
<link rel="manifest" href="./manifest.json">
|
||||||
<link rel="icon" type="image/png" href="./assets/favicon.png">
|
<link rel="icon" type="image/png" href="./assets/favicon.png">
|
||||||
|
<link rel="stylesheet" type="text/css" href="./user.css" />
|
||||||
<script type="text/javascript" src="./lib/litegraph.core.js"></script>
|
<script type="text/javascript" src="./lib/litegraph.core.js"></script>
|
||||||
<script type="text/javascript" src="./lib/litegraph.extensions.js" defer></script>
|
<script type="text/javascript" src="./lib/litegraph.extensions.js" defer></script>
|
||||||
<script type="module">
|
<script type="module">
|
||||||
|
|||||||
@ -6233,11 +6233,17 @@ LGraphNode.prototype.executeAction = function(action)
|
|||||||
,posAdd:[!mClikSlot_isOut?-30:30, -alphaPosY*130] //-alphaPosY*30]
|
,posAdd:[!mClikSlot_isOut?-30:30, -alphaPosY*130] //-alphaPosY*30]
|
||||||
,posSizeFix:[!mClikSlot_isOut?-1:0, 0] //-alphaPosY*2*/
|
,posSizeFix:[!mClikSlot_isOut?-1:0, 0] //-alphaPosY*2*/
|
||||||
});
|
});
|
||||||
|
skip_action = true;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (!skip_action && this.allow_dragcanvas) {
|
||||||
|
//console.log("pointerevents: dragging_canvas start from middle button");
|
||||||
|
this.dragging_canvas = true;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
} else if (e.which == 3 || this.pointer_is_double) {
|
} else if (e.which == 3 || this.pointer_is_double) {
|
||||||
|
|
||||||
@ -9766,6 +9772,7 @@ LGraphNode.prototype.executeAction = function(action)
|
|||||||
|
|
||||||
switch (w.type) {
|
switch (w.type) {
|
||||||
case "button":
|
case "button":
|
||||||
|
ctx.fillStyle = background_color;
|
||||||
if (w.clicked) {
|
if (w.clicked) {
|
||||||
ctx.fillStyle = "#AAA";
|
ctx.fillStyle = "#AAA";
|
||||||
w.clicked = false;
|
w.clicked = false;
|
||||||
@ -9835,7 +9842,11 @@ LGraphNode.prototype.executeAction = function(action)
|
|||||||
ctx.textAlign = "center";
|
ctx.textAlign = "center";
|
||||||
ctx.fillStyle = text_color;
|
ctx.fillStyle = text_color;
|
||||||
ctx.fillText(
|
ctx.fillText(
|
||||||
w.label || w.name + " " + Number(w.value).toFixed(3),
|
w.label || w.name + " " + Number(w.value).toFixed(
|
||||||
|
w.options.precision != null
|
||||||
|
? w.options.precision
|
||||||
|
: 3
|
||||||
|
),
|
||||||
widget_width * 0.5,
|
widget_width * 0.5,
|
||||||
y + H * 0.7
|
y + H * 0.7
|
||||||
);
|
);
|
||||||
@ -13835,7 +13846,7 @@ LGraphNode.prototype.executeAction = function(action)
|
|||||||
if (!disabled) {
|
if (!disabled) {
|
||||||
element.addEventListener("click", inner_onclick);
|
element.addEventListener("click", inner_onclick);
|
||||||
}
|
}
|
||||||
if (options.autoopen) {
|
if (!disabled && options.autoopen) {
|
||||||
LiteGraph.pointerListenerAdd(element,"enter",inner_over);
|
LiteGraph.pointerListenerAdd(element,"enter",inner_over);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@ -264,6 +264,15 @@ class ComfyApi extends EventTarget {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Gets system & device stats
|
||||||
|
* @returns System stats such as python version, OS, per device info
|
||||||
|
*/
|
||||||
|
async getSystemStats() {
|
||||||
|
const res = await this.fetchApi("/system_stats");
|
||||||
|
return await res.json();
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Sends a POST request to the API
|
* Sends a POST request to the API
|
||||||
* @param {*} type The endpoint to post to
|
* @param {*} type The endpoint to post to
|
||||||
|
|||||||
@ -1,3 +1,4 @@
|
|||||||
|
import { ComfyLogging } from "./logging.js";
|
||||||
import { ComfyWidgets } from "./widgets.js";
|
import { ComfyWidgets } from "./widgets.js";
|
||||||
import { ComfyUI, $el } from "./ui.js";
|
import { ComfyUI, $el } from "./ui.js";
|
||||||
import { api } from "./api.js";
|
import { api } from "./api.js";
|
||||||
@ -31,6 +32,7 @@ export class ComfyApp {
|
|||||||
|
|
||||||
constructor() {
|
constructor() {
|
||||||
this.ui = new ComfyUI(this);
|
this.ui = new ComfyUI(this);
|
||||||
|
this.logging = new ComfyLogging(this);
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* List of extensions that are registered with the app
|
* List of extensions that are registered with the app
|
||||||
@ -282,6 +284,11 @@ export class ComfyApp {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
options.push({
|
||||||
|
content: "Bypass",
|
||||||
|
callback: (obj) => { if (this.mode === 4) this.mode = 0; else this.mode = 4; this.graph.change(); }
|
||||||
|
});
|
||||||
|
|
||||||
// prevent conflict of clipspace content
|
// prevent conflict of clipspace content
|
||||||
if(!ComfyApp.clipspace_return_node) {
|
if(!ComfyApp.clipspace_return_node) {
|
||||||
options.push({
|
options.push({
|
||||||
@ -768,6 +775,19 @@ export class ComfyApp {
|
|||||||
}
|
}
|
||||||
block_default = true;
|
block_default = true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (e.keyCode == 66 && e.ctrlKey) {
|
||||||
|
if (this.selected_nodes) {
|
||||||
|
for (var i in this.selected_nodes) {
|
||||||
|
if (this.selected_nodes[i].mode === 4) { // never
|
||||||
|
this.selected_nodes[i].mode = 0; // always
|
||||||
|
} else {
|
||||||
|
this.selected_nodes[i].mode = 4; // never
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
block_default = true;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
this.graph.change();
|
this.graph.change();
|
||||||
@ -914,14 +934,21 @@ export class ComfyApp {
|
|||||||
const origDrawNode = LGraphCanvas.prototype.drawNode;
|
const origDrawNode = LGraphCanvas.prototype.drawNode;
|
||||||
LGraphCanvas.prototype.drawNode = function (node, ctx) {
|
LGraphCanvas.prototype.drawNode = function (node, ctx) {
|
||||||
var editor_alpha = this.editor_alpha;
|
var editor_alpha = this.editor_alpha;
|
||||||
|
var old_color = node.bgcolor;
|
||||||
|
|
||||||
if (node.mode === 2) { // never
|
if (node.mode === 2) { // never
|
||||||
this.editor_alpha = 0.4;
|
this.editor_alpha = 0.4;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (node.mode === 4) { // never
|
||||||
|
node.bgcolor = "#FF00FF";
|
||||||
|
this.editor_alpha = 0.2;
|
||||||
|
}
|
||||||
|
|
||||||
const res = origDrawNode.apply(this, arguments);
|
const res = origDrawNode.apply(this, arguments);
|
||||||
|
|
||||||
this.editor_alpha = editor_alpha;
|
this.editor_alpha = editor_alpha;
|
||||||
|
node.bgcolor = old_color;
|
||||||
|
|
||||||
return res;
|
return res;
|
||||||
};
|
};
|
||||||
@ -999,17 +1026,21 @@ export class ComfyApp {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Loads all extensions from the API into the window
|
* Loads all extensions from the API into the window in parallel
|
||||||
*/
|
*/
|
||||||
async #loadExtensions() {
|
async #loadExtensions() {
|
||||||
const extensions = await api.getExtensions();
|
const extensions = await api.getExtensions();
|
||||||
for (const ext of extensions) {
|
this.logging.addEntry("Comfy.App", "debug", { Extensions: extensions });
|
||||||
try {
|
|
||||||
await import(api.apiURL(ext));
|
const extensionPromises = extensions.map(async ext => {
|
||||||
} catch (error) {
|
try {
|
||||||
console.error("Error loading extension", ext, error);
|
await import(api.apiURL(ext));
|
||||||
}
|
} catch (error) {
|
||||||
}
|
console.error("Error loading extension", ext, error);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
await Promise.all(extensionPromises);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@ -1286,6 +1317,9 @@ export class ComfyApp {
|
|||||||
(t) => `<li>${t}</li>`
|
(t) => `<li>${t}</li>`
|
||||||
).join("")}</ul>Nodes that have failed to load will show as red on the graph.`
|
).join("")}</ul>Nodes that have failed to load will show as red on the graph.`
|
||||||
);
|
);
|
||||||
|
this.logging.addEntry("Comfy.App", "warn", {
|
||||||
|
MissingNodes: missingNodeTypes,
|
||||||
|
});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -1308,7 +1342,7 @@ export class ComfyApp {
|
|||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (node.mode === 2) {
|
if (node.mode === 2 || node.mode === 4) {
|
||||||
// Don't serialize muted nodes
|
// Don't serialize muted nodes
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
@ -1331,12 +1365,36 @@ export class ComfyApp {
|
|||||||
let parent = node.getInputNode(i);
|
let parent = node.getInputNode(i);
|
||||||
if (parent) {
|
if (parent) {
|
||||||
let link = node.getInputLink(i);
|
let link = node.getInputLink(i);
|
||||||
while (parent && parent.isVirtualNode) {
|
while (parent.mode === 4 || parent.isVirtualNode) {
|
||||||
link = parent.getInputLink(link.origin_slot);
|
let found = false;
|
||||||
if (link) {
|
if (parent.isVirtualNode) {
|
||||||
parent = parent.getInputNode(link.origin_slot);
|
link = parent.getInputLink(link.origin_slot);
|
||||||
} else {
|
if (link) {
|
||||||
parent = null;
|
parent = parent.getInputNode(link.target_slot);
|
||||||
|
if (parent) {
|
||||||
|
found = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else if (link && parent.mode === 4) {
|
||||||
|
let all_inputs = [link.origin_slot];
|
||||||
|
if (parent.inputs) {
|
||||||
|
all_inputs = all_inputs.concat(Object.keys(parent.inputs))
|
||||||
|
for (let parent_input in all_inputs) {
|
||||||
|
parent_input = all_inputs[parent_input];
|
||||||
|
if (parent.inputs[parent_input].type === node.inputs[i].type) {
|
||||||
|
link = parent.getInputLink(parent_input);
|
||||||
|
if (link) {
|
||||||
|
parent = parent.getInputNode(parent_input);
|
||||||
|
}
|
||||||
|
found = true;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!found) {
|
||||||
|
break;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
367
web/scripts/logging.js
Normal file
367
web/scripts/logging.js
Normal file
@ -0,0 +1,367 @@
|
|||||||
|
import { $el, ComfyDialog } from "./ui.js";
|
||||||
|
import { api } from "./api.js";
|
||||||
|
|
||||||
|
$el("style", {
|
||||||
|
textContent: `
|
||||||
|
.comfy-logging-logs {
|
||||||
|
display: grid;
|
||||||
|
color: var(--fg-color);
|
||||||
|
white-space: pre-wrap;
|
||||||
|
}
|
||||||
|
.comfy-logging-log {
|
||||||
|
display: contents;
|
||||||
|
}
|
||||||
|
.comfy-logging-title {
|
||||||
|
background: var(--tr-even-bg-color);
|
||||||
|
font-weight: bold;
|
||||||
|
margin-bottom: 5px;
|
||||||
|
text-align: center;
|
||||||
|
}
|
||||||
|
.comfy-logging-log div {
|
||||||
|
background: var(--row-bg);
|
||||||
|
padding: 5px;
|
||||||
|
}
|
||||||
|
`,
|
||||||
|
parent: document.body,
|
||||||
|
});
|
||||||
|
|
||||||
|
// Stringify function supporting max depth and removal of circular references
|
||||||
|
// https://stackoverflow.com/a/57193345
|
||||||
|
function stringify(val, depth, replacer, space, onGetObjID) {
|
||||||
|
depth = isNaN(+depth) ? 1 : depth;
|
||||||
|
var recursMap = new WeakMap();
|
||||||
|
function _build(val, depth, o, a, r) {
|
||||||
|
// (JSON.stringify() has it's own rules, which we respect here by using it for property iteration)
|
||||||
|
return !val || typeof val != "object"
|
||||||
|
? val
|
||||||
|
: ((r = recursMap.has(val)),
|
||||||
|
recursMap.set(val, true),
|
||||||
|
(a = Array.isArray(val)),
|
||||||
|
r
|
||||||
|
? (o = (onGetObjID && onGetObjID(val)) || null)
|
||||||
|
: JSON.stringify(val, function (k, v) {
|
||||||
|
if (a || depth > 0) {
|
||||||
|
if (replacer) v = replacer(k, v);
|
||||||
|
if (!k) return (a = Array.isArray(v)), (val = v);
|
||||||
|
!o && (o = a ? [] : {});
|
||||||
|
o[k] = _build(v, a ? depth : depth - 1);
|
||||||
|
}
|
||||||
|
}),
|
||||||
|
o === void 0 ? (a ? [] : {}) : o);
|
||||||
|
}
|
||||||
|
return JSON.stringify(_build(val, depth), null, space);
|
||||||
|
}
|
||||||
|
|
||||||
|
const jsonReplacer = (k, v, ui) => {
|
||||||
|
if (v instanceof Array && v.length === 1) {
|
||||||
|
v = v[0];
|
||||||
|
}
|
||||||
|
if (v instanceof Date) {
|
||||||
|
v = v.toISOString();
|
||||||
|
if (ui) {
|
||||||
|
v = v.split("T")[1];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (v instanceof Error) {
|
||||||
|
let err = "";
|
||||||
|
if (v.name) err += v.name + "\n";
|
||||||
|
if (v.message) err += v.message + "\n";
|
||||||
|
if (v.stack) err += v.stack + "\n";
|
||||||
|
if (!err) {
|
||||||
|
err = v.toString();
|
||||||
|
}
|
||||||
|
v = err;
|
||||||
|
}
|
||||||
|
return v;
|
||||||
|
};
|
||||||
|
|
||||||
|
const fileInput = $el("input", {
|
||||||
|
type: "file",
|
||||||
|
accept: ".json",
|
||||||
|
style: { display: "none" },
|
||||||
|
parent: document.body,
|
||||||
|
});
|
||||||
|
|
||||||
|
class ComfyLoggingDialog extends ComfyDialog {
|
||||||
|
constructor(logging) {
|
||||||
|
super();
|
||||||
|
this.logging = logging;
|
||||||
|
}
|
||||||
|
|
||||||
|
clear() {
|
||||||
|
this.logging.clear();
|
||||||
|
this.show();
|
||||||
|
}
|
||||||
|
|
||||||
|
export() {
|
||||||
|
const blob = new Blob([stringify([...this.logging.entries], 20, jsonReplacer, "\t")], {
|
||||||
|
type: "application/json",
|
||||||
|
});
|
||||||
|
const url = URL.createObjectURL(blob);
|
||||||
|
const a = $el("a", {
|
||||||
|
href: url,
|
||||||
|
download: `comfyui-logs-${Date.now()}.json`,
|
||||||
|
style: { display: "none" },
|
||||||
|
parent: document.body,
|
||||||
|
});
|
||||||
|
a.click();
|
||||||
|
setTimeout(function () {
|
||||||
|
a.remove();
|
||||||
|
window.URL.revokeObjectURL(url);
|
||||||
|
}, 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
import() {
|
||||||
|
fileInput.onchange = () => {
|
||||||
|
const reader = new FileReader();
|
||||||
|
reader.onload = () => {
|
||||||
|
fileInput.remove();
|
||||||
|
try {
|
||||||
|
const obj = JSON.parse(reader.result);
|
||||||
|
if (obj instanceof Array) {
|
||||||
|
this.show(obj);
|
||||||
|
} else {
|
||||||
|
throw new Error("Invalid file selected.");
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
alert("Unable to load logs: " + error.message);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
reader.readAsText(fileInput.files[0]);
|
||||||
|
};
|
||||||
|
fileInput.click();
|
||||||
|
}
|
||||||
|
|
||||||
|
createButtons() {
|
||||||
|
return [
|
||||||
|
$el("button", {
|
||||||
|
type: "button",
|
||||||
|
textContent: "Clear",
|
||||||
|
onclick: () => this.clear(),
|
||||||
|
}),
|
||||||
|
$el("button", {
|
||||||
|
type: "button",
|
||||||
|
textContent: "Export logs...",
|
||||||
|
onclick: () => this.export(),
|
||||||
|
}),
|
||||||
|
$el("button", {
|
||||||
|
type: "button",
|
||||||
|
textContent: "View exported logs...",
|
||||||
|
onclick: () => this.import(),
|
||||||
|
}),
|
||||||
|
...super.createButtons(),
|
||||||
|
];
|
||||||
|
}
|
||||||
|
|
||||||
|
getTypeColor(type) {
|
||||||
|
switch (type) {
|
||||||
|
case "error":
|
||||||
|
return "red";
|
||||||
|
case "warn":
|
||||||
|
return "orange";
|
||||||
|
case "debug":
|
||||||
|
return "dodgerblue";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
show(entries) {
|
||||||
|
if (!entries) entries = this.logging.entries;
|
||||||
|
this.element.style.width = "100%";
|
||||||
|
const cols = {
|
||||||
|
source: "Source",
|
||||||
|
type: "Type",
|
||||||
|
timestamp: "Timestamp",
|
||||||
|
message: "Message",
|
||||||
|
};
|
||||||
|
const keys = Object.keys(cols);
|
||||||
|
const headers = Object.values(cols).map((title) =>
|
||||||
|
$el("div.comfy-logging-title", {
|
||||||
|
textContent: title,
|
||||||
|
})
|
||||||
|
);
|
||||||
|
const rows = entries.map((entry, i) => {
|
||||||
|
return $el(
|
||||||
|
"div.comfy-logging-log",
|
||||||
|
{
|
||||||
|
$: (el) => el.style.setProperty("--row-bg", `var(--tr-${i % 2 ? "even" : "odd"}-bg-color)`),
|
||||||
|
},
|
||||||
|
keys.map((key) => {
|
||||||
|
let v = entry[key];
|
||||||
|
let color;
|
||||||
|
if (key === "type") {
|
||||||
|
color = this.getTypeColor(v);
|
||||||
|
} else {
|
||||||
|
v = jsonReplacer(key, v, true);
|
||||||
|
|
||||||
|
if (typeof v === "object") {
|
||||||
|
v = stringify(v, 5, jsonReplacer, " ");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return $el("div", {
|
||||||
|
style: {
|
||||||
|
color,
|
||||||
|
},
|
||||||
|
textContent: v,
|
||||||
|
});
|
||||||
|
})
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
const grid = $el(
|
||||||
|
"div.comfy-logging-logs",
|
||||||
|
{
|
||||||
|
style: {
|
||||||
|
gridTemplateColumns: `repeat(${headers.length}, 1fr)`,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
[...headers, ...rows]
|
||||||
|
);
|
||||||
|
const els = [grid];
|
||||||
|
if (!this.logging.enabled) {
|
||||||
|
els.unshift(
|
||||||
|
$el("h3", {
|
||||||
|
style: { textAlign: "center" },
|
||||||
|
textContent: "Logging is disabled",
|
||||||
|
})
|
||||||
|
);
|
||||||
|
}
|
||||||
|
super.show($el("div", els));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export class ComfyLogging {
|
||||||
|
/**
|
||||||
|
* @type Array<{ source: string, type: string, timestamp: Date, message: any }>
|
||||||
|
*/
|
||||||
|
entries = [];
|
||||||
|
|
||||||
|
#enabled;
|
||||||
|
#console = {};
|
||||||
|
|
||||||
|
get enabled() {
|
||||||
|
return this.#enabled;
|
||||||
|
}
|
||||||
|
|
||||||
|
set enabled(value) {
|
||||||
|
if (value === this.#enabled) return;
|
||||||
|
if (value) {
|
||||||
|
this.patchConsole();
|
||||||
|
} else {
|
||||||
|
this.unpatchConsole();
|
||||||
|
}
|
||||||
|
this.#enabled = value;
|
||||||
|
}
|
||||||
|
|
||||||
|
constructor(app) {
|
||||||
|
this.app = app;
|
||||||
|
|
||||||
|
this.dialog = new ComfyLoggingDialog(this);
|
||||||
|
this.addSetting();
|
||||||
|
this.catchUnhandled();
|
||||||
|
this.addInitData();
|
||||||
|
}
|
||||||
|
|
||||||
|
addSetting() {
|
||||||
|
const settingId = "Comfy.Logging.Enabled";
|
||||||
|
const htmlSettingId = settingId.replaceAll(".", "-");
|
||||||
|
const setting = this.app.ui.settings.addSetting({
|
||||||
|
id: settingId,
|
||||||
|
name: settingId,
|
||||||
|
defaultValue: true,
|
||||||
|
type: (name, setter, value) => {
|
||||||
|
return $el("tr", [
|
||||||
|
$el("td", [
|
||||||
|
$el("label", {
|
||||||
|
textContent: "Logging",
|
||||||
|
for: htmlSettingId,
|
||||||
|
}),
|
||||||
|
]),
|
||||||
|
$el("td", [
|
||||||
|
$el("input", {
|
||||||
|
id: htmlSettingId,
|
||||||
|
type: "checkbox",
|
||||||
|
checked: value,
|
||||||
|
onchange: (event) => {
|
||||||
|
setter((this.enabled = event.target.checked));
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
$el("button", {
|
||||||
|
textContent: "View Logs",
|
||||||
|
onclick: () => {
|
||||||
|
this.app.ui.settings.element.close();
|
||||||
|
this.dialog.show();
|
||||||
|
},
|
||||||
|
style: {
|
||||||
|
fontSize: "14px",
|
||||||
|
display: "block",
|
||||||
|
marginTop: "5px",
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
]),
|
||||||
|
]);
|
||||||
|
},
|
||||||
|
});
|
||||||
|
this.enabled = setting.value;
|
||||||
|
}
|
||||||
|
|
||||||
|
patchConsole() {
|
||||||
|
// Capture common console outputs
|
||||||
|
const self = this;
|
||||||
|
for (const type of ["log", "warn", "error", "debug"]) {
|
||||||
|
const orig = console[type];
|
||||||
|
this.#console[type] = orig;
|
||||||
|
console[type] = function () {
|
||||||
|
orig.apply(console, arguments);
|
||||||
|
self.addEntry("console", type, ...arguments);
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
unpatchConsole() {
|
||||||
|
// Restore original console functions
|
||||||
|
for (const type of Object.keys(this.#console)) {
|
||||||
|
console[type] = this.#console[type];
|
||||||
|
}
|
||||||
|
this.#console = {};
|
||||||
|
}
|
||||||
|
|
||||||
|
catchUnhandled() {
|
||||||
|
// Capture uncaught errors
|
||||||
|
window.addEventListener("error", (e) => {
|
||||||
|
this.addEntry("window", "error", e.error ?? "Unknown error");
|
||||||
|
return false;
|
||||||
|
});
|
||||||
|
|
||||||
|
window.addEventListener("unhandledrejection", (e) => {
|
||||||
|
this.addEntry("unhandledrejection", "error", e.reason ?? "Unknown error");
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
clear() {
|
||||||
|
this.entries = [];
|
||||||
|
}
|
||||||
|
|
||||||
|
addEntry(source, type, ...args) {
|
||||||
|
if (this.enabled) {
|
||||||
|
this.entries.push({
|
||||||
|
source,
|
||||||
|
type,
|
||||||
|
timestamp: new Date(),
|
||||||
|
message: args,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
log(source, ...args) {
|
||||||
|
this.addEntry(source, "log", ...args);
|
||||||
|
}
|
||||||
|
|
||||||
|
async addInitData() {
|
||||||
|
if (!this.enabled) return;
|
||||||
|
const source = "ComfyUI.Logging";
|
||||||
|
this.addEntry(source, "debug", { UserAgent: navigator.userAgent });
|
||||||
|
const systemStats = await api.getSystemStats();
|
||||||
|
this.addEntry(source, "debug", systemStats);
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -234,7 +234,7 @@ class ComfySettingsDialog extends ComfyDialog {
|
|||||||
localStorage[settingId] = JSON.stringify(value);
|
localStorage[settingId] = JSON.stringify(value);
|
||||||
}
|
}
|
||||||
|
|
||||||
addSetting({id, name, type, defaultValue, onChange, attrs = {}, tooltip = "",}) {
|
addSetting({id, name, type, defaultValue, onChange, attrs = {}, tooltip = "", options = undefined}) {
|
||||||
if (!id) {
|
if (!id) {
|
||||||
throw new Error("Settings must have an ID");
|
throw new Error("Settings must have an ID");
|
||||||
}
|
}
|
||||||
@ -347,6 +347,32 @@ class ComfySettingsDialog extends ComfyDialog {
|
|||||||
]),
|
]),
|
||||||
]);
|
]);
|
||||||
break;
|
break;
|
||||||
|
case "combo":
|
||||||
|
element = $el("tr", [
|
||||||
|
labelCell,
|
||||||
|
$el("td", [
|
||||||
|
$el(
|
||||||
|
"select",
|
||||||
|
{
|
||||||
|
oninput: (e) => {
|
||||||
|
setter(e.target.value);
|
||||||
|
},
|
||||||
|
},
|
||||||
|
(typeof options === "function" ? options(value) : options || []).map((opt) => {
|
||||||
|
if (typeof opt === "string") {
|
||||||
|
opt = { text: opt };
|
||||||
|
}
|
||||||
|
const v = opt.value ?? opt.text;
|
||||||
|
return $el("option", {
|
||||||
|
value: v,
|
||||||
|
textContent: opt.text,
|
||||||
|
selected: value + "" === v + "",
|
||||||
|
});
|
||||||
|
})
|
||||||
|
),
|
||||||
|
]),
|
||||||
|
]);
|
||||||
|
break;
|
||||||
case "text":
|
case "text":
|
||||||
default:
|
default:
|
||||||
if (type !== "text") {
|
if (type !== "text") {
|
||||||
@ -480,7 +506,7 @@ class ComfyList {
|
|||||||
|
|
||||||
hide() {
|
hide() {
|
||||||
this.element.style.display = "none";
|
this.element.style.display = "none";
|
||||||
this.button.textContent = "See " + this.#text;
|
this.button.textContent = "View " + this.#text;
|
||||||
}
|
}
|
||||||
|
|
||||||
toggle() {
|
toggle() {
|
||||||
@ -542,6 +568,13 @@ export class ComfyUI {
|
|||||||
defaultValue: "",
|
defaultValue: "",
|
||||||
});
|
});
|
||||||
|
|
||||||
|
this.settings.addSetting({
|
||||||
|
id: "Comfy.DisableSliders",
|
||||||
|
name: "Disable sliders.",
|
||||||
|
type: "boolean",
|
||||||
|
defaultValue: false,
|
||||||
|
});
|
||||||
|
|
||||||
const fileInput = $el("input", {
|
const fileInput = $el("input", {
|
||||||
id: "comfy-file-input",
|
id: "comfy-file-input",
|
||||||
type: "file",
|
type: "file",
|
||||||
|
|||||||
@ -79,8 +79,8 @@ export function addValueControlWidget(node, targetWidget, defaultValue = "random
|
|||||||
return valueControl;
|
return valueControl;
|
||||||
};
|
};
|
||||||
|
|
||||||
function seedWidget(node, inputName, inputData) {
|
function seedWidget(node, inputName, inputData, app) {
|
||||||
const seed = ComfyWidgets.INT(node, inputName, inputData);
|
const seed = ComfyWidgets.INT(node, inputName, inputData, app);
|
||||||
const seedControl = addValueControlWidget(node, seed.widget, "randomize");
|
const seedControl = addValueControlWidget(node, seed.widget, "randomize");
|
||||||
|
|
||||||
seed.widget.linkedWidgets = [seedControl];
|
seed.widget.linkedWidgets = [seedControl];
|
||||||
@ -250,19 +250,29 @@ function addMultilineWidget(node, name, opts, app) {
|
|||||||
return { minWidth: 400, minHeight: 200, widget };
|
return { minWidth: 400, minHeight: 200, widget };
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function isSlider(display, app) {
|
||||||
|
if (app.ui.settings.getSettingValue("Comfy.DisableSliders")) {
|
||||||
|
return "number"
|
||||||
|
}
|
||||||
|
|
||||||
|
return (display==="slider") ? "slider" : "number"
|
||||||
|
}
|
||||||
|
|
||||||
export const ComfyWidgets = {
|
export const ComfyWidgets = {
|
||||||
"INT:seed": seedWidget,
|
"INT:seed": seedWidget,
|
||||||
"INT:noise_seed": seedWidget,
|
"INT:noise_seed": seedWidget,
|
||||||
FLOAT(node, inputName, inputData) {
|
FLOAT(node, inputName, inputData, app) {
|
||||||
|
let widgetType = isSlider(inputData[1]["display"], app);
|
||||||
const { val, config } = getNumberDefaults(inputData, 0.5);
|
const { val, config } = getNumberDefaults(inputData, 0.5);
|
||||||
return { widget: node.addWidget("number", inputName, val, () => {}, config) };
|
return { widget: node.addWidget(widgetType, inputName, val, () => {}, config) };
|
||||||
},
|
},
|
||||||
INT(node, inputName, inputData) {
|
INT(node, inputName, inputData, app) {
|
||||||
|
let widgetType = isSlider(inputData[1]["display"], app);
|
||||||
const { val, config } = getNumberDefaults(inputData, 1);
|
const { val, config } = getNumberDefaults(inputData, 1);
|
||||||
Object.assign(config, { precision: 0 });
|
Object.assign(config, { precision: 0 });
|
||||||
return {
|
return {
|
||||||
widget: node.addWidget(
|
widget: node.addWidget(
|
||||||
"number",
|
widgetType,
|
||||||
inputName,
|
inputName,
|
||||||
val,
|
val,
|
||||||
function (v) {
|
function (v) {
|
||||||
@ -273,15 +283,33 @@ export const ComfyWidgets = {
|
|||||||
),
|
),
|
||||||
};
|
};
|
||||||
},
|
},
|
||||||
|
BOOLEAN(node, inputName, inputData) {
|
||||||
|
let defaultVal = inputData[1]["default"];
|
||||||
|
return {
|
||||||
|
widget: node.addWidget(
|
||||||
|
"toggle",
|
||||||
|
inputName,
|
||||||
|
defaultVal,
|
||||||
|
() => {},
|
||||||
|
{"on": inputData[1].label_on, "off": inputData[1].label_off}
|
||||||
|
)
|
||||||
|
};
|
||||||
|
},
|
||||||
STRING(node, inputName, inputData, app) {
|
STRING(node, inputName, inputData, app) {
|
||||||
const defaultVal = inputData[1].default || "";
|
const defaultVal = inputData[1].default || "";
|
||||||
const multiline = !!inputData[1].multiline;
|
const multiline = !!inputData[1].multiline;
|
||||||
|
|
||||||
|
let res;
|
||||||
if (multiline) {
|
if (multiline) {
|
||||||
return addMultilineWidget(node, inputName, { defaultVal, ...inputData[1] }, app);
|
res = addMultilineWidget(node, inputName, { defaultVal, ...inputData[1] }, app);
|
||||||
} else {
|
} else {
|
||||||
return { widget: node.addWidget("text", inputName, defaultVal, () => {}, {}) };
|
res = { widget: node.addWidget("text", inputName, defaultVal, () => {}, {}) };
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if(inputData[1].dynamicPrompts != undefined)
|
||||||
|
res.widget.dynamicPrompts = inputData[1].dynamicPrompts;
|
||||||
|
|
||||||
|
return res;
|
||||||
},
|
},
|
||||||
COMBO(node, inputName, inputData) {
|
COMBO(node, inputName, inputData) {
|
||||||
const type = inputData[0];
|
const type = inputData[0];
|
||||||
@ -411,7 +439,7 @@ export const ComfyWidgets = {
|
|||||||
// Add handler to check if an image is being dragged over our node
|
// Add handler to check if an image is being dragged over our node
|
||||||
node.onDragOver = function (e) {
|
node.onDragOver = function (e) {
|
||||||
if (e.dataTransfer && e.dataTransfer.items) {
|
if (e.dataTransfer && e.dataTransfer.items) {
|
||||||
const image = [...e.dataTransfer.items].find((f) => f.kind === "file" && f.type.startsWith("image/"));
|
const image = [...e.dataTransfer.items].find((f) => f.kind === "file");
|
||||||
return !!image;
|
return !!image;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
1
web/user.css
Normal file
1
web/user.css
Normal file
@ -0,0 +1 @@
|
|||||||
|
/* Put custom styles here */
|
||||||
Loading…
x
Reference in New Issue
Block a user