mirror of
https://git.datalinker.icu/comfyanonymous/ComfyUI
synced 2026-11-21 15:27:02 +08:00
Merge branch 'comfyanonymous:master' into dpr
This commit is contained in:
commit
e8a8ae03c8
@ -1,3 +0,0 @@
|
|||||||
..\python_embeded\python.exe .\update.py ..\ComfyUI\
|
|
||||||
..\python_embeded\python.exe -s -m pip install --upgrade --pre torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/nightly/cu121 -r ../ComfyUI/requirements.txt pygit2
|
|
||||||
pause
|
|
||||||
@ -1,2 +0,0 @@
|
|||||||
.\python_embeded\python.exe -s ComfyUI\main.py --windows-standalone-build --use-pytorch-cross-attention
|
|
||||||
pause
|
|
||||||
2
.github/workflows/test-ui.yaml
vendored
2
.github/workflows/test-ui.yaml
vendored
@ -22,5 +22,5 @@ jobs:
|
|||||||
run: |
|
run: |
|
||||||
npm ci
|
npm ci
|
||||||
npm run test:generate
|
npm run test:generate
|
||||||
npm test
|
npm test -- --verbose
|
||||||
working-directory: ./tests-ui
|
working-directory: ./tests-ui
|
||||||
|
|||||||
@ -2,6 +2,24 @@ name: "Windows Release Nightly pytorch"
|
|||||||
|
|
||||||
on:
|
on:
|
||||||
workflow_dispatch:
|
workflow_dispatch:
|
||||||
|
inputs:
|
||||||
|
cu:
|
||||||
|
description: 'cuda version'
|
||||||
|
required: true
|
||||||
|
type: string
|
||||||
|
default: "121"
|
||||||
|
|
||||||
|
python_minor:
|
||||||
|
description: 'python minor version'
|
||||||
|
required: true
|
||||||
|
type: string
|
||||||
|
default: "12"
|
||||||
|
|
||||||
|
python_patch:
|
||||||
|
description: 'python patch version'
|
||||||
|
required: true
|
||||||
|
type: string
|
||||||
|
default: "1"
|
||||||
# push:
|
# push:
|
||||||
# branches:
|
# branches:
|
||||||
# - master
|
# - master
|
||||||
@ -20,21 +38,21 @@ jobs:
|
|||||||
persist-credentials: false
|
persist-credentials: false
|
||||||
- uses: actions/setup-python@v4
|
- uses: actions/setup-python@v4
|
||||||
with:
|
with:
|
||||||
python-version: '3.11.6'
|
python-version: 3.${{ inputs.python_minor }}.${{ inputs.python_patch }}
|
||||||
- shell: bash
|
- shell: bash
|
||||||
run: |
|
run: |
|
||||||
cd ..
|
cd ..
|
||||||
cp -r ComfyUI ComfyUI_copy
|
cp -r ComfyUI ComfyUI_copy
|
||||||
curl https://www.python.org/ftp/python/3.11.6/python-3.11.6-embed-amd64.zip -o python_embeded.zip
|
curl https://www.python.org/ftp/python/3.${{ inputs.python_minor }}.${{ inputs.python_patch }}/python-3.${{ inputs.python_minor }}.${{ inputs.python_patch }}-embed-amd64.zip -o python_embeded.zip
|
||||||
unzip python_embeded.zip -d python_embeded
|
unzip python_embeded.zip -d python_embeded
|
||||||
cd python_embeded
|
cd python_embeded
|
||||||
echo 'import site' >> ./python311._pth
|
echo 'import site' >> ./python3${{ inputs.python_minor }}._pth
|
||||||
curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py
|
curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py
|
||||||
./python.exe get-pip.py
|
./python.exe get-pip.py
|
||||||
python -m pip wheel torch torchvision torchaudio aiohttp==3.8.5 --pre --extra-index-url https://download.pytorch.org/whl/nightly/cu121 -r ../ComfyUI/requirements.txt pygit2 -w ../temp_wheel_dir
|
python -m pip wheel torch torchvision torchaudio --pre --extra-index-url https://download.pytorch.org/whl/nightly/cu${{ inputs.cu }} -r ../ComfyUI/requirements.txt pygit2 -w ../temp_wheel_dir
|
||||||
ls ../temp_wheel_dir
|
ls ../temp_wheel_dir
|
||||||
./python.exe -s -m pip install --pre ../temp_wheel_dir/*
|
./python.exe -s -m pip install --pre ../temp_wheel_dir/*
|
||||||
sed -i '1i../ComfyUI' ./python311._pth
|
sed -i '1i../ComfyUI' ./python3${{ inputs.python_minor }}._pth
|
||||||
cd ..
|
cd ..
|
||||||
|
|
||||||
git clone https://github.com/comfyanonymous/taesd
|
git clone https://github.com/comfyanonymous/taesd
|
||||||
@ -49,9 +67,10 @@ jobs:
|
|||||||
mkdir update
|
mkdir update
|
||||||
cp -r ComfyUI/.ci/update_windows/* ./update/
|
cp -r ComfyUI/.ci/update_windows/* ./update/
|
||||||
cp -r ComfyUI/.ci/windows_base_files/* ./
|
cp -r ComfyUI/.ci/windows_base_files/* ./
|
||||||
cp -r ComfyUI/.ci/nightly/update_windows/* ./update/
|
|
||||||
cp -r ComfyUI/.ci/nightly/windows_base_files/* ./
|
|
||||||
|
|
||||||
|
echo "..\python_embeded\python.exe .\update.py ..\ComfyUI\\
|
||||||
|
..\python_embeded\python.exe -s -m pip install --upgrade --pre torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/nightly/cu${{ inputs.cu }} -r ../ComfyUI/requirements.txt pygit2
|
||||||
|
pause" > ./update/update_comfyui_and_python_dependencies.bat
|
||||||
cd ..
|
cd ..
|
||||||
|
|
||||||
"C:\Program Files\7-Zip\7z.exe" a -t7z -m0=lzma -mx=8 -mfb=64 -md=32m -ms=on -mf=BCJ2 ComfyUI_windows_portable_nightly_pytorch.7z ComfyUI_windows_portable_nightly_pytorch
|
"C:\Program Files\7-Zip\7z.exe" a -t7z -m0=lzma -mx=8 -mfb=64 -md=32m -ms=on -mf=BCJ2 ComfyUI_windows_portable_nightly_pytorch.7z ComfyUI_windows_portable_nightly_pytorch
|
||||||
|
|||||||
18
README.md
18
README.md
@ -11,7 +11,7 @@ This ui will let you design and execute advanced stable diffusion pipelines usin
|
|||||||
|
|
||||||
## Features
|
## Features
|
||||||
- Nodes/graph/flowchart interface to experiment and create complex Stable Diffusion workflows without needing to code anything.
|
- Nodes/graph/flowchart interface to experiment and create complex Stable Diffusion workflows without needing to code anything.
|
||||||
- Fully supports SD1.x, SD2.x and SDXL
|
- Fully supports SD1.x, SD2.x, [SDXL](https://comfyanonymous.github.io/ComfyUI_examples/sdxl/) and [Stable Video Diffusion](https://comfyanonymous.github.io/ComfyUI_examples/video/)
|
||||||
- Asynchronous Queue system
|
- Asynchronous Queue system
|
||||||
- Many optimizations: Only re-executes the parts of the workflow that changes between executions.
|
- Many optimizations: Only re-executes the parts of the workflow that changes between executions.
|
||||||
- Command line option: ```--lowvram``` to make it work on GPUs with less than 3GB vram (enabled automatically on GPUs with low vram)
|
- Command line option: ```--lowvram``` to make it work on GPUs with less than 3GB vram (enabled automatically on GPUs with low vram)
|
||||||
@ -30,6 +30,8 @@ This ui will let you design and execute advanced stable diffusion pipelines usin
|
|||||||
- [unCLIP Models](https://comfyanonymous.github.io/ComfyUI_examples/unclip/)
|
- [unCLIP Models](https://comfyanonymous.github.io/ComfyUI_examples/unclip/)
|
||||||
- [GLIGEN](https://comfyanonymous.github.io/ComfyUI_examples/gligen/)
|
- [GLIGEN](https://comfyanonymous.github.io/ComfyUI_examples/gligen/)
|
||||||
- [Model Merging](https://comfyanonymous.github.io/ComfyUI_examples/model_merging/)
|
- [Model Merging](https://comfyanonymous.github.io/ComfyUI_examples/model_merging/)
|
||||||
|
- [LCM models and Loras](https://comfyanonymous.github.io/ComfyUI_examples/lcm/)
|
||||||
|
- [SDXL Turbo](https://comfyanonymous.github.io/ComfyUI_examples/sdturbo/)
|
||||||
- Latent previews with [TAESD](#how-to-show-high-quality-previews)
|
- Latent previews with [TAESD](#how-to-show-high-quality-previews)
|
||||||
- Starts up very fast.
|
- Starts up very fast.
|
||||||
- Works fully offline: will never download anything.
|
- Works fully offline: will never download anything.
|
||||||
@ -43,6 +45,7 @@ Workflow examples can be found on the [Examples page](https://comfyanonymous.git
|
|||||||
|---------------------------|--------------------------------------------------------------------------------------------------------------------|
|
|---------------------------|--------------------------------------------------------------------------------------------------------------------|
|
||||||
| Ctrl + Enter | Queue up current graph for generation |
|
| Ctrl + Enter | Queue up current graph for generation |
|
||||||
| Ctrl + Shift + Enter | Queue up current graph as first for generation |
|
| Ctrl + Shift + Enter | Queue up current graph as first for generation |
|
||||||
|
| Ctrl + Z/Ctrl + Y | Undo/Redo |
|
||||||
| Ctrl + S | Save workflow |
|
| Ctrl + S | Save workflow |
|
||||||
| Ctrl + O | Load workflow |
|
| Ctrl + O | Load workflow |
|
||||||
| Ctrl + A | Select all nodes |
|
| Ctrl + A | Select all nodes |
|
||||||
@ -90,22 +93,27 @@ Put your SD checkpoints (the huge ckpt/safetensors files) in: models/checkpoints
|
|||||||
|
|
||||||
Put your VAE in: models/vae
|
Put your VAE in: models/vae
|
||||||
|
|
||||||
Note: pytorch does not support python 3.12 yet so make sure your python version is 3.11 or earlier.
|
Note: pytorch stable does not support python 3.12 yet. If you have python 3.12 you will have to use the nightly version of pytorch. If you run into issues you should try python 3.11 instead.
|
||||||
|
|
||||||
### AMD GPUs (Linux only)
|
### AMD GPUs (Linux only)
|
||||||
AMD users can install rocm and pytorch with pip if you don't have it already installed, this is the command to install the stable version:
|
AMD users can install rocm and pytorch with pip if you don't have it already installed, this is the command to install the stable version:
|
||||||
|
|
||||||
```pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/rocm5.6```
|
```pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/rocm5.6```
|
||||||
|
|
||||||
This is the command to install the nightly with ROCm 5.7 that might have some performance improvements:
|
This is the command to install the nightly with ROCm 5.7 which has a python 3.12 package and might have some performance improvements:
|
||||||
|
|
||||||
```pip install --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/rocm5.7```
|
```pip install --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/rocm5.7```
|
||||||
|
|
||||||
### NVIDIA
|
### NVIDIA
|
||||||
|
|
||||||
Nvidia users should install pytorch using this command:
|
Nvidia users should install stable pytorch using this command:
|
||||||
|
|
||||||
```pip install torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cu121```
|
```pip install torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cu121```
|
||||||
|
|
||||||
|
This is the command to install pytorch nightly instead which has a python 3.12 package and might have performance improvements:
|
||||||
|
|
||||||
|
```pip install --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/cu121```
|
||||||
|
|
||||||
#### Troubleshooting
|
#### Troubleshooting
|
||||||
|
|
||||||
If you get the "Torch not compiled with CUDA enabled" error, uninstall torch with:
|
If you get the "Torch not compiled with CUDA enabled" error, uninstall torch with:
|
||||||
@ -190,7 +198,7 @@ To use a textual inversion concepts/embeddings in a text prompt put them in the
|
|||||||
|
|
||||||
Make sure you use the regular loaders/Load Checkpoint node to load checkpoints. It will auto pick the right settings depending on your GPU.
|
Make sure you use the regular loaders/Load Checkpoint node to load checkpoints. It will auto pick the right settings depending on your GPU.
|
||||||
|
|
||||||
You can set this command line setting to disable the upcasting to fp32 in some cross attention operations which will increase your speed. Note that this will very likely give you black images on SD2.x models. If you use xformers this option does not do anything.
|
You can set this command line setting to disable the upcasting to fp32 in some cross attention operations which will increase your speed. Note that this will very likely give you black images on SD2.x models. If you use xformers or pytorch attention this option does not do anything.
|
||||||
|
|
||||||
```--dont-upcast-attention```
|
```--dont-upcast-attention```
|
||||||
|
|
||||||
|
|||||||
@ -53,7 +53,8 @@ class ControlNet(nn.Module):
|
|||||||
transformer_depth_middle=None,
|
transformer_depth_middle=None,
|
||||||
transformer_depth_output=None,
|
transformer_depth_output=None,
|
||||||
device=None,
|
device=None,
|
||||||
operations=comfy.ops,
|
operations=comfy.ops.disable_weight_init,
|
||||||
|
**kwargs,
|
||||||
):
|
):
|
||||||
super().__init__()
|
super().__init__()
|
||||||
assert use_spatial_transformer == True, "use_spatial_transformer has to be true"
|
assert use_spatial_transformer == True, "use_spatial_transformer has to be true"
|
||||||
@ -140,24 +141,24 @@ class ControlNet(nn.Module):
|
|||||||
)
|
)
|
||||||
]
|
]
|
||||||
)
|
)
|
||||||
self.zero_convs = nn.ModuleList([self.make_zero_conv(model_channels, operations=operations)])
|
self.zero_convs = nn.ModuleList([self.make_zero_conv(model_channels, operations=operations, dtype=self.dtype, device=device)])
|
||||||
|
|
||||||
self.input_hint_block = TimestepEmbedSequential(
|
self.input_hint_block = TimestepEmbedSequential(
|
||||||
operations.conv_nd(dims, hint_channels, 16, 3, padding=1),
|
operations.conv_nd(dims, hint_channels, 16, 3, padding=1, dtype=self.dtype, device=device),
|
||||||
nn.SiLU(),
|
nn.SiLU(),
|
||||||
operations.conv_nd(dims, 16, 16, 3, padding=1),
|
operations.conv_nd(dims, 16, 16, 3, padding=1, dtype=self.dtype, device=device),
|
||||||
nn.SiLU(),
|
nn.SiLU(),
|
||||||
operations.conv_nd(dims, 16, 32, 3, padding=1, stride=2),
|
operations.conv_nd(dims, 16, 32, 3, padding=1, stride=2, dtype=self.dtype, device=device),
|
||||||
nn.SiLU(),
|
nn.SiLU(),
|
||||||
operations.conv_nd(dims, 32, 32, 3, padding=1),
|
operations.conv_nd(dims, 32, 32, 3, padding=1, dtype=self.dtype, device=device),
|
||||||
nn.SiLU(),
|
nn.SiLU(),
|
||||||
operations.conv_nd(dims, 32, 96, 3, padding=1, stride=2),
|
operations.conv_nd(dims, 32, 96, 3, padding=1, stride=2, dtype=self.dtype, device=device),
|
||||||
nn.SiLU(),
|
nn.SiLU(),
|
||||||
operations.conv_nd(dims, 96, 96, 3, padding=1),
|
operations.conv_nd(dims, 96, 96, 3, padding=1, dtype=self.dtype, device=device),
|
||||||
nn.SiLU(),
|
nn.SiLU(),
|
||||||
operations.conv_nd(dims, 96, 256, 3, padding=1, stride=2),
|
operations.conv_nd(dims, 96, 256, 3, padding=1, stride=2, dtype=self.dtype, device=device),
|
||||||
nn.SiLU(),
|
nn.SiLU(),
|
||||||
zero_module(operations.conv_nd(dims, 256, model_channels, 3, padding=1))
|
operations.conv_nd(dims, 256, model_channels, 3, padding=1, dtype=self.dtype, device=device)
|
||||||
)
|
)
|
||||||
|
|
||||||
self._feature_size = model_channels
|
self._feature_size = model_channels
|
||||||
@ -205,7 +206,7 @@ class ControlNet(nn.Module):
|
|||||||
)
|
)
|
||||||
)
|
)
|
||||||
self.input_blocks.append(TimestepEmbedSequential(*layers))
|
self.input_blocks.append(TimestepEmbedSequential(*layers))
|
||||||
self.zero_convs.append(self.make_zero_conv(ch, operations=operations))
|
self.zero_convs.append(self.make_zero_conv(ch, operations=operations, dtype=self.dtype, device=device))
|
||||||
self._feature_size += ch
|
self._feature_size += ch
|
||||||
input_block_chans.append(ch)
|
input_block_chans.append(ch)
|
||||||
if level != len(channel_mult) - 1:
|
if level != len(channel_mult) - 1:
|
||||||
@ -233,7 +234,7 @@ class ControlNet(nn.Module):
|
|||||||
)
|
)
|
||||||
ch = out_ch
|
ch = out_ch
|
||||||
input_block_chans.append(ch)
|
input_block_chans.append(ch)
|
||||||
self.zero_convs.append(self.make_zero_conv(ch, operations=operations))
|
self.zero_convs.append(self.make_zero_conv(ch, operations=operations, dtype=self.dtype, device=device))
|
||||||
ds *= 2
|
ds *= 2
|
||||||
self._feature_size += ch
|
self._feature_size += ch
|
||||||
|
|
||||||
@ -275,14 +276,14 @@ class ControlNet(nn.Module):
|
|||||||
operations=operations
|
operations=operations
|
||||||
)]
|
)]
|
||||||
self.middle_block = TimestepEmbedSequential(*mid_block)
|
self.middle_block = TimestepEmbedSequential(*mid_block)
|
||||||
self.middle_block_out = self.make_zero_conv(ch, operations=operations)
|
self.middle_block_out = self.make_zero_conv(ch, operations=operations, dtype=self.dtype, device=device)
|
||||||
self._feature_size += ch
|
self._feature_size += ch
|
||||||
|
|
||||||
def make_zero_conv(self, channels, operations=None):
|
def make_zero_conv(self, channels, operations=None, dtype=None, device=None):
|
||||||
return TimestepEmbedSequential(zero_module(operations.conv_nd(self.dims, channels, channels, 1, padding=0)))
|
return TimestepEmbedSequential(operations.conv_nd(self.dims, channels, channels, 1, padding=0, dtype=dtype, device=device))
|
||||||
|
|
||||||
def forward(self, x, hint, timesteps, context, y=None, **kwargs):
|
def forward(self, x, hint, timesteps, context, y=None, **kwargs):
|
||||||
t_emb = timestep_embedding(timesteps, self.model_channels, repeat_only=False).to(self.dtype)
|
t_emb = timestep_embedding(timesteps, self.model_channels, repeat_only=False).to(x.dtype)
|
||||||
emb = self.time_embed(t_emb)
|
emb = self.time_embed(t_emb)
|
||||||
|
|
||||||
guided_hint = self.input_hint_block(hint, emb, context)
|
guided_hint = self.input_hint_block(hint, emb, context)
|
||||||
@ -294,7 +295,7 @@ class ControlNet(nn.Module):
|
|||||||
assert y.shape[0] == x.shape[0]
|
assert y.shape[0] == x.shape[0]
|
||||||
emb = emb + self.label_emb(y)
|
emb = emb + self.label_emb(y)
|
||||||
|
|
||||||
h = x.type(self.dtype)
|
h = x
|
||||||
for module, zero_conv in zip(self.input_blocks, self.zero_convs):
|
for module, zero_conv in zip(self.input_blocks, self.zero_convs):
|
||||||
if guided_hint is not None:
|
if guided_hint is not None:
|
||||||
h = module(h, emb, context)
|
h = module(h, emb, context)
|
||||||
|
|||||||
@ -55,13 +55,26 @@ fp_group = parser.add_mutually_exclusive_group()
|
|||||||
fp_group.add_argument("--force-fp32", action="store_true", help="Force fp32 (If this makes your GPU work better please report it).")
|
fp_group.add_argument("--force-fp32", action="store_true", help="Force fp32 (If this makes your GPU work better please report it).")
|
||||||
fp_group.add_argument("--force-fp16", action="store_true", help="Force fp16.")
|
fp_group.add_argument("--force-fp16", action="store_true", help="Force fp16.")
|
||||||
|
|
||||||
parser.add_argument("--bf16-unet", action="store_true", help="Run the UNET in bf16. This should only be used for testing stuff.")
|
fpunet_group = parser.add_mutually_exclusive_group()
|
||||||
|
fpunet_group.add_argument("--bf16-unet", action="store_true", help="Run the UNET in bf16. This should only be used for testing stuff.")
|
||||||
|
fpunet_group.add_argument("--fp16-unet", action="store_true", help="Store unet weights in fp16.")
|
||||||
|
fpunet_group.add_argument("--fp8_e4m3fn-unet", action="store_true", help="Store unet weights in fp8_e4m3fn.")
|
||||||
|
fpunet_group.add_argument("--fp8_e5m2-unet", action="store_true", help="Store unet weights in fp8_e5m2.")
|
||||||
|
|
||||||
fpvae_group = parser.add_mutually_exclusive_group()
|
fpvae_group = parser.add_mutually_exclusive_group()
|
||||||
fpvae_group.add_argument("--fp16-vae", action="store_true", help="Run the VAE in fp16, might cause black images.")
|
fpvae_group.add_argument("--fp16-vae", action="store_true", help="Run the VAE in fp16, might cause black images.")
|
||||||
fpvae_group.add_argument("--fp32-vae", action="store_true", help="Run the VAE in full precision fp32.")
|
fpvae_group.add_argument("--fp32-vae", action="store_true", help="Run the VAE in full precision fp32.")
|
||||||
fpvae_group.add_argument("--bf16-vae", action="store_true", help="Run the VAE in bf16.")
|
fpvae_group.add_argument("--bf16-vae", action="store_true", help="Run the VAE in bf16.")
|
||||||
|
|
||||||
|
parser.add_argument("--cpu-vae", action="store_true", help="Run the VAE on the CPU.")
|
||||||
|
|
||||||
|
fpte_group = parser.add_mutually_exclusive_group()
|
||||||
|
fpte_group.add_argument("--fp8_e4m3fn-text-enc", action="store_true", help="Store text encoder weights in fp8 (e4m3fn variant).")
|
||||||
|
fpte_group.add_argument("--fp8_e5m2-text-enc", action="store_true", help="Store text encoder weights in fp8 (e5m2 variant).")
|
||||||
|
fpte_group.add_argument("--fp16-text-enc", action="store_true", help="Store text encoder weights in fp16.")
|
||||||
|
fpte_group.add_argument("--fp32-text-enc", action="store_true", help="Store text encoder weights in fp32.")
|
||||||
|
|
||||||
|
|
||||||
parser.add_argument("--directml", type=int, nargs="?", metavar="DIRECTML_DEVICE", const=-1, help="Use torch-directml.")
|
parser.add_argument("--directml", type=int, nargs="?", metavar="DIRECTML_DEVICE", const=-1, help="Use torch-directml.")
|
||||||
|
|
||||||
parser.add_argument("--disable-ipex-optimize", action="store_true", help="Disables ipex.optimize when loading models with Intel GPUs.")
|
parser.add_argument("--disable-ipex-optimize", action="store_true", help="Disables ipex.optimize when loading models with Intel GPUs.")
|
||||||
@ -91,7 +104,7 @@ vram_group.add_argument("--cpu", action="store_true", help="To use the CPU for e
|
|||||||
|
|
||||||
|
|
||||||
parser.add_argument("--disable-smart-memory", action="store_true", help="Force ComfyUI to agressively offload to regular ram instead of keeping models in vram when it can.")
|
parser.add_argument("--disable-smart-memory", action="store_true", help="Force ComfyUI to agressively offload to regular ram instead of keeping models in vram when it can.")
|
||||||
|
parser.add_argument("--deterministic", action="store_true", help="Make pytorch use slower deterministic algorithms when it can. Note that this might not make images deterministic in all cases.")
|
||||||
|
|
||||||
parser.add_argument("--dont-print-server", action="store_true", help="Don't print server output.")
|
parser.add_argument("--dont-print-server", action="store_true", help="Don't print server output.")
|
||||||
parser.add_argument("--quick-test-for-ci", action="store_true", help="Quick test for CI.")
|
parser.add_argument("--quick-test-for-ci", action="store_true", help="Quick test for CI.")
|
||||||
|
|||||||
188
comfy/clip_model.py
Normal file
188
comfy/clip_model.py
Normal file
@ -0,0 +1,188 @@
|
|||||||
|
import torch
|
||||||
|
from comfy.ldm.modules.attention import optimized_attention_for_device
|
||||||
|
|
||||||
|
class CLIPAttention(torch.nn.Module):
|
||||||
|
def __init__(self, embed_dim, heads, dtype, device, operations):
|
||||||
|
super().__init__()
|
||||||
|
|
||||||
|
self.heads = heads
|
||||||
|
self.q_proj = operations.Linear(embed_dim, embed_dim, bias=True, dtype=dtype, device=device)
|
||||||
|
self.k_proj = operations.Linear(embed_dim, embed_dim, bias=True, dtype=dtype, device=device)
|
||||||
|
self.v_proj = operations.Linear(embed_dim, embed_dim, bias=True, dtype=dtype, device=device)
|
||||||
|
|
||||||
|
self.out_proj = operations.Linear(embed_dim, embed_dim, bias=True, dtype=dtype, device=device)
|
||||||
|
|
||||||
|
def forward(self, x, mask=None, optimized_attention=None):
|
||||||
|
q = self.q_proj(x)
|
||||||
|
k = self.k_proj(x)
|
||||||
|
v = self.v_proj(x)
|
||||||
|
|
||||||
|
out = optimized_attention(q, k, v, self.heads, mask)
|
||||||
|
return self.out_proj(out)
|
||||||
|
|
||||||
|
ACTIVATIONS = {"quick_gelu": lambda a: a * torch.sigmoid(1.702 * a),
|
||||||
|
"gelu": torch.nn.functional.gelu,
|
||||||
|
}
|
||||||
|
|
||||||
|
class CLIPMLP(torch.nn.Module):
|
||||||
|
def __init__(self, embed_dim, intermediate_size, activation, dtype, device, operations):
|
||||||
|
super().__init__()
|
||||||
|
self.fc1 = operations.Linear(embed_dim, intermediate_size, bias=True, dtype=dtype, device=device)
|
||||||
|
self.activation = ACTIVATIONS[activation]
|
||||||
|
self.fc2 = operations.Linear(intermediate_size, embed_dim, bias=True, dtype=dtype, device=device)
|
||||||
|
|
||||||
|
def forward(self, x):
|
||||||
|
x = self.fc1(x)
|
||||||
|
x = self.activation(x)
|
||||||
|
x = self.fc2(x)
|
||||||
|
return x
|
||||||
|
|
||||||
|
class CLIPLayer(torch.nn.Module):
|
||||||
|
def __init__(self, embed_dim, heads, intermediate_size, intermediate_activation, dtype, device, operations):
|
||||||
|
super().__init__()
|
||||||
|
self.layer_norm1 = operations.LayerNorm(embed_dim, dtype=dtype, device=device)
|
||||||
|
self.self_attn = CLIPAttention(embed_dim, heads, dtype, device, operations)
|
||||||
|
self.layer_norm2 = operations.LayerNorm(embed_dim, dtype=dtype, device=device)
|
||||||
|
self.mlp = CLIPMLP(embed_dim, intermediate_size, intermediate_activation, dtype, device, operations)
|
||||||
|
|
||||||
|
def forward(self, x, mask=None, optimized_attention=None):
|
||||||
|
x += self.self_attn(self.layer_norm1(x), mask, optimized_attention)
|
||||||
|
x += self.mlp(self.layer_norm2(x))
|
||||||
|
return x
|
||||||
|
|
||||||
|
|
||||||
|
class CLIPEncoder(torch.nn.Module):
|
||||||
|
def __init__(self, num_layers, embed_dim, heads, intermediate_size, intermediate_activation, dtype, device, operations):
|
||||||
|
super().__init__()
|
||||||
|
self.layers = torch.nn.ModuleList([CLIPLayer(embed_dim, heads, intermediate_size, intermediate_activation, dtype, device, operations) for i in range(num_layers)])
|
||||||
|
|
||||||
|
def forward(self, x, mask=None, intermediate_output=None):
|
||||||
|
optimized_attention = optimized_attention_for_device(x.device, mask=mask is not None)
|
||||||
|
|
||||||
|
if intermediate_output is not None:
|
||||||
|
if intermediate_output < 0:
|
||||||
|
intermediate_output = len(self.layers) + intermediate_output
|
||||||
|
|
||||||
|
intermediate = None
|
||||||
|
for i, l in enumerate(self.layers):
|
||||||
|
x = l(x, mask, optimized_attention)
|
||||||
|
if i == intermediate_output:
|
||||||
|
intermediate = x.clone()
|
||||||
|
return x, intermediate
|
||||||
|
|
||||||
|
class CLIPEmbeddings(torch.nn.Module):
|
||||||
|
def __init__(self, embed_dim, vocab_size=49408, num_positions=77, dtype=None, device=None):
|
||||||
|
super().__init__()
|
||||||
|
self.token_embedding = torch.nn.Embedding(vocab_size, embed_dim, dtype=dtype, device=device)
|
||||||
|
self.position_embedding = torch.nn.Embedding(num_positions, embed_dim, dtype=dtype, device=device)
|
||||||
|
|
||||||
|
def forward(self, input_tokens):
|
||||||
|
return self.token_embedding(input_tokens) + self.position_embedding.weight
|
||||||
|
|
||||||
|
|
||||||
|
class CLIPTextModel_(torch.nn.Module):
|
||||||
|
def __init__(self, config_dict, dtype, device, operations):
|
||||||
|
num_layers = config_dict["num_hidden_layers"]
|
||||||
|
embed_dim = config_dict["hidden_size"]
|
||||||
|
heads = config_dict["num_attention_heads"]
|
||||||
|
intermediate_size = config_dict["intermediate_size"]
|
||||||
|
intermediate_activation = config_dict["hidden_act"]
|
||||||
|
|
||||||
|
super().__init__()
|
||||||
|
self.embeddings = CLIPEmbeddings(embed_dim, dtype=torch.float32, device=device)
|
||||||
|
self.encoder = CLIPEncoder(num_layers, embed_dim, heads, intermediate_size, intermediate_activation, dtype, device, operations)
|
||||||
|
self.final_layer_norm = operations.LayerNorm(embed_dim, dtype=dtype, device=device)
|
||||||
|
|
||||||
|
def forward(self, input_tokens, attention_mask=None, intermediate_output=None, final_layer_norm_intermediate=True):
|
||||||
|
x = self.embeddings(input_tokens)
|
||||||
|
mask = None
|
||||||
|
if attention_mask is not None:
|
||||||
|
mask = 1.0 - attention_mask.to(x.dtype).unsqueeze(1).unsqueeze(1).expand(attention_mask.shape[0], 1, attention_mask.shape[-1], attention_mask.shape[-1])
|
||||||
|
mask = mask.masked_fill(mask.to(torch.bool), float("-inf"))
|
||||||
|
|
||||||
|
causal_mask = torch.empty(x.shape[1], x.shape[1], dtype=x.dtype, device=x.device).fill_(float("-inf")).triu_(1)
|
||||||
|
if mask is not None:
|
||||||
|
mask += causal_mask
|
||||||
|
else:
|
||||||
|
mask = causal_mask
|
||||||
|
|
||||||
|
x, i = self.encoder(x, mask=mask, intermediate_output=intermediate_output)
|
||||||
|
x = self.final_layer_norm(x)
|
||||||
|
if i is not None and final_layer_norm_intermediate:
|
||||||
|
i = self.final_layer_norm(i)
|
||||||
|
|
||||||
|
pooled_output = x[torch.arange(x.shape[0], device=x.device), input_tokens.to(dtype=torch.int, device=x.device).argmax(dim=-1),]
|
||||||
|
return x, i, pooled_output
|
||||||
|
|
||||||
|
class CLIPTextModel(torch.nn.Module):
|
||||||
|
def __init__(self, config_dict, dtype, device, operations):
|
||||||
|
super().__init__()
|
||||||
|
self.num_layers = config_dict["num_hidden_layers"]
|
||||||
|
self.text_model = CLIPTextModel_(config_dict, dtype, device, operations)
|
||||||
|
self.dtype = dtype
|
||||||
|
|
||||||
|
def get_input_embeddings(self):
|
||||||
|
return self.text_model.embeddings.token_embedding
|
||||||
|
|
||||||
|
def set_input_embeddings(self, embeddings):
|
||||||
|
self.text_model.embeddings.token_embedding = embeddings
|
||||||
|
|
||||||
|
def forward(self, *args, **kwargs):
|
||||||
|
return self.text_model(*args, **kwargs)
|
||||||
|
|
||||||
|
class CLIPVisionEmbeddings(torch.nn.Module):
|
||||||
|
def __init__(self, embed_dim, num_channels=3, patch_size=14, image_size=224, dtype=None, device=None, operations=None):
|
||||||
|
super().__init__()
|
||||||
|
self.class_embedding = torch.nn.Parameter(torch.empty(embed_dim, dtype=dtype, device=device))
|
||||||
|
|
||||||
|
self.patch_embedding = operations.Conv2d(
|
||||||
|
in_channels=num_channels,
|
||||||
|
out_channels=embed_dim,
|
||||||
|
kernel_size=patch_size,
|
||||||
|
stride=patch_size,
|
||||||
|
bias=False,
|
||||||
|
dtype=dtype,
|
||||||
|
device=device
|
||||||
|
)
|
||||||
|
|
||||||
|
num_patches = (image_size // patch_size) ** 2
|
||||||
|
num_positions = num_patches + 1
|
||||||
|
self.position_embedding = torch.nn.Embedding(num_positions, embed_dim, dtype=dtype, device=device)
|
||||||
|
|
||||||
|
def forward(self, pixel_values):
|
||||||
|
embeds = self.patch_embedding(pixel_values).flatten(2).transpose(1, 2)
|
||||||
|
return torch.cat([self.class_embedding.to(embeds.device).expand(pixel_values.shape[0], 1, -1), embeds], dim=1) + self.position_embedding.weight.to(embeds.device)
|
||||||
|
|
||||||
|
|
||||||
|
class CLIPVision(torch.nn.Module):
|
||||||
|
def __init__(self, config_dict, dtype, device, operations):
|
||||||
|
super().__init__()
|
||||||
|
num_layers = config_dict["num_hidden_layers"]
|
||||||
|
embed_dim = config_dict["hidden_size"]
|
||||||
|
heads = config_dict["num_attention_heads"]
|
||||||
|
intermediate_size = config_dict["intermediate_size"]
|
||||||
|
intermediate_activation = config_dict["hidden_act"]
|
||||||
|
|
||||||
|
self.embeddings = CLIPVisionEmbeddings(embed_dim, config_dict["num_channels"], config_dict["patch_size"], config_dict["image_size"], dtype=torch.float32, device=device, operations=operations)
|
||||||
|
self.pre_layrnorm = operations.LayerNorm(embed_dim)
|
||||||
|
self.encoder = CLIPEncoder(num_layers, embed_dim, heads, intermediate_size, intermediate_activation, dtype, device, operations)
|
||||||
|
self.post_layernorm = operations.LayerNorm(embed_dim)
|
||||||
|
|
||||||
|
def forward(self, pixel_values, attention_mask=None, intermediate_output=None):
|
||||||
|
x = self.embeddings(pixel_values)
|
||||||
|
x = self.pre_layrnorm(x)
|
||||||
|
#TODO: attention_mask?
|
||||||
|
x, i = self.encoder(x, mask=None, intermediate_output=intermediate_output)
|
||||||
|
pooled_output = self.post_layernorm(x[:, 0, :])
|
||||||
|
return x, i, pooled_output
|
||||||
|
|
||||||
|
class CLIPVisionModelProjection(torch.nn.Module):
|
||||||
|
def __init__(self, config_dict, dtype, device, operations):
|
||||||
|
super().__init__()
|
||||||
|
self.vision_model = CLIPVision(config_dict, dtype, device, operations)
|
||||||
|
self.visual_projection = operations.Linear(config_dict["hidden_size"], config_dict["projection_dim"], bias=False)
|
||||||
|
|
||||||
|
def forward(self, *args, **kwargs):
|
||||||
|
x = self.vision_model(*args, **kwargs)
|
||||||
|
out = self.visual_projection(x[2])
|
||||||
|
return (x[0], x[1], out)
|
||||||
@ -1,38 +1,44 @@
|
|||||||
from transformers import CLIPVisionModelWithProjection, CLIPVisionConfig, modeling_utils
|
|
||||||
from .utils import load_torch_file, transformers_convert, common_upscale
|
from .utils import load_torch_file, transformers_convert, common_upscale
|
||||||
import os
|
import os
|
||||||
import torch
|
import torch
|
||||||
import contextlib
|
import contextlib
|
||||||
|
import json
|
||||||
|
|
||||||
import comfy.ops
|
import comfy.ops
|
||||||
import comfy.model_patcher
|
import comfy.model_patcher
|
||||||
import comfy.model_management
|
import comfy.model_management
|
||||||
import comfy.utils
|
import comfy.utils
|
||||||
|
import comfy.clip_model
|
||||||
|
|
||||||
|
class Output:
|
||||||
|
def __getitem__(self, key):
|
||||||
|
return getattr(self, key)
|
||||||
|
def __setitem__(self, key, item):
|
||||||
|
setattr(self, key, item)
|
||||||
|
|
||||||
def clip_preprocess(image, size=224):
|
def clip_preprocess(image, size=224):
|
||||||
mean = torch.tensor([ 0.48145466,0.4578275,0.40821073], device=image.device, dtype=image.dtype)
|
mean = torch.tensor([ 0.48145466,0.4578275,0.40821073], device=image.device, dtype=image.dtype)
|
||||||
std = torch.tensor([0.26862954,0.26130258,0.27577711], device=image.device, dtype=image.dtype)
|
std = torch.tensor([0.26862954,0.26130258,0.27577711], device=image.device, dtype=image.dtype)
|
||||||
scale = (size / min(image.shape[1], image.shape[2]))
|
image = image.movedim(-1, 1)
|
||||||
image = torch.nn.functional.interpolate(image.movedim(-1, 1), size=(round(scale * image.shape[1]), round(scale * image.shape[2])), mode="bicubic", antialias=True)
|
if not (image.shape[2] == size and image.shape[3] == size):
|
||||||
h = (image.shape[2] - size)//2
|
scale = (size / min(image.shape[2], image.shape[3]))
|
||||||
w = (image.shape[3] - size)//2
|
image = torch.nn.functional.interpolate(image, size=(round(scale * image.shape[2]), round(scale * image.shape[3])), mode="bicubic", antialias=True)
|
||||||
image = image[:,:,h:h+size,w:w+size]
|
h = (image.shape[2] - size)//2
|
||||||
|
w = (image.shape[3] - size)//2
|
||||||
|
image = image[:,:,h:h+size,w:w+size]
|
||||||
image = torch.clip((255. * image), 0, 255).round() / 255.0
|
image = torch.clip((255. * image), 0, 255).round() / 255.0
|
||||||
return (image - mean.view([3,1,1])) / std.view([3,1,1])
|
return (image - mean.view([3,1,1])) / std.view([3,1,1])
|
||||||
|
|
||||||
class ClipVisionModel():
|
class ClipVisionModel():
|
||||||
def __init__(self, json_config):
|
def __init__(self, json_config):
|
||||||
config = CLIPVisionConfig.from_json_file(json_config)
|
with open(json_config) as f:
|
||||||
|
config = json.load(f)
|
||||||
|
|
||||||
self.load_device = comfy.model_management.text_encoder_device()
|
self.load_device = comfy.model_management.text_encoder_device()
|
||||||
offload_device = comfy.model_management.text_encoder_offload_device()
|
offload_device = comfy.model_management.text_encoder_offload_device()
|
||||||
self.dtype = torch.float32
|
self.dtype = comfy.model_management.text_encoder_dtype(self.load_device)
|
||||||
if comfy.model_management.should_use_fp16(self.load_device, prioritize_performance=False):
|
self.model = comfy.clip_model.CLIPVisionModelProjection(config, self.dtype, offload_device, comfy.ops.manual_cast)
|
||||||
self.dtype = torch.float16
|
self.model.eval()
|
||||||
|
|
||||||
with comfy.ops.use_comfy_ops(offload_device, self.dtype):
|
|
||||||
with modeling_utils.no_init_weights():
|
|
||||||
self.model = CLIPVisionModelWithProjection(config)
|
|
||||||
self.model.to(self.dtype)
|
|
||||||
|
|
||||||
self.patcher = comfy.model_patcher.ModelPatcher(self.model, load_device=self.load_device, offload_device=offload_device)
|
self.patcher = comfy.model_patcher.ModelPatcher(self.model, load_device=self.load_device, offload_device=offload_device)
|
||||||
def load_sd(self, sd):
|
def load_sd(self, sd):
|
||||||
@ -40,25 +46,13 @@ class ClipVisionModel():
|
|||||||
|
|
||||||
def encode_image(self, image):
|
def encode_image(self, image):
|
||||||
comfy.model_management.load_model_gpu(self.patcher)
|
comfy.model_management.load_model_gpu(self.patcher)
|
||||||
pixel_values = clip_preprocess(image.to(self.load_device))
|
pixel_values = clip_preprocess(image.to(self.load_device)).float()
|
||||||
|
out = self.model(pixel_values=pixel_values, intermediate_output=-2)
|
||||||
if self.dtype != torch.float32:
|
|
||||||
precision_scope = torch.autocast
|
|
||||||
else:
|
|
||||||
precision_scope = lambda a, b: contextlib.nullcontext(a)
|
|
||||||
|
|
||||||
with precision_scope(comfy.model_management.get_autocast_device(self.load_device), torch.float32):
|
|
||||||
outputs = self.model(pixel_values=pixel_values, output_hidden_states=True)
|
|
||||||
|
|
||||||
for k in outputs:
|
|
||||||
t = outputs[k]
|
|
||||||
if t is not None:
|
|
||||||
if k == 'hidden_states':
|
|
||||||
outputs["penultimate_hidden_states"] = t[-2].cpu()
|
|
||||||
outputs["hidden_states"] = None
|
|
||||||
else:
|
|
||||||
outputs[k] = t.cpu()
|
|
||||||
|
|
||||||
|
outputs = Output()
|
||||||
|
outputs["last_hidden_state"] = out[0].to(comfy.model_management.intermediate_device())
|
||||||
|
outputs["image_embeds"] = out[2].to(comfy.model_management.intermediate_device())
|
||||||
|
outputs["penultimate_hidden_states"] = out[1].to(comfy.model_management.intermediate_device())
|
||||||
return outputs
|
return outputs
|
||||||
|
|
||||||
def convert_to_transformers(sd, prefix):
|
def convert_to_transformers(sd, prefix):
|
||||||
|
|||||||
@ -1,10 +1,12 @@
|
|||||||
import torch
|
import torch
|
||||||
import math
|
import math
|
||||||
import os
|
import os
|
||||||
|
import contextlib
|
||||||
import comfy.utils
|
import comfy.utils
|
||||||
import comfy.model_management
|
import comfy.model_management
|
||||||
import comfy.model_detection
|
import comfy.model_detection
|
||||||
import comfy.model_patcher
|
import comfy.model_patcher
|
||||||
|
import comfy.ops
|
||||||
|
|
||||||
import comfy.cldm.cldm
|
import comfy.cldm.cldm
|
||||||
import comfy.t2i_adapter.adapter
|
import comfy.t2i_adapter.adapter
|
||||||
@ -33,16 +35,16 @@ class ControlBase:
|
|||||||
self.cond_hint_original = None
|
self.cond_hint_original = None
|
||||||
self.cond_hint = None
|
self.cond_hint = None
|
||||||
self.strength = 1.0
|
self.strength = 1.0
|
||||||
self.timestep_percent_range = (1.0, 0.0)
|
self.timestep_percent_range = (0.0, 1.0)
|
||||||
|
self.global_average_pooling = False
|
||||||
self.timestep_range = None
|
self.timestep_range = None
|
||||||
|
|
||||||
if device is None:
|
if device is None:
|
||||||
device = comfy.model_management.get_torch_device()
|
device = comfy.model_management.get_torch_device()
|
||||||
self.device = device
|
self.device = device
|
||||||
self.previous_controlnet = None
|
self.previous_controlnet = None
|
||||||
self.global_average_pooling = False
|
|
||||||
|
|
||||||
def set_cond_hint(self, cond_hint, strength=1.0, timestep_percent_range=(1.0, 0.0)):
|
def set_cond_hint(self, cond_hint, strength=1.0, timestep_percent_range=(0.0, 1.0)):
|
||||||
self.cond_hint_original = cond_hint
|
self.cond_hint_original = cond_hint
|
||||||
self.strength = strength
|
self.strength = strength
|
||||||
self.timestep_percent_range = timestep_percent_range
|
self.timestep_percent_range = timestep_percent_range
|
||||||
@ -75,6 +77,7 @@ class ControlBase:
|
|||||||
c.cond_hint_original = self.cond_hint_original
|
c.cond_hint_original = self.cond_hint_original
|
||||||
c.strength = self.strength
|
c.strength = self.strength
|
||||||
c.timestep_percent_range = self.timestep_percent_range
|
c.timestep_percent_range = self.timestep_percent_range
|
||||||
|
c.global_average_pooling = self.global_average_pooling
|
||||||
|
|
||||||
def inference_memory_requirements(self, dtype):
|
def inference_memory_requirements(self, dtype):
|
||||||
if self.previous_controlnet is not None:
|
if self.previous_controlnet is not None:
|
||||||
@ -127,12 +130,14 @@ class ControlBase:
|
|||||||
return out
|
return out
|
||||||
|
|
||||||
class ControlNet(ControlBase):
|
class ControlNet(ControlBase):
|
||||||
def __init__(self, control_model, global_average_pooling=False, device=None):
|
def __init__(self, control_model, global_average_pooling=False, device=None, load_device=None, manual_cast_dtype=None):
|
||||||
super().__init__(device)
|
super().__init__(device)
|
||||||
self.control_model = control_model
|
self.control_model = control_model
|
||||||
self.control_model_wrapped = comfy.model_patcher.ModelPatcher(self.control_model, load_device=comfy.model_management.get_torch_device(), offload_device=comfy.model_management.unet_offload_device())
|
self.load_device = load_device
|
||||||
|
self.control_model_wrapped = comfy.model_patcher.ModelPatcher(self.control_model, load_device=load_device, offload_device=comfy.model_management.unet_offload_device())
|
||||||
self.global_average_pooling = global_average_pooling
|
self.global_average_pooling = global_average_pooling
|
||||||
self.model_sampling_current = None
|
self.model_sampling_current = None
|
||||||
|
self.manual_cast_dtype = manual_cast_dtype
|
||||||
|
|
||||||
def get_control(self, x_noisy, t, cond, batched_number):
|
def get_control(self, x_noisy, t, cond, batched_number):
|
||||||
control_prev = None
|
control_prev = None
|
||||||
@ -146,28 +151,31 @@ class ControlNet(ControlBase):
|
|||||||
else:
|
else:
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
dtype = self.control_model.dtype
|
||||||
|
if self.manual_cast_dtype is not None:
|
||||||
|
dtype = self.manual_cast_dtype
|
||||||
|
|
||||||
output_dtype = x_noisy.dtype
|
output_dtype = x_noisy.dtype
|
||||||
if self.cond_hint is None or x_noisy.shape[2] * 8 != self.cond_hint.shape[2] or x_noisy.shape[3] * 8 != self.cond_hint.shape[3]:
|
if self.cond_hint is None or x_noisy.shape[2] * 8 != self.cond_hint.shape[2] or x_noisy.shape[3] * 8 != self.cond_hint.shape[3]:
|
||||||
if self.cond_hint is not None:
|
if self.cond_hint is not None:
|
||||||
del self.cond_hint
|
del self.cond_hint
|
||||||
self.cond_hint = None
|
self.cond_hint = None
|
||||||
self.cond_hint = comfy.utils.common_upscale(self.cond_hint_original, x_noisy.shape[3] * 8, x_noisy.shape[2] * 8, 'nearest-exact', "center").to(self.control_model.dtype).to(self.device)
|
self.cond_hint = comfy.utils.common_upscale(self.cond_hint_original, x_noisy.shape[3] * 8, x_noisy.shape[2] * 8, 'nearest-exact', "center").to(dtype).to(self.device)
|
||||||
if x_noisy.shape[0] != self.cond_hint.shape[0]:
|
if x_noisy.shape[0] != self.cond_hint.shape[0]:
|
||||||
self.cond_hint = broadcast_image_to(self.cond_hint, x_noisy.shape[0], batched_number)
|
self.cond_hint = broadcast_image_to(self.cond_hint, x_noisy.shape[0], batched_number)
|
||||||
|
|
||||||
|
|
||||||
context = cond['c_crossattn']
|
context = cond['c_crossattn']
|
||||||
y = cond.get('y', None)
|
y = cond.get('y', None)
|
||||||
if y is not None:
|
if y is not None:
|
||||||
y = y.to(self.control_model.dtype)
|
y = y.to(dtype)
|
||||||
timestep = self.model_sampling_current.timestep(t)
|
timestep = self.model_sampling_current.timestep(t)
|
||||||
x_noisy = self.model_sampling_current.calculate_input(t, x_noisy)
|
x_noisy = self.model_sampling_current.calculate_input(t, x_noisy)
|
||||||
|
|
||||||
control = self.control_model(x=x_noisy.to(self.control_model.dtype), hint=self.cond_hint, timesteps=timestep.float(), context=context.to(self.control_model.dtype), y=y)
|
control = self.control_model(x=x_noisy.to(dtype), hint=self.cond_hint, timesteps=timestep.float(), context=context.to(dtype), y=y)
|
||||||
return self.control_merge(None, control, control_prev, output_dtype)
|
return self.control_merge(None, control, control_prev, output_dtype)
|
||||||
|
|
||||||
def copy(self):
|
def copy(self):
|
||||||
c = ControlNet(self.control_model, global_average_pooling=self.global_average_pooling)
|
c = ControlNet(self.control_model, global_average_pooling=self.global_average_pooling, load_device=self.load_device, manual_cast_dtype=self.manual_cast_dtype)
|
||||||
self.copy_to(c)
|
self.copy_to(c)
|
||||||
return c
|
return c
|
||||||
|
|
||||||
@ -198,10 +206,11 @@ class ControlLoraOps:
|
|||||||
self.bias = None
|
self.bias = None
|
||||||
|
|
||||||
def forward(self, input):
|
def forward(self, input):
|
||||||
|
weight, bias = comfy.ops.cast_bias_weight(self, input)
|
||||||
if self.up is not None:
|
if self.up is not None:
|
||||||
return torch.nn.functional.linear(input, self.weight.to(input.device) + (torch.mm(self.up.flatten(start_dim=1), self.down.flatten(start_dim=1))).reshape(self.weight.shape).type(input.dtype), self.bias)
|
return torch.nn.functional.linear(input, weight + (torch.mm(self.up.flatten(start_dim=1), self.down.flatten(start_dim=1))).reshape(self.weight.shape).type(input.dtype), bias)
|
||||||
else:
|
else:
|
||||||
return torch.nn.functional.linear(input, self.weight.to(input.device), self.bias)
|
return torch.nn.functional.linear(input, weight, bias)
|
||||||
|
|
||||||
class Conv2d(torch.nn.Module):
|
class Conv2d(torch.nn.Module):
|
||||||
def __init__(
|
def __init__(
|
||||||
@ -237,16 +246,11 @@ class ControlLoraOps:
|
|||||||
|
|
||||||
|
|
||||||
def forward(self, input):
|
def forward(self, input):
|
||||||
|
weight, bias = comfy.ops.cast_bias_weight(self, input)
|
||||||
if self.up is not None:
|
if self.up is not None:
|
||||||
return torch.nn.functional.conv2d(input, self.weight.to(input.device) + (torch.mm(self.up.flatten(start_dim=1), self.down.flatten(start_dim=1))).reshape(self.weight.shape).type(input.dtype), self.bias, self.stride, self.padding, self.dilation, self.groups)
|
return torch.nn.functional.conv2d(input, weight + (torch.mm(self.up.flatten(start_dim=1), self.down.flatten(start_dim=1))).reshape(self.weight.shape).type(input.dtype), bias, self.stride, self.padding, self.dilation, self.groups)
|
||||||
else:
|
else:
|
||||||
return torch.nn.functional.conv2d(input, self.weight.to(input.device), self.bias, self.stride, self.padding, self.dilation, self.groups)
|
return torch.nn.functional.conv2d(input, weight, bias, self.stride, self.padding, self.dilation, self.groups)
|
||||||
|
|
||||||
def conv_nd(self, dims, *args, **kwargs):
|
|
||||||
if dims == 2:
|
|
||||||
return self.Conv2d(*args, **kwargs)
|
|
||||||
else:
|
|
||||||
raise ValueError(f"unsupported dimensions: {dims}")
|
|
||||||
|
|
||||||
|
|
||||||
class ControlLora(ControlNet):
|
class ControlLora(ControlNet):
|
||||||
@ -260,17 +264,26 @@ class ControlLora(ControlNet):
|
|||||||
controlnet_config = model.model_config.unet_config.copy()
|
controlnet_config = model.model_config.unet_config.copy()
|
||||||
controlnet_config.pop("out_channels")
|
controlnet_config.pop("out_channels")
|
||||||
controlnet_config["hint_channels"] = self.control_weights["input_hint_block.0.weight"].shape[1]
|
controlnet_config["hint_channels"] = self.control_weights["input_hint_block.0.weight"].shape[1]
|
||||||
controlnet_config["operations"] = ControlLoraOps()
|
self.manual_cast_dtype = model.manual_cast_dtype
|
||||||
self.control_model = comfy.cldm.cldm.ControlNet(**controlnet_config)
|
|
||||||
dtype = model.get_dtype()
|
dtype = model.get_dtype()
|
||||||
self.control_model.to(dtype)
|
if self.manual_cast_dtype is None:
|
||||||
|
class control_lora_ops(ControlLoraOps, comfy.ops.disable_weight_init):
|
||||||
|
pass
|
||||||
|
else:
|
||||||
|
class control_lora_ops(ControlLoraOps, comfy.ops.manual_cast):
|
||||||
|
pass
|
||||||
|
dtype = self.manual_cast_dtype
|
||||||
|
|
||||||
|
controlnet_config["operations"] = control_lora_ops
|
||||||
|
controlnet_config["dtype"] = dtype
|
||||||
|
self.control_model = comfy.cldm.cldm.ControlNet(**controlnet_config)
|
||||||
self.control_model.to(comfy.model_management.get_torch_device())
|
self.control_model.to(comfy.model_management.get_torch_device())
|
||||||
diffusion_model = model.diffusion_model
|
diffusion_model = model.diffusion_model
|
||||||
sd = diffusion_model.state_dict()
|
sd = diffusion_model.state_dict()
|
||||||
cm = self.control_model.state_dict()
|
cm = self.control_model.state_dict()
|
||||||
|
|
||||||
for k in sd:
|
for k in sd:
|
||||||
weight = comfy.model_management.resolve_lowvram_weight(sd[k], diffusion_model, k)
|
weight = sd[k]
|
||||||
try:
|
try:
|
||||||
comfy.utils.set_attr(self.control_model, k, weight)
|
comfy.utils.set_attr(self.control_model, k, weight)
|
||||||
except:
|
except:
|
||||||
@ -367,6 +380,10 @@ def load_controlnet(ckpt_path, model=None):
|
|||||||
if controlnet_config is None:
|
if controlnet_config is None:
|
||||||
unet_dtype = comfy.model_management.unet_dtype()
|
unet_dtype = comfy.model_management.unet_dtype()
|
||||||
controlnet_config = comfy.model_detection.model_config_from_unet(controlnet_data, prefix, unet_dtype, True).unet_config
|
controlnet_config = comfy.model_detection.model_config_from_unet(controlnet_data, prefix, unet_dtype, True).unet_config
|
||||||
|
load_device = comfy.model_management.get_torch_device()
|
||||||
|
manual_cast_dtype = comfy.model_management.unet_manual_cast(unet_dtype, load_device)
|
||||||
|
if manual_cast_dtype is not None:
|
||||||
|
controlnet_config["operations"] = comfy.ops.manual_cast
|
||||||
controlnet_config.pop("out_channels")
|
controlnet_config.pop("out_channels")
|
||||||
controlnet_config["hint_channels"] = controlnet_data["{}input_hint_block.0.weight".format(prefix)].shape[1]
|
controlnet_config["hint_channels"] = controlnet_data["{}input_hint_block.0.weight".format(prefix)].shape[1]
|
||||||
control_model = comfy.cldm.cldm.ControlNet(**controlnet_config)
|
control_model = comfy.cldm.cldm.ControlNet(**controlnet_config)
|
||||||
@ -395,14 +412,12 @@ def load_controlnet(ckpt_path, model=None):
|
|||||||
missing, unexpected = control_model.load_state_dict(controlnet_data, strict=False)
|
missing, unexpected = control_model.load_state_dict(controlnet_data, strict=False)
|
||||||
print(missing, unexpected)
|
print(missing, unexpected)
|
||||||
|
|
||||||
control_model = control_model.to(unet_dtype)
|
|
||||||
|
|
||||||
global_average_pooling = False
|
global_average_pooling = False
|
||||||
filename = os.path.splitext(ckpt_path)[0]
|
filename = os.path.splitext(ckpt_path)[0]
|
||||||
if filename.endswith("_shuffle") or filename.endswith("_shuffle_fp16"): #TODO: smarter way of enabling global_average_pooling
|
if filename.endswith("_shuffle") or filename.endswith("_shuffle_fp16"): #TODO: smarter way of enabling global_average_pooling
|
||||||
global_average_pooling = True
|
global_average_pooling = True
|
||||||
|
|
||||||
control = ControlNet(control_model, global_average_pooling=global_average_pooling)
|
control = ControlNet(control_model, global_average_pooling=global_average_pooling, load_device=load_device, manual_cast_dtype=manual_cast_dtype)
|
||||||
return control
|
return control
|
||||||
|
|
||||||
class T2IAdapter(ControlBase):
|
class T2IAdapter(ControlBase):
|
||||||
|
|||||||
@ -858,7 +858,7 @@ def predict_eps_sigma(model, input, sigma_in, **kwargs):
|
|||||||
return (input - model(input, sigma_in, **kwargs)) / sigma
|
return (input - model(input, sigma_in, **kwargs)) / sigma
|
||||||
|
|
||||||
|
|
||||||
def sample_unipc(model, noise, image, sigmas, sampling_function, max_denoise, extra_args=None, callback=None, disable=False, noise_mask=None, variant='bh1'):
|
def sample_unipc(model, noise, image, sigmas, max_denoise, extra_args=None, callback=None, disable=False, noise_mask=None, variant='bh1'):
|
||||||
timesteps = sigmas.clone()
|
timesteps = sigmas.clone()
|
||||||
if sigmas[-1] == 0:
|
if sigmas[-1] == 0:
|
||||||
timesteps = sigmas[:]
|
timesteps = sigmas[:]
|
||||||
|
|||||||
@ -750,3 +750,61 @@ def sample_lcm(model, x, sigmas, extra_args=None, callback=None, disable=None, n
|
|||||||
if sigmas[i + 1] > 0:
|
if sigmas[i + 1] > 0:
|
||||||
x += sigmas[i + 1] * noise_sampler(sigmas[i], sigmas[i + 1])
|
x += sigmas[i + 1] * noise_sampler(sigmas[i], sigmas[i + 1])
|
||||||
return x
|
return x
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
@torch.no_grad()
|
||||||
|
def sample_heunpp2(model, x, sigmas, extra_args=None, callback=None, disable=None, s_churn=0., s_tmin=0., s_tmax=float('inf'), s_noise=1.):
|
||||||
|
# From MIT licensed: https://github.com/Carzit/sd-webui-samplers-scheduler/
|
||||||
|
extra_args = {} if extra_args is None else extra_args
|
||||||
|
s_in = x.new_ones([x.shape[0]])
|
||||||
|
s_end = sigmas[-1]
|
||||||
|
for i in trange(len(sigmas) - 1, disable=disable):
|
||||||
|
gamma = min(s_churn / (len(sigmas) - 1), 2 ** 0.5 - 1) if s_tmin <= sigmas[i] <= s_tmax else 0.
|
||||||
|
eps = torch.randn_like(x) * s_noise
|
||||||
|
sigma_hat = sigmas[i] * (gamma + 1)
|
||||||
|
if gamma > 0:
|
||||||
|
x = x + eps * (sigma_hat ** 2 - sigmas[i] ** 2) ** 0.5
|
||||||
|
denoised = model(x, sigma_hat * s_in, **extra_args)
|
||||||
|
d = to_d(x, sigma_hat, denoised)
|
||||||
|
if callback is not None:
|
||||||
|
callback({'x': x, 'i': i, 'sigma': sigmas[i], 'sigma_hat': sigma_hat, 'denoised': denoised})
|
||||||
|
dt = sigmas[i + 1] - sigma_hat
|
||||||
|
if sigmas[i + 1] == s_end:
|
||||||
|
# Euler method
|
||||||
|
x = x + d * dt
|
||||||
|
elif sigmas[i + 2] == s_end:
|
||||||
|
|
||||||
|
# Heun's method
|
||||||
|
x_2 = x + d * dt
|
||||||
|
denoised_2 = model(x_2, sigmas[i + 1] * s_in, **extra_args)
|
||||||
|
d_2 = to_d(x_2, sigmas[i + 1], denoised_2)
|
||||||
|
|
||||||
|
w = 2 * sigmas[0]
|
||||||
|
w2 = sigmas[i+1]/w
|
||||||
|
w1 = 1 - w2
|
||||||
|
|
||||||
|
d_prime = d * w1 + d_2 * w2
|
||||||
|
|
||||||
|
|
||||||
|
x = x + d_prime * dt
|
||||||
|
|
||||||
|
else:
|
||||||
|
# Heun++
|
||||||
|
x_2 = x + d * dt
|
||||||
|
denoised_2 = model(x_2, sigmas[i + 1] * s_in, **extra_args)
|
||||||
|
d_2 = to_d(x_2, sigmas[i + 1], denoised_2)
|
||||||
|
dt_2 = sigmas[i + 2] - sigmas[i + 1]
|
||||||
|
|
||||||
|
x_3 = x_2 + d_2 * dt_2
|
||||||
|
denoised_3 = model(x_3, sigmas[i + 2] * s_in, **extra_args)
|
||||||
|
d_3 = to_d(x_3, sigmas[i + 2], denoised_3)
|
||||||
|
|
||||||
|
w = 3 * sigmas[0]
|
||||||
|
w2 = sigmas[i + 1] / w
|
||||||
|
w3 = sigmas[i + 2] / w
|
||||||
|
w1 = 1 - w2 - w3
|
||||||
|
|
||||||
|
d_prime = w1 * d + w2 * d_2 + w3 * d_3
|
||||||
|
x = x + d_prime * dt
|
||||||
|
return x
|
||||||
|
|||||||
@ -8,6 +8,7 @@ from comfy.ldm.modules.distributions.distributions import DiagonalGaussianDistri
|
|||||||
|
|
||||||
from comfy.ldm.util import instantiate_from_config
|
from comfy.ldm.util import instantiate_from_config
|
||||||
from comfy.ldm.modules.ema import LitEma
|
from comfy.ldm.modules.ema import LitEma
|
||||||
|
import comfy.ops
|
||||||
|
|
||||||
class DiagonalGaussianRegularizer(torch.nn.Module):
|
class DiagonalGaussianRegularizer(torch.nn.Module):
|
||||||
def __init__(self, sample: bool = True):
|
def __init__(self, sample: bool = True):
|
||||||
@ -161,12 +162,12 @@ class AutoencodingEngineLegacy(AutoencodingEngine):
|
|||||||
},
|
},
|
||||||
**kwargs,
|
**kwargs,
|
||||||
)
|
)
|
||||||
self.quant_conv = torch.nn.Conv2d(
|
self.quant_conv = comfy.ops.disable_weight_init.Conv2d(
|
||||||
(1 + ddconfig["double_z"]) * ddconfig["z_channels"],
|
(1 + ddconfig["double_z"]) * ddconfig["z_channels"],
|
||||||
(1 + ddconfig["double_z"]) * embed_dim,
|
(1 + ddconfig["double_z"]) * embed_dim,
|
||||||
1,
|
1,
|
||||||
)
|
)
|
||||||
self.post_quant_conv = torch.nn.Conv2d(embed_dim, ddconfig["z_channels"], 1)
|
self.post_quant_conv = comfy.ops.disable_weight_init.Conv2d(embed_dim, ddconfig["z_channels"], 1)
|
||||||
self.embed_dim = embed_dim
|
self.embed_dim = embed_dim
|
||||||
|
|
||||||
def get_autoencoder_params(self) -> list:
|
def get_autoencoder_params(self) -> list:
|
||||||
|
|||||||
@ -5,8 +5,10 @@ import torch.nn.functional as F
|
|||||||
from torch import nn, einsum
|
from torch import nn, einsum
|
||||||
from einops import rearrange, repeat
|
from einops import rearrange, repeat
|
||||||
from typing import Optional, Any
|
from typing import Optional, Any
|
||||||
|
from functools import partial
|
||||||
|
|
||||||
from .diffusionmodules.util import checkpoint
|
|
||||||
|
from .diffusionmodules.util import checkpoint, AlphaBlender, timestep_embedding
|
||||||
from .sub_quadratic_attention import efficient_dot_product_attention
|
from .sub_quadratic_attention import efficient_dot_product_attention
|
||||||
|
|
||||||
from comfy import model_management
|
from comfy import model_management
|
||||||
@ -17,6 +19,7 @@ if model_management.xformers_enabled():
|
|||||||
|
|
||||||
from comfy.cli_args import args
|
from comfy.cli_args import args
|
||||||
import comfy.ops
|
import comfy.ops
|
||||||
|
ops = comfy.ops.disable_weight_init
|
||||||
|
|
||||||
# CrossAttn precision handling
|
# CrossAttn precision handling
|
||||||
if args.dont_upcast_attention:
|
if args.dont_upcast_attention:
|
||||||
@ -53,7 +56,7 @@ def init_(tensor):
|
|||||||
|
|
||||||
# feedforward
|
# feedforward
|
||||||
class GEGLU(nn.Module):
|
class GEGLU(nn.Module):
|
||||||
def __init__(self, dim_in, dim_out, dtype=None, device=None, operations=comfy.ops):
|
def __init__(self, dim_in, dim_out, dtype=None, device=None, operations=ops):
|
||||||
super().__init__()
|
super().__init__()
|
||||||
self.proj = operations.Linear(dim_in, dim_out * 2, dtype=dtype, device=device)
|
self.proj = operations.Linear(dim_in, dim_out * 2, dtype=dtype, device=device)
|
||||||
|
|
||||||
@ -63,7 +66,7 @@ class GEGLU(nn.Module):
|
|||||||
|
|
||||||
|
|
||||||
class FeedForward(nn.Module):
|
class FeedForward(nn.Module):
|
||||||
def __init__(self, dim, dim_out=None, mult=4, glu=False, dropout=0., dtype=None, device=None, operations=comfy.ops):
|
def __init__(self, dim, dim_out=None, mult=4, glu=False, dropout=0., dtype=None, device=None, operations=ops):
|
||||||
super().__init__()
|
super().__init__()
|
||||||
inner_dim = int(dim * mult)
|
inner_dim = int(dim * mult)
|
||||||
dim_out = default(dim_out, dim)
|
dim_out = default(dim_out, dim)
|
||||||
@ -81,16 +84,6 @@ class FeedForward(nn.Module):
|
|||||||
def forward(self, x):
|
def forward(self, x):
|
||||||
return self.net(x)
|
return self.net(x)
|
||||||
|
|
||||||
|
|
||||||
def zero_module(module):
|
|
||||||
"""
|
|
||||||
Zero out the parameters of a module and return it.
|
|
||||||
"""
|
|
||||||
for p in module.parameters():
|
|
||||||
p.detach().zero_()
|
|
||||||
return module
|
|
||||||
|
|
||||||
|
|
||||||
def Normalize(in_channels, dtype=None, device=None):
|
def Normalize(in_channels, dtype=None, device=None):
|
||||||
return torch.nn.GroupNorm(num_groups=32, num_channels=in_channels, eps=1e-6, affine=True, dtype=dtype, device=device)
|
return torch.nn.GroupNorm(num_groups=32, num_channels=in_channels, eps=1e-6, affine=True, dtype=dtype, device=device)
|
||||||
|
|
||||||
@ -111,19 +104,20 @@ def attention_basic(q, k, v, heads, mask=None):
|
|||||||
|
|
||||||
# force cast to fp32 to avoid overflowing
|
# force cast to fp32 to avoid overflowing
|
||||||
if _ATTN_PRECISION =="fp32":
|
if _ATTN_PRECISION =="fp32":
|
||||||
with torch.autocast(enabled=False, device_type = 'cuda'):
|
sim = einsum('b i d, b j d -> b i j', q.float(), k.float()) * scale
|
||||||
q, k = q.float(), k.float()
|
|
||||||
sim = einsum('b i d, b j d -> b i j', q, k) * scale
|
|
||||||
else:
|
else:
|
||||||
sim = einsum('b i d, b j d -> b i j', q, k) * scale
|
sim = einsum('b i d, b j d -> b i j', q, k) * scale
|
||||||
|
|
||||||
del q, k
|
del q, k
|
||||||
|
|
||||||
if exists(mask):
|
if exists(mask):
|
||||||
mask = rearrange(mask, 'b ... -> b (...)')
|
if mask.dtype == torch.bool:
|
||||||
max_neg_value = -torch.finfo(sim.dtype).max
|
mask = rearrange(mask, 'b ... -> b (...)') #TODO: check if this bool part matches pytorch attention
|
||||||
mask = repeat(mask, 'b j -> (b h) () j', h=h)
|
max_neg_value = -torch.finfo(sim.dtype).max
|
||||||
sim.masked_fill_(~mask, max_neg_value)
|
mask = repeat(mask, 'b j -> (b h) () j', h=h)
|
||||||
|
sim.masked_fill_(~mask, max_neg_value)
|
||||||
|
else:
|
||||||
|
sim += mask
|
||||||
|
|
||||||
# attention, what we cannot get enough of
|
# attention, what we cannot get enough of
|
||||||
sim = sim.softmax(dim=-1)
|
sim = sim.softmax(dim=-1)
|
||||||
@ -276,9 +270,20 @@ def attention_split(q, k, v, heads, mask=None):
|
|||||||
)
|
)
|
||||||
return r1
|
return r1
|
||||||
|
|
||||||
|
BROKEN_XFORMERS = False
|
||||||
|
try:
|
||||||
|
x_vers = xformers.__version__
|
||||||
|
#I think 0.0.23 is also broken (q with bs bigger than 65535 gives CUDA error)
|
||||||
|
BROKEN_XFORMERS = x_vers.startswith("0.0.21") or x_vers.startswith("0.0.22") or x_vers.startswith("0.0.23")
|
||||||
|
except:
|
||||||
|
pass
|
||||||
|
|
||||||
def attention_xformers(q, k, v, heads, mask=None):
|
def attention_xformers(q, k, v, heads, mask=None):
|
||||||
b, _, dim_head = q.shape
|
b, _, dim_head = q.shape
|
||||||
dim_head //= heads
|
dim_head //= heads
|
||||||
|
if BROKEN_XFORMERS:
|
||||||
|
if b * heads > 65535:
|
||||||
|
return attention_pytorch(q, k, v, heads, mask)
|
||||||
|
|
||||||
q, k, v = map(
|
q, k, v = map(
|
||||||
lambda t: t.unsqueeze(3)
|
lambda t: t.unsqueeze(3)
|
||||||
@ -337,8 +342,20 @@ else:
|
|||||||
if model_management.pytorch_attention_enabled():
|
if model_management.pytorch_attention_enabled():
|
||||||
optimized_attention_masked = attention_pytorch
|
optimized_attention_masked = attention_pytorch
|
||||||
|
|
||||||
|
def optimized_attention_for_device(device, mask=False):
|
||||||
|
if device == torch.device("cpu"): #TODO
|
||||||
|
if model_management.pytorch_attention_enabled():
|
||||||
|
return attention_pytorch
|
||||||
|
else:
|
||||||
|
return attention_basic
|
||||||
|
if mask:
|
||||||
|
return optimized_attention_masked
|
||||||
|
|
||||||
|
return optimized_attention
|
||||||
|
|
||||||
|
|
||||||
class CrossAttention(nn.Module):
|
class CrossAttention(nn.Module):
|
||||||
def __init__(self, query_dim, context_dim=None, heads=8, dim_head=64, dropout=0., dtype=None, device=None, operations=comfy.ops):
|
def __init__(self, query_dim, context_dim=None, heads=8, dim_head=64, dropout=0., dtype=None, device=None, operations=ops):
|
||||||
super().__init__()
|
super().__init__()
|
||||||
inner_dim = dim_head * heads
|
inner_dim = dim_head * heads
|
||||||
context_dim = default(context_dim, query_dim)
|
context_dim = default(context_dim, query_dim)
|
||||||
@ -370,53 +387,72 @@ class CrossAttention(nn.Module):
|
|||||||
|
|
||||||
|
|
||||||
class BasicTransformerBlock(nn.Module):
|
class BasicTransformerBlock(nn.Module):
|
||||||
def __init__(self, dim, n_heads, d_head, dropout=0., context_dim=None, gated_ff=True, checkpoint=True,
|
def __init__(self, dim, n_heads, d_head, dropout=0., context_dim=None, gated_ff=True, checkpoint=True, ff_in=False, inner_dim=None,
|
||||||
disable_self_attn=False, dtype=None, device=None, operations=comfy.ops):
|
disable_self_attn=False, disable_temporal_crossattention=False, switch_temporal_ca_to_sa=False, dtype=None, device=None, operations=ops):
|
||||||
super().__init__()
|
super().__init__()
|
||||||
|
|
||||||
|
self.ff_in = ff_in or inner_dim is not None
|
||||||
|
if inner_dim is None:
|
||||||
|
inner_dim = dim
|
||||||
|
|
||||||
|
self.is_res = inner_dim == dim
|
||||||
|
|
||||||
|
if self.ff_in:
|
||||||
|
self.norm_in = operations.LayerNorm(dim, dtype=dtype, device=device)
|
||||||
|
self.ff_in = FeedForward(dim, dim_out=inner_dim, dropout=dropout, glu=gated_ff, dtype=dtype, device=device, operations=operations)
|
||||||
|
|
||||||
self.disable_self_attn = disable_self_attn
|
self.disable_self_attn = disable_self_attn
|
||||||
self.attn1 = CrossAttention(query_dim=dim, heads=n_heads, dim_head=d_head, dropout=dropout,
|
self.attn1 = CrossAttention(query_dim=inner_dim, heads=n_heads, dim_head=d_head, dropout=dropout,
|
||||||
context_dim=context_dim if self.disable_self_attn else None, dtype=dtype, device=device, operations=operations) # is a self-attention if not self.disable_self_attn
|
context_dim=context_dim if self.disable_self_attn else None, dtype=dtype, device=device, operations=operations) # is a self-attention if not self.disable_self_attn
|
||||||
self.ff = FeedForward(dim, dropout=dropout, glu=gated_ff, dtype=dtype, device=device, operations=operations)
|
self.ff = FeedForward(inner_dim, dim_out=dim, dropout=dropout, glu=gated_ff, dtype=dtype, device=device, operations=operations)
|
||||||
self.attn2 = CrossAttention(query_dim=dim, context_dim=context_dim,
|
|
||||||
heads=n_heads, dim_head=d_head, dropout=dropout, dtype=dtype, device=device, operations=operations) # is self-attn if context is none
|
if disable_temporal_crossattention:
|
||||||
self.norm1 = nn.LayerNorm(dim, dtype=dtype, device=device)
|
if switch_temporal_ca_to_sa:
|
||||||
self.norm2 = nn.LayerNorm(dim, dtype=dtype, device=device)
|
raise ValueError
|
||||||
self.norm3 = nn.LayerNorm(dim, dtype=dtype, device=device)
|
else:
|
||||||
|
self.attn2 = None
|
||||||
|
else:
|
||||||
|
context_dim_attn2 = None
|
||||||
|
if not switch_temporal_ca_to_sa:
|
||||||
|
context_dim_attn2 = context_dim
|
||||||
|
|
||||||
|
self.attn2 = CrossAttention(query_dim=inner_dim, context_dim=context_dim_attn2,
|
||||||
|
heads=n_heads, dim_head=d_head, dropout=dropout, dtype=dtype, device=device, operations=operations) # is self-attn if context is none
|
||||||
|
self.norm2 = operations.LayerNorm(inner_dim, dtype=dtype, device=device)
|
||||||
|
|
||||||
|
self.norm1 = operations.LayerNorm(inner_dim, dtype=dtype, device=device)
|
||||||
|
self.norm3 = operations.LayerNorm(inner_dim, dtype=dtype, device=device)
|
||||||
self.checkpoint = checkpoint
|
self.checkpoint = checkpoint
|
||||||
self.n_heads = n_heads
|
self.n_heads = n_heads
|
||||||
self.d_head = d_head
|
self.d_head = d_head
|
||||||
|
self.switch_temporal_ca_to_sa = switch_temporal_ca_to_sa
|
||||||
|
|
||||||
def forward(self, x, context=None, transformer_options={}):
|
def forward(self, x, context=None, transformer_options={}):
|
||||||
return checkpoint(self._forward, (x, context, transformer_options), self.parameters(), self.checkpoint)
|
return checkpoint(self._forward, (x, context, transformer_options), self.parameters(), self.checkpoint)
|
||||||
|
|
||||||
def _forward(self, x, context=None, transformer_options={}):
|
def _forward(self, x, context=None, transformer_options={}):
|
||||||
extra_options = {}
|
extra_options = {}
|
||||||
block = None
|
block = transformer_options.get("block", None)
|
||||||
block_index = 0
|
block_index = transformer_options.get("block_index", 0)
|
||||||
if "current_index" in transformer_options:
|
transformer_patches = {}
|
||||||
extra_options["transformer_index"] = transformer_options["current_index"]
|
transformer_patches_replace = {}
|
||||||
if "block_index" in transformer_options:
|
|
||||||
block_index = transformer_options["block_index"]
|
for k in transformer_options:
|
||||||
extra_options["block_index"] = block_index
|
if k == "patches":
|
||||||
if "original_shape" in transformer_options:
|
transformer_patches = transformer_options[k]
|
||||||
extra_options["original_shape"] = transformer_options["original_shape"]
|
elif k == "patches_replace":
|
||||||
if "block" in transformer_options:
|
transformer_patches_replace = transformer_options[k]
|
||||||
block = transformer_options["block"]
|
else:
|
||||||
extra_options["block"] = block
|
extra_options[k] = transformer_options[k]
|
||||||
if "cond_or_uncond" in transformer_options:
|
|
||||||
extra_options["cond_or_uncond"] = transformer_options["cond_or_uncond"]
|
|
||||||
if "patches" in transformer_options:
|
|
||||||
transformer_patches = transformer_options["patches"]
|
|
||||||
else:
|
|
||||||
transformer_patches = {}
|
|
||||||
|
|
||||||
extra_options["n_heads"] = self.n_heads
|
extra_options["n_heads"] = self.n_heads
|
||||||
extra_options["dim_head"] = self.d_head
|
extra_options["dim_head"] = self.d_head
|
||||||
|
|
||||||
if "patches_replace" in transformer_options:
|
if self.ff_in:
|
||||||
transformer_patches_replace = transformer_options["patches_replace"]
|
x_skip = x
|
||||||
else:
|
x = self.ff_in(self.norm_in(x))
|
||||||
transformer_patches_replace = {}
|
if self.is_res:
|
||||||
|
x += x_skip
|
||||||
|
|
||||||
n = self.norm1(x)
|
n = self.norm1(x)
|
||||||
if self.disable_self_attn:
|
if self.disable_self_attn:
|
||||||
@ -465,31 +501,34 @@ class BasicTransformerBlock(nn.Module):
|
|||||||
for p in patch:
|
for p in patch:
|
||||||
x = p(x, extra_options)
|
x = p(x, extra_options)
|
||||||
|
|
||||||
n = self.norm2(x)
|
if self.attn2 is not None:
|
||||||
|
n = self.norm2(x)
|
||||||
context_attn2 = context
|
if self.switch_temporal_ca_to_sa:
|
||||||
value_attn2 = None
|
context_attn2 = n
|
||||||
if "attn2_patch" in transformer_patches:
|
else:
|
||||||
patch = transformer_patches["attn2_patch"]
|
context_attn2 = context
|
||||||
value_attn2 = context_attn2
|
value_attn2 = None
|
||||||
for p in patch:
|
if "attn2_patch" in transformer_patches:
|
||||||
n, context_attn2, value_attn2 = p(n, context_attn2, value_attn2, extra_options)
|
patch = transformer_patches["attn2_patch"]
|
||||||
|
|
||||||
attn2_replace_patch = transformer_patches_replace.get("attn2", {})
|
|
||||||
block_attn2 = transformer_block
|
|
||||||
if block_attn2 not in attn2_replace_patch:
|
|
||||||
block_attn2 = block
|
|
||||||
|
|
||||||
if block_attn2 in attn2_replace_patch:
|
|
||||||
if value_attn2 is None:
|
|
||||||
value_attn2 = context_attn2
|
value_attn2 = context_attn2
|
||||||
n = self.attn2.to_q(n)
|
for p in patch:
|
||||||
context_attn2 = self.attn2.to_k(context_attn2)
|
n, context_attn2, value_attn2 = p(n, context_attn2, value_attn2, extra_options)
|
||||||
value_attn2 = self.attn2.to_v(value_attn2)
|
|
||||||
n = attn2_replace_patch[block_attn2](n, context_attn2, value_attn2, extra_options)
|
attn2_replace_patch = transformer_patches_replace.get("attn2", {})
|
||||||
n = self.attn2.to_out(n)
|
block_attn2 = transformer_block
|
||||||
else:
|
if block_attn2 not in attn2_replace_patch:
|
||||||
n = self.attn2(n, context=context_attn2, value=value_attn2)
|
block_attn2 = block
|
||||||
|
|
||||||
|
if block_attn2 in attn2_replace_patch:
|
||||||
|
if value_attn2 is None:
|
||||||
|
value_attn2 = context_attn2
|
||||||
|
n = self.attn2.to_q(n)
|
||||||
|
context_attn2 = self.attn2.to_k(context_attn2)
|
||||||
|
value_attn2 = self.attn2.to_v(value_attn2)
|
||||||
|
n = attn2_replace_patch[block_attn2](n, context_attn2, value_attn2, extra_options)
|
||||||
|
n = self.attn2.to_out(n)
|
||||||
|
else:
|
||||||
|
n = self.attn2(n, context=context_attn2, value=value_attn2)
|
||||||
|
|
||||||
if "attn2_output_patch" in transformer_patches:
|
if "attn2_output_patch" in transformer_patches:
|
||||||
patch = transformer_patches["attn2_output_patch"]
|
patch = transformer_patches["attn2_output_patch"]
|
||||||
@ -497,7 +536,12 @@ class BasicTransformerBlock(nn.Module):
|
|||||||
n = p(n, extra_options)
|
n = p(n, extra_options)
|
||||||
|
|
||||||
x += n
|
x += n
|
||||||
x = self.ff(self.norm3(x)) + x
|
if self.is_res:
|
||||||
|
x_skip = x
|
||||||
|
x = self.ff(self.norm3(x))
|
||||||
|
if self.is_res:
|
||||||
|
x += x_skip
|
||||||
|
|
||||||
return x
|
return x
|
||||||
|
|
||||||
|
|
||||||
@ -513,13 +557,13 @@ class SpatialTransformer(nn.Module):
|
|||||||
def __init__(self, in_channels, n_heads, d_head,
|
def __init__(self, in_channels, n_heads, d_head,
|
||||||
depth=1, dropout=0., context_dim=None,
|
depth=1, dropout=0., context_dim=None,
|
||||||
disable_self_attn=False, use_linear=False,
|
disable_self_attn=False, use_linear=False,
|
||||||
use_checkpoint=True, dtype=None, device=None, operations=comfy.ops):
|
use_checkpoint=True, dtype=None, device=None, operations=ops):
|
||||||
super().__init__()
|
super().__init__()
|
||||||
if exists(context_dim) and not isinstance(context_dim, list):
|
if exists(context_dim) and not isinstance(context_dim, list):
|
||||||
context_dim = [context_dim] * depth
|
context_dim = [context_dim] * depth
|
||||||
self.in_channels = in_channels
|
self.in_channels = in_channels
|
||||||
inner_dim = n_heads * d_head
|
inner_dim = n_heads * d_head
|
||||||
self.norm = Normalize(in_channels, dtype=dtype, device=device)
|
self.norm = operations.GroupNorm(num_groups=32, num_channels=in_channels, eps=1e-6, affine=True, dtype=dtype, device=device)
|
||||||
if not use_linear:
|
if not use_linear:
|
||||||
self.proj_in = operations.Conv2d(in_channels,
|
self.proj_in = operations.Conv2d(in_channels,
|
||||||
inner_dim,
|
inner_dim,
|
||||||
@ -565,3 +609,164 @@ class SpatialTransformer(nn.Module):
|
|||||||
x = self.proj_out(x)
|
x = self.proj_out(x)
|
||||||
return x + x_in
|
return x + x_in
|
||||||
|
|
||||||
|
|
||||||
|
class SpatialVideoTransformer(SpatialTransformer):
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
in_channels,
|
||||||
|
n_heads,
|
||||||
|
d_head,
|
||||||
|
depth=1,
|
||||||
|
dropout=0.0,
|
||||||
|
use_linear=False,
|
||||||
|
context_dim=None,
|
||||||
|
use_spatial_context=False,
|
||||||
|
timesteps=None,
|
||||||
|
merge_strategy: str = "fixed",
|
||||||
|
merge_factor: float = 0.5,
|
||||||
|
time_context_dim=None,
|
||||||
|
ff_in=False,
|
||||||
|
checkpoint=False,
|
||||||
|
time_depth=1,
|
||||||
|
disable_self_attn=False,
|
||||||
|
disable_temporal_crossattention=False,
|
||||||
|
max_time_embed_period: int = 10000,
|
||||||
|
dtype=None, device=None, operations=ops
|
||||||
|
):
|
||||||
|
super().__init__(
|
||||||
|
in_channels,
|
||||||
|
n_heads,
|
||||||
|
d_head,
|
||||||
|
depth=depth,
|
||||||
|
dropout=dropout,
|
||||||
|
use_checkpoint=checkpoint,
|
||||||
|
context_dim=context_dim,
|
||||||
|
use_linear=use_linear,
|
||||||
|
disable_self_attn=disable_self_attn,
|
||||||
|
dtype=dtype, device=device, operations=operations
|
||||||
|
)
|
||||||
|
self.time_depth = time_depth
|
||||||
|
self.depth = depth
|
||||||
|
self.max_time_embed_period = max_time_embed_period
|
||||||
|
|
||||||
|
time_mix_d_head = d_head
|
||||||
|
n_time_mix_heads = n_heads
|
||||||
|
|
||||||
|
time_mix_inner_dim = int(time_mix_d_head * n_time_mix_heads)
|
||||||
|
|
||||||
|
inner_dim = n_heads * d_head
|
||||||
|
if use_spatial_context:
|
||||||
|
time_context_dim = context_dim
|
||||||
|
|
||||||
|
self.time_stack = nn.ModuleList(
|
||||||
|
[
|
||||||
|
BasicTransformerBlock(
|
||||||
|
inner_dim,
|
||||||
|
n_time_mix_heads,
|
||||||
|
time_mix_d_head,
|
||||||
|
dropout=dropout,
|
||||||
|
context_dim=time_context_dim,
|
||||||
|
# timesteps=timesteps,
|
||||||
|
checkpoint=checkpoint,
|
||||||
|
ff_in=ff_in,
|
||||||
|
inner_dim=time_mix_inner_dim,
|
||||||
|
disable_self_attn=disable_self_attn,
|
||||||
|
disable_temporal_crossattention=disable_temporal_crossattention,
|
||||||
|
dtype=dtype, device=device, operations=operations
|
||||||
|
)
|
||||||
|
for _ in range(self.depth)
|
||||||
|
]
|
||||||
|
)
|
||||||
|
|
||||||
|
assert len(self.time_stack) == len(self.transformer_blocks)
|
||||||
|
|
||||||
|
self.use_spatial_context = use_spatial_context
|
||||||
|
self.in_channels = in_channels
|
||||||
|
|
||||||
|
time_embed_dim = self.in_channels * 4
|
||||||
|
self.time_pos_embed = nn.Sequential(
|
||||||
|
operations.Linear(self.in_channels, time_embed_dim, dtype=dtype, device=device),
|
||||||
|
nn.SiLU(),
|
||||||
|
operations.Linear(time_embed_dim, self.in_channels, dtype=dtype, device=device),
|
||||||
|
)
|
||||||
|
|
||||||
|
self.time_mixer = AlphaBlender(
|
||||||
|
alpha=merge_factor, merge_strategy=merge_strategy
|
||||||
|
)
|
||||||
|
|
||||||
|
def forward(
|
||||||
|
self,
|
||||||
|
x: torch.Tensor,
|
||||||
|
context: Optional[torch.Tensor] = None,
|
||||||
|
time_context: Optional[torch.Tensor] = None,
|
||||||
|
timesteps: Optional[int] = None,
|
||||||
|
image_only_indicator: Optional[torch.Tensor] = None,
|
||||||
|
transformer_options={}
|
||||||
|
) -> torch.Tensor:
|
||||||
|
_, _, h, w = x.shape
|
||||||
|
x_in = x
|
||||||
|
spatial_context = None
|
||||||
|
if exists(context):
|
||||||
|
spatial_context = context
|
||||||
|
|
||||||
|
if self.use_spatial_context:
|
||||||
|
assert (
|
||||||
|
context.ndim == 3
|
||||||
|
), f"n dims of spatial context should be 3 but are {context.ndim}"
|
||||||
|
|
||||||
|
if time_context is None:
|
||||||
|
time_context = context
|
||||||
|
time_context_first_timestep = time_context[::timesteps]
|
||||||
|
time_context = repeat(
|
||||||
|
time_context_first_timestep, "b ... -> (b n) ...", n=h * w
|
||||||
|
)
|
||||||
|
elif time_context is not None and not self.use_spatial_context:
|
||||||
|
time_context = repeat(time_context, "b ... -> (b n) ...", n=h * w)
|
||||||
|
if time_context.ndim == 2:
|
||||||
|
time_context = rearrange(time_context, "b c -> b 1 c")
|
||||||
|
|
||||||
|
x = self.norm(x)
|
||||||
|
if not self.use_linear:
|
||||||
|
x = self.proj_in(x)
|
||||||
|
x = rearrange(x, "b c h w -> b (h w) c")
|
||||||
|
if self.use_linear:
|
||||||
|
x = self.proj_in(x)
|
||||||
|
|
||||||
|
num_frames = torch.arange(timesteps, device=x.device)
|
||||||
|
num_frames = repeat(num_frames, "t -> b t", b=x.shape[0] // timesteps)
|
||||||
|
num_frames = rearrange(num_frames, "b t -> (b t)")
|
||||||
|
t_emb = timestep_embedding(num_frames, self.in_channels, repeat_only=False, max_period=self.max_time_embed_period).to(x.dtype)
|
||||||
|
emb = self.time_pos_embed(t_emb)
|
||||||
|
emb = emb[:, None, :]
|
||||||
|
|
||||||
|
for it_, (block, mix_block) in enumerate(
|
||||||
|
zip(self.transformer_blocks, self.time_stack)
|
||||||
|
):
|
||||||
|
transformer_options["block_index"] = it_
|
||||||
|
x = block(
|
||||||
|
x,
|
||||||
|
context=spatial_context,
|
||||||
|
transformer_options=transformer_options,
|
||||||
|
)
|
||||||
|
|
||||||
|
x_mix = x
|
||||||
|
x_mix = x_mix + emb
|
||||||
|
|
||||||
|
B, S, C = x_mix.shape
|
||||||
|
x_mix = rearrange(x_mix, "(b t) s c -> (b s) t c", t=timesteps)
|
||||||
|
x_mix = mix_block(x_mix, context=time_context) #TODO: transformer_options
|
||||||
|
x_mix = rearrange(
|
||||||
|
x_mix, "(b s) t c -> (b t) s c", s=S, b=B // timesteps, c=C, t=timesteps
|
||||||
|
)
|
||||||
|
|
||||||
|
x = self.time_mixer(x_spatial=x, x_temporal=x_mix, image_only_indicator=image_only_indicator)
|
||||||
|
|
||||||
|
if self.use_linear:
|
||||||
|
x = self.proj_out(x)
|
||||||
|
x = rearrange(x, "b (h w) c -> b c h w", h=h, w=w)
|
||||||
|
if not self.use_linear:
|
||||||
|
x = self.proj_out(x)
|
||||||
|
out = x + x_in
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@ -8,6 +8,7 @@ from typing import Optional, Any
|
|||||||
|
|
||||||
from comfy import model_management
|
from comfy import model_management
|
||||||
import comfy.ops
|
import comfy.ops
|
||||||
|
ops = comfy.ops.disable_weight_init
|
||||||
|
|
||||||
if model_management.xformers_enabled_vae():
|
if model_management.xformers_enabled_vae():
|
||||||
import xformers
|
import xformers
|
||||||
@ -40,7 +41,7 @@ def nonlinearity(x):
|
|||||||
|
|
||||||
|
|
||||||
def Normalize(in_channels, num_groups=32):
|
def Normalize(in_channels, num_groups=32):
|
||||||
return torch.nn.GroupNorm(num_groups=num_groups, num_channels=in_channels, eps=1e-6, affine=True)
|
return ops.GroupNorm(num_groups=num_groups, num_channels=in_channels, eps=1e-6, affine=True)
|
||||||
|
|
||||||
|
|
||||||
class Upsample(nn.Module):
|
class Upsample(nn.Module):
|
||||||
@ -48,7 +49,7 @@ class Upsample(nn.Module):
|
|||||||
super().__init__()
|
super().__init__()
|
||||||
self.with_conv = with_conv
|
self.with_conv = with_conv
|
||||||
if self.with_conv:
|
if self.with_conv:
|
||||||
self.conv = comfy.ops.Conv2d(in_channels,
|
self.conv = ops.Conv2d(in_channels,
|
||||||
in_channels,
|
in_channels,
|
||||||
kernel_size=3,
|
kernel_size=3,
|
||||||
stride=1,
|
stride=1,
|
||||||
@ -78,7 +79,7 @@ class Downsample(nn.Module):
|
|||||||
self.with_conv = with_conv
|
self.with_conv = with_conv
|
||||||
if self.with_conv:
|
if self.with_conv:
|
||||||
# no asymmetric padding in torch conv, must do it ourselves
|
# no asymmetric padding in torch conv, must do it ourselves
|
||||||
self.conv = comfy.ops.Conv2d(in_channels,
|
self.conv = ops.Conv2d(in_channels,
|
||||||
in_channels,
|
in_channels,
|
||||||
kernel_size=3,
|
kernel_size=3,
|
||||||
stride=2,
|
stride=2,
|
||||||
@ -105,30 +106,30 @@ class ResnetBlock(nn.Module):
|
|||||||
|
|
||||||
self.swish = torch.nn.SiLU(inplace=True)
|
self.swish = torch.nn.SiLU(inplace=True)
|
||||||
self.norm1 = Normalize(in_channels)
|
self.norm1 = Normalize(in_channels)
|
||||||
self.conv1 = comfy.ops.Conv2d(in_channels,
|
self.conv1 = ops.Conv2d(in_channels,
|
||||||
out_channels,
|
out_channels,
|
||||||
kernel_size=3,
|
kernel_size=3,
|
||||||
stride=1,
|
stride=1,
|
||||||
padding=1)
|
padding=1)
|
||||||
if temb_channels > 0:
|
if temb_channels > 0:
|
||||||
self.temb_proj = comfy.ops.Linear(temb_channels,
|
self.temb_proj = ops.Linear(temb_channels,
|
||||||
out_channels)
|
out_channels)
|
||||||
self.norm2 = Normalize(out_channels)
|
self.norm2 = Normalize(out_channels)
|
||||||
self.dropout = torch.nn.Dropout(dropout, inplace=True)
|
self.dropout = torch.nn.Dropout(dropout, inplace=True)
|
||||||
self.conv2 = comfy.ops.Conv2d(out_channels,
|
self.conv2 = ops.Conv2d(out_channels,
|
||||||
out_channels,
|
out_channels,
|
||||||
kernel_size=3,
|
kernel_size=3,
|
||||||
stride=1,
|
stride=1,
|
||||||
padding=1)
|
padding=1)
|
||||||
if self.in_channels != self.out_channels:
|
if self.in_channels != self.out_channels:
|
||||||
if self.use_conv_shortcut:
|
if self.use_conv_shortcut:
|
||||||
self.conv_shortcut = comfy.ops.Conv2d(in_channels,
|
self.conv_shortcut = ops.Conv2d(in_channels,
|
||||||
out_channels,
|
out_channels,
|
||||||
kernel_size=3,
|
kernel_size=3,
|
||||||
stride=1,
|
stride=1,
|
||||||
padding=1)
|
padding=1)
|
||||||
else:
|
else:
|
||||||
self.nin_shortcut = comfy.ops.Conv2d(in_channels,
|
self.nin_shortcut = ops.Conv2d(in_channels,
|
||||||
out_channels,
|
out_channels,
|
||||||
kernel_size=1,
|
kernel_size=1,
|
||||||
stride=1,
|
stride=1,
|
||||||
@ -245,22 +246,22 @@ class AttnBlock(nn.Module):
|
|||||||
self.in_channels = in_channels
|
self.in_channels = in_channels
|
||||||
|
|
||||||
self.norm = Normalize(in_channels)
|
self.norm = Normalize(in_channels)
|
||||||
self.q = comfy.ops.Conv2d(in_channels,
|
self.q = ops.Conv2d(in_channels,
|
||||||
in_channels,
|
in_channels,
|
||||||
kernel_size=1,
|
kernel_size=1,
|
||||||
stride=1,
|
stride=1,
|
||||||
padding=0)
|
padding=0)
|
||||||
self.k = comfy.ops.Conv2d(in_channels,
|
self.k = ops.Conv2d(in_channels,
|
||||||
in_channels,
|
in_channels,
|
||||||
kernel_size=1,
|
kernel_size=1,
|
||||||
stride=1,
|
stride=1,
|
||||||
padding=0)
|
padding=0)
|
||||||
self.v = comfy.ops.Conv2d(in_channels,
|
self.v = ops.Conv2d(in_channels,
|
||||||
in_channels,
|
in_channels,
|
||||||
kernel_size=1,
|
kernel_size=1,
|
||||||
stride=1,
|
stride=1,
|
||||||
padding=0)
|
padding=0)
|
||||||
self.proj_out = comfy.ops.Conv2d(in_channels,
|
self.proj_out = ops.Conv2d(in_channels,
|
||||||
in_channels,
|
in_channels,
|
||||||
kernel_size=1,
|
kernel_size=1,
|
||||||
stride=1,
|
stride=1,
|
||||||
@ -312,14 +313,14 @@ class Model(nn.Module):
|
|||||||
# timestep embedding
|
# timestep embedding
|
||||||
self.temb = nn.Module()
|
self.temb = nn.Module()
|
||||||
self.temb.dense = nn.ModuleList([
|
self.temb.dense = nn.ModuleList([
|
||||||
comfy.ops.Linear(self.ch,
|
ops.Linear(self.ch,
|
||||||
self.temb_ch),
|
self.temb_ch),
|
||||||
comfy.ops.Linear(self.temb_ch,
|
ops.Linear(self.temb_ch,
|
||||||
self.temb_ch),
|
self.temb_ch),
|
||||||
])
|
])
|
||||||
|
|
||||||
# downsampling
|
# downsampling
|
||||||
self.conv_in = comfy.ops.Conv2d(in_channels,
|
self.conv_in = ops.Conv2d(in_channels,
|
||||||
self.ch,
|
self.ch,
|
||||||
kernel_size=3,
|
kernel_size=3,
|
||||||
stride=1,
|
stride=1,
|
||||||
@ -388,7 +389,7 @@ class Model(nn.Module):
|
|||||||
|
|
||||||
# end
|
# end
|
||||||
self.norm_out = Normalize(block_in)
|
self.norm_out = Normalize(block_in)
|
||||||
self.conv_out = comfy.ops.Conv2d(block_in,
|
self.conv_out = ops.Conv2d(block_in,
|
||||||
out_ch,
|
out_ch,
|
||||||
kernel_size=3,
|
kernel_size=3,
|
||||||
stride=1,
|
stride=1,
|
||||||
@ -461,7 +462,7 @@ class Encoder(nn.Module):
|
|||||||
self.in_channels = in_channels
|
self.in_channels = in_channels
|
||||||
|
|
||||||
# downsampling
|
# downsampling
|
||||||
self.conv_in = comfy.ops.Conv2d(in_channels,
|
self.conv_in = ops.Conv2d(in_channels,
|
||||||
self.ch,
|
self.ch,
|
||||||
kernel_size=3,
|
kernel_size=3,
|
||||||
stride=1,
|
stride=1,
|
||||||
@ -506,7 +507,7 @@ class Encoder(nn.Module):
|
|||||||
|
|
||||||
# end
|
# end
|
||||||
self.norm_out = Normalize(block_in)
|
self.norm_out = Normalize(block_in)
|
||||||
self.conv_out = comfy.ops.Conv2d(block_in,
|
self.conv_out = ops.Conv2d(block_in,
|
||||||
2*z_channels if double_z else z_channels,
|
2*z_channels if double_z else z_channels,
|
||||||
kernel_size=3,
|
kernel_size=3,
|
||||||
stride=1,
|
stride=1,
|
||||||
@ -541,7 +542,7 @@ class Decoder(nn.Module):
|
|||||||
def __init__(self, *, ch, out_ch, ch_mult=(1,2,4,8), num_res_blocks,
|
def __init__(self, *, ch, out_ch, ch_mult=(1,2,4,8), num_res_blocks,
|
||||||
attn_resolutions, dropout=0.0, resamp_with_conv=True, in_channels,
|
attn_resolutions, dropout=0.0, resamp_with_conv=True, in_channels,
|
||||||
resolution, z_channels, give_pre_end=False, tanh_out=False, use_linear_attn=False,
|
resolution, z_channels, give_pre_end=False, tanh_out=False, use_linear_attn=False,
|
||||||
conv_out_op=comfy.ops.Conv2d,
|
conv_out_op=ops.Conv2d,
|
||||||
resnet_op=ResnetBlock,
|
resnet_op=ResnetBlock,
|
||||||
attn_op=AttnBlock,
|
attn_op=AttnBlock,
|
||||||
**ignorekwargs):
|
**ignorekwargs):
|
||||||
@ -565,7 +566,7 @@ class Decoder(nn.Module):
|
|||||||
self.z_shape, np.prod(self.z_shape)))
|
self.z_shape, np.prod(self.z_shape)))
|
||||||
|
|
||||||
# z to block_in
|
# z to block_in
|
||||||
self.conv_in = comfy.ops.Conv2d(z_channels,
|
self.conv_in = ops.Conv2d(z_channels,
|
||||||
block_in,
|
block_in,
|
||||||
kernel_size=3,
|
kernel_size=3,
|
||||||
stride=1,
|
stride=1,
|
||||||
|
|||||||
@ -5,17 +5,20 @@ import numpy as np
|
|||||||
import torch as th
|
import torch as th
|
||||||
import torch.nn as nn
|
import torch.nn as nn
|
||||||
import torch.nn.functional as F
|
import torch.nn.functional as F
|
||||||
|
from einops import rearrange
|
||||||
|
from functools import partial
|
||||||
|
|
||||||
from .util import (
|
from .util import (
|
||||||
checkpoint,
|
checkpoint,
|
||||||
avg_pool_nd,
|
avg_pool_nd,
|
||||||
zero_module,
|
zero_module,
|
||||||
normalization,
|
|
||||||
timestep_embedding,
|
timestep_embedding,
|
||||||
|
AlphaBlender,
|
||||||
)
|
)
|
||||||
from ..attention import SpatialTransformer
|
from ..attention import SpatialTransformer, SpatialVideoTransformer, default
|
||||||
from comfy.ldm.util import exists
|
from comfy.ldm.util import exists
|
||||||
import comfy.ops
|
import comfy.ops
|
||||||
|
ops = comfy.ops.disable_weight_init
|
||||||
|
|
||||||
class TimestepBlock(nn.Module):
|
class TimestepBlock(nn.Module):
|
||||||
"""
|
"""
|
||||||
@ -28,6 +31,26 @@ class TimestepBlock(nn.Module):
|
|||||||
Apply the module to `x` given `emb` timestep embeddings.
|
Apply the module to `x` given `emb` timestep embeddings.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
#This is needed because accelerate makes a copy of transformer_options which breaks "transformer_index"
|
||||||
|
def forward_timestep_embed(ts, x, emb, context=None, transformer_options={}, output_shape=None, time_context=None, num_video_frames=None, image_only_indicator=None):
|
||||||
|
for layer in ts:
|
||||||
|
if isinstance(layer, VideoResBlock):
|
||||||
|
x = layer(x, emb, num_video_frames, image_only_indicator)
|
||||||
|
elif isinstance(layer, TimestepBlock):
|
||||||
|
x = layer(x, emb)
|
||||||
|
elif isinstance(layer, SpatialVideoTransformer):
|
||||||
|
x = layer(x, context, time_context, num_video_frames, image_only_indicator, transformer_options)
|
||||||
|
if "transformer_index" in transformer_options:
|
||||||
|
transformer_options["transformer_index"] += 1
|
||||||
|
elif isinstance(layer, SpatialTransformer):
|
||||||
|
x = layer(x, context, transformer_options)
|
||||||
|
if "transformer_index" in transformer_options:
|
||||||
|
transformer_options["transformer_index"] += 1
|
||||||
|
elif isinstance(layer, Upsample):
|
||||||
|
x = layer(x, output_shape=output_shape)
|
||||||
|
else:
|
||||||
|
x = layer(x)
|
||||||
|
return x
|
||||||
|
|
||||||
class TimestepEmbedSequential(nn.Sequential, TimestepBlock):
|
class TimestepEmbedSequential(nn.Sequential, TimestepBlock):
|
||||||
"""
|
"""
|
||||||
@ -35,31 +58,8 @@ class TimestepEmbedSequential(nn.Sequential, TimestepBlock):
|
|||||||
support it as an extra input.
|
support it as an extra input.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
def forward(self, x, emb, context=None, transformer_options={}, output_shape=None):
|
def forward(self, *args, **kwargs):
|
||||||
for layer in self:
|
return forward_timestep_embed(self, *args, **kwargs)
|
||||||
if isinstance(layer, TimestepBlock):
|
|
||||||
x = layer(x, emb)
|
|
||||||
elif isinstance(layer, SpatialTransformer):
|
|
||||||
x = layer(x, context, transformer_options)
|
|
||||||
elif isinstance(layer, Upsample):
|
|
||||||
x = layer(x, output_shape=output_shape)
|
|
||||||
else:
|
|
||||||
x = layer(x)
|
|
||||||
return x
|
|
||||||
|
|
||||||
#This is needed because accelerate makes a copy of transformer_options which breaks "current_index"
|
|
||||||
def forward_timestep_embed(ts, x, emb, context=None, transformer_options={}, output_shape=None):
|
|
||||||
for layer in ts:
|
|
||||||
if isinstance(layer, TimestepBlock):
|
|
||||||
x = layer(x, emb)
|
|
||||||
elif isinstance(layer, SpatialTransformer):
|
|
||||||
x = layer(x, context, transformer_options)
|
|
||||||
transformer_options["current_index"] += 1
|
|
||||||
elif isinstance(layer, Upsample):
|
|
||||||
x = layer(x, output_shape=output_shape)
|
|
||||||
else:
|
|
||||||
x = layer(x)
|
|
||||||
return x
|
|
||||||
|
|
||||||
class Upsample(nn.Module):
|
class Upsample(nn.Module):
|
||||||
"""
|
"""
|
||||||
@ -70,7 +70,7 @@ class Upsample(nn.Module):
|
|||||||
upsampling occurs in the inner-two dimensions.
|
upsampling occurs in the inner-two dimensions.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
def __init__(self, channels, use_conv, dims=2, out_channels=None, padding=1, dtype=None, device=None, operations=comfy.ops):
|
def __init__(self, channels, use_conv, dims=2, out_channels=None, padding=1, dtype=None, device=None, operations=ops):
|
||||||
super().__init__()
|
super().__init__()
|
||||||
self.channels = channels
|
self.channels = channels
|
||||||
self.out_channels = out_channels or channels
|
self.out_channels = out_channels or channels
|
||||||
@ -106,7 +106,7 @@ class Downsample(nn.Module):
|
|||||||
downsampling occurs in the inner-two dimensions.
|
downsampling occurs in the inner-two dimensions.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
def __init__(self, channels, use_conv, dims=2, out_channels=None, padding=1, dtype=None, device=None, operations=comfy.ops):
|
def __init__(self, channels, use_conv, dims=2, out_channels=None, padding=1, dtype=None, device=None, operations=ops):
|
||||||
super().__init__()
|
super().__init__()
|
||||||
self.channels = channels
|
self.channels = channels
|
||||||
self.out_channels = out_channels or channels
|
self.out_channels = out_channels or channels
|
||||||
@ -154,9 +154,12 @@ class ResBlock(TimestepBlock):
|
|||||||
use_checkpoint=False,
|
use_checkpoint=False,
|
||||||
up=False,
|
up=False,
|
||||||
down=False,
|
down=False,
|
||||||
|
kernel_size=3,
|
||||||
|
exchange_temb_dims=False,
|
||||||
|
skip_t_emb=False,
|
||||||
dtype=None,
|
dtype=None,
|
||||||
device=None,
|
device=None,
|
||||||
operations=comfy.ops
|
operations=ops
|
||||||
):
|
):
|
||||||
super().__init__()
|
super().__init__()
|
||||||
self.channels = channels
|
self.channels = channels
|
||||||
@ -166,11 +169,17 @@ class ResBlock(TimestepBlock):
|
|||||||
self.use_conv = use_conv
|
self.use_conv = use_conv
|
||||||
self.use_checkpoint = use_checkpoint
|
self.use_checkpoint = use_checkpoint
|
||||||
self.use_scale_shift_norm = use_scale_shift_norm
|
self.use_scale_shift_norm = use_scale_shift_norm
|
||||||
|
self.exchange_temb_dims = exchange_temb_dims
|
||||||
|
|
||||||
|
if isinstance(kernel_size, list):
|
||||||
|
padding = [k // 2 for k in kernel_size]
|
||||||
|
else:
|
||||||
|
padding = kernel_size // 2
|
||||||
|
|
||||||
self.in_layers = nn.Sequential(
|
self.in_layers = nn.Sequential(
|
||||||
nn.GroupNorm(32, channels, dtype=dtype, device=device),
|
operations.GroupNorm(32, channels, dtype=dtype, device=device),
|
||||||
nn.SiLU(),
|
nn.SiLU(),
|
||||||
operations.conv_nd(dims, channels, self.out_channels, 3, padding=1, dtype=dtype, device=device),
|
operations.conv_nd(dims, channels, self.out_channels, kernel_size, padding=padding, dtype=dtype, device=device),
|
||||||
)
|
)
|
||||||
|
|
||||||
self.updown = up or down
|
self.updown = up or down
|
||||||
@ -184,27 +193,31 @@ class ResBlock(TimestepBlock):
|
|||||||
else:
|
else:
|
||||||
self.h_upd = self.x_upd = nn.Identity()
|
self.h_upd = self.x_upd = nn.Identity()
|
||||||
|
|
||||||
self.emb_layers = nn.Sequential(
|
self.skip_t_emb = skip_t_emb
|
||||||
nn.SiLU(),
|
if self.skip_t_emb:
|
||||||
operations.Linear(
|
self.emb_layers = None
|
||||||
emb_channels,
|
self.exchange_temb_dims = False
|
||||||
2 * self.out_channels if use_scale_shift_norm else self.out_channels, dtype=dtype, device=device
|
else:
|
||||||
),
|
self.emb_layers = nn.Sequential(
|
||||||
)
|
nn.SiLU(),
|
||||||
|
operations.Linear(
|
||||||
|
emb_channels,
|
||||||
|
2 * self.out_channels if use_scale_shift_norm else self.out_channels, dtype=dtype, device=device
|
||||||
|
),
|
||||||
|
)
|
||||||
self.out_layers = nn.Sequential(
|
self.out_layers = nn.Sequential(
|
||||||
nn.GroupNorm(32, self.out_channels, dtype=dtype, device=device),
|
operations.GroupNorm(32, self.out_channels, dtype=dtype, device=device),
|
||||||
nn.SiLU(),
|
nn.SiLU(),
|
||||||
nn.Dropout(p=dropout),
|
nn.Dropout(p=dropout),
|
||||||
zero_module(
|
operations.conv_nd(dims, self.out_channels, self.out_channels, kernel_size, padding=padding, dtype=dtype, device=device)
|
||||||
operations.conv_nd(dims, self.out_channels, self.out_channels, 3, padding=1, dtype=dtype, device=device)
|
,
|
||||||
),
|
|
||||||
)
|
)
|
||||||
|
|
||||||
if self.out_channels == channels:
|
if self.out_channels == channels:
|
||||||
self.skip_connection = nn.Identity()
|
self.skip_connection = nn.Identity()
|
||||||
elif use_conv:
|
elif use_conv:
|
||||||
self.skip_connection = operations.conv_nd(
|
self.skip_connection = operations.conv_nd(
|
||||||
dims, channels, self.out_channels, 3, padding=1, dtype=dtype, device=device
|
dims, channels, self.out_channels, kernel_size, padding=padding, dtype=dtype, device=device
|
||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
self.skip_connection = operations.conv_nd(dims, channels, self.out_channels, 1, dtype=dtype, device=device)
|
self.skip_connection = operations.conv_nd(dims, channels, self.out_channels, 1, dtype=dtype, device=device)
|
||||||
@ -230,19 +243,110 @@ class ResBlock(TimestepBlock):
|
|||||||
h = in_conv(h)
|
h = in_conv(h)
|
||||||
else:
|
else:
|
||||||
h = self.in_layers(x)
|
h = self.in_layers(x)
|
||||||
emb_out = self.emb_layers(emb).type(h.dtype)
|
|
||||||
while len(emb_out.shape) < len(h.shape):
|
emb_out = None
|
||||||
emb_out = emb_out[..., None]
|
if not self.skip_t_emb:
|
||||||
|
emb_out = self.emb_layers(emb).type(h.dtype)
|
||||||
|
while len(emb_out.shape) < len(h.shape):
|
||||||
|
emb_out = emb_out[..., None]
|
||||||
if self.use_scale_shift_norm:
|
if self.use_scale_shift_norm:
|
||||||
out_norm, out_rest = self.out_layers[0], self.out_layers[1:]
|
out_norm, out_rest = self.out_layers[0], self.out_layers[1:]
|
||||||
scale, shift = th.chunk(emb_out, 2, dim=1)
|
h = out_norm(h)
|
||||||
h = out_norm(h) * (1 + scale) + shift
|
if emb_out is not None:
|
||||||
|
scale, shift = th.chunk(emb_out, 2, dim=1)
|
||||||
|
h *= (1 + scale)
|
||||||
|
h += shift
|
||||||
h = out_rest(h)
|
h = out_rest(h)
|
||||||
else:
|
else:
|
||||||
h = h + emb_out
|
if emb_out is not None:
|
||||||
|
if self.exchange_temb_dims:
|
||||||
|
emb_out = rearrange(emb_out, "b t c ... -> b c t ...")
|
||||||
|
h = h + emb_out
|
||||||
h = self.out_layers(h)
|
h = self.out_layers(h)
|
||||||
return self.skip_connection(x) + h
|
return self.skip_connection(x) + h
|
||||||
|
|
||||||
|
|
||||||
|
class VideoResBlock(ResBlock):
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
channels: int,
|
||||||
|
emb_channels: int,
|
||||||
|
dropout: float,
|
||||||
|
video_kernel_size=3,
|
||||||
|
merge_strategy: str = "fixed",
|
||||||
|
merge_factor: float = 0.5,
|
||||||
|
out_channels=None,
|
||||||
|
use_conv: bool = False,
|
||||||
|
use_scale_shift_norm: bool = False,
|
||||||
|
dims: int = 2,
|
||||||
|
use_checkpoint: bool = False,
|
||||||
|
up: bool = False,
|
||||||
|
down: bool = False,
|
||||||
|
dtype=None,
|
||||||
|
device=None,
|
||||||
|
operations=ops
|
||||||
|
):
|
||||||
|
super().__init__(
|
||||||
|
channels,
|
||||||
|
emb_channels,
|
||||||
|
dropout,
|
||||||
|
out_channels=out_channels,
|
||||||
|
use_conv=use_conv,
|
||||||
|
use_scale_shift_norm=use_scale_shift_norm,
|
||||||
|
dims=dims,
|
||||||
|
use_checkpoint=use_checkpoint,
|
||||||
|
up=up,
|
||||||
|
down=down,
|
||||||
|
dtype=dtype,
|
||||||
|
device=device,
|
||||||
|
operations=operations
|
||||||
|
)
|
||||||
|
|
||||||
|
self.time_stack = ResBlock(
|
||||||
|
default(out_channels, channels),
|
||||||
|
emb_channels,
|
||||||
|
dropout=dropout,
|
||||||
|
dims=3,
|
||||||
|
out_channels=default(out_channels, channels),
|
||||||
|
use_scale_shift_norm=False,
|
||||||
|
use_conv=False,
|
||||||
|
up=False,
|
||||||
|
down=False,
|
||||||
|
kernel_size=video_kernel_size,
|
||||||
|
use_checkpoint=use_checkpoint,
|
||||||
|
exchange_temb_dims=True,
|
||||||
|
dtype=dtype,
|
||||||
|
device=device,
|
||||||
|
operations=operations
|
||||||
|
)
|
||||||
|
self.time_mixer = AlphaBlender(
|
||||||
|
alpha=merge_factor,
|
||||||
|
merge_strategy=merge_strategy,
|
||||||
|
rearrange_pattern="b t -> b 1 t 1 1",
|
||||||
|
)
|
||||||
|
|
||||||
|
def forward(
|
||||||
|
self,
|
||||||
|
x: th.Tensor,
|
||||||
|
emb: th.Tensor,
|
||||||
|
num_video_frames: int,
|
||||||
|
image_only_indicator = None,
|
||||||
|
) -> th.Tensor:
|
||||||
|
x = super().forward(x, emb)
|
||||||
|
|
||||||
|
x_mix = rearrange(x, "(b t) c h w -> b c t h w", t=num_video_frames)
|
||||||
|
x = rearrange(x, "(b t) c h w -> b c t h w", t=num_video_frames)
|
||||||
|
|
||||||
|
x = self.time_stack(
|
||||||
|
x, rearrange(emb, "(b t) ... -> b t ...", t=num_video_frames)
|
||||||
|
)
|
||||||
|
x = self.time_mixer(
|
||||||
|
x_spatial=x_mix, x_temporal=x, image_only_indicator=image_only_indicator
|
||||||
|
)
|
||||||
|
x = rearrange(x, "b c t h w -> (b t) c h w")
|
||||||
|
return x
|
||||||
|
|
||||||
|
|
||||||
class Timestep(nn.Module):
|
class Timestep(nn.Module):
|
||||||
def __init__(self, dim):
|
def __init__(self, dim):
|
||||||
super().__init__()
|
super().__init__()
|
||||||
@ -255,7 +359,10 @@ def apply_control(h, control, name):
|
|||||||
if control is not None and name in control and len(control[name]) > 0:
|
if control is not None and name in control and len(control[name]) > 0:
|
||||||
ctrl = control[name].pop()
|
ctrl = control[name].pop()
|
||||||
if ctrl is not None:
|
if ctrl is not None:
|
||||||
h += ctrl
|
try:
|
||||||
|
h += ctrl
|
||||||
|
except:
|
||||||
|
print("warning control could not be applied", h.shape, ctrl.shape)
|
||||||
return h
|
return h
|
||||||
|
|
||||||
class UNetModel(nn.Module):
|
class UNetModel(nn.Module):
|
||||||
@ -316,8 +423,18 @@ class UNetModel(nn.Module):
|
|||||||
adm_in_channels=None,
|
adm_in_channels=None,
|
||||||
transformer_depth_middle=None,
|
transformer_depth_middle=None,
|
||||||
transformer_depth_output=None,
|
transformer_depth_output=None,
|
||||||
|
use_temporal_resblock=False,
|
||||||
|
use_temporal_attention=False,
|
||||||
|
time_context_dim=None,
|
||||||
|
extra_ff_mix_layer=False,
|
||||||
|
use_spatial_context=False,
|
||||||
|
merge_strategy=None,
|
||||||
|
merge_factor=0.0,
|
||||||
|
video_kernel_size=None,
|
||||||
|
disable_temporal_crossattention=False,
|
||||||
|
max_ddpm_temb_period=10000,
|
||||||
device=None,
|
device=None,
|
||||||
operations=comfy.ops,
|
operations=ops,
|
||||||
):
|
):
|
||||||
super().__init__()
|
super().__init__()
|
||||||
assert use_spatial_transformer == True, "use_spatial_transformer has to be true"
|
assert use_spatial_transformer == True, "use_spatial_transformer has to be true"
|
||||||
@ -370,8 +487,12 @@ class UNetModel(nn.Module):
|
|||||||
self.num_heads = num_heads
|
self.num_heads = num_heads
|
||||||
self.num_head_channels = num_head_channels
|
self.num_head_channels = num_head_channels
|
||||||
self.num_heads_upsample = num_heads_upsample
|
self.num_heads_upsample = num_heads_upsample
|
||||||
|
self.use_temporal_resblocks = use_temporal_resblock
|
||||||
self.predict_codebook_ids = n_embed is not None
|
self.predict_codebook_ids = n_embed is not None
|
||||||
|
|
||||||
|
self.default_num_video_frames = None
|
||||||
|
self.default_image_only_indicator = None
|
||||||
|
|
||||||
time_embed_dim = model_channels * 4
|
time_embed_dim = model_channels * 4
|
||||||
self.time_embed = nn.Sequential(
|
self.time_embed = nn.Sequential(
|
||||||
operations.Linear(model_channels, time_embed_dim, dtype=self.dtype, device=device),
|
operations.Linear(model_channels, time_embed_dim, dtype=self.dtype, device=device),
|
||||||
@ -408,13 +529,104 @@ class UNetModel(nn.Module):
|
|||||||
input_block_chans = [model_channels]
|
input_block_chans = [model_channels]
|
||||||
ch = model_channels
|
ch = model_channels
|
||||||
ds = 1
|
ds = 1
|
||||||
|
|
||||||
|
def get_attention_layer(
|
||||||
|
ch,
|
||||||
|
num_heads,
|
||||||
|
dim_head,
|
||||||
|
depth=1,
|
||||||
|
context_dim=None,
|
||||||
|
use_checkpoint=False,
|
||||||
|
disable_self_attn=False,
|
||||||
|
):
|
||||||
|
if use_temporal_attention:
|
||||||
|
return SpatialVideoTransformer(
|
||||||
|
ch,
|
||||||
|
num_heads,
|
||||||
|
dim_head,
|
||||||
|
depth=depth,
|
||||||
|
context_dim=context_dim,
|
||||||
|
time_context_dim=time_context_dim,
|
||||||
|
dropout=dropout,
|
||||||
|
ff_in=extra_ff_mix_layer,
|
||||||
|
use_spatial_context=use_spatial_context,
|
||||||
|
merge_strategy=merge_strategy,
|
||||||
|
merge_factor=merge_factor,
|
||||||
|
checkpoint=use_checkpoint,
|
||||||
|
use_linear=use_linear_in_transformer,
|
||||||
|
disable_self_attn=disable_self_attn,
|
||||||
|
disable_temporal_crossattention=disable_temporal_crossattention,
|
||||||
|
max_time_embed_period=max_ddpm_temb_period,
|
||||||
|
dtype=self.dtype, device=device, operations=operations
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
return SpatialTransformer(
|
||||||
|
ch, num_heads, dim_head, depth=depth, context_dim=context_dim,
|
||||||
|
disable_self_attn=disable_self_attn, use_linear=use_linear_in_transformer,
|
||||||
|
use_checkpoint=use_checkpoint, dtype=self.dtype, device=device, operations=operations
|
||||||
|
)
|
||||||
|
|
||||||
|
def get_resblock(
|
||||||
|
merge_factor,
|
||||||
|
merge_strategy,
|
||||||
|
video_kernel_size,
|
||||||
|
ch,
|
||||||
|
time_embed_dim,
|
||||||
|
dropout,
|
||||||
|
out_channels,
|
||||||
|
dims,
|
||||||
|
use_checkpoint,
|
||||||
|
use_scale_shift_norm,
|
||||||
|
down=False,
|
||||||
|
up=False,
|
||||||
|
dtype=None,
|
||||||
|
device=None,
|
||||||
|
operations=ops
|
||||||
|
):
|
||||||
|
if self.use_temporal_resblocks:
|
||||||
|
return VideoResBlock(
|
||||||
|
merge_factor=merge_factor,
|
||||||
|
merge_strategy=merge_strategy,
|
||||||
|
video_kernel_size=video_kernel_size,
|
||||||
|
channels=ch,
|
||||||
|
emb_channels=time_embed_dim,
|
||||||
|
dropout=dropout,
|
||||||
|
out_channels=out_channels,
|
||||||
|
dims=dims,
|
||||||
|
use_checkpoint=use_checkpoint,
|
||||||
|
use_scale_shift_norm=use_scale_shift_norm,
|
||||||
|
down=down,
|
||||||
|
up=up,
|
||||||
|
dtype=dtype,
|
||||||
|
device=device,
|
||||||
|
operations=operations
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
return ResBlock(
|
||||||
|
channels=ch,
|
||||||
|
emb_channels=time_embed_dim,
|
||||||
|
dropout=dropout,
|
||||||
|
out_channels=out_channels,
|
||||||
|
use_checkpoint=use_checkpoint,
|
||||||
|
dims=dims,
|
||||||
|
use_scale_shift_norm=use_scale_shift_norm,
|
||||||
|
down=down,
|
||||||
|
up=up,
|
||||||
|
dtype=dtype,
|
||||||
|
device=device,
|
||||||
|
operations=operations
|
||||||
|
)
|
||||||
|
|
||||||
for level, mult in enumerate(channel_mult):
|
for level, mult in enumerate(channel_mult):
|
||||||
for nr in range(self.num_res_blocks[level]):
|
for nr in range(self.num_res_blocks[level]):
|
||||||
layers = [
|
layers = [
|
||||||
ResBlock(
|
get_resblock(
|
||||||
ch,
|
merge_factor=merge_factor,
|
||||||
time_embed_dim,
|
merge_strategy=merge_strategy,
|
||||||
dropout,
|
video_kernel_size=video_kernel_size,
|
||||||
|
ch=ch,
|
||||||
|
time_embed_dim=time_embed_dim,
|
||||||
|
dropout=dropout,
|
||||||
out_channels=mult * model_channels,
|
out_channels=mult * model_channels,
|
||||||
dims=dims,
|
dims=dims,
|
||||||
use_checkpoint=use_checkpoint,
|
use_checkpoint=use_checkpoint,
|
||||||
@ -441,11 +653,9 @@ class UNetModel(nn.Module):
|
|||||||
disabled_sa = False
|
disabled_sa = False
|
||||||
|
|
||||||
if not exists(num_attention_blocks) or nr < num_attention_blocks[level]:
|
if not exists(num_attention_blocks) or nr < num_attention_blocks[level]:
|
||||||
layers.append(SpatialTransformer(
|
layers.append(get_attention_layer(
|
||||||
ch, num_heads, dim_head, depth=num_transformers, context_dim=context_dim,
|
ch, num_heads, dim_head, depth=num_transformers, context_dim=context_dim,
|
||||||
disable_self_attn=disabled_sa, use_linear=use_linear_in_transformer,
|
disable_self_attn=disabled_sa, use_checkpoint=use_checkpoint)
|
||||||
use_checkpoint=use_checkpoint, dtype=self.dtype, device=device, operations=operations
|
|
||||||
)
|
|
||||||
)
|
)
|
||||||
self.input_blocks.append(TimestepEmbedSequential(*layers))
|
self.input_blocks.append(TimestepEmbedSequential(*layers))
|
||||||
self._feature_size += ch
|
self._feature_size += ch
|
||||||
@ -454,10 +664,13 @@ class UNetModel(nn.Module):
|
|||||||
out_ch = ch
|
out_ch = ch
|
||||||
self.input_blocks.append(
|
self.input_blocks.append(
|
||||||
TimestepEmbedSequential(
|
TimestepEmbedSequential(
|
||||||
ResBlock(
|
get_resblock(
|
||||||
ch,
|
merge_factor=merge_factor,
|
||||||
time_embed_dim,
|
merge_strategy=merge_strategy,
|
||||||
dropout,
|
video_kernel_size=video_kernel_size,
|
||||||
|
ch=ch,
|
||||||
|
time_embed_dim=time_embed_dim,
|
||||||
|
dropout=dropout,
|
||||||
out_channels=out_ch,
|
out_channels=out_ch,
|
||||||
dims=dims,
|
dims=dims,
|
||||||
use_checkpoint=use_checkpoint,
|
use_checkpoint=use_checkpoint,
|
||||||
@ -487,10 +700,14 @@ class UNetModel(nn.Module):
|
|||||||
#num_heads = 1
|
#num_heads = 1
|
||||||
dim_head = ch // num_heads if use_spatial_transformer else num_head_channels
|
dim_head = ch // num_heads if use_spatial_transformer else num_head_channels
|
||||||
mid_block = [
|
mid_block = [
|
||||||
ResBlock(
|
get_resblock(
|
||||||
ch,
|
merge_factor=merge_factor,
|
||||||
time_embed_dim,
|
merge_strategy=merge_strategy,
|
||||||
dropout,
|
video_kernel_size=video_kernel_size,
|
||||||
|
ch=ch,
|
||||||
|
time_embed_dim=time_embed_dim,
|
||||||
|
dropout=dropout,
|
||||||
|
out_channels=None,
|
||||||
dims=dims,
|
dims=dims,
|
||||||
use_checkpoint=use_checkpoint,
|
use_checkpoint=use_checkpoint,
|
||||||
use_scale_shift_norm=use_scale_shift_norm,
|
use_scale_shift_norm=use_scale_shift_norm,
|
||||||
@ -499,15 +716,18 @@ class UNetModel(nn.Module):
|
|||||||
operations=operations
|
operations=operations
|
||||||
)]
|
)]
|
||||||
if transformer_depth_middle >= 0:
|
if transformer_depth_middle >= 0:
|
||||||
mid_block += [SpatialTransformer( # always uses a self-attn
|
mid_block += [get_attention_layer( # always uses a self-attn
|
||||||
ch, num_heads, dim_head, depth=transformer_depth_middle, context_dim=context_dim,
|
ch, num_heads, dim_head, depth=transformer_depth_middle, context_dim=context_dim,
|
||||||
disable_self_attn=disable_middle_self_attn, use_linear=use_linear_in_transformer,
|
disable_self_attn=disable_middle_self_attn, use_checkpoint=use_checkpoint
|
||||||
use_checkpoint=use_checkpoint, dtype=self.dtype, device=device, operations=operations
|
|
||||||
),
|
),
|
||||||
ResBlock(
|
get_resblock(
|
||||||
ch,
|
merge_factor=merge_factor,
|
||||||
time_embed_dim,
|
merge_strategy=merge_strategy,
|
||||||
dropout,
|
video_kernel_size=video_kernel_size,
|
||||||
|
ch=ch,
|
||||||
|
time_embed_dim=time_embed_dim,
|
||||||
|
dropout=dropout,
|
||||||
|
out_channels=None,
|
||||||
dims=dims,
|
dims=dims,
|
||||||
use_checkpoint=use_checkpoint,
|
use_checkpoint=use_checkpoint,
|
||||||
use_scale_shift_norm=use_scale_shift_norm,
|
use_scale_shift_norm=use_scale_shift_norm,
|
||||||
@ -523,10 +743,13 @@ class UNetModel(nn.Module):
|
|||||||
for i in range(self.num_res_blocks[level] + 1):
|
for i in range(self.num_res_blocks[level] + 1):
|
||||||
ich = input_block_chans.pop()
|
ich = input_block_chans.pop()
|
||||||
layers = [
|
layers = [
|
||||||
ResBlock(
|
get_resblock(
|
||||||
ch + ich,
|
merge_factor=merge_factor,
|
||||||
time_embed_dim,
|
merge_strategy=merge_strategy,
|
||||||
dropout,
|
video_kernel_size=video_kernel_size,
|
||||||
|
ch=ch + ich,
|
||||||
|
time_embed_dim=time_embed_dim,
|
||||||
|
dropout=dropout,
|
||||||
out_channels=model_channels * mult,
|
out_channels=model_channels * mult,
|
||||||
dims=dims,
|
dims=dims,
|
||||||
use_checkpoint=use_checkpoint,
|
use_checkpoint=use_checkpoint,
|
||||||
@ -554,19 +777,21 @@ class UNetModel(nn.Module):
|
|||||||
|
|
||||||
if not exists(num_attention_blocks) or i < num_attention_blocks[level]:
|
if not exists(num_attention_blocks) or i < num_attention_blocks[level]:
|
||||||
layers.append(
|
layers.append(
|
||||||
SpatialTransformer(
|
get_attention_layer(
|
||||||
ch, num_heads, dim_head, depth=num_transformers, context_dim=context_dim,
|
ch, num_heads, dim_head, depth=num_transformers, context_dim=context_dim,
|
||||||
disable_self_attn=disabled_sa, use_linear=use_linear_in_transformer,
|
disable_self_attn=disabled_sa, use_checkpoint=use_checkpoint
|
||||||
use_checkpoint=use_checkpoint, dtype=self.dtype, device=device, operations=operations
|
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
if level and i == self.num_res_blocks[level]:
|
if level and i == self.num_res_blocks[level]:
|
||||||
out_ch = ch
|
out_ch = ch
|
||||||
layers.append(
|
layers.append(
|
||||||
ResBlock(
|
get_resblock(
|
||||||
ch,
|
merge_factor=merge_factor,
|
||||||
time_embed_dim,
|
merge_strategy=merge_strategy,
|
||||||
dropout,
|
video_kernel_size=video_kernel_size,
|
||||||
|
ch=ch,
|
||||||
|
time_embed_dim=time_embed_dim,
|
||||||
|
dropout=dropout,
|
||||||
out_channels=out_ch,
|
out_channels=out_ch,
|
||||||
dims=dims,
|
dims=dims,
|
||||||
use_checkpoint=use_checkpoint,
|
use_checkpoint=use_checkpoint,
|
||||||
@ -584,13 +809,13 @@ class UNetModel(nn.Module):
|
|||||||
self._feature_size += ch
|
self._feature_size += ch
|
||||||
|
|
||||||
self.out = nn.Sequential(
|
self.out = nn.Sequential(
|
||||||
nn.GroupNorm(32, ch, dtype=self.dtype, device=device),
|
operations.GroupNorm(32, ch, dtype=self.dtype, device=device),
|
||||||
nn.SiLU(),
|
nn.SiLU(),
|
||||||
zero_module(operations.conv_nd(dims, model_channels, out_channels, 3, padding=1, dtype=self.dtype, device=device)),
|
zero_module(operations.conv_nd(dims, model_channels, out_channels, 3, padding=1, dtype=self.dtype, device=device)),
|
||||||
)
|
)
|
||||||
if self.predict_codebook_ids:
|
if self.predict_codebook_ids:
|
||||||
self.id_predictor = nn.Sequential(
|
self.id_predictor = nn.Sequential(
|
||||||
nn.GroupNorm(32, ch, dtype=self.dtype, device=device),
|
operations.GroupNorm(32, ch, dtype=self.dtype, device=device),
|
||||||
operations.conv_nd(dims, model_channels, n_embed, 1, dtype=self.dtype, device=device),
|
operations.conv_nd(dims, model_channels, n_embed, 1, dtype=self.dtype, device=device),
|
||||||
#nn.LogSoftmax(dim=1) # change to cross_entropy and produce non-normalized logits
|
#nn.LogSoftmax(dim=1) # change to cross_entropy and produce non-normalized logits
|
||||||
)
|
)
|
||||||
@ -605,31 +830,45 @@ class UNetModel(nn.Module):
|
|||||||
:return: an [N x C x ...] Tensor of outputs.
|
:return: an [N x C x ...] Tensor of outputs.
|
||||||
"""
|
"""
|
||||||
transformer_options["original_shape"] = list(x.shape)
|
transformer_options["original_shape"] = list(x.shape)
|
||||||
transformer_options["current_index"] = 0
|
transformer_options["transformer_index"] = 0
|
||||||
transformer_patches = transformer_options.get("patches", {})
|
transformer_patches = transformer_options.get("patches", {})
|
||||||
|
|
||||||
|
num_video_frames = kwargs.get("num_video_frames", self.default_num_video_frames)
|
||||||
|
image_only_indicator = kwargs.get("image_only_indicator", self.default_image_only_indicator)
|
||||||
|
time_context = kwargs.get("time_context", None)
|
||||||
|
|
||||||
assert (y is not None) == (
|
assert (y is not None) == (
|
||||||
self.num_classes is not None
|
self.num_classes is not None
|
||||||
), "must specify y if and only if the model is class-conditional"
|
), "must specify y if and only if the model is class-conditional"
|
||||||
hs = []
|
hs = []
|
||||||
t_emb = timestep_embedding(timesteps, self.model_channels, repeat_only=False).to(self.dtype)
|
t_emb = timestep_embedding(timesteps, self.model_channels, repeat_only=False).to(x.dtype)
|
||||||
emb = self.time_embed(t_emb)
|
emb = self.time_embed(t_emb)
|
||||||
|
|
||||||
if self.num_classes is not None:
|
if self.num_classes is not None:
|
||||||
assert y.shape[0] == x.shape[0]
|
assert y.shape[0] == x.shape[0]
|
||||||
emb = emb + self.label_emb(y)
|
emb = emb + self.label_emb(y)
|
||||||
|
|
||||||
h = x.type(self.dtype)
|
h = x
|
||||||
for id, module in enumerate(self.input_blocks):
|
for id, module in enumerate(self.input_blocks):
|
||||||
transformer_options["block"] = ("input", id)
|
transformer_options["block"] = ("input", id)
|
||||||
h = forward_timestep_embed(module, h, emb, context, transformer_options)
|
h = forward_timestep_embed(module, h, emb, context, transformer_options, time_context=time_context, num_video_frames=num_video_frames, image_only_indicator=image_only_indicator)
|
||||||
h = apply_control(h, control, 'input')
|
h = apply_control(h, control, 'input')
|
||||||
|
if "input_block_patch" in transformer_patches:
|
||||||
|
patch = transformer_patches["input_block_patch"]
|
||||||
|
for p in patch:
|
||||||
|
h = p(h, transformer_options)
|
||||||
|
|
||||||
hs.append(h)
|
hs.append(h)
|
||||||
|
if "input_block_patch_after_skip" in transformer_patches:
|
||||||
|
patch = transformer_patches["input_block_patch_after_skip"]
|
||||||
|
for p in patch:
|
||||||
|
h = p(h, transformer_options)
|
||||||
|
|
||||||
transformer_options["block"] = ("middle", 0)
|
transformer_options["block"] = ("middle", 0)
|
||||||
h = forward_timestep_embed(self.middle_block, h, emb, context, transformer_options)
|
h = forward_timestep_embed(self.middle_block, h, emb, context, transformer_options, time_context=time_context, num_video_frames=num_video_frames, image_only_indicator=image_only_indicator)
|
||||||
h = apply_control(h, control, 'middle')
|
h = apply_control(h, control, 'middle')
|
||||||
|
|
||||||
|
|
||||||
for id, module in enumerate(self.output_blocks):
|
for id, module in enumerate(self.output_blocks):
|
||||||
transformer_options["block"] = ("output", id)
|
transformer_options["block"] = ("output", id)
|
||||||
hsp = hs.pop()
|
hsp = hs.pop()
|
||||||
@ -646,7 +885,7 @@ class UNetModel(nn.Module):
|
|||||||
output_shape = hs[-1].shape
|
output_shape = hs[-1].shape
|
||||||
else:
|
else:
|
||||||
output_shape = None
|
output_shape = None
|
||||||
h = forward_timestep_embed(module, h, emb, context, transformer_options, output_shape)
|
h = forward_timestep_embed(module, h, emb, context, transformer_options, output_shape, time_context=time_context, num_video_frames=num_video_frames, image_only_indicator=image_only_indicator)
|
||||||
h = h.type(x.dtype)
|
h = h.type(x.dtype)
|
||||||
if self.predict_codebook_ids:
|
if self.predict_codebook_ids:
|
||||||
return self.id_predictor(h)
|
return self.id_predictor(h)
|
||||||
|
|||||||
@ -43,8 +43,8 @@ class AbstractLowScaleModel(nn.Module):
|
|||||||
|
|
||||||
def q_sample(self, x_start, t, noise=None):
|
def q_sample(self, x_start, t, noise=None):
|
||||||
noise = default(noise, lambda: torch.randn_like(x_start))
|
noise = default(noise, lambda: torch.randn_like(x_start))
|
||||||
return (extract_into_tensor(self.sqrt_alphas_cumprod, t, x_start.shape) * x_start +
|
return (extract_into_tensor(self.sqrt_alphas_cumprod.to(x_start.device), t, x_start.shape) * x_start +
|
||||||
extract_into_tensor(self.sqrt_one_minus_alphas_cumprod, t, x_start.shape) * noise)
|
extract_into_tensor(self.sqrt_one_minus_alphas_cumprod.to(x_start.device), t, x_start.shape) * noise)
|
||||||
|
|
||||||
def forward(self, x):
|
def forward(self, x):
|
||||||
return x, None
|
return x, None
|
||||||
|
|||||||
@ -13,10 +13,76 @@ import math
|
|||||||
import torch
|
import torch
|
||||||
import torch.nn as nn
|
import torch.nn as nn
|
||||||
import numpy as np
|
import numpy as np
|
||||||
from einops import repeat
|
from einops import repeat, rearrange
|
||||||
|
|
||||||
from comfy.ldm.util import instantiate_from_config
|
from comfy.ldm.util import instantiate_from_config
|
||||||
import comfy.ops
|
|
||||||
|
class AlphaBlender(nn.Module):
|
||||||
|
strategies = ["learned", "fixed", "learned_with_images"]
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
alpha: float,
|
||||||
|
merge_strategy: str = "learned_with_images",
|
||||||
|
rearrange_pattern: str = "b t -> (b t) 1 1",
|
||||||
|
):
|
||||||
|
super().__init__()
|
||||||
|
self.merge_strategy = merge_strategy
|
||||||
|
self.rearrange_pattern = rearrange_pattern
|
||||||
|
|
||||||
|
assert (
|
||||||
|
merge_strategy in self.strategies
|
||||||
|
), f"merge_strategy needs to be in {self.strategies}"
|
||||||
|
|
||||||
|
if self.merge_strategy == "fixed":
|
||||||
|
self.register_buffer("mix_factor", torch.Tensor([alpha]))
|
||||||
|
elif (
|
||||||
|
self.merge_strategy == "learned"
|
||||||
|
or self.merge_strategy == "learned_with_images"
|
||||||
|
):
|
||||||
|
self.register_parameter(
|
||||||
|
"mix_factor", torch.nn.Parameter(torch.Tensor([alpha]))
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
raise ValueError(f"unknown merge strategy {self.merge_strategy}")
|
||||||
|
|
||||||
|
def get_alpha(self, image_only_indicator: torch.Tensor) -> torch.Tensor:
|
||||||
|
# skip_time_mix = rearrange(repeat(skip_time_mix, 'b -> (b t) () () ()', t=t), '(b t) 1 ... -> b 1 t ...', t=t)
|
||||||
|
if self.merge_strategy == "fixed":
|
||||||
|
# make shape compatible
|
||||||
|
# alpha = repeat(self.mix_factor, '1 -> b () t () ()', t=t, b=bs)
|
||||||
|
alpha = self.mix_factor.to(image_only_indicator.device)
|
||||||
|
elif self.merge_strategy == "learned":
|
||||||
|
alpha = torch.sigmoid(self.mix_factor.to(image_only_indicator.device))
|
||||||
|
# make shape compatible
|
||||||
|
# alpha = repeat(alpha, '1 -> s () ()', s = t * bs)
|
||||||
|
elif self.merge_strategy == "learned_with_images":
|
||||||
|
assert image_only_indicator is not None, "need image_only_indicator ..."
|
||||||
|
alpha = torch.where(
|
||||||
|
image_only_indicator.bool(),
|
||||||
|
torch.ones(1, 1, device=image_only_indicator.device),
|
||||||
|
rearrange(torch.sigmoid(self.mix_factor.to(image_only_indicator.device)), "... -> ... 1"),
|
||||||
|
)
|
||||||
|
alpha = rearrange(alpha, self.rearrange_pattern)
|
||||||
|
# make shape compatible
|
||||||
|
# alpha = repeat(alpha, '1 -> s () ()', s = t * bs)
|
||||||
|
else:
|
||||||
|
raise NotImplementedError()
|
||||||
|
return alpha
|
||||||
|
|
||||||
|
def forward(
|
||||||
|
self,
|
||||||
|
x_spatial,
|
||||||
|
x_temporal,
|
||||||
|
image_only_indicator=None,
|
||||||
|
) -> torch.Tensor:
|
||||||
|
alpha = self.get_alpha(image_only_indicator)
|
||||||
|
x = (
|
||||||
|
alpha.to(x_spatial.dtype) * x_spatial
|
||||||
|
+ (1.0 - alpha).to(x_spatial.dtype) * x_temporal
|
||||||
|
)
|
||||||
|
return x
|
||||||
|
|
||||||
|
|
||||||
def make_beta_schedule(schedule, n_timestep, linear_start=1e-4, linear_end=2e-2, cosine_s=8e-3):
|
def make_beta_schedule(schedule, n_timestep, linear_start=1e-4, linear_end=2e-2, cosine_s=8e-3):
|
||||||
if schedule == "linear":
|
if schedule == "linear":
|
||||||
@ -206,46 +272,6 @@ def mean_flat(tensor):
|
|||||||
return tensor.mean(dim=list(range(1, len(tensor.shape))))
|
return tensor.mean(dim=list(range(1, len(tensor.shape))))
|
||||||
|
|
||||||
|
|
||||||
def normalization(channels, dtype=None):
|
|
||||||
"""
|
|
||||||
Make a standard normalization layer.
|
|
||||||
:param channels: number of input channels.
|
|
||||||
:return: an nn.Module for normalization.
|
|
||||||
"""
|
|
||||||
return GroupNorm32(32, channels, dtype=dtype)
|
|
||||||
|
|
||||||
|
|
||||||
# PyTorch 1.7 has SiLU, but we support PyTorch 1.5.
|
|
||||||
class SiLU(nn.Module):
|
|
||||||
def forward(self, x):
|
|
||||||
return x * torch.sigmoid(x)
|
|
||||||
|
|
||||||
|
|
||||||
class GroupNorm32(nn.GroupNorm):
|
|
||||||
def forward(self, x):
|
|
||||||
return super().forward(x.float()).type(x.dtype)
|
|
||||||
|
|
||||||
|
|
||||||
def conv_nd(dims, *args, **kwargs):
|
|
||||||
"""
|
|
||||||
Create a 1D, 2D, or 3D convolution module.
|
|
||||||
"""
|
|
||||||
if dims == 1:
|
|
||||||
return nn.Conv1d(*args, **kwargs)
|
|
||||||
elif dims == 2:
|
|
||||||
return comfy.ops.Conv2d(*args, **kwargs)
|
|
||||||
elif dims == 3:
|
|
||||||
return nn.Conv3d(*args, **kwargs)
|
|
||||||
raise ValueError(f"unsupported dimensions: {dims}")
|
|
||||||
|
|
||||||
|
|
||||||
def linear(*args, **kwargs):
|
|
||||||
"""
|
|
||||||
Create a linear module.
|
|
||||||
"""
|
|
||||||
return comfy.ops.Linear(*args, **kwargs)
|
|
||||||
|
|
||||||
|
|
||||||
def avg_pool_nd(dims, *args, **kwargs):
|
def avg_pool_nd(dims, *args, **kwargs):
|
||||||
"""
|
"""
|
||||||
Create a 1D, 2D, or 3D average pooling module.
|
Create a 1D, 2D, or 3D average pooling module.
|
||||||
|
|||||||
@ -15,12 +15,12 @@ class CLIPEmbeddingNoiseAugmentation(ImageConcatWithNoiseAugmentation):
|
|||||||
|
|
||||||
def scale(self, x):
|
def scale(self, x):
|
||||||
# re-normalize to centered mean and unit variance
|
# re-normalize to centered mean and unit variance
|
||||||
x = (x - self.data_mean) * 1. / self.data_std
|
x = (x - self.data_mean.to(x.device)) * 1. / self.data_std.to(x.device)
|
||||||
return x
|
return x
|
||||||
|
|
||||||
def unscale(self, x):
|
def unscale(self, x):
|
||||||
# back to original data stats
|
# back to original data stats
|
||||||
x = (x * self.data_std) + self.data_mean
|
x = (x * self.data_std.to(x.device)) + self.data_mean.to(x.device)
|
||||||
return x
|
return x
|
||||||
|
|
||||||
def forward(self, x, noise_level=None):
|
def forward(self, x, noise_level=None):
|
||||||
|
|||||||
245
comfy/ldm/modules/temporal_ae.py
Normal file
245
comfy/ldm/modules/temporal_ae.py
Normal file
@ -0,0 +1,245 @@
|
|||||||
|
import functools
|
||||||
|
from typing import Callable, Iterable, Union
|
||||||
|
|
||||||
|
import torch
|
||||||
|
from einops import rearrange, repeat
|
||||||
|
|
||||||
|
import comfy.ops
|
||||||
|
ops = comfy.ops.disable_weight_init
|
||||||
|
|
||||||
|
from .diffusionmodules.model import (
|
||||||
|
AttnBlock,
|
||||||
|
Decoder,
|
||||||
|
ResnetBlock,
|
||||||
|
)
|
||||||
|
from .diffusionmodules.openaimodel import ResBlock, timestep_embedding
|
||||||
|
from .attention import BasicTransformerBlock
|
||||||
|
|
||||||
|
def partialclass(cls, *args, **kwargs):
|
||||||
|
class NewCls(cls):
|
||||||
|
__init__ = functools.partialmethod(cls.__init__, *args, **kwargs)
|
||||||
|
|
||||||
|
return NewCls
|
||||||
|
|
||||||
|
|
||||||
|
class VideoResBlock(ResnetBlock):
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
out_channels,
|
||||||
|
*args,
|
||||||
|
dropout=0.0,
|
||||||
|
video_kernel_size=3,
|
||||||
|
alpha=0.0,
|
||||||
|
merge_strategy="learned",
|
||||||
|
**kwargs,
|
||||||
|
):
|
||||||
|
super().__init__(out_channels=out_channels, dropout=dropout, *args, **kwargs)
|
||||||
|
if video_kernel_size is None:
|
||||||
|
video_kernel_size = [3, 1, 1]
|
||||||
|
self.time_stack = ResBlock(
|
||||||
|
channels=out_channels,
|
||||||
|
emb_channels=0,
|
||||||
|
dropout=dropout,
|
||||||
|
dims=3,
|
||||||
|
use_scale_shift_norm=False,
|
||||||
|
use_conv=False,
|
||||||
|
up=False,
|
||||||
|
down=False,
|
||||||
|
kernel_size=video_kernel_size,
|
||||||
|
use_checkpoint=False,
|
||||||
|
skip_t_emb=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
self.merge_strategy = merge_strategy
|
||||||
|
if self.merge_strategy == "fixed":
|
||||||
|
self.register_buffer("mix_factor", torch.Tensor([alpha]))
|
||||||
|
elif self.merge_strategy == "learned":
|
||||||
|
self.register_parameter(
|
||||||
|
"mix_factor", torch.nn.Parameter(torch.Tensor([alpha]))
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
raise ValueError(f"unknown merge strategy {self.merge_strategy}")
|
||||||
|
|
||||||
|
def get_alpha(self, bs):
|
||||||
|
if self.merge_strategy == "fixed":
|
||||||
|
return self.mix_factor
|
||||||
|
elif self.merge_strategy == "learned":
|
||||||
|
return torch.sigmoid(self.mix_factor)
|
||||||
|
else:
|
||||||
|
raise NotImplementedError()
|
||||||
|
|
||||||
|
def forward(self, x, temb, skip_video=False, timesteps=None):
|
||||||
|
b, c, h, w = x.shape
|
||||||
|
if timesteps is None:
|
||||||
|
timesteps = b
|
||||||
|
|
||||||
|
x = super().forward(x, temb)
|
||||||
|
|
||||||
|
if not skip_video:
|
||||||
|
x_mix = rearrange(x, "(b t) c h w -> b c t h w", t=timesteps)
|
||||||
|
|
||||||
|
x = rearrange(x, "(b t) c h w -> b c t h w", t=timesteps)
|
||||||
|
|
||||||
|
x = self.time_stack(x, temb)
|
||||||
|
|
||||||
|
alpha = self.get_alpha(bs=b // timesteps).to(x.device)
|
||||||
|
x = alpha * x + (1.0 - alpha) * x_mix
|
||||||
|
|
||||||
|
x = rearrange(x, "b c t h w -> (b t) c h w")
|
||||||
|
return x
|
||||||
|
|
||||||
|
|
||||||
|
class AE3DConv(ops.Conv2d):
|
||||||
|
def __init__(self, in_channels, out_channels, video_kernel_size=3, *args, **kwargs):
|
||||||
|
super().__init__(in_channels, out_channels, *args, **kwargs)
|
||||||
|
if isinstance(video_kernel_size, Iterable):
|
||||||
|
padding = [int(k // 2) for k in video_kernel_size]
|
||||||
|
else:
|
||||||
|
padding = int(video_kernel_size // 2)
|
||||||
|
|
||||||
|
self.time_mix_conv = ops.Conv3d(
|
||||||
|
in_channels=out_channels,
|
||||||
|
out_channels=out_channels,
|
||||||
|
kernel_size=video_kernel_size,
|
||||||
|
padding=padding,
|
||||||
|
)
|
||||||
|
|
||||||
|
def forward(self, input, timesteps=None, skip_video=False):
|
||||||
|
if timesteps is None:
|
||||||
|
timesteps = input.shape[0]
|
||||||
|
x = super().forward(input)
|
||||||
|
if skip_video:
|
||||||
|
return x
|
||||||
|
x = rearrange(x, "(b t) c h w -> b c t h w", t=timesteps)
|
||||||
|
x = self.time_mix_conv(x)
|
||||||
|
return rearrange(x, "b c t h w -> (b t) c h w")
|
||||||
|
|
||||||
|
|
||||||
|
class AttnVideoBlock(AttnBlock):
|
||||||
|
def __init__(
|
||||||
|
self, in_channels: int, alpha: float = 0, merge_strategy: str = "learned"
|
||||||
|
):
|
||||||
|
super().__init__(in_channels)
|
||||||
|
# no context, single headed, as in base class
|
||||||
|
self.time_mix_block = BasicTransformerBlock(
|
||||||
|
dim=in_channels,
|
||||||
|
n_heads=1,
|
||||||
|
d_head=in_channels,
|
||||||
|
checkpoint=False,
|
||||||
|
ff_in=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
time_embed_dim = self.in_channels * 4
|
||||||
|
self.video_time_embed = torch.nn.Sequential(
|
||||||
|
ops.Linear(self.in_channels, time_embed_dim),
|
||||||
|
torch.nn.SiLU(),
|
||||||
|
ops.Linear(time_embed_dim, self.in_channels),
|
||||||
|
)
|
||||||
|
|
||||||
|
self.merge_strategy = merge_strategy
|
||||||
|
if self.merge_strategy == "fixed":
|
||||||
|
self.register_buffer("mix_factor", torch.Tensor([alpha]))
|
||||||
|
elif self.merge_strategy == "learned":
|
||||||
|
self.register_parameter(
|
||||||
|
"mix_factor", torch.nn.Parameter(torch.Tensor([alpha]))
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
raise ValueError(f"unknown merge strategy {self.merge_strategy}")
|
||||||
|
|
||||||
|
def forward(self, x, timesteps=None, skip_time_block=False):
|
||||||
|
if skip_time_block:
|
||||||
|
return super().forward(x)
|
||||||
|
|
||||||
|
if timesteps is None:
|
||||||
|
timesteps = x.shape[0]
|
||||||
|
|
||||||
|
x_in = x
|
||||||
|
x = self.attention(x)
|
||||||
|
h, w = x.shape[2:]
|
||||||
|
x = rearrange(x, "b c h w -> b (h w) c")
|
||||||
|
|
||||||
|
x_mix = x
|
||||||
|
num_frames = torch.arange(timesteps, device=x.device)
|
||||||
|
num_frames = repeat(num_frames, "t -> b t", b=x.shape[0] // timesteps)
|
||||||
|
num_frames = rearrange(num_frames, "b t -> (b t)")
|
||||||
|
t_emb = timestep_embedding(num_frames, self.in_channels, repeat_only=False)
|
||||||
|
emb = self.video_time_embed(t_emb) # b, n_channels
|
||||||
|
emb = emb[:, None, :]
|
||||||
|
x_mix = x_mix + emb
|
||||||
|
|
||||||
|
alpha = self.get_alpha().to(x.device)
|
||||||
|
x_mix = self.time_mix_block(x_mix, timesteps=timesteps)
|
||||||
|
x = alpha * x + (1.0 - alpha) * x_mix # alpha merge
|
||||||
|
|
||||||
|
x = rearrange(x, "b (h w) c -> b c h w", h=h, w=w)
|
||||||
|
x = self.proj_out(x)
|
||||||
|
|
||||||
|
return x_in + x
|
||||||
|
|
||||||
|
def get_alpha(
|
||||||
|
self,
|
||||||
|
):
|
||||||
|
if self.merge_strategy == "fixed":
|
||||||
|
return self.mix_factor
|
||||||
|
elif self.merge_strategy == "learned":
|
||||||
|
return torch.sigmoid(self.mix_factor)
|
||||||
|
else:
|
||||||
|
raise NotImplementedError(f"unknown merge strategy {self.merge_strategy}")
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
def make_time_attn(
|
||||||
|
in_channels,
|
||||||
|
attn_type="vanilla",
|
||||||
|
attn_kwargs=None,
|
||||||
|
alpha: float = 0,
|
||||||
|
merge_strategy: str = "learned",
|
||||||
|
):
|
||||||
|
return partialclass(
|
||||||
|
AttnVideoBlock, in_channels, alpha=alpha, merge_strategy=merge_strategy
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class Conv2DWrapper(torch.nn.Conv2d):
|
||||||
|
def forward(self, input: torch.Tensor, **kwargs) -> torch.Tensor:
|
||||||
|
return super().forward(input)
|
||||||
|
|
||||||
|
|
||||||
|
class VideoDecoder(Decoder):
|
||||||
|
available_time_modes = ["all", "conv-only", "attn-only"]
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
*args,
|
||||||
|
video_kernel_size: Union[int, list] = 3,
|
||||||
|
alpha: float = 0.0,
|
||||||
|
merge_strategy: str = "learned",
|
||||||
|
time_mode: str = "conv-only",
|
||||||
|
**kwargs,
|
||||||
|
):
|
||||||
|
self.video_kernel_size = video_kernel_size
|
||||||
|
self.alpha = alpha
|
||||||
|
self.merge_strategy = merge_strategy
|
||||||
|
self.time_mode = time_mode
|
||||||
|
assert (
|
||||||
|
self.time_mode in self.available_time_modes
|
||||||
|
), f"time_mode parameter has to be in {self.available_time_modes}"
|
||||||
|
|
||||||
|
if self.time_mode != "attn-only":
|
||||||
|
kwargs["conv_out_op"] = partialclass(AE3DConv, video_kernel_size=self.video_kernel_size)
|
||||||
|
if self.time_mode not in ["conv-only", "only-last-conv"]:
|
||||||
|
kwargs["attn_op"] = partialclass(make_time_attn, alpha=self.alpha, merge_strategy=self.merge_strategy)
|
||||||
|
if self.time_mode not in ["attn-only", "only-last-conv"]:
|
||||||
|
kwargs["resnet_op"] = partialclass(VideoResBlock, video_kernel_size=self.video_kernel_size, alpha=self.alpha, merge_strategy=self.merge_strategy)
|
||||||
|
|
||||||
|
super().__init__(*args, **kwargs)
|
||||||
|
|
||||||
|
def get_last_layer(self, skip_time_mix=False, **kwargs):
|
||||||
|
if self.time_mode == "attn-only":
|
||||||
|
raise NotImplementedError("TODO")
|
||||||
|
else:
|
||||||
|
return (
|
||||||
|
self.conv_out.time_mix_conv.weight
|
||||||
|
if not skip_time_mix
|
||||||
|
else self.conv_out.weight
|
||||||
|
)
|
||||||
@ -43,7 +43,7 @@ def load_lora(lora, to_load):
|
|||||||
if mid_name is not None and mid_name in lora.keys():
|
if mid_name is not None and mid_name in lora.keys():
|
||||||
mid = lora[mid_name]
|
mid = lora[mid_name]
|
||||||
loaded_keys.add(mid_name)
|
loaded_keys.add(mid_name)
|
||||||
patch_dict[to_load[x]] = (lora[A_name], lora[B_name], alpha, mid)
|
patch_dict[to_load[x]] = ("lora", (lora[A_name], lora[B_name], alpha, mid))
|
||||||
loaded_keys.add(A_name)
|
loaded_keys.add(A_name)
|
||||||
loaded_keys.add(B_name)
|
loaded_keys.add(B_name)
|
||||||
|
|
||||||
@ -64,7 +64,7 @@ def load_lora(lora, to_load):
|
|||||||
loaded_keys.add(hada_t1_name)
|
loaded_keys.add(hada_t1_name)
|
||||||
loaded_keys.add(hada_t2_name)
|
loaded_keys.add(hada_t2_name)
|
||||||
|
|
||||||
patch_dict[to_load[x]] = (lora[hada_w1_a_name], lora[hada_w1_b_name], alpha, lora[hada_w2_a_name], lora[hada_w2_b_name], hada_t1, hada_t2)
|
patch_dict[to_load[x]] = ("loha", (lora[hada_w1_a_name], lora[hada_w1_b_name], alpha, lora[hada_w2_a_name], lora[hada_w2_b_name], hada_t1, hada_t2))
|
||||||
loaded_keys.add(hada_w1_a_name)
|
loaded_keys.add(hada_w1_a_name)
|
||||||
loaded_keys.add(hada_w1_b_name)
|
loaded_keys.add(hada_w1_b_name)
|
||||||
loaded_keys.add(hada_w2_a_name)
|
loaded_keys.add(hada_w2_a_name)
|
||||||
@ -116,8 +116,19 @@ def load_lora(lora, to_load):
|
|||||||
loaded_keys.add(lokr_t2_name)
|
loaded_keys.add(lokr_t2_name)
|
||||||
|
|
||||||
if (lokr_w1 is not None) or (lokr_w2 is not None) or (lokr_w1_a is not None) or (lokr_w2_a is not None):
|
if (lokr_w1 is not None) or (lokr_w2 is not None) or (lokr_w1_a is not None) or (lokr_w2_a is not None):
|
||||||
patch_dict[to_load[x]] = (lokr_w1, lokr_w2, alpha, lokr_w1_a, lokr_w1_b, lokr_w2_a, lokr_w2_b, lokr_t2)
|
patch_dict[to_load[x]] = ("lokr", (lokr_w1, lokr_w2, alpha, lokr_w1_a, lokr_w1_b, lokr_w2_a, lokr_w2_b, lokr_t2))
|
||||||
|
|
||||||
|
#glora
|
||||||
|
a1_name = "{}.a1.weight".format(x)
|
||||||
|
a2_name = "{}.a2.weight".format(x)
|
||||||
|
b1_name = "{}.b1.weight".format(x)
|
||||||
|
b2_name = "{}.b2.weight".format(x)
|
||||||
|
if a1_name in lora:
|
||||||
|
patch_dict[to_load[x]] = ("glora", (lora[a1_name], lora[a2_name], lora[b1_name], lora[b2_name], alpha))
|
||||||
|
loaded_keys.add(a1_name)
|
||||||
|
loaded_keys.add(a2_name)
|
||||||
|
loaded_keys.add(b1_name)
|
||||||
|
loaded_keys.add(b2_name)
|
||||||
|
|
||||||
w_norm_name = "{}.w_norm".format(x)
|
w_norm_name = "{}.w_norm".format(x)
|
||||||
b_norm_name = "{}.b_norm".format(x)
|
b_norm_name = "{}.b_norm".format(x)
|
||||||
@ -126,21 +137,21 @@ def load_lora(lora, to_load):
|
|||||||
|
|
||||||
if w_norm is not None:
|
if w_norm is not None:
|
||||||
loaded_keys.add(w_norm_name)
|
loaded_keys.add(w_norm_name)
|
||||||
patch_dict[to_load[x]] = (w_norm,)
|
patch_dict[to_load[x]] = ("diff", (w_norm,))
|
||||||
if b_norm is not None:
|
if b_norm is not None:
|
||||||
loaded_keys.add(b_norm_name)
|
loaded_keys.add(b_norm_name)
|
||||||
patch_dict["{}.bias".format(to_load[x][:-len(".weight")])] = (b_norm,)
|
patch_dict["{}.bias".format(to_load[x][:-len(".weight")])] = ("diff", (b_norm,))
|
||||||
|
|
||||||
diff_name = "{}.diff".format(x)
|
diff_name = "{}.diff".format(x)
|
||||||
diff_weight = lora.get(diff_name, None)
|
diff_weight = lora.get(diff_name, None)
|
||||||
if diff_weight is not None:
|
if diff_weight is not None:
|
||||||
patch_dict[to_load[x]] = (diff_weight,)
|
patch_dict[to_load[x]] = ("diff", (diff_weight,))
|
||||||
loaded_keys.add(diff_name)
|
loaded_keys.add(diff_name)
|
||||||
|
|
||||||
diff_bias_name = "{}.diff_b".format(x)
|
diff_bias_name = "{}.diff_b".format(x)
|
||||||
diff_bias = lora.get(diff_bias_name, None)
|
diff_bias = lora.get(diff_bias_name, None)
|
||||||
if diff_bias is not None:
|
if diff_bias is not None:
|
||||||
patch_dict["{}.bias".format(to_load[x][:-len(".weight")])] = (diff_bias,)
|
patch_dict["{}.bias".format(to_load[x][:-len(".weight")])] = ("diff", (diff_bias,))
|
||||||
loaded_keys.add(diff_bias_name)
|
loaded_keys.add(diff_bias_name)
|
||||||
|
|
||||||
for x in lora.keys():
|
for x in lora.keys():
|
||||||
|
|||||||
@ -4,23 +4,30 @@ from comfy.ldm.modules.encoders.noise_aug_modules import CLIPEmbeddingNoiseAugme
|
|||||||
from comfy.ldm.modules.diffusionmodules.openaimodel import Timestep
|
from comfy.ldm.modules.diffusionmodules.openaimodel import Timestep
|
||||||
import comfy.model_management
|
import comfy.model_management
|
||||||
import comfy.conds
|
import comfy.conds
|
||||||
|
import comfy.ops
|
||||||
from enum import Enum
|
from enum import Enum
|
||||||
|
import contextlib
|
||||||
from . import utils
|
from . import utils
|
||||||
|
|
||||||
class ModelType(Enum):
|
class ModelType(Enum):
|
||||||
EPS = 1
|
EPS = 1
|
||||||
V_PREDICTION = 2
|
V_PREDICTION = 2
|
||||||
|
V_PREDICTION_EDM = 3
|
||||||
|
|
||||||
|
|
||||||
from comfy.model_sampling import EPS, V_PREDICTION, ModelSamplingDiscrete
|
from comfy.model_sampling import EPS, V_PREDICTION, ModelSamplingDiscrete, ModelSamplingContinuousEDM
|
||||||
|
|
||||||
|
|
||||||
def model_sampling(model_config, model_type):
|
def model_sampling(model_config, model_type):
|
||||||
|
s = ModelSamplingDiscrete
|
||||||
|
|
||||||
if model_type == ModelType.EPS:
|
if model_type == ModelType.EPS:
|
||||||
c = EPS
|
c = EPS
|
||||||
elif model_type == ModelType.V_PREDICTION:
|
elif model_type == ModelType.V_PREDICTION:
|
||||||
c = V_PREDICTION
|
c = V_PREDICTION
|
||||||
|
elif model_type == ModelType.V_PREDICTION_EDM:
|
||||||
s = ModelSamplingDiscrete
|
c = V_PREDICTION
|
||||||
|
s = ModelSamplingContinuousEDM
|
||||||
|
|
||||||
class ModelSampling(s, c):
|
class ModelSampling(s, c):
|
||||||
pass
|
pass
|
||||||
@ -35,9 +42,14 @@ class BaseModel(torch.nn.Module):
|
|||||||
unet_config = model_config.unet_config
|
unet_config = model_config.unet_config
|
||||||
self.latent_format = model_config.latent_format
|
self.latent_format = model_config.latent_format
|
||||||
self.model_config = model_config
|
self.model_config = model_config
|
||||||
|
self.manual_cast_dtype = model_config.manual_cast_dtype
|
||||||
|
|
||||||
if not unet_config.get("disable_unet_model_creation", False):
|
if not unet_config.get("disable_unet_model_creation", False):
|
||||||
self.diffusion_model = UNetModel(**unet_config, device=device)
|
if self.manual_cast_dtype is not None:
|
||||||
|
operations = comfy.ops.manual_cast
|
||||||
|
else:
|
||||||
|
operations = comfy.ops.disable_weight_init
|
||||||
|
self.diffusion_model = UNetModel(**unet_config, device=device, operations=operations)
|
||||||
self.model_type = model_type
|
self.model_type = model_type
|
||||||
self.model_sampling = model_sampling(model_config, model_type)
|
self.model_sampling = model_sampling(model_config, model_type)
|
||||||
|
|
||||||
@ -56,6 +68,10 @@ class BaseModel(torch.nn.Module):
|
|||||||
|
|
||||||
context = c_crossattn
|
context = c_crossattn
|
||||||
dtype = self.get_dtype()
|
dtype = self.get_dtype()
|
||||||
|
|
||||||
|
if self.manual_cast_dtype is not None:
|
||||||
|
dtype = self.manual_cast_dtype
|
||||||
|
|
||||||
xc = xc.to(dtype)
|
xc = xc.to(dtype)
|
||||||
t = self.model_sampling.timestep(t).float()
|
t = self.model_sampling.timestep(t).float()
|
||||||
context = context.to(dtype)
|
context = context.to(dtype)
|
||||||
@ -65,6 +81,7 @@ class BaseModel(torch.nn.Module):
|
|||||||
if hasattr(extra, "to"):
|
if hasattr(extra, "to"):
|
||||||
extra = extra.to(dtype)
|
extra = extra.to(dtype)
|
||||||
extra_conds[o] = extra
|
extra_conds[o] = extra
|
||||||
|
|
||||||
model_output = self.diffusion_model(xc, t, context=context, control=control, transformer_options=transformer_options, **extra_conds).float()
|
model_output = self.diffusion_model(xc, t, context=context, control=control, transformer_options=transformer_options, **extra_conds).float()
|
||||||
return self.model_sampling.calculate_denoised(sigma, model_output, x)
|
return self.model_sampling.calculate_denoised(sigma, model_output, x)
|
||||||
|
|
||||||
@ -109,9 +126,15 @@ class BaseModel(torch.nn.Module):
|
|||||||
cond_concat.append(blank_inpaint_image_like(noise))
|
cond_concat.append(blank_inpaint_image_like(noise))
|
||||||
data = torch.cat(cond_concat, dim=1)
|
data = torch.cat(cond_concat, dim=1)
|
||||||
out['c_concat'] = comfy.conds.CONDNoiseShape(data)
|
out['c_concat'] = comfy.conds.CONDNoiseShape(data)
|
||||||
|
|
||||||
adm = self.encode_adm(**kwargs)
|
adm = self.encode_adm(**kwargs)
|
||||||
if adm is not None:
|
if adm is not None:
|
||||||
out['y'] = comfy.conds.CONDRegular(adm)
|
out['y'] = comfy.conds.CONDRegular(adm)
|
||||||
|
|
||||||
|
cross_attn = kwargs.get("cross_attn", None)
|
||||||
|
if cross_attn is not None:
|
||||||
|
out['c_crossattn'] = comfy.conds.CONDCrossAttn(cross_attn)
|
||||||
|
|
||||||
return out
|
return out
|
||||||
|
|
||||||
def load_model_weights(self, sd, unet_prefix=""):
|
def load_model_weights(self, sd, unet_prefix=""):
|
||||||
@ -121,6 +144,7 @@ class BaseModel(torch.nn.Module):
|
|||||||
if k.startswith(unet_prefix):
|
if k.startswith(unet_prefix):
|
||||||
to_load[k[len(unet_prefix):]] = sd.pop(k)
|
to_load[k[len(unet_prefix):]] = sd.pop(k)
|
||||||
|
|
||||||
|
to_load = self.model_config.process_unet_state_dict(to_load)
|
||||||
m, u = self.diffusion_model.load_state_dict(to_load, strict=False)
|
m, u = self.diffusion_model.load_state_dict(to_load, strict=False)
|
||||||
if len(m) > 0:
|
if len(m) > 0:
|
||||||
print("unet missing:", m)
|
print("unet missing:", m)
|
||||||
@ -138,11 +162,7 @@ class BaseModel(torch.nn.Module):
|
|||||||
|
|
||||||
def state_dict_for_saving(self, clip_state_dict, vae_state_dict):
|
def state_dict_for_saving(self, clip_state_dict, vae_state_dict):
|
||||||
clip_state_dict = self.model_config.process_clip_state_dict_for_saving(clip_state_dict)
|
clip_state_dict = self.model_config.process_clip_state_dict_for_saving(clip_state_dict)
|
||||||
unet_sd = self.diffusion_model.state_dict()
|
unet_state_dict = self.diffusion_model.state_dict()
|
||||||
unet_state_dict = {}
|
|
||||||
for k in unet_sd:
|
|
||||||
unet_state_dict[k] = comfy.model_management.resolve_lowvram_weight(unet_sd[k], self.diffusion_model, k)
|
|
||||||
|
|
||||||
unet_state_dict = self.model_config.process_unet_state_dict_for_saving(unet_state_dict)
|
unet_state_dict = self.model_config.process_unet_state_dict_for_saving(unet_state_dict)
|
||||||
vae_state_dict = self.model_config.process_vae_state_dict_for_saving(vae_state_dict)
|
vae_state_dict = self.model_config.process_vae_state_dict_for_saving(vae_state_dict)
|
||||||
if self.get_dtype() == torch.float16:
|
if self.get_dtype() == torch.float16:
|
||||||
@ -157,6 +177,20 @@ class BaseModel(torch.nn.Module):
|
|||||||
def set_inpaint(self):
|
def set_inpaint(self):
|
||||||
self.inpaint_model = True
|
self.inpaint_model = True
|
||||||
|
|
||||||
|
def memory_required(self, input_shape):
|
||||||
|
if comfy.model_management.xformers_enabled() or comfy.model_management.pytorch_attention_flash_attention():
|
||||||
|
dtype = self.get_dtype()
|
||||||
|
if self.manual_cast_dtype is not None:
|
||||||
|
dtype = self.manual_cast_dtype
|
||||||
|
#TODO: this needs to be tweaked
|
||||||
|
area = input_shape[0] * input_shape[2] * input_shape[3]
|
||||||
|
return (area * comfy.model_management.dtype_size(dtype) / 50) * (1024 * 1024)
|
||||||
|
else:
|
||||||
|
#TODO: this formula might be too aggressive since I tweaked the sub-quad and split algorithms to use less memory.
|
||||||
|
area = input_shape[0] * input_shape[2] * input_shape[3]
|
||||||
|
return (((area * 0.6) / 0.9) + 1024) * (1024 * 1024)
|
||||||
|
|
||||||
|
|
||||||
def unclip_adm(unclip_conditioning, device, noise_augmentor, noise_augment_merge=0.0):
|
def unclip_adm(unclip_conditioning, device, noise_augmentor, noise_augment_merge=0.0):
|
||||||
adm_inputs = []
|
adm_inputs = []
|
||||||
weights = []
|
weights = []
|
||||||
@ -251,3 +285,82 @@ class SDXL(BaseModel):
|
|||||||
out.append(self.embedder(torch.Tensor([target_width])))
|
out.append(self.embedder(torch.Tensor([target_width])))
|
||||||
flat = torch.flatten(torch.cat(out)).unsqueeze(dim=0).repeat(clip_pooled.shape[0], 1)
|
flat = torch.flatten(torch.cat(out)).unsqueeze(dim=0).repeat(clip_pooled.shape[0], 1)
|
||||||
return torch.cat((clip_pooled.to(flat.device), flat), dim=1)
|
return torch.cat((clip_pooled.to(flat.device), flat), dim=1)
|
||||||
|
|
||||||
|
class SVD_img2vid(BaseModel):
|
||||||
|
def __init__(self, model_config, model_type=ModelType.V_PREDICTION_EDM, device=None):
|
||||||
|
super().__init__(model_config, model_type, device=device)
|
||||||
|
self.embedder = Timestep(256)
|
||||||
|
|
||||||
|
def encode_adm(self, **kwargs):
|
||||||
|
fps_id = kwargs.get("fps", 6) - 1
|
||||||
|
motion_bucket_id = kwargs.get("motion_bucket_id", 127)
|
||||||
|
augmentation = kwargs.get("augmentation_level", 0)
|
||||||
|
|
||||||
|
out = []
|
||||||
|
out.append(self.embedder(torch.Tensor([fps_id])))
|
||||||
|
out.append(self.embedder(torch.Tensor([motion_bucket_id])))
|
||||||
|
out.append(self.embedder(torch.Tensor([augmentation])))
|
||||||
|
|
||||||
|
flat = torch.flatten(torch.cat(out)).unsqueeze(dim=0)
|
||||||
|
return flat
|
||||||
|
|
||||||
|
def extra_conds(self, **kwargs):
|
||||||
|
out = {}
|
||||||
|
adm = self.encode_adm(**kwargs)
|
||||||
|
if adm is not None:
|
||||||
|
out['y'] = comfy.conds.CONDRegular(adm)
|
||||||
|
|
||||||
|
latent_image = kwargs.get("concat_latent_image", None)
|
||||||
|
noise = kwargs.get("noise", None)
|
||||||
|
device = kwargs["device"]
|
||||||
|
|
||||||
|
if latent_image is None:
|
||||||
|
latent_image = torch.zeros_like(noise)
|
||||||
|
|
||||||
|
if latent_image.shape[1:] != noise.shape[1:]:
|
||||||
|
latent_image = utils.common_upscale(latent_image, noise.shape[-1], noise.shape[-2], "bilinear", "center")
|
||||||
|
|
||||||
|
latent_image = utils.resize_to_batch_size(latent_image, noise.shape[0])
|
||||||
|
|
||||||
|
out['c_concat'] = comfy.conds.CONDNoiseShape(latent_image)
|
||||||
|
|
||||||
|
cross_attn = kwargs.get("cross_attn", None)
|
||||||
|
if cross_attn is not None:
|
||||||
|
out['c_crossattn'] = comfy.conds.CONDCrossAttn(cross_attn)
|
||||||
|
|
||||||
|
if "time_conditioning" in kwargs:
|
||||||
|
out["time_context"] = comfy.conds.CONDCrossAttn(kwargs["time_conditioning"])
|
||||||
|
|
||||||
|
out['image_only_indicator'] = comfy.conds.CONDConstant(torch.zeros((1,), device=device))
|
||||||
|
out['num_video_frames'] = comfy.conds.CONDConstant(noise.shape[0])
|
||||||
|
return out
|
||||||
|
|
||||||
|
class Stable_Zero123(BaseModel):
|
||||||
|
def __init__(self, model_config, model_type=ModelType.EPS, device=None, cc_projection_weight=None, cc_projection_bias=None):
|
||||||
|
super().__init__(model_config, model_type, device=device)
|
||||||
|
self.cc_projection = comfy.ops.manual_cast.Linear(cc_projection_weight.shape[1], cc_projection_weight.shape[0], dtype=self.get_dtype(), device=device)
|
||||||
|
self.cc_projection.weight.copy_(cc_projection_weight)
|
||||||
|
self.cc_projection.bias.copy_(cc_projection_bias)
|
||||||
|
|
||||||
|
def extra_conds(self, **kwargs):
|
||||||
|
out = {}
|
||||||
|
|
||||||
|
latent_image = kwargs.get("concat_latent_image", None)
|
||||||
|
noise = kwargs.get("noise", None)
|
||||||
|
|
||||||
|
if latent_image is None:
|
||||||
|
latent_image = torch.zeros_like(noise)
|
||||||
|
|
||||||
|
if latent_image.shape[1:] != noise.shape[1:]:
|
||||||
|
latent_image = utils.common_upscale(latent_image, noise.shape[-1], noise.shape[-2], "bilinear", "center")
|
||||||
|
|
||||||
|
latent_image = utils.resize_to_batch_size(latent_image, noise.shape[0])
|
||||||
|
|
||||||
|
out['c_concat'] = comfy.conds.CONDNoiseShape(latent_image)
|
||||||
|
|
||||||
|
cross_attn = kwargs.get("cross_attn", None)
|
||||||
|
if cross_attn is not None:
|
||||||
|
if cross_attn.shape[-1] != 768:
|
||||||
|
cross_attn = self.cc_projection(cross_attn)
|
||||||
|
out['c_crossattn'] = comfy.conds.CONDCrossAttn(cross_attn)
|
||||||
|
return out
|
||||||
|
|||||||
@ -24,7 +24,8 @@ def calculate_transformer_depth(prefix, state_dict_keys, state_dict):
|
|||||||
last_transformer_depth = count_blocks(state_dict_keys, transformer_prefix + '{}')
|
last_transformer_depth = count_blocks(state_dict_keys, transformer_prefix + '{}')
|
||||||
context_dim = state_dict['{}0.attn2.to_k.weight'.format(transformer_prefix)].shape[1]
|
context_dim = state_dict['{}0.attn2.to_k.weight'.format(transformer_prefix)].shape[1]
|
||||||
use_linear_in_transformer = len(state_dict['{}1.proj_in.weight'.format(prefix)].shape) == 2
|
use_linear_in_transformer = len(state_dict['{}1.proj_in.weight'.format(prefix)].shape) == 2
|
||||||
return last_transformer_depth, context_dim, use_linear_in_transformer
|
time_stack = '{}1.time_stack.0.attn1.to_q.weight'.format(prefix) in state_dict or '{}1.time_mix_blocks.0.attn1.to_q.weight'.format(prefix) in state_dict
|
||||||
|
return last_transformer_depth, context_dim, use_linear_in_transformer, time_stack
|
||||||
return None
|
return None
|
||||||
|
|
||||||
def detect_unet_config(state_dict, key_prefix, dtype):
|
def detect_unet_config(state_dict, key_prefix, dtype):
|
||||||
@ -57,6 +58,7 @@ def detect_unet_config(state_dict, key_prefix, dtype):
|
|||||||
context_dim = None
|
context_dim = None
|
||||||
use_linear_in_transformer = False
|
use_linear_in_transformer = False
|
||||||
|
|
||||||
|
video_model = False
|
||||||
|
|
||||||
current_res = 1
|
current_res = 1
|
||||||
count = 0
|
count = 0
|
||||||
@ -99,6 +101,7 @@ def detect_unet_config(state_dict, key_prefix, dtype):
|
|||||||
if context_dim is None:
|
if context_dim is None:
|
||||||
context_dim = out[1]
|
context_dim = out[1]
|
||||||
use_linear_in_transformer = out[2]
|
use_linear_in_transformer = out[2]
|
||||||
|
video_model = out[3]
|
||||||
else:
|
else:
|
||||||
transformer_depth.append(0)
|
transformer_depth.append(0)
|
||||||
|
|
||||||
@ -127,6 +130,19 @@ def detect_unet_config(state_dict, key_prefix, dtype):
|
|||||||
unet_config["transformer_depth_middle"] = transformer_depth_middle
|
unet_config["transformer_depth_middle"] = transformer_depth_middle
|
||||||
unet_config['use_linear_in_transformer'] = use_linear_in_transformer
|
unet_config['use_linear_in_transformer'] = use_linear_in_transformer
|
||||||
unet_config["context_dim"] = context_dim
|
unet_config["context_dim"] = context_dim
|
||||||
|
|
||||||
|
if video_model:
|
||||||
|
unet_config["extra_ff_mix_layer"] = True
|
||||||
|
unet_config["use_spatial_context"] = True
|
||||||
|
unet_config["merge_strategy"] = "learned_with_images"
|
||||||
|
unet_config["merge_factor"] = 0.0
|
||||||
|
unet_config["video_kernel_size"] = [3, 1, 1]
|
||||||
|
unet_config["use_temporal_resblock"] = True
|
||||||
|
unet_config["use_temporal_attention"] = True
|
||||||
|
else:
|
||||||
|
unet_config["use_temporal_resblock"] = False
|
||||||
|
unet_config["use_temporal_attention"] = False
|
||||||
|
|
||||||
return unet_config
|
return unet_config
|
||||||
|
|
||||||
def model_config_from_unet_config(unet_config):
|
def model_config_from_unet_config(unet_config):
|
||||||
@ -186,17 +202,24 @@ def convert_config(unet_config):
|
|||||||
|
|
||||||
def unet_config_from_diffusers_unet(state_dict, dtype):
|
def unet_config_from_diffusers_unet(state_dict, dtype):
|
||||||
match = {}
|
match = {}
|
||||||
attention_resolutions = []
|
transformer_depth = []
|
||||||
|
|
||||||
attn_res = 1
|
attn_res = 1
|
||||||
for i in range(5):
|
down_blocks = count_blocks(state_dict, "down_blocks.{}")
|
||||||
k = "down_blocks.{}.attentions.1.transformer_blocks.0.attn2.to_k.weight".format(i)
|
for i in range(down_blocks):
|
||||||
if k in state_dict:
|
attn_blocks = count_blocks(state_dict, "down_blocks.{}.attentions.".format(i) + '{}')
|
||||||
match["context_dim"] = state_dict[k].shape[1]
|
for ab in range(attn_blocks):
|
||||||
attention_resolutions.append(attn_res)
|
transformer_count = count_blocks(state_dict, "down_blocks.{}.attentions.{}.transformer_blocks.".format(i, ab) + '{}')
|
||||||
attn_res *= 2
|
transformer_depth.append(transformer_count)
|
||||||
|
if transformer_count > 0:
|
||||||
|
match["context_dim"] = state_dict["down_blocks.{}.attentions.{}.transformer_blocks.0.attn2.to_k.weight".format(i, ab)].shape[1]
|
||||||
|
|
||||||
match["attention_resolutions"] = attention_resolutions
|
attn_res *= 2
|
||||||
|
if attn_blocks == 0:
|
||||||
|
transformer_depth.append(0)
|
||||||
|
transformer_depth.append(0)
|
||||||
|
|
||||||
|
match["transformer_depth"] = transformer_depth
|
||||||
|
|
||||||
match["model_channels"] = state_dict["conv_in.weight"].shape[0]
|
match["model_channels"] = state_dict["conv_in.weight"].shape[0]
|
||||||
match["in_channels"] = state_dict["conv_in.weight"].shape[1]
|
match["in_channels"] = state_dict["conv_in.weight"].shape[1]
|
||||||
@ -208,50 +231,71 @@ def unet_config_from_diffusers_unet(state_dict, dtype):
|
|||||||
|
|
||||||
SDXL = {'use_checkpoint': False, 'image_size': 32, 'out_channels': 4, 'use_spatial_transformer': True, 'legacy': False,
|
SDXL = {'use_checkpoint': False, 'image_size': 32, 'out_channels': 4, 'use_spatial_transformer': True, 'legacy': False,
|
||||||
'num_classes': 'sequential', 'adm_in_channels': 2816, 'dtype': dtype, 'in_channels': 4, 'model_channels': 320,
|
'num_classes': 'sequential', 'adm_in_channels': 2816, 'dtype': dtype, 'in_channels': 4, 'model_channels': 320,
|
||||||
'num_res_blocks': 2, 'attention_resolutions': [2, 4], 'transformer_depth': [0, 2, 10], 'channel_mult': [1, 2, 4],
|
'num_res_blocks': [2, 2, 2], 'transformer_depth': [0, 0, 2, 2, 10, 10], 'channel_mult': [1, 2, 4], 'transformer_depth_middle': 10,
|
||||||
'transformer_depth_middle': 10, 'use_linear_in_transformer': True, 'context_dim': 2048, "num_head_channels": 64}
|
'use_linear_in_transformer': True, 'context_dim': 2048, 'num_head_channels': 64, 'transformer_depth_output': [0, 0, 0, 2, 2, 2, 10, 10, 10],
|
||||||
|
'use_temporal_attention': False, 'use_temporal_resblock': False}
|
||||||
|
|
||||||
SDXL_refiner = {'use_checkpoint': False, 'image_size': 32, 'out_channels': 4, 'use_spatial_transformer': True, 'legacy': False,
|
SDXL_refiner = {'use_checkpoint': False, 'image_size': 32, 'out_channels': 4, 'use_spatial_transformer': True, 'legacy': False,
|
||||||
'num_classes': 'sequential', 'adm_in_channels': 2560, 'dtype': dtype, 'in_channels': 4, 'model_channels': 384,
|
'num_classes': 'sequential', 'adm_in_channels': 2560, 'dtype': dtype, 'in_channels': 4, 'model_channels': 384,
|
||||||
'num_res_blocks': 2, 'attention_resolutions': [2, 4], 'transformer_depth': [0, 4, 4, 0], 'channel_mult': [1, 2, 4, 4],
|
'num_res_blocks': [2, 2, 2, 2], 'transformer_depth': [0, 0, 4, 4, 4, 4, 0, 0], 'channel_mult': [1, 2, 4, 4], 'transformer_depth_middle': 4,
|
||||||
'transformer_depth_middle': 4, 'use_linear_in_transformer': True, 'context_dim': 1280, "num_head_channels": 64}
|
'use_linear_in_transformer': True, 'context_dim': 1280, 'num_head_channels': 64, 'transformer_depth_output': [0, 0, 0, 4, 4, 4, 4, 4, 4, 0, 0, 0],
|
||||||
|
'use_temporal_attention': False, 'use_temporal_resblock': False}
|
||||||
|
|
||||||
SD21 = {'use_checkpoint': False, 'image_size': 32, 'out_channels': 4, 'use_spatial_transformer': True, 'legacy': False,
|
SD21 = {'use_checkpoint': False, 'image_size': 32, 'out_channels': 4, 'use_spatial_transformer': True, 'legacy': False,
|
||||||
'adm_in_channels': None, 'dtype': dtype, 'in_channels': 4, 'model_channels': 320, 'num_res_blocks': 2,
|
'adm_in_channels': None, 'dtype': dtype, 'in_channels': 4, 'model_channels': 320, 'num_res_blocks': [2, 2, 2, 2],
|
||||||
'attention_resolutions': [1, 2, 4], 'transformer_depth': [1, 1, 1, 0], 'channel_mult': [1, 2, 4, 4],
|
'transformer_depth': [1, 1, 1, 1, 1, 1, 0, 0], 'channel_mult': [1, 2, 4, 4], 'transformer_depth_middle': 1, 'use_linear_in_transformer': True,
|
||||||
'transformer_depth_middle': 1, 'use_linear_in_transformer': True, 'context_dim': 1024, "num_head_channels": 64}
|
'context_dim': 1024, 'num_head_channels': 64, 'transformer_depth_output': [1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0],
|
||||||
|
'use_temporal_attention': False, 'use_temporal_resblock': False}
|
||||||
|
|
||||||
SD21_uncliph = {'use_checkpoint': False, 'image_size': 32, 'out_channels': 4, 'use_spatial_transformer': True, 'legacy': False,
|
SD21_uncliph = {'use_checkpoint': False, 'image_size': 32, 'out_channels': 4, 'use_spatial_transformer': True, 'legacy': False,
|
||||||
'num_classes': 'sequential', 'adm_in_channels': 2048, 'dtype': dtype, 'in_channels': 4, 'model_channels': 320,
|
'num_classes': 'sequential', 'adm_in_channels': 2048, 'dtype': dtype, 'in_channels': 4, 'model_channels': 320,
|
||||||
'num_res_blocks': 2, 'attention_resolutions': [1, 2, 4], 'transformer_depth': [1, 1, 1, 0], 'channel_mult': [1, 2, 4, 4],
|
'num_res_blocks': [2, 2, 2, 2], 'transformer_depth': [1, 1, 1, 1, 1, 1, 0, 0], 'channel_mult': [1, 2, 4, 4], 'transformer_depth_middle': 1,
|
||||||
'transformer_depth_middle': 1, 'use_linear_in_transformer': True, 'context_dim': 1024, "num_head_channels": 64}
|
'use_linear_in_transformer': True, 'context_dim': 1024, 'num_head_channels': 64, 'transformer_depth_output': [1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0],
|
||||||
|
'use_temporal_attention': False, 'use_temporal_resblock': False}
|
||||||
|
|
||||||
SD21_unclipl = {'use_checkpoint': False, 'image_size': 32, 'out_channels': 4, 'use_spatial_transformer': True, 'legacy': False,
|
SD21_unclipl = {'use_checkpoint': False, 'image_size': 32, 'out_channels': 4, 'use_spatial_transformer': True, 'legacy': False,
|
||||||
'num_classes': 'sequential', 'adm_in_channels': 1536, 'dtype': dtype, 'in_channels': 4, 'model_channels': 320,
|
'num_classes': 'sequential', 'adm_in_channels': 1536, 'dtype': dtype, 'in_channels': 4, 'model_channels': 320,
|
||||||
'num_res_blocks': 2, 'attention_resolutions': [1, 2, 4], 'transformer_depth': [1, 1, 1, 0], 'channel_mult': [1, 2, 4, 4],
|
'num_res_blocks': [2, 2, 2, 2], 'transformer_depth': [1, 1, 1, 1, 1, 1, 0, 0], 'channel_mult': [1, 2, 4, 4], 'transformer_depth_middle': 1,
|
||||||
'transformer_depth_middle': 1, 'use_linear_in_transformer': True, 'context_dim': 1024}
|
'use_linear_in_transformer': True, 'context_dim': 1024, 'num_head_channels': 64, 'transformer_depth_output': [1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0],
|
||||||
|
'use_temporal_attention': False, 'use_temporal_resblock': False}
|
||||||
|
|
||||||
SD15 = {'use_checkpoint': False, 'image_size': 32, 'out_channels': 4, 'use_spatial_transformer': True, 'legacy': False,
|
SD15 = {'use_checkpoint': False, 'image_size': 32, 'out_channels': 4, 'use_spatial_transformer': True, 'legacy': False, 'adm_in_channels': None,
|
||||||
'adm_in_channels': None, 'dtype': dtype, 'in_channels': 4, 'model_channels': 320, 'num_res_blocks': 2,
|
'dtype': dtype, 'in_channels': 4, 'model_channels': 320, 'num_res_blocks': [2, 2, 2, 2], 'transformer_depth': [1, 1, 1, 1, 1, 1, 0, 0],
|
||||||
'attention_resolutions': [1, 2, 4], 'transformer_depth': [1, 1, 1, 0], 'channel_mult': [1, 2, 4, 4],
|
'channel_mult': [1, 2, 4, 4], 'transformer_depth_middle': 1, 'use_linear_in_transformer': False, 'context_dim': 768, 'num_heads': 8,
|
||||||
'transformer_depth_middle': 1, 'use_linear_in_transformer': False, 'context_dim': 768, "num_heads": 8}
|
'transformer_depth_output': [1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0],
|
||||||
|
'use_temporal_attention': False, 'use_temporal_resblock': False}
|
||||||
|
|
||||||
SDXL_mid_cnet = {'use_checkpoint': False, 'image_size': 32, 'out_channels': 4, 'use_spatial_transformer': True, 'legacy': False,
|
SDXL_mid_cnet = {'use_checkpoint': False, 'image_size': 32, 'out_channels': 4, 'use_spatial_transformer': True, 'legacy': False,
|
||||||
'num_classes': 'sequential', 'adm_in_channels': 2816, 'dtype': dtype, 'in_channels': 4, 'model_channels': 320,
|
'num_classes': 'sequential', 'adm_in_channels': 2816, 'dtype': dtype, 'in_channels': 4, 'model_channels': 320,
|
||||||
'num_res_blocks': 2, 'attention_resolutions': [4], 'transformer_depth': [0, 0, 1], 'channel_mult': [1, 2, 4],
|
'num_res_blocks': [2, 2, 2], 'transformer_depth': [0, 0, 0, 0, 1, 1], 'channel_mult': [1, 2, 4], 'transformer_depth_middle': 1,
|
||||||
'transformer_depth_middle': 1, 'use_linear_in_transformer': True, 'context_dim': 2048, "num_head_channels": 64}
|
'use_linear_in_transformer': True, 'context_dim': 2048, 'num_head_channels': 64, 'transformer_depth_output': [0, 0, 0, 0, 0, 0, 1, 1, 1],
|
||||||
|
'use_temporal_attention': False, 'use_temporal_resblock': False}
|
||||||
|
|
||||||
SDXL_small_cnet = {'use_checkpoint': False, 'image_size': 32, 'out_channels': 4, 'use_spatial_transformer': True, 'legacy': False,
|
SDXL_small_cnet = {'use_checkpoint': False, 'image_size': 32, 'out_channels': 4, 'use_spatial_transformer': True, 'legacy': False,
|
||||||
'num_classes': 'sequential', 'adm_in_channels': 2816, 'dtype': dtype, 'in_channels': 4, 'model_channels': 320,
|
'num_classes': 'sequential', 'adm_in_channels': 2816, 'dtype': dtype, 'in_channels': 4, 'model_channels': 320,
|
||||||
'num_res_blocks': 2, 'attention_resolutions': [], 'transformer_depth': [0, 0, 0], 'channel_mult': [1, 2, 4],
|
'num_res_blocks': [2, 2, 2], 'transformer_depth': [0, 0, 0, 0, 0, 0], 'channel_mult': [1, 2, 4], 'transformer_depth_middle': 0,
|
||||||
'transformer_depth_middle': 0, 'use_linear_in_transformer': True, "num_head_channels": 64, 'context_dim': 1}
|
'use_linear_in_transformer': True, 'num_head_channels': 64, 'context_dim': 1, 'transformer_depth_output': [0, 0, 0, 0, 0, 0, 0, 0, 0],
|
||||||
|
'use_temporal_attention': False, 'use_temporal_resblock': False}
|
||||||
|
|
||||||
SDXL_diffusers_inpaint = {'use_checkpoint': False, 'image_size': 32, 'out_channels': 4, 'use_spatial_transformer': True, 'legacy': False,
|
SDXL_diffusers_inpaint = {'use_checkpoint': False, 'image_size': 32, 'out_channels': 4, 'use_spatial_transformer': True, 'legacy': False,
|
||||||
'num_classes': 'sequential', 'adm_in_channels': 2816, 'dtype': dtype, 'in_channels': 9, 'model_channels': 320,
|
'num_classes': 'sequential', 'adm_in_channels': 2816, 'dtype': dtype, 'in_channels': 9, 'model_channels': 320,
|
||||||
'num_res_blocks': 2, 'attention_resolutions': [2, 4], 'transformer_depth': [0, 2, 10], 'channel_mult': [1, 2, 4],
|
'num_res_blocks': [2, 2, 2], 'transformer_depth': [0, 0, 2, 2, 10, 10], 'channel_mult': [1, 2, 4], 'transformer_depth_middle': 10,
|
||||||
'transformer_depth_middle': 10, 'use_linear_in_transformer': True, 'context_dim': 2048, "num_head_channels": 64}
|
'use_linear_in_transformer': True, 'context_dim': 2048, 'num_head_channels': 64, 'transformer_depth_output': [0, 0, 0, 2, 2, 2, 10, 10, 10],
|
||||||
|
'use_temporal_attention': False, 'use_temporal_resblock': False}
|
||||||
|
|
||||||
supported_models = [SDXL, SDXL_refiner, SD21, SD15, SD21_uncliph, SD21_unclipl, SDXL_mid_cnet, SDXL_small_cnet, SDXL_diffusers_inpaint]
|
SSD_1B = {'use_checkpoint': False, 'image_size': 32, 'out_channels': 4, 'use_spatial_transformer': True, 'legacy': False,
|
||||||
|
'num_classes': 'sequential', 'adm_in_channels': 2816, 'dtype': dtype, 'in_channels': 4, 'model_channels': 320,
|
||||||
|
'num_res_blocks': [2, 2, 2], 'transformer_depth': [0, 0, 2, 2, 4, 4], 'transformer_depth_output': [0, 0, 0, 1, 1, 2, 10, 4, 4],
|
||||||
|
'channel_mult': [1, 2, 4], 'transformer_depth_middle': -1, 'use_linear_in_transformer': True, 'context_dim': 2048, 'num_head_channels': 64,
|
||||||
|
'use_temporal_attention': False, 'use_temporal_resblock': False}
|
||||||
|
|
||||||
|
Segmind_Vega = {'use_checkpoint': False, 'image_size': 32, 'out_channels': 4, 'use_spatial_transformer': True, 'legacy': False,
|
||||||
|
'num_classes': 'sequential', 'adm_in_channels': 2816, 'dtype': dtype, 'in_channels': 4, 'model_channels': 320,
|
||||||
|
'num_res_blocks': [2, 2, 2], 'transformer_depth': [0, 0, 1, 1, 2, 2], 'transformer_depth_output': [0, 0, 0, 1, 1, 1, 2, 2, 2],
|
||||||
|
'channel_mult': [1, 2, 4], 'transformer_depth_middle': -1, 'use_linear_in_transformer': True, 'context_dim': 2048, 'num_head_channels': 64,
|
||||||
|
'use_temporal_attention': False, 'use_temporal_resblock': False}
|
||||||
|
|
||||||
|
supported_models = [SDXL, SDXL_refiner, SD21, SD15, SD21_uncliph, SD21_unclipl, SDXL_mid_cnet, SDXL_small_cnet, SDXL_diffusers_inpaint, SSD_1B, Segmind_Vega]
|
||||||
|
|
||||||
for unet_config in supported_models:
|
for unet_config in supported_models:
|
||||||
matches = True
|
matches = True
|
||||||
|
|||||||
@ -28,6 +28,10 @@ total_vram = 0
|
|||||||
lowvram_available = True
|
lowvram_available = True
|
||||||
xpu_available = False
|
xpu_available = False
|
||||||
|
|
||||||
|
if args.deterministic:
|
||||||
|
print("Using deterministic algorithms for pytorch")
|
||||||
|
torch.use_deterministic_algorithms(True, warn_only=True)
|
||||||
|
|
||||||
directml_enabled = False
|
directml_enabled = False
|
||||||
if args.directml is not None:
|
if args.directml is not None:
|
||||||
import torch_directml
|
import torch_directml
|
||||||
@ -133,6 +137,10 @@ else:
|
|||||||
import xformers
|
import xformers
|
||||||
import xformers.ops
|
import xformers.ops
|
||||||
XFORMERS_IS_AVAILABLE = True
|
XFORMERS_IS_AVAILABLE = True
|
||||||
|
try:
|
||||||
|
XFORMERS_IS_AVAILABLE = xformers._has_cpp_library
|
||||||
|
except:
|
||||||
|
pass
|
||||||
try:
|
try:
|
||||||
XFORMERS_VERSION = xformers.version.__version__
|
XFORMERS_VERSION = xformers.version.__version__
|
||||||
print("xformers version:", XFORMERS_VERSION)
|
print("xformers version:", XFORMERS_VERSION)
|
||||||
@ -178,6 +186,9 @@ except:
|
|||||||
if is_intel_xpu():
|
if is_intel_xpu():
|
||||||
VAE_DTYPE = torch.bfloat16
|
VAE_DTYPE = torch.bfloat16
|
||||||
|
|
||||||
|
if args.cpu_vae:
|
||||||
|
VAE_DTYPE = torch.float32
|
||||||
|
|
||||||
if args.fp16_vae:
|
if args.fp16_vae:
|
||||||
VAE_DTYPE = torch.float16
|
VAE_DTYPE = torch.float16
|
||||||
elif args.bf16_vae:
|
elif args.bf16_vae:
|
||||||
@ -210,15 +221,8 @@ if args.force_fp16:
|
|||||||
FORCE_FP16 = True
|
FORCE_FP16 = True
|
||||||
|
|
||||||
if lowvram_available:
|
if lowvram_available:
|
||||||
try:
|
if set_vram_to in (VRAMState.LOW_VRAM, VRAMState.NO_VRAM):
|
||||||
import accelerate
|
vram_state = set_vram_to
|
||||||
if set_vram_to in (VRAMState.LOW_VRAM, VRAMState.NO_VRAM):
|
|
||||||
vram_state = set_vram_to
|
|
||||||
except Exception as e:
|
|
||||||
import traceback
|
|
||||||
print(traceback.format_exc())
|
|
||||||
print("ERROR: LOW VRAM MODE NEEDS accelerate.")
|
|
||||||
lowvram_available = False
|
|
||||||
|
|
||||||
|
|
||||||
if cpu_state != CPUState.GPU:
|
if cpu_state != CPUState.GPU:
|
||||||
@ -258,6 +262,14 @@ print("VAE dtype:", VAE_DTYPE)
|
|||||||
|
|
||||||
current_loaded_models = []
|
current_loaded_models = []
|
||||||
|
|
||||||
|
def module_size(module):
|
||||||
|
module_mem = 0
|
||||||
|
sd = module.state_dict()
|
||||||
|
for k in sd:
|
||||||
|
t = sd[k]
|
||||||
|
module_mem += t.nelement() * t.element_size()
|
||||||
|
return module_mem
|
||||||
|
|
||||||
class LoadedModel:
|
class LoadedModel:
|
||||||
def __init__(self, model):
|
def __init__(self, model):
|
||||||
self.model = model
|
self.model = model
|
||||||
@ -290,8 +302,20 @@ class LoadedModel:
|
|||||||
|
|
||||||
if lowvram_model_memory > 0:
|
if lowvram_model_memory > 0:
|
||||||
print("loading in lowvram mode", lowvram_model_memory/(1024 * 1024))
|
print("loading in lowvram mode", lowvram_model_memory/(1024 * 1024))
|
||||||
device_map = accelerate.infer_auto_device_map(self.real_model, max_memory={0: "{}MiB".format(lowvram_model_memory // (1024 * 1024)), "cpu": "16GiB"})
|
mem_counter = 0
|
||||||
accelerate.dispatch_model(self.real_model, device_map=device_map, main_device=self.device)
|
for m in self.real_model.modules():
|
||||||
|
if hasattr(m, "comfy_cast_weights"):
|
||||||
|
m.prev_comfy_cast_weights = m.comfy_cast_weights
|
||||||
|
m.comfy_cast_weights = True
|
||||||
|
module_mem = module_size(m)
|
||||||
|
if mem_counter + module_mem < lowvram_model_memory:
|
||||||
|
m.to(self.device)
|
||||||
|
mem_counter += module_mem
|
||||||
|
elif hasattr(m, "weight"): #only modules with comfy_cast_weights can be set to lowvram mode
|
||||||
|
m.to(self.device)
|
||||||
|
mem_counter += module_size(m)
|
||||||
|
print("lowvram: loaded module regularly", m)
|
||||||
|
|
||||||
self.model_accelerated = True
|
self.model_accelerated = True
|
||||||
|
|
||||||
if is_intel_xpu() and not args.disable_ipex_optimize:
|
if is_intel_xpu() and not args.disable_ipex_optimize:
|
||||||
@ -301,7 +325,11 @@ class LoadedModel:
|
|||||||
|
|
||||||
def model_unload(self):
|
def model_unload(self):
|
||||||
if self.model_accelerated:
|
if self.model_accelerated:
|
||||||
accelerate.hooks.remove_hook_from_submodules(self.real_model)
|
for m in self.real_model.modules():
|
||||||
|
if hasattr(m, "prev_comfy_cast_weights"):
|
||||||
|
m.comfy_cast_weights = m.prev_comfy_cast_weights
|
||||||
|
del m.prev_comfy_cast_weights
|
||||||
|
|
||||||
self.model_accelerated = False
|
self.model_accelerated = False
|
||||||
|
|
||||||
self.model.unpatch_model(self.model.offload_device)
|
self.model.unpatch_model(self.model.offload_device)
|
||||||
@ -394,14 +422,14 @@ def load_models_gpu(models, memory_required=0):
|
|||||||
if lowvram_available and (vram_set_state == VRAMState.LOW_VRAM or vram_set_state == VRAMState.NORMAL_VRAM):
|
if lowvram_available and (vram_set_state == VRAMState.LOW_VRAM or vram_set_state == VRAMState.NORMAL_VRAM):
|
||||||
model_size = loaded_model.model_memory_required(torch_dev)
|
model_size = loaded_model.model_memory_required(torch_dev)
|
||||||
current_free_mem = get_free_memory(torch_dev)
|
current_free_mem = get_free_memory(torch_dev)
|
||||||
lowvram_model_memory = int(max(256 * (1024 * 1024), (current_free_mem - 1024 * (1024 * 1024)) / 1.3 ))
|
lowvram_model_memory = int(max(64 * (1024 * 1024), (current_free_mem - 1024 * (1024 * 1024)) / 1.3 ))
|
||||||
if model_size > (current_free_mem - inference_memory): #only switch to lowvram if really necessary
|
if model_size > (current_free_mem - inference_memory): #only switch to lowvram if really necessary
|
||||||
vram_set_state = VRAMState.LOW_VRAM
|
vram_set_state = VRAMState.LOW_VRAM
|
||||||
else:
|
else:
|
||||||
lowvram_model_memory = 0
|
lowvram_model_memory = 0
|
||||||
|
|
||||||
if vram_set_state == VRAMState.NO_VRAM:
|
if vram_set_state == VRAMState.NO_VRAM:
|
||||||
lowvram_model_memory = 256 * 1024 * 1024
|
lowvram_model_memory = 64 * 1024 * 1024
|
||||||
|
|
||||||
cur_loaded_model = loaded_model.model_load(lowvram_model_memory)
|
cur_loaded_model = loaded_model.model_load(lowvram_model_memory)
|
||||||
current_loaded_models.insert(0, loaded_model)
|
current_loaded_models.insert(0, loaded_model)
|
||||||
@ -426,6 +454,13 @@ def dtype_size(dtype):
|
|||||||
dtype_size = 4
|
dtype_size = 4
|
||||||
if dtype == torch.float16 or dtype == torch.bfloat16:
|
if dtype == torch.float16 or dtype == torch.bfloat16:
|
||||||
dtype_size = 2
|
dtype_size = 2
|
||||||
|
elif dtype == torch.float32:
|
||||||
|
dtype_size = 4
|
||||||
|
else:
|
||||||
|
try:
|
||||||
|
dtype_size = dtype.itemsize
|
||||||
|
except: #Old pytorch doesn't have .itemsize
|
||||||
|
pass
|
||||||
return dtype_size
|
return dtype_size
|
||||||
|
|
||||||
def unet_offload_device():
|
def unet_offload_device():
|
||||||
@ -455,10 +490,30 @@ def unet_inital_load_device(parameters, dtype):
|
|||||||
def unet_dtype(device=None, model_params=0):
|
def unet_dtype(device=None, model_params=0):
|
||||||
if args.bf16_unet:
|
if args.bf16_unet:
|
||||||
return torch.bfloat16
|
return torch.bfloat16
|
||||||
|
if args.fp16_unet:
|
||||||
|
return torch.float16
|
||||||
|
if args.fp8_e4m3fn_unet:
|
||||||
|
return torch.float8_e4m3fn
|
||||||
|
if args.fp8_e5m2_unet:
|
||||||
|
return torch.float8_e5m2
|
||||||
if should_use_fp16(device=device, model_params=model_params):
|
if should_use_fp16(device=device, model_params=model_params):
|
||||||
return torch.float16
|
return torch.float16
|
||||||
return torch.float32
|
return torch.float32
|
||||||
|
|
||||||
|
# None means no manual cast
|
||||||
|
def unet_manual_cast(weight_dtype, inference_device):
|
||||||
|
if weight_dtype == torch.float32:
|
||||||
|
return None
|
||||||
|
|
||||||
|
fp16_supported = comfy.model_management.should_use_fp16(inference_device, prioritize_performance=False)
|
||||||
|
if fp16_supported and weight_dtype == torch.float16:
|
||||||
|
return None
|
||||||
|
|
||||||
|
if fp16_supported:
|
||||||
|
return torch.float16
|
||||||
|
else:
|
||||||
|
return torch.float32
|
||||||
|
|
||||||
def text_encoder_offload_device():
|
def text_encoder_offload_device():
|
||||||
if args.gpu_only:
|
if args.gpu_only:
|
||||||
return get_torch_device()
|
return get_torch_device()
|
||||||
@ -478,7 +533,33 @@ def text_encoder_device():
|
|||||||
else:
|
else:
|
||||||
return torch.device("cpu")
|
return torch.device("cpu")
|
||||||
|
|
||||||
|
def text_encoder_dtype(device=None):
|
||||||
|
if args.fp8_e4m3fn_text_enc:
|
||||||
|
return torch.float8_e4m3fn
|
||||||
|
elif args.fp8_e5m2_text_enc:
|
||||||
|
return torch.float8_e5m2
|
||||||
|
elif args.fp16_text_enc:
|
||||||
|
return torch.float16
|
||||||
|
elif args.fp32_text_enc:
|
||||||
|
return torch.float32
|
||||||
|
|
||||||
|
if is_device_cpu(device):
|
||||||
|
return torch.float16
|
||||||
|
|
||||||
|
if should_use_fp16(device, prioritize_performance=False):
|
||||||
|
return torch.float16
|
||||||
|
else:
|
||||||
|
return torch.float32
|
||||||
|
|
||||||
|
def intermediate_device():
|
||||||
|
if args.gpu_only:
|
||||||
|
return get_torch_device()
|
||||||
|
else:
|
||||||
|
return torch.device("cpu")
|
||||||
|
|
||||||
def vae_device():
|
def vae_device():
|
||||||
|
if args.cpu_vae:
|
||||||
|
return torch.device("cpu")
|
||||||
return get_torch_device()
|
return get_torch_device()
|
||||||
|
|
||||||
def vae_offload_device():
|
def vae_offload_device():
|
||||||
@ -496,6 +577,22 @@ def get_autocast_device(dev):
|
|||||||
return dev.type
|
return dev.type
|
||||||
return "cuda"
|
return "cuda"
|
||||||
|
|
||||||
|
def supports_dtype(device, dtype): #TODO
|
||||||
|
if dtype == torch.float32:
|
||||||
|
return True
|
||||||
|
if is_device_cpu(device):
|
||||||
|
return False
|
||||||
|
if dtype == torch.float16:
|
||||||
|
return True
|
||||||
|
if dtype == torch.bfloat16:
|
||||||
|
return True
|
||||||
|
return False
|
||||||
|
|
||||||
|
def device_supports_non_blocking(device):
|
||||||
|
if is_device_mps(device):
|
||||||
|
return False #pytorch bug? mps doesn't support non blocking
|
||||||
|
return True
|
||||||
|
|
||||||
def cast_to_device(tensor, device, dtype, copy=False):
|
def cast_to_device(tensor, device, dtype, copy=False):
|
||||||
device_supports_cast = False
|
device_supports_cast = False
|
||||||
if tensor.dtype == torch.float32 or tensor.dtype == torch.float16:
|
if tensor.dtype == torch.float32 or tensor.dtype == torch.float16:
|
||||||
@ -506,15 +603,17 @@ def cast_to_device(tensor, device, dtype, copy=False):
|
|||||||
elif is_intel_xpu():
|
elif is_intel_xpu():
|
||||||
device_supports_cast = True
|
device_supports_cast = True
|
||||||
|
|
||||||
|
non_blocking = device_supports_non_blocking(device)
|
||||||
|
|
||||||
if device_supports_cast:
|
if device_supports_cast:
|
||||||
if copy:
|
if copy:
|
||||||
if tensor.device == device:
|
if tensor.device == device:
|
||||||
return tensor.to(dtype, copy=copy)
|
return tensor.to(dtype, copy=copy, non_blocking=non_blocking)
|
||||||
return tensor.to(device, copy=copy).to(dtype)
|
return tensor.to(device, copy=copy, non_blocking=non_blocking).to(dtype, non_blocking=non_blocking)
|
||||||
else:
|
else:
|
||||||
return tensor.to(device).to(dtype)
|
return tensor.to(device, non_blocking=non_blocking).to(dtype, non_blocking=non_blocking)
|
||||||
else:
|
else:
|
||||||
return tensor.to(dtype).to(device, copy=copy)
|
return tensor.to(device, dtype, copy=copy, non_blocking=non_blocking)
|
||||||
|
|
||||||
def xformers_enabled():
|
def xformers_enabled():
|
||||||
global directml_enabled
|
global directml_enabled
|
||||||
@ -579,27 +678,6 @@ def get_free_memory(dev=None, torch_free_too=False):
|
|||||||
else:
|
else:
|
||||||
return mem_free_total
|
return mem_free_total
|
||||||
|
|
||||||
def batch_area_memory(area):
|
|
||||||
if xformers_enabled() or pytorch_attention_flash_attention():
|
|
||||||
#TODO: these formulas are copied from maximum_batch_area below
|
|
||||||
return (area / 20) * (1024 * 1024)
|
|
||||||
else:
|
|
||||||
return (((area * 0.6) / 0.9) + 1024) * (1024 * 1024)
|
|
||||||
|
|
||||||
def maximum_batch_area():
|
|
||||||
global vram_state
|
|
||||||
if vram_state == VRAMState.NO_VRAM:
|
|
||||||
return 0
|
|
||||||
|
|
||||||
memory_free = get_free_memory() / (1024 * 1024)
|
|
||||||
if xformers_enabled() or pytorch_attention_flash_attention():
|
|
||||||
#TODO: this needs to be tweaked
|
|
||||||
area = 20 * memory_free
|
|
||||||
else:
|
|
||||||
#TODO: this formula is because AMD sucks and has memory management issues which might be fixed in the future
|
|
||||||
area = ((memory_free - 1024) * 0.9) / (0.6)
|
|
||||||
return int(max(area, 0))
|
|
||||||
|
|
||||||
def cpu_mode():
|
def cpu_mode():
|
||||||
global cpu_state
|
global cpu_state
|
||||||
return cpu_state == CPUState.CPU
|
return cpu_state == CPUState.CPU
|
||||||
@ -689,11 +767,11 @@ def soft_empty_cache(force=False):
|
|||||||
torch.cuda.empty_cache()
|
torch.cuda.empty_cache()
|
||||||
torch.cuda.ipc_collect()
|
torch.cuda.ipc_collect()
|
||||||
|
|
||||||
def resolve_lowvram_weight(weight, model, key):
|
def unload_all_models():
|
||||||
if weight.device == torch.device("meta"): #lowvram NOTE: this depends on the inner working of the accelerate library so it might break.
|
free_memory(1e30, get_torch_device())
|
||||||
key_split = key.split('.') # I have no idea why they don't just leave the weight there instead of using the meta device.
|
|
||||||
op = comfy.utils.get_attr(model, '.'.join(key_split[:-1]))
|
|
||||||
weight = op._hf_hook.weights_map[key_split[-1]]
|
def resolve_lowvram_weight(weight, model, key): #TODO: remove
|
||||||
return weight
|
return weight
|
||||||
|
|
||||||
#TODO: might be cleaner to put this somewhere else
|
#TODO: might be cleaner to put this somewhere else
|
||||||
|
|||||||
@ -28,16 +28,12 @@ class ModelPatcher:
|
|||||||
if self.size > 0:
|
if self.size > 0:
|
||||||
return self.size
|
return self.size
|
||||||
model_sd = self.model.state_dict()
|
model_sd = self.model.state_dict()
|
||||||
size = 0
|
self.size = comfy.model_management.module_size(self.model)
|
||||||
for k in model_sd:
|
|
||||||
t = model_sd[k]
|
|
||||||
size += t.nelement() * t.element_size()
|
|
||||||
self.size = size
|
|
||||||
self.model_keys = set(model_sd.keys())
|
self.model_keys = set(model_sd.keys())
|
||||||
return size
|
return self.size
|
||||||
|
|
||||||
def clone(self):
|
def clone(self):
|
||||||
n = ModelPatcher(self.model, self.load_device, self.offload_device, self.size, self.current_device)
|
n = ModelPatcher(self.model, self.load_device, self.offload_device, self.size, self.current_device, weight_inplace_update=self.weight_inplace_update)
|
||||||
n.patches = {}
|
n.patches = {}
|
||||||
for k in self.patches:
|
for k in self.patches:
|
||||||
n.patches[k] = self.patches[k][:]
|
n.patches[k] = self.patches[k][:]
|
||||||
@ -52,11 +48,21 @@ class ModelPatcher:
|
|||||||
return True
|
return True
|
||||||
return False
|
return False
|
||||||
|
|
||||||
def set_model_sampler_cfg_function(self, sampler_cfg_function):
|
def memory_required(self, input_shape):
|
||||||
|
return self.model.memory_required(input_shape=input_shape)
|
||||||
|
|
||||||
|
def set_model_sampler_cfg_function(self, sampler_cfg_function, disable_cfg1_optimization=False):
|
||||||
if len(inspect.signature(sampler_cfg_function).parameters) == 3:
|
if len(inspect.signature(sampler_cfg_function).parameters) == 3:
|
||||||
self.model_options["sampler_cfg_function"] = lambda args: sampler_cfg_function(args["cond"], args["uncond"], args["cond_scale"]) #Old way
|
self.model_options["sampler_cfg_function"] = lambda args: sampler_cfg_function(args["cond"], args["uncond"], args["cond_scale"]) #Old way
|
||||||
else:
|
else:
|
||||||
self.model_options["sampler_cfg_function"] = sampler_cfg_function
|
self.model_options["sampler_cfg_function"] = sampler_cfg_function
|
||||||
|
if disable_cfg1_optimization:
|
||||||
|
self.model_options["disable_cfg1_optimization"] = True
|
||||||
|
|
||||||
|
def set_model_sampler_post_cfg_function(self, post_cfg_function, disable_cfg1_optimization=False):
|
||||||
|
self.model_options["sampler_post_cfg_function"] = self.model_options.get("sampler_post_cfg_function", []) + [post_cfg_function]
|
||||||
|
if disable_cfg1_optimization:
|
||||||
|
self.model_options["disable_cfg1_optimization"] = True
|
||||||
|
|
||||||
def set_model_unet_function_wrapper(self, unet_wrapper_function):
|
def set_model_unet_function_wrapper(self, unet_wrapper_function):
|
||||||
self.model_options["model_function_wrapper"] = unet_wrapper_function
|
self.model_options["model_function_wrapper"] = unet_wrapper_function
|
||||||
@ -67,13 +73,17 @@ class ModelPatcher:
|
|||||||
to["patches"] = {}
|
to["patches"] = {}
|
||||||
to["patches"][name] = to["patches"].get(name, []) + [patch]
|
to["patches"][name] = to["patches"].get(name, []) + [patch]
|
||||||
|
|
||||||
def set_model_patch_replace(self, patch, name, block_name, number):
|
def set_model_patch_replace(self, patch, name, block_name, number, transformer_index=None):
|
||||||
to = self.model_options["transformer_options"]
|
to = self.model_options["transformer_options"]
|
||||||
if "patches_replace" not in to:
|
if "patches_replace" not in to:
|
||||||
to["patches_replace"] = {}
|
to["patches_replace"] = {}
|
||||||
if name not in to["patches_replace"]:
|
if name not in to["patches_replace"]:
|
||||||
to["patches_replace"][name] = {}
|
to["patches_replace"][name] = {}
|
||||||
to["patches_replace"][name][(block_name, number)] = patch
|
if transformer_index is not None:
|
||||||
|
block = (block_name, number, transformer_index)
|
||||||
|
else:
|
||||||
|
block = (block_name, number)
|
||||||
|
to["patches_replace"][name][block] = patch
|
||||||
|
|
||||||
def set_model_attn1_patch(self, patch):
|
def set_model_attn1_patch(self, patch):
|
||||||
self.set_model_patch(patch, "attn1_patch")
|
self.set_model_patch(patch, "attn1_patch")
|
||||||
@ -81,11 +91,11 @@ class ModelPatcher:
|
|||||||
def set_model_attn2_patch(self, patch):
|
def set_model_attn2_patch(self, patch):
|
||||||
self.set_model_patch(patch, "attn2_patch")
|
self.set_model_patch(patch, "attn2_patch")
|
||||||
|
|
||||||
def set_model_attn1_replace(self, patch, block_name, number):
|
def set_model_attn1_replace(self, patch, block_name, number, transformer_index=None):
|
||||||
self.set_model_patch_replace(patch, "attn1", block_name, number)
|
self.set_model_patch_replace(patch, "attn1", block_name, number, transformer_index)
|
||||||
|
|
||||||
def set_model_attn2_replace(self, patch, block_name, number):
|
def set_model_attn2_replace(self, patch, block_name, number, transformer_index=None):
|
||||||
self.set_model_patch_replace(patch, "attn2", block_name, number)
|
self.set_model_patch_replace(patch, "attn2", block_name, number, transformer_index)
|
||||||
|
|
||||||
def set_model_attn1_output_patch(self, patch):
|
def set_model_attn1_output_patch(self, patch):
|
||||||
self.set_model_patch(patch, "attn1_output_patch")
|
self.set_model_patch(patch, "attn1_output_patch")
|
||||||
@ -93,6 +103,12 @@ class ModelPatcher:
|
|||||||
def set_model_attn2_output_patch(self, patch):
|
def set_model_attn2_output_patch(self, patch):
|
||||||
self.set_model_patch(patch, "attn2_output_patch")
|
self.set_model_patch(patch, "attn2_output_patch")
|
||||||
|
|
||||||
|
def set_model_input_block_patch(self, patch):
|
||||||
|
self.set_model_patch(patch, "input_block_patch")
|
||||||
|
|
||||||
|
def set_model_input_block_patch_after_skip(self, patch):
|
||||||
|
self.set_model_patch(patch, "input_block_patch_after_skip")
|
||||||
|
|
||||||
def set_model_output_block_patch(self, patch):
|
def set_model_output_block_patch(self, patch):
|
||||||
self.set_model_patch(patch, "output_block_patch")
|
self.set_model_patch(patch, "output_block_patch")
|
||||||
|
|
||||||
@ -176,7 +192,7 @@ class ModelPatcher:
|
|||||||
inplace_update = self.weight_inplace_update
|
inplace_update = self.weight_inplace_update
|
||||||
|
|
||||||
if key not in self.backup:
|
if key not in self.backup:
|
||||||
self.backup[key] = weight.to(device=device_to, copy=inplace_update)
|
self.backup[key] = weight.to(device=self.offload_device, copy=inplace_update)
|
||||||
|
|
||||||
if device_to is not None:
|
if device_to is not None:
|
||||||
temp_weight = comfy.model_management.cast_to_device(weight, device_to, torch.float32, copy=True)
|
temp_weight = comfy.model_management.cast_to_device(weight, device_to, torch.float32, copy=True)
|
||||||
@ -208,13 +224,19 @@ class ModelPatcher:
|
|||||||
v = (self.calculate_weight(v[1:], v[0].clone(), key), )
|
v = (self.calculate_weight(v[1:], v[0].clone(), key), )
|
||||||
|
|
||||||
if len(v) == 1:
|
if len(v) == 1:
|
||||||
|
patch_type = "diff"
|
||||||
|
elif len(v) == 2:
|
||||||
|
patch_type = v[0]
|
||||||
|
v = v[1]
|
||||||
|
|
||||||
|
if patch_type == "diff":
|
||||||
w1 = v[0]
|
w1 = v[0]
|
||||||
if alpha != 0.0:
|
if alpha != 0.0:
|
||||||
if w1.shape != weight.shape:
|
if w1.shape != weight.shape:
|
||||||
print("WARNING SHAPE MISMATCH {} WEIGHT NOT MERGED {} != {}".format(key, w1.shape, weight.shape))
|
print("WARNING SHAPE MISMATCH {} WEIGHT NOT MERGED {} != {}".format(key, w1.shape, weight.shape))
|
||||||
else:
|
else:
|
||||||
weight += alpha * comfy.model_management.cast_to_device(w1, weight.device, weight.dtype)
|
weight += alpha * comfy.model_management.cast_to_device(w1, weight.device, weight.dtype)
|
||||||
elif len(v) == 4: #lora/locon
|
elif patch_type == "lora": #lora/locon
|
||||||
mat1 = comfy.model_management.cast_to_device(v[0], weight.device, torch.float32)
|
mat1 = comfy.model_management.cast_to_device(v[0], weight.device, torch.float32)
|
||||||
mat2 = comfy.model_management.cast_to_device(v[1], weight.device, torch.float32)
|
mat2 = comfy.model_management.cast_to_device(v[1], weight.device, torch.float32)
|
||||||
if v[2] is not None:
|
if v[2] is not None:
|
||||||
@ -228,7 +250,7 @@ class ModelPatcher:
|
|||||||
weight += (alpha * torch.mm(mat1.flatten(start_dim=1), mat2.flatten(start_dim=1))).reshape(weight.shape).type(weight.dtype)
|
weight += (alpha * torch.mm(mat1.flatten(start_dim=1), mat2.flatten(start_dim=1))).reshape(weight.shape).type(weight.dtype)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
print("ERROR", key, e)
|
print("ERROR", key, e)
|
||||||
elif len(v) == 8: #lokr
|
elif patch_type == "lokr":
|
||||||
w1 = v[0]
|
w1 = v[0]
|
||||||
w2 = v[1]
|
w2 = v[1]
|
||||||
w1_a = v[3]
|
w1_a = v[3]
|
||||||
@ -267,7 +289,7 @@ class ModelPatcher:
|
|||||||
weight += alpha * torch.kron(w1, w2).reshape(weight.shape).type(weight.dtype)
|
weight += alpha * torch.kron(w1, w2).reshape(weight.shape).type(weight.dtype)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
print("ERROR", key, e)
|
print("ERROR", key, e)
|
||||||
else: #loha
|
elif patch_type == "loha":
|
||||||
w1a = v[0]
|
w1a = v[0]
|
||||||
w1b = v[1]
|
w1b = v[1]
|
||||||
if v[2] is not None:
|
if v[2] is not None:
|
||||||
@ -296,6 +318,18 @@ class ModelPatcher:
|
|||||||
weight += (alpha * m1 * m2).reshape(weight.shape).type(weight.dtype)
|
weight += (alpha * m1 * m2).reshape(weight.shape).type(weight.dtype)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
print("ERROR", key, e)
|
print("ERROR", key, e)
|
||||||
|
elif patch_type == "glora":
|
||||||
|
if v[4] is not None:
|
||||||
|
alpha *= v[4] / v[0].shape[0]
|
||||||
|
|
||||||
|
a1 = comfy.model_management.cast_to_device(v[0].flatten(start_dim=1), weight.device, torch.float32)
|
||||||
|
a2 = comfy.model_management.cast_to_device(v[1].flatten(start_dim=1), weight.device, torch.float32)
|
||||||
|
b1 = comfy.model_management.cast_to_device(v[2].flatten(start_dim=1), weight.device, torch.float32)
|
||||||
|
b2 = comfy.model_management.cast_to_device(v[3].flatten(start_dim=1), weight.device, torch.float32)
|
||||||
|
|
||||||
|
weight += ((torch.mm(b2, b1) + torch.mm(torch.mm(weight.flatten(start_dim=1), a2), a1)) * alpha).reshape(weight.shape).type(weight.dtype)
|
||||||
|
else:
|
||||||
|
print("patch type not recognized", patch_type, key)
|
||||||
|
|
||||||
return weight
|
return weight
|
||||||
|
|
||||||
|
|||||||
@ -1,7 +1,7 @@
|
|||||||
import torch
|
import torch
|
||||||
import numpy as np
|
import numpy as np
|
||||||
from comfy.ldm.modules.diffusionmodules.util import make_beta_schedule
|
from comfy.ldm.modules.diffusionmodules.util import make_beta_schedule
|
||||||
|
import math
|
||||||
|
|
||||||
class EPS:
|
class EPS:
|
||||||
def calculate_input(self, sigma, noise):
|
def calculate_input(self, sigma, noise):
|
||||||
@ -22,10 +22,17 @@ class V_PREDICTION(EPS):
|
|||||||
class ModelSamplingDiscrete(torch.nn.Module):
|
class ModelSamplingDiscrete(torch.nn.Module):
|
||||||
def __init__(self, model_config=None):
|
def __init__(self, model_config=None):
|
||||||
super().__init__()
|
super().__init__()
|
||||||
beta_schedule = "linear"
|
|
||||||
if model_config is not None:
|
if model_config is not None:
|
||||||
beta_schedule = model_config.beta_schedule
|
sampling_settings = model_config.sampling_settings
|
||||||
self._register_schedule(given_betas=None, beta_schedule=beta_schedule, timesteps=1000, linear_start=0.00085, linear_end=0.012, cosine_s=8e-3)
|
else:
|
||||||
|
sampling_settings = {}
|
||||||
|
|
||||||
|
beta_schedule = sampling_settings.get("beta_schedule", "linear")
|
||||||
|
linear_start = sampling_settings.get("linear_start", 0.00085)
|
||||||
|
linear_end = sampling_settings.get("linear_end", 0.012)
|
||||||
|
|
||||||
|
self._register_schedule(given_betas=None, beta_schedule=beta_schedule, timesteps=1000, linear_start=linear_start, linear_end=linear_end, cosine_s=8e-3)
|
||||||
self.sigma_data = 1.0
|
self.sigma_data = 1.0
|
||||||
|
|
||||||
def _register_schedule(self, given_betas=None, beta_schedule="linear", timesteps=1000,
|
def _register_schedule(self, given_betas=None, beta_schedule="linear", timesteps=1000,
|
||||||
@ -65,16 +72,65 @@ class ModelSamplingDiscrete(torch.nn.Module):
|
|||||||
def timestep(self, sigma):
|
def timestep(self, sigma):
|
||||||
log_sigma = sigma.log()
|
log_sigma = sigma.log()
|
||||||
dists = log_sigma.to(self.log_sigmas.device) - self.log_sigmas[:, None]
|
dists = log_sigma.to(self.log_sigmas.device) - self.log_sigmas[:, None]
|
||||||
return dists.abs().argmin(dim=0).view(sigma.shape)
|
return dists.abs().argmin(dim=0).view(sigma.shape).to(sigma.device)
|
||||||
|
|
||||||
def sigma(self, timestep):
|
def sigma(self, timestep):
|
||||||
t = torch.clamp(timestep.float(), min=0, max=(len(self.sigmas) - 1))
|
t = torch.clamp(timestep.float().to(self.log_sigmas.device), min=0, max=(len(self.sigmas) - 1))
|
||||||
low_idx = t.floor().long()
|
low_idx = t.floor().long()
|
||||||
high_idx = t.ceil().long()
|
high_idx = t.ceil().long()
|
||||||
w = t.frac()
|
w = t.frac()
|
||||||
log_sigma = (1 - w) * self.log_sigmas[low_idx] + w * self.log_sigmas[high_idx]
|
log_sigma = (1 - w) * self.log_sigmas[low_idx] + w * self.log_sigmas[high_idx]
|
||||||
return log_sigma.exp()
|
return log_sigma.exp().to(timestep.device)
|
||||||
|
|
||||||
def percent_to_sigma(self, percent):
|
def percent_to_sigma(self, percent):
|
||||||
return self.sigma(torch.tensor(percent * 999.0))
|
if percent <= 0.0:
|
||||||
|
return 999999999.9
|
||||||
|
if percent >= 1.0:
|
||||||
|
return 0.0
|
||||||
|
percent = 1.0 - percent
|
||||||
|
return self.sigma(torch.tensor(percent * 999.0)).item()
|
||||||
|
|
||||||
|
|
||||||
|
class ModelSamplingContinuousEDM(torch.nn.Module):
|
||||||
|
def __init__(self, model_config=None):
|
||||||
|
super().__init__()
|
||||||
|
self.sigma_data = 1.0
|
||||||
|
|
||||||
|
if model_config is not None:
|
||||||
|
sampling_settings = model_config.sampling_settings
|
||||||
|
else:
|
||||||
|
sampling_settings = {}
|
||||||
|
|
||||||
|
sigma_min = sampling_settings.get("sigma_min", 0.002)
|
||||||
|
sigma_max = sampling_settings.get("sigma_max", 120.0)
|
||||||
|
self.set_sigma_range(sigma_min, sigma_max)
|
||||||
|
|
||||||
|
def set_sigma_range(self, sigma_min, sigma_max):
|
||||||
|
sigmas = torch.linspace(math.log(sigma_min), math.log(sigma_max), 1000).exp()
|
||||||
|
|
||||||
|
self.register_buffer('sigmas', sigmas) #for compatibility with some schedulers
|
||||||
|
self.register_buffer('log_sigmas', sigmas.log())
|
||||||
|
|
||||||
|
@property
|
||||||
|
def sigma_min(self):
|
||||||
|
return self.sigmas[0]
|
||||||
|
|
||||||
|
@property
|
||||||
|
def sigma_max(self):
|
||||||
|
return self.sigmas[-1]
|
||||||
|
|
||||||
|
def timestep(self, sigma):
|
||||||
|
return 0.25 * sigma.log()
|
||||||
|
|
||||||
|
def sigma(self, timestep):
|
||||||
|
return (timestep / 0.25).exp()
|
||||||
|
|
||||||
|
def percent_to_sigma(self, percent):
|
||||||
|
if percent <= 0.0:
|
||||||
|
return 999999999.9
|
||||||
|
if percent >= 1.0:
|
||||||
|
return 0.0
|
||||||
|
percent = 1.0 - percent
|
||||||
|
|
||||||
|
log_sigma_min = math.log(self.sigma_min)
|
||||||
|
return math.exp((math.log(self.sigma_max) - log_sigma_min) * percent + log_sigma_min)
|
||||||
|
|||||||
139
comfy/ops.py
139
comfy/ops.py
@ -1,40 +1,115 @@
|
|||||||
import torch
|
import torch
|
||||||
from contextlib import contextmanager
|
from contextlib import contextmanager
|
||||||
|
import comfy.model_management
|
||||||
|
|
||||||
class Linear(torch.nn.Linear):
|
def cast_bias_weight(s, input):
|
||||||
def reset_parameters(self):
|
bias = None
|
||||||
return None
|
non_blocking = comfy.model_management.device_supports_non_blocking(input.device)
|
||||||
|
if s.bias is not None:
|
||||||
|
bias = s.bias.to(device=input.device, dtype=input.dtype, non_blocking=non_blocking)
|
||||||
|
weight = s.weight.to(device=input.device, dtype=input.dtype, non_blocking=non_blocking)
|
||||||
|
return weight, bias
|
||||||
|
|
||||||
class Conv2d(torch.nn.Conv2d):
|
|
||||||
def reset_parameters(self):
|
|
||||||
return None
|
|
||||||
|
|
||||||
class Conv3d(torch.nn.Conv3d):
|
class disable_weight_init:
|
||||||
def reset_parameters(self):
|
class Linear(torch.nn.Linear):
|
||||||
return None
|
comfy_cast_weights = False
|
||||||
|
def reset_parameters(self):
|
||||||
|
return None
|
||||||
|
|
||||||
def conv_nd(dims, *args, **kwargs):
|
def forward_comfy_cast_weights(self, input):
|
||||||
if dims == 2:
|
weight, bias = cast_bias_weight(self, input)
|
||||||
return Conv2d(*args, **kwargs)
|
return torch.nn.functional.linear(input, weight, bias)
|
||||||
elif dims == 3:
|
|
||||||
return Conv3d(*args, **kwargs)
|
|
||||||
else:
|
|
||||||
raise ValueError(f"unsupported dimensions: {dims}")
|
|
||||||
|
|
||||||
@contextmanager
|
def forward(self, *args, **kwargs):
|
||||||
def use_comfy_ops(device=None, dtype=None): # Kind of an ugly hack but I can't think of a better way
|
if self.comfy_cast_weights:
|
||||||
old_torch_nn_linear = torch.nn.Linear
|
return self.forward_comfy_cast_weights(*args, **kwargs)
|
||||||
force_device = device
|
else:
|
||||||
force_dtype = dtype
|
return super().forward(*args, **kwargs)
|
||||||
def linear_with_dtype(in_features: int, out_features: int, bias: bool = True, device=None, dtype=None):
|
|
||||||
if force_device is not None:
|
|
||||||
device = force_device
|
|
||||||
if force_dtype is not None:
|
|
||||||
dtype = force_dtype
|
|
||||||
return Linear(in_features, out_features, bias=bias, device=device, dtype=dtype)
|
|
||||||
|
|
||||||
torch.nn.Linear = linear_with_dtype
|
class Conv2d(torch.nn.Conv2d):
|
||||||
try:
|
comfy_cast_weights = False
|
||||||
yield
|
def reset_parameters(self):
|
||||||
finally:
|
return None
|
||||||
torch.nn.Linear = old_torch_nn_linear
|
|
||||||
|
def forward_comfy_cast_weights(self, input):
|
||||||
|
weight, bias = cast_bias_weight(self, input)
|
||||||
|
return self._conv_forward(input, weight, bias)
|
||||||
|
|
||||||
|
def forward(self, *args, **kwargs):
|
||||||
|
if self.comfy_cast_weights:
|
||||||
|
return self.forward_comfy_cast_weights(*args, **kwargs)
|
||||||
|
else:
|
||||||
|
return super().forward(*args, **kwargs)
|
||||||
|
|
||||||
|
class Conv3d(torch.nn.Conv3d):
|
||||||
|
comfy_cast_weights = False
|
||||||
|
def reset_parameters(self):
|
||||||
|
return None
|
||||||
|
|
||||||
|
def forward_comfy_cast_weights(self, input):
|
||||||
|
weight, bias = cast_bias_weight(self, input)
|
||||||
|
return self._conv_forward(input, weight, bias)
|
||||||
|
|
||||||
|
def forward(self, *args, **kwargs):
|
||||||
|
if self.comfy_cast_weights:
|
||||||
|
return self.forward_comfy_cast_weights(*args, **kwargs)
|
||||||
|
else:
|
||||||
|
return super().forward(*args, **kwargs)
|
||||||
|
|
||||||
|
class GroupNorm(torch.nn.GroupNorm):
|
||||||
|
comfy_cast_weights = False
|
||||||
|
def reset_parameters(self):
|
||||||
|
return None
|
||||||
|
|
||||||
|
def forward_comfy_cast_weights(self, input):
|
||||||
|
weight, bias = cast_bias_weight(self, input)
|
||||||
|
return torch.nn.functional.group_norm(input, self.num_groups, weight, bias, self.eps)
|
||||||
|
|
||||||
|
def forward(self, *args, **kwargs):
|
||||||
|
if self.comfy_cast_weights:
|
||||||
|
return self.forward_comfy_cast_weights(*args, **kwargs)
|
||||||
|
else:
|
||||||
|
return super().forward(*args, **kwargs)
|
||||||
|
|
||||||
|
|
||||||
|
class LayerNorm(torch.nn.LayerNorm):
|
||||||
|
comfy_cast_weights = False
|
||||||
|
def reset_parameters(self):
|
||||||
|
return None
|
||||||
|
|
||||||
|
def forward_comfy_cast_weights(self, input):
|
||||||
|
weight, bias = cast_bias_weight(self, input)
|
||||||
|
return torch.nn.functional.layer_norm(input, self.normalized_shape, weight, bias, self.eps)
|
||||||
|
|
||||||
|
def forward(self, *args, **kwargs):
|
||||||
|
if self.comfy_cast_weights:
|
||||||
|
return self.forward_comfy_cast_weights(*args, **kwargs)
|
||||||
|
else:
|
||||||
|
return super().forward(*args, **kwargs)
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def conv_nd(s, dims, *args, **kwargs):
|
||||||
|
if dims == 2:
|
||||||
|
return s.Conv2d(*args, **kwargs)
|
||||||
|
elif dims == 3:
|
||||||
|
return s.Conv3d(*args, **kwargs)
|
||||||
|
else:
|
||||||
|
raise ValueError(f"unsupported dimensions: {dims}")
|
||||||
|
|
||||||
|
|
||||||
|
class manual_cast(disable_weight_init):
|
||||||
|
class Linear(disable_weight_init.Linear):
|
||||||
|
comfy_cast_weights = True
|
||||||
|
|
||||||
|
class Conv2d(disable_weight_init.Conv2d):
|
||||||
|
comfy_cast_weights = True
|
||||||
|
|
||||||
|
class Conv3d(disable_weight_init.Conv3d):
|
||||||
|
comfy_cast_weights = True
|
||||||
|
|
||||||
|
class GroupNorm(disable_weight_init.GroupNorm):
|
||||||
|
comfy_cast_weights = True
|
||||||
|
|
||||||
|
class LayerNorm(disable_weight_init.LayerNorm):
|
||||||
|
comfy_cast_weights = True
|
||||||
|
|||||||
@ -47,7 +47,8 @@ def convert_cond(cond):
|
|||||||
temp = c[1].copy()
|
temp = c[1].copy()
|
||||||
model_conds = temp.get("model_conds", {})
|
model_conds = temp.get("model_conds", {})
|
||||||
if c[0] is not None:
|
if c[0] is not None:
|
||||||
model_conds["c_crossattn"] = comfy.conds.CONDCrossAttn(c[0])
|
model_conds["c_crossattn"] = comfy.conds.CONDCrossAttn(c[0]) #TODO: remove
|
||||||
|
temp["cross_attn"] = c[0]
|
||||||
temp["model_conds"] = model_conds
|
temp["model_conds"] = model_conds
|
||||||
out.append(temp)
|
out.append(temp)
|
||||||
return out
|
return out
|
||||||
@ -83,7 +84,7 @@ def prepare_sampling(model, noise_shape, positive, negative, noise_mask):
|
|||||||
|
|
||||||
real_model = None
|
real_model = None
|
||||||
models, inference_memory = get_additional_models(positive, negative, model.model_dtype())
|
models, inference_memory = get_additional_models(positive, negative, model.model_dtype())
|
||||||
comfy.model_management.load_models_gpu([model] + models, comfy.model_management.batch_area_memory(noise_shape[0] * noise_shape[2] * noise_shape[3]) + inference_memory)
|
comfy.model_management.load_models_gpu([model] + models, model.memory_required([noise_shape[0] * 2] + list(noise_shape[1:])) + inference_memory)
|
||||||
real_model = model.model
|
real_model = model.model
|
||||||
|
|
||||||
return real_model, positive, negative, noise_mask, models
|
return real_model, positive, negative, noise_mask, models
|
||||||
@ -98,10 +99,10 @@ def sample(model, noise, steps, cfg, sampler_name, scheduler, positive, negative
|
|||||||
sampler = comfy.samplers.KSampler(real_model, steps=steps, device=model.load_device, sampler=sampler_name, scheduler=scheduler, denoise=denoise, model_options=model.model_options)
|
sampler = comfy.samplers.KSampler(real_model, steps=steps, device=model.load_device, sampler=sampler_name, scheduler=scheduler, denoise=denoise, model_options=model.model_options)
|
||||||
|
|
||||||
samples = sampler.sample(noise, positive_copy, negative_copy, cfg=cfg, latent_image=latent_image, start_step=start_step, last_step=last_step, force_full_denoise=force_full_denoise, denoise_mask=noise_mask, sigmas=sigmas, callback=callback, disable_pbar=disable_pbar, seed=seed)
|
samples = sampler.sample(noise, positive_copy, negative_copy, cfg=cfg, latent_image=latent_image, start_step=start_step, last_step=last_step, force_full_denoise=force_full_denoise, denoise_mask=noise_mask, sigmas=sigmas, callback=callback, disable_pbar=disable_pbar, seed=seed)
|
||||||
samples = samples.cpu()
|
samples = samples.to(comfy.model_management.intermediate_device())
|
||||||
|
|
||||||
cleanup_additional_models(models)
|
cleanup_additional_models(models)
|
||||||
cleanup_additional_models(set(get_models_from_cond(positive, "control") + get_models_from_cond(negative, "control")))
|
cleanup_additional_models(set(get_models_from_cond(positive_copy, "control") + get_models_from_cond(negative_copy, "control")))
|
||||||
return samples
|
return samples
|
||||||
|
|
||||||
def sample_custom(model, noise, cfg, sampler, sigmas, positive, negative, latent_image, noise_mask=None, callback=None, disable_pbar=False, seed=None):
|
def sample_custom(model, noise, cfg, sampler, sigmas, positive, negative, latent_image, noise_mask=None, callback=None, disable_pbar=False, seed=None):
|
||||||
@ -111,8 +112,8 @@ def sample_custom(model, noise, cfg, sampler, sigmas, positive, negative, latent
|
|||||||
sigmas = sigmas.to(model.load_device)
|
sigmas = sigmas.to(model.load_device)
|
||||||
|
|
||||||
samples = comfy.samplers.sample(real_model, noise, positive_copy, negative_copy, cfg, model.load_device, sampler, sigmas, model_options=model.model_options, latent_image=latent_image, denoise_mask=noise_mask, callback=callback, disable_pbar=disable_pbar, seed=seed)
|
samples = comfy.samplers.sample(real_model, noise, positive_copy, negative_copy, cfg, model.load_device, sampler, sigmas, model_options=model.model_options, latent_image=latent_image, denoise_mask=noise_mask, callback=callback, disable_pbar=disable_pbar, seed=seed)
|
||||||
samples = samples.cpu()
|
samples = samples.to(comfy.model_management.intermediate_device())
|
||||||
cleanup_additional_models(models)
|
cleanup_additional_models(models)
|
||||||
cleanup_additional_models(set(get_models_from_cond(positive, "control") + get_models_from_cond(negative, "control")))
|
cleanup_additional_models(set(get_models_from_cond(positive_copy, "control") + get_models_from_cond(negative_copy, "control")))
|
||||||
return samples
|
return samples
|
||||||
|
|
||||||
|
|||||||
@ -2,263 +2,274 @@ from .k_diffusion import sampling as k_diffusion_sampling
|
|||||||
from .extra_samplers import uni_pc
|
from .extra_samplers import uni_pc
|
||||||
import torch
|
import torch
|
||||||
import enum
|
import enum
|
||||||
|
import collections
|
||||||
from comfy import model_management
|
from comfy import model_management
|
||||||
import math
|
import math
|
||||||
from comfy import model_base
|
from comfy import model_base
|
||||||
import comfy.utils
|
import comfy.utils
|
||||||
import comfy.conds
|
import comfy.conds
|
||||||
|
|
||||||
|
def get_area_and_mult(conds, x_in, timestep_in):
|
||||||
|
area = (x_in.shape[2], x_in.shape[3], 0, 0)
|
||||||
|
strength = 1.0
|
||||||
|
|
||||||
|
if 'timestep_start' in conds:
|
||||||
|
timestep_start = conds['timestep_start']
|
||||||
|
if timestep_in[0] > timestep_start:
|
||||||
|
return None
|
||||||
|
if 'timestep_end' in conds:
|
||||||
|
timestep_end = conds['timestep_end']
|
||||||
|
if timestep_in[0] < timestep_end:
|
||||||
|
return None
|
||||||
|
if 'area' in conds:
|
||||||
|
area = conds['area']
|
||||||
|
if 'strength' in conds:
|
||||||
|
strength = conds['strength']
|
||||||
|
|
||||||
|
input_x = x_in[:,:,area[2]:area[0] + area[2],area[3]:area[1] + area[3]]
|
||||||
|
if 'mask' in conds:
|
||||||
|
# Scale the mask to the size of the input
|
||||||
|
# The mask should have been resized as we began the sampling process
|
||||||
|
mask_strength = 1.0
|
||||||
|
if "mask_strength" in conds:
|
||||||
|
mask_strength = conds["mask_strength"]
|
||||||
|
mask = conds['mask']
|
||||||
|
assert(mask.shape[1] == x_in.shape[2])
|
||||||
|
assert(mask.shape[2] == x_in.shape[3])
|
||||||
|
mask = mask[:,area[2]:area[0] + area[2],area[3]:area[1] + area[3]] * mask_strength
|
||||||
|
mask = mask.unsqueeze(1).repeat(input_x.shape[0] // mask.shape[0], input_x.shape[1], 1, 1)
|
||||||
|
else:
|
||||||
|
mask = torch.ones_like(input_x)
|
||||||
|
mult = mask * strength
|
||||||
|
|
||||||
|
if 'mask' not in conds:
|
||||||
|
rr = 8
|
||||||
|
if area[2] != 0:
|
||||||
|
for t in range(rr):
|
||||||
|
mult[:,:,t:1+t,:] *= ((1.0/rr) * (t + 1))
|
||||||
|
if (area[0] + area[2]) < x_in.shape[2]:
|
||||||
|
for t in range(rr):
|
||||||
|
mult[:,:,area[0] - 1 - t:area[0] - t,:] *= ((1.0/rr) * (t + 1))
|
||||||
|
if area[3] != 0:
|
||||||
|
for t in range(rr):
|
||||||
|
mult[:,:,:,t:1+t] *= ((1.0/rr) * (t + 1))
|
||||||
|
if (area[1] + area[3]) < x_in.shape[3]:
|
||||||
|
for t in range(rr):
|
||||||
|
mult[:,:,:,area[1] - 1 - t:area[1] - t] *= ((1.0/rr) * (t + 1))
|
||||||
|
|
||||||
|
conditioning = {}
|
||||||
|
model_conds = conds["model_conds"]
|
||||||
|
for c in model_conds:
|
||||||
|
conditioning[c] = model_conds[c].process_cond(batch_size=x_in.shape[0], device=x_in.device, area=area)
|
||||||
|
|
||||||
|
control = conds.get('control', None)
|
||||||
|
|
||||||
|
patches = None
|
||||||
|
if 'gligen' in conds:
|
||||||
|
gligen = conds['gligen']
|
||||||
|
patches = {}
|
||||||
|
gligen_type = gligen[0]
|
||||||
|
gligen_model = gligen[1]
|
||||||
|
if gligen_type == "position":
|
||||||
|
gligen_patch = gligen_model.model.set_position(input_x.shape, gligen[2], input_x.device)
|
||||||
|
else:
|
||||||
|
gligen_patch = gligen_model.model.set_empty(input_x.shape, input_x.device)
|
||||||
|
|
||||||
|
patches['middle_patch'] = [gligen_patch]
|
||||||
|
|
||||||
|
cond_obj = collections.namedtuple('cond_obj', ['input_x', 'mult', 'conditioning', 'area', 'control', 'patches'])
|
||||||
|
return cond_obj(input_x, mult, conditioning, area, control, patches)
|
||||||
|
|
||||||
|
def cond_equal_size(c1, c2):
|
||||||
|
if c1 is c2:
|
||||||
|
return True
|
||||||
|
if c1.keys() != c2.keys():
|
||||||
|
return False
|
||||||
|
for k in c1:
|
||||||
|
if not c1[k].can_concat(c2[k]):
|
||||||
|
return False
|
||||||
|
return True
|
||||||
|
|
||||||
|
def can_concat_cond(c1, c2):
|
||||||
|
if c1.input_x.shape != c2.input_x.shape:
|
||||||
|
return False
|
||||||
|
|
||||||
|
def objects_concatable(obj1, obj2):
|
||||||
|
if (obj1 is None) != (obj2 is None):
|
||||||
|
return False
|
||||||
|
if obj1 is not None:
|
||||||
|
if obj1 is not obj2:
|
||||||
|
return False
|
||||||
|
return True
|
||||||
|
|
||||||
|
if not objects_concatable(c1.control, c2.control):
|
||||||
|
return False
|
||||||
|
|
||||||
|
if not objects_concatable(c1.patches, c2.patches):
|
||||||
|
return False
|
||||||
|
|
||||||
|
return cond_equal_size(c1.conditioning, c2.conditioning)
|
||||||
|
|
||||||
|
def cond_cat(c_list):
|
||||||
|
c_crossattn = []
|
||||||
|
c_concat = []
|
||||||
|
c_adm = []
|
||||||
|
crossattn_max_len = 0
|
||||||
|
|
||||||
|
temp = {}
|
||||||
|
for x in c_list:
|
||||||
|
for k in x:
|
||||||
|
cur = temp.get(k, [])
|
||||||
|
cur.append(x[k])
|
||||||
|
temp[k] = cur
|
||||||
|
|
||||||
|
out = {}
|
||||||
|
for k in temp:
|
||||||
|
conds = temp[k]
|
||||||
|
out[k] = conds[0].concat(conds[1:])
|
||||||
|
|
||||||
|
return out
|
||||||
|
|
||||||
|
def calc_cond_uncond_batch(model, cond, uncond, x_in, timestep, model_options):
|
||||||
|
out_cond = torch.zeros_like(x_in)
|
||||||
|
out_count = torch.ones_like(x_in) * 1e-37
|
||||||
|
|
||||||
|
out_uncond = torch.zeros_like(x_in)
|
||||||
|
out_uncond_count = torch.ones_like(x_in) * 1e-37
|
||||||
|
|
||||||
|
COND = 0
|
||||||
|
UNCOND = 1
|
||||||
|
|
||||||
|
to_run = []
|
||||||
|
for x in cond:
|
||||||
|
p = get_area_and_mult(x, x_in, timestep)
|
||||||
|
if p is None:
|
||||||
|
continue
|
||||||
|
|
||||||
|
to_run += [(p, COND)]
|
||||||
|
if uncond is not None:
|
||||||
|
for x in uncond:
|
||||||
|
p = get_area_and_mult(x, x_in, timestep)
|
||||||
|
if p is None:
|
||||||
|
continue
|
||||||
|
|
||||||
|
to_run += [(p, UNCOND)]
|
||||||
|
|
||||||
|
while len(to_run) > 0:
|
||||||
|
first = to_run[0]
|
||||||
|
first_shape = first[0][0].shape
|
||||||
|
to_batch_temp = []
|
||||||
|
for x in range(len(to_run)):
|
||||||
|
if can_concat_cond(to_run[x][0], first[0]):
|
||||||
|
to_batch_temp += [x]
|
||||||
|
|
||||||
|
to_batch_temp.reverse()
|
||||||
|
to_batch = to_batch_temp[:1]
|
||||||
|
|
||||||
|
free_memory = model_management.get_free_memory(x_in.device)
|
||||||
|
for i in range(1, len(to_batch_temp) + 1):
|
||||||
|
batch_amount = to_batch_temp[:len(to_batch_temp)//i]
|
||||||
|
input_shape = [len(batch_amount) * first_shape[0]] + list(first_shape)[1:]
|
||||||
|
if model.memory_required(input_shape) < free_memory:
|
||||||
|
to_batch = batch_amount
|
||||||
|
break
|
||||||
|
|
||||||
|
input_x = []
|
||||||
|
mult = []
|
||||||
|
c = []
|
||||||
|
cond_or_uncond = []
|
||||||
|
area = []
|
||||||
|
control = None
|
||||||
|
patches = None
|
||||||
|
for x in to_batch:
|
||||||
|
o = to_run.pop(x)
|
||||||
|
p = o[0]
|
||||||
|
input_x.append(p.input_x)
|
||||||
|
mult.append(p.mult)
|
||||||
|
c.append(p.conditioning)
|
||||||
|
area.append(p.area)
|
||||||
|
cond_or_uncond.append(o[1])
|
||||||
|
control = p.control
|
||||||
|
patches = p.patches
|
||||||
|
|
||||||
|
batch_chunks = len(cond_or_uncond)
|
||||||
|
input_x = torch.cat(input_x)
|
||||||
|
c = cond_cat(c)
|
||||||
|
timestep_ = torch.cat([timestep] * batch_chunks)
|
||||||
|
|
||||||
|
if control is not None:
|
||||||
|
c['control'] = control.get_control(input_x, timestep_, c, len(cond_or_uncond))
|
||||||
|
|
||||||
|
transformer_options = {}
|
||||||
|
if 'transformer_options' in model_options:
|
||||||
|
transformer_options = model_options['transformer_options'].copy()
|
||||||
|
|
||||||
|
if patches is not None:
|
||||||
|
if "patches" in transformer_options:
|
||||||
|
cur_patches = transformer_options["patches"].copy()
|
||||||
|
for p in patches:
|
||||||
|
if p in cur_patches:
|
||||||
|
cur_patches[p] = cur_patches[p] + patches[p]
|
||||||
|
else:
|
||||||
|
cur_patches[p] = patches[p]
|
||||||
|
else:
|
||||||
|
transformer_options["patches"] = patches
|
||||||
|
|
||||||
|
transformer_options["cond_or_uncond"] = cond_or_uncond[:]
|
||||||
|
transformer_options["sigmas"] = timestep
|
||||||
|
|
||||||
|
c['transformer_options'] = transformer_options
|
||||||
|
|
||||||
|
if 'model_function_wrapper' in model_options:
|
||||||
|
output = model_options['model_function_wrapper'](model.apply_model, {"input": input_x, "timestep": timestep_, "c": c, "cond_or_uncond": cond_or_uncond}).chunk(batch_chunks)
|
||||||
|
else:
|
||||||
|
output = model.apply_model(input_x, timestep_, **c).chunk(batch_chunks)
|
||||||
|
del input_x
|
||||||
|
|
||||||
|
for o in range(batch_chunks):
|
||||||
|
if cond_or_uncond[o] == COND:
|
||||||
|
out_cond[:,:,area[o][2]:area[o][0] + area[o][2],area[o][3]:area[o][1] + area[o][3]] += output[o] * mult[o]
|
||||||
|
out_count[:,:,area[o][2]:area[o][0] + area[o][2],area[o][3]:area[o][1] + area[o][3]] += mult[o]
|
||||||
|
else:
|
||||||
|
out_uncond[:,:,area[o][2]:area[o][0] + area[o][2],area[o][3]:area[o][1] + area[o][3]] += output[o] * mult[o]
|
||||||
|
out_uncond_count[:,:,area[o][2]:area[o][0] + area[o][2],area[o][3]:area[o][1] + area[o][3]] += mult[o]
|
||||||
|
del mult
|
||||||
|
|
||||||
|
out_cond /= out_count
|
||||||
|
del out_count
|
||||||
|
out_uncond /= out_uncond_count
|
||||||
|
del out_uncond_count
|
||||||
|
return out_cond, out_uncond
|
||||||
|
|
||||||
#The main sampling function shared by all the samplers
|
#The main sampling function shared by all the samplers
|
||||||
#Returns denoised
|
#Returns denoised
|
||||||
def sampling_function(model_function, x, timestep, uncond, cond, cond_scale, model_options={}, seed=None):
|
def sampling_function(model, x, timestep, uncond, cond, cond_scale, model_options={}, seed=None):
|
||||||
def get_area_and_mult(conds, x_in, timestep_in):
|
if math.isclose(cond_scale, 1.0) and model_options.get("disable_cfg1_optimization", False) == False:
|
||||||
area = (x_in.shape[2], x_in.shape[3], 0, 0)
|
uncond_ = None
|
||||||
strength = 1.0
|
|
||||||
|
|
||||||
if 'timestep_start' in conds:
|
|
||||||
timestep_start = conds['timestep_start']
|
|
||||||
if timestep_in[0] > timestep_start:
|
|
||||||
return None
|
|
||||||
if 'timestep_end' in conds:
|
|
||||||
timestep_end = conds['timestep_end']
|
|
||||||
if timestep_in[0] < timestep_end:
|
|
||||||
return None
|
|
||||||
if 'area' in conds:
|
|
||||||
area = conds['area']
|
|
||||||
if 'strength' in conds:
|
|
||||||
strength = conds['strength']
|
|
||||||
|
|
||||||
input_x = x_in[:,:,area[2]:area[0] + area[2],area[3]:area[1] + area[3]]
|
|
||||||
if 'mask' in conds:
|
|
||||||
# Scale the mask to the size of the input
|
|
||||||
# The mask should have been resized as we began the sampling process
|
|
||||||
mask_strength = 1.0
|
|
||||||
if "mask_strength" in conds:
|
|
||||||
mask_strength = conds["mask_strength"]
|
|
||||||
mask = conds['mask']
|
|
||||||
assert(mask.shape[1] == x_in.shape[2])
|
|
||||||
assert(mask.shape[2] == x_in.shape[3])
|
|
||||||
mask = mask[:,area[2]:area[0] + area[2],area[3]:area[1] + area[3]] * mask_strength
|
|
||||||
mask = mask.unsqueeze(1).repeat(input_x.shape[0] // mask.shape[0], input_x.shape[1], 1, 1)
|
|
||||||
else:
|
|
||||||
mask = torch.ones_like(input_x)
|
|
||||||
mult = mask * strength
|
|
||||||
|
|
||||||
if 'mask' not in conds:
|
|
||||||
rr = 8
|
|
||||||
if area[2] != 0:
|
|
||||||
for t in range(rr):
|
|
||||||
mult[:,:,t:1+t,:] *= ((1.0/rr) * (t + 1))
|
|
||||||
if (area[0] + area[2]) < x_in.shape[2]:
|
|
||||||
for t in range(rr):
|
|
||||||
mult[:,:,area[0] - 1 - t:area[0] - t,:] *= ((1.0/rr) * (t + 1))
|
|
||||||
if area[3] != 0:
|
|
||||||
for t in range(rr):
|
|
||||||
mult[:,:,:,t:1+t] *= ((1.0/rr) * (t + 1))
|
|
||||||
if (area[1] + area[3]) < x_in.shape[3]:
|
|
||||||
for t in range(rr):
|
|
||||||
mult[:,:,:,area[1] - 1 - t:area[1] - t] *= ((1.0/rr) * (t + 1))
|
|
||||||
|
|
||||||
conditionning = {}
|
|
||||||
model_conds = conds["model_conds"]
|
|
||||||
for c in model_conds:
|
|
||||||
conditionning[c] = model_conds[c].process_cond(batch_size=x_in.shape[0], device=x_in.device, area=area)
|
|
||||||
|
|
||||||
control = None
|
|
||||||
if 'control' in conds:
|
|
||||||
control = conds['control']
|
|
||||||
|
|
||||||
patches = None
|
|
||||||
if 'gligen' in conds:
|
|
||||||
gligen = conds['gligen']
|
|
||||||
patches = {}
|
|
||||||
gligen_type = gligen[0]
|
|
||||||
gligen_model = gligen[1]
|
|
||||||
if gligen_type == "position":
|
|
||||||
gligen_patch = gligen_model.model.set_position(input_x.shape, gligen[2], input_x.device)
|
|
||||||
else:
|
|
||||||
gligen_patch = gligen_model.model.set_empty(input_x.shape, input_x.device)
|
|
||||||
|
|
||||||
patches['middle_patch'] = [gligen_patch]
|
|
||||||
|
|
||||||
return (input_x, mult, conditionning, area, control, patches)
|
|
||||||
|
|
||||||
def cond_equal_size(c1, c2):
|
|
||||||
if c1 is c2:
|
|
||||||
return True
|
|
||||||
if c1.keys() != c2.keys():
|
|
||||||
return False
|
|
||||||
for k in c1:
|
|
||||||
if not c1[k].can_concat(c2[k]):
|
|
||||||
return False
|
|
||||||
return True
|
|
||||||
|
|
||||||
def can_concat_cond(c1, c2):
|
|
||||||
if c1[0].shape != c2[0].shape:
|
|
||||||
return False
|
|
||||||
|
|
||||||
#control
|
|
||||||
if (c1[4] is None) != (c2[4] is None):
|
|
||||||
return False
|
|
||||||
if c1[4] is not None:
|
|
||||||
if c1[4] is not c2[4]:
|
|
||||||
return False
|
|
||||||
|
|
||||||
#patches
|
|
||||||
if (c1[5] is None) != (c2[5] is None):
|
|
||||||
return False
|
|
||||||
if (c1[5] is not None):
|
|
||||||
if c1[5] is not c2[5]:
|
|
||||||
return False
|
|
||||||
|
|
||||||
return cond_equal_size(c1[2], c2[2])
|
|
||||||
|
|
||||||
def cond_cat(c_list):
|
|
||||||
c_crossattn = []
|
|
||||||
c_concat = []
|
|
||||||
c_adm = []
|
|
||||||
crossattn_max_len = 0
|
|
||||||
|
|
||||||
temp = {}
|
|
||||||
for x in c_list:
|
|
||||||
for k in x:
|
|
||||||
cur = temp.get(k, [])
|
|
||||||
cur.append(x[k])
|
|
||||||
temp[k] = cur
|
|
||||||
|
|
||||||
out = {}
|
|
||||||
for k in temp:
|
|
||||||
conds = temp[k]
|
|
||||||
out[k] = conds[0].concat(conds[1:])
|
|
||||||
|
|
||||||
return out
|
|
||||||
|
|
||||||
def calc_cond_uncond_batch(model_function, cond, uncond, x_in, timestep, max_total_area, model_options):
|
|
||||||
out_cond = torch.zeros_like(x_in)
|
|
||||||
out_count = torch.ones_like(x_in) * 1e-37
|
|
||||||
|
|
||||||
out_uncond = torch.zeros_like(x_in)
|
|
||||||
out_uncond_count = torch.ones_like(x_in) * 1e-37
|
|
||||||
|
|
||||||
COND = 0
|
|
||||||
UNCOND = 1
|
|
||||||
|
|
||||||
to_run = []
|
|
||||||
for x in cond:
|
|
||||||
p = get_area_and_mult(x, x_in, timestep)
|
|
||||||
if p is None:
|
|
||||||
continue
|
|
||||||
|
|
||||||
to_run += [(p, COND)]
|
|
||||||
if uncond is not None:
|
|
||||||
for x in uncond:
|
|
||||||
p = get_area_and_mult(x, x_in, timestep)
|
|
||||||
if p is None:
|
|
||||||
continue
|
|
||||||
|
|
||||||
to_run += [(p, UNCOND)]
|
|
||||||
|
|
||||||
while len(to_run) > 0:
|
|
||||||
first = to_run[0]
|
|
||||||
first_shape = first[0][0].shape
|
|
||||||
to_batch_temp = []
|
|
||||||
for x in range(len(to_run)):
|
|
||||||
if can_concat_cond(to_run[x][0], first[0]):
|
|
||||||
to_batch_temp += [x]
|
|
||||||
|
|
||||||
to_batch_temp.reverse()
|
|
||||||
to_batch = to_batch_temp[:1]
|
|
||||||
|
|
||||||
for i in range(1, len(to_batch_temp) + 1):
|
|
||||||
batch_amount = to_batch_temp[:len(to_batch_temp)//i]
|
|
||||||
if (len(batch_amount) * first_shape[0] * first_shape[2] * first_shape[3] < max_total_area):
|
|
||||||
to_batch = batch_amount
|
|
||||||
break
|
|
||||||
|
|
||||||
input_x = []
|
|
||||||
mult = []
|
|
||||||
c = []
|
|
||||||
cond_or_uncond = []
|
|
||||||
area = []
|
|
||||||
control = None
|
|
||||||
patches = None
|
|
||||||
for x in to_batch:
|
|
||||||
o = to_run.pop(x)
|
|
||||||
p = o[0]
|
|
||||||
input_x += [p[0]]
|
|
||||||
mult += [p[1]]
|
|
||||||
c += [p[2]]
|
|
||||||
area += [p[3]]
|
|
||||||
cond_or_uncond += [o[1]]
|
|
||||||
control = p[4]
|
|
||||||
patches = p[5]
|
|
||||||
|
|
||||||
batch_chunks = len(cond_or_uncond)
|
|
||||||
input_x = torch.cat(input_x)
|
|
||||||
c = cond_cat(c)
|
|
||||||
timestep_ = torch.cat([timestep] * batch_chunks)
|
|
||||||
|
|
||||||
if control is not None:
|
|
||||||
c['control'] = control.get_control(input_x, timestep_, c, len(cond_or_uncond))
|
|
||||||
|
|
||||||
transformer_options = {}
|
|
||||||
if 'transformer_options' in model_options:
|
|
||||||
transformer_options = model_options['transformer_options'].copy()
|
|
||||||
|
|
||||||
if patches is not None:
|
|
||||||
if "patches" in transformer_options:
|
|
||||||
cur_patches = transformer_options["patches"].copy()
|
|
||||||
for p in patches:
|
|
||||||
if p in cur_patches:
|
|
||||||
cur_patches[p] = cur_patches[p] + patches[p]
|
|
||||||
else:
|
|
||||||
cur_patches[p] = patches[p]
|
|
||||||
else:
|
|
||||||
transformer_options["patches"] = patches
|
|
||||||
|
|
||||||
transformer_options["cond_or_uncond"] = cond_or_uncond[:]
|
|
||||||
c['transformer_options'] = transformer_options
|
|
||||||
|
|
||||||
if 'model_function_wrapper' in model_options:
|
|
||||||
output = model_options['model_function_wrapper'](model_function, {"input": input_x, "timestep": timestep_, "c": c, "cond_or_uncond": cond_or_uncond}).chunk(batch_chunks)
|
|
||||||
else:
|
|
||||||
output = model_function(input_x, timestep_, **c).chunk(batch_chunks)
|
|
||||||
del input_x
|
|
||||||
|
|
||||||
for o in range(batch_chunks):
|
|
||||||
if cond_or_uncond[o] == COND:
|
|
||||||
out_cond[:,:,area[o][2]:area[o][0] + area[o][2],area[o][3]:area[o][1] + area[o][3]] += output[o] * mult[o]
|
|
||||||
out_count[:,:,area[o][2]:area[o][0] + area[o][2],area[o][3]:area[o][1] + area[o][3]] += mult[o]
|
|
||||||
else:
|
|
||||||
out_uncond[:,:,area[o][2]:area[o][0] + area[o][2],area[o][3]:area[o][1] + area[o][3]] += output[o] * mult[o]
|
|
||||||
out_uncond_count[:,:,area[o][2]:area[o][0] + area[o][2],area[o][3]:area[o][1] + area[o][3]] += mult[o]
|
|
||||||
del mult
|
|
||||||
|
|
||||||
out_cond /= out_count
|
|
||||||
del out_count
|
|
||||||
out_uncond /= out_uncond_count
|
|
||||||
del out_uncond_count
|
|
||||||
return out_cond, out_uncond
|
|
||||||
|
|
||||||
|
|
||||||
max_total_area = model_management.maximum_batch_area()
|
|
||||||
if math.isclose(cond_scale, 1.0):
|
|
||||||
uncond = None
|
|
||||||
|
|
||||||
cond, uncond = calc_cond_uncond_batch(model_function, cond, uncond, x, timestep, max_total_area, model_options)
|
|
||||||
if "sampler_cfg_function" in model_options:
|
|
||||||
args = {"cond": x - cond, "uncond": x - uncond, "cond_scale": cond_scale, "timestep": timestep, "input": x, "sigma": timestep}
|
|
||||||
return x - model_options["sampler_cfg_function"](args)
|
|
||||||
else:
|
else:
|
||||||
return uncond + (cond - uncond) * cond_scale
|
uncond_ = uncond
|
||||||
|
|
||||||
|
cond_pred, uncond_pred = calc_cond_uncond_batch(model, cond, uncond_, x, timestep, model_options)
|
||||||
|
if "sampler_cfg_function" in model_options:
|
||||||
|
args = {"cond": x - cond_pred, "uncond": x - uncond_pred, "cond_scale": cond_scale, "timestep": timestep, "input": x, "sigma": timestep,
|
||||||
|
"cond_denoised": cond_pred, "uncond_denoised": uncond_pred, "model": model, "model_options": model_options}
|
||||||
|
cfg_result = x - model_options["sampler_cfg_function"](args)
|
||||||
|
else:
|
||||||
|
cfg_result = uncond_pred + (cond_pred - uncond_pred) * cond_scale
|
||||||
|
|
||||||
|
for fn in model_options.get("sampler_post_cfg_function", []):
|
||||||
|
args = {"denoised": cfg_result, "cond": cond, "uncond": uncond, "model": model, "uncond_denoised": uncond_pred, "cond_denoised": cond_pred,
|
||||||
|
"sigma": timestep, "model_options": model_options, "input": x}
|
||||||
|
cfg_result = fn(args)
|
||||||
|
|
||||||
|
return cfg_result
|
||||||
|
|
||||||
class CFGNoisePredictor(torch.nn.Module):
|
class CFGNoisePredictor(torch.nn.Module):
|
||||||
def __init__(self, model):
|
def __init__(self, model):
|
||||||
super().__init__()
|
super().__init__()
|
||||||
self.inner_model = model
|
self.inner_model = model
|
||||||
def apply_model(self, x, timestep, cond, uncond, cond_scale, model_options={}, seed=None):
|
def apply_model(self, x, timestep, cond, uncond, cond_scale, model_options={}, seed=None):
|
||||||
out = sampling_function(self.inner_model.apply_model, x, timestep, uncond, cond, cond_scale, model_options=model_options, seed=seed)
|
out = sampling_function(self.inner_model, x, timestep, uncond, cond, cond_scale, model_options=model_options, seed=seed)
|
||||||
return out
|
return out
|
||||||
def forward(self, *args, **kwargs):
|
def forward(self, *args, **kwargs):
|
||||||
return self.apply_model(*args, **kwargs)
|
return self.apply_model(*args, **kwargs)
|
||||||
@ -273,10 +284,7 @@ class KSamplerX0Inpaint(torch.nn.Module):
|
|||||||
x = x * denoise_mask + (self.latent_image + self.noise * sigma.reshape([sigma.shape[0]] + [1] * (len(self.noise.shape) - 1))) * latent_mask
|
x = x * denoise_mask + (self.latent_image + self.noise * sigma.reshape([sigma.shape[0]] + [1] * (len(self.noise.shape) - 1))) * latent_mask
|
||||||
out = self.inner_model(x, sigma, cond=cond, uncond=uncond, cond_scale=cond_scale, model_options=model_options, seed=seed)
|
out = self.inner_model(x, sigma, cond=cond, uncond=uncond, cond_scale=cond_scale, model_options=model_options, seed=seed)
|
||||||
if denoise_mask is not None:
|
if denoise_mask is not None:
|
||||||
out *= denoise_mask
|
out = out * denoise_mask + self.latent_image * latent_mask
|
||||||
|
|
||||||
if denoise_mask is not None:
|
|
||||||
out += self.latent_image * latent_mask
|
|
||||||
return out
|
return out
|
||||||
|
|
||||||
def simple_scheduler(model, steps):
|
def simple_scheduler(model, steps):
|
||||||
@ -511,52 +519,69 @@ class Sampler:
|
|||||||
|
|
||||||
class UNIPC(Sampler):
|
class UNIPC(Sampler):
|
||||||
def sample(self, model_wrap, sigmas, extra_args, callback, noise, latent_image=None, denoise_mask=None, disable_pbar=False):
|
def sample(self, model_wrap, sigmas, extra_args, callback, noise, latent_image=None, denoise_mask=None, disable_pbar=False):
|
||||||
return uni_pc.sample_unipc(model_wrap, noise, latent_image, sigmas, sampling_function=sampling_function, max_denoise=self.max_denoise(model_wrap, sigmas), extra_args=extra_args, noise_mask=denoise_mask, callback=callback, disable=disable_pbar)
|
return uni_pc.sample_unipc(model_wrap, noise, latent_image, sigmas, max_denoise=self.max_denoise(model_wrap, sigmas), extra_args=extra_args, noise_mask=denoise_mask, callback=callback, disable=disable_pbar)
|
||||||
|
|
||||||
class UNIPCBH2(Sampler):
|
class UNIPCBH2(Sampler):
|
||||||
def sample(self, model_wrap, sigmas, extra_args, callback, noise, latent_image=None, denoise_mask=None, disable_pbar=False):
|
def sample(self, model_wrap, sigmas, extra_args, callback, noise, latent_image=None, denoise_mask=None, disable_pbar=False):
|
||||||
return uni_pc.sample_unipc(model_wrap, noise, latent_image, sigmas, sampling_function=sampling_function, max_denoise=self.max_denoise(model_wrap, sigmas), extra_args=extra_args, noise_mask=denoise_mask, callback=callback, variant='bh2', disable=disable_pbar)
|
return uni_pc.sample_unipc(model_wrap, noise, latent_image, sigmas, max_denoise=self.max_denoise(model_wrap, sigmas), extra_args=extra_args, noise_mask=denoise_mask, callback=callback, variant='bh2', disable=disable_pbar)
|
||||||
|
|
||||||
KSAMPLER_NAMES = ["euler", "euler_ancestral", "heun", "dpm_2", "dpm_2_ancestral",
|
KSAMPLER_NAMES = ["euler", "euler_ancestral", "heun", "heunpp2","dpm_2", "dpm_2_ancestral",
|
||||||
"lms", "dpm_fast", "dpm_adaptive", "dpmpp_2s_ancestral", "dpmpp_sde", "dpmpp_sde_gpu",
|
"lms", "dpm_fast", "dpm_adaptive", "dpmpp_2s_ancestral", "dpmpp_sde", "dpmpp_sde_gpu",
|
||||||
"dpmpp_2m", "dpmpp_2m_sde", "dpmpp_2m_sde_gpu", "dpmpp_3m_sde", "dpmpp_3m_sde_gpu", "ddpm", "lcm"]
|
"dpmpp_2m", "dpmpp_2m_sde", "dpmpp_2m_sde_gpu", "dpmpp_3m_sde", "dpmpp_3m_sde_gpu", "ddpm", "lcm"]
|
||||||
|
|
||||||
|
class KSAMPLER(Sampler):
|
||||||
|
def __init__(self, sampler_function, extra_options={}, inpaint_options={}):
|
||||||
|
self.sampler_function = sampler_function
|
||||||
|
self.extra_options = extra_options
|
||||||
|
self.inpaint_options = inpaint_options
|
||||||
|
|
||||||
|
def sample(self, model_wrap, sigmas, extra_args, callback, noise, latent_image=None, denoise_mask=None, disable_pbar=False):
|
||||||
|
extra_args["denoise_mask"] = denoise_mask
|
||||||
|
model_k = KSamplerX0Inpaint(model_wrap)
|
||||||
|
model_k.latent_image = latent_image
|
||||||
|
if self.inpaint_options.get("random", False): #TODO: Should this be the default?
|
||||||
|
generator = torch.manual_seed(extra_args.get("seed", 41) + 1)
|
||||||
|
model_k.noise = torch.randn(noise.shape, generator=generator, device="cpu").to(noise.dtype).to(noise.device)
|
||||||
|
else:
|
||||||
|
model_k.noise = noise
|
||||||
|
|
||||||
|
if self.max_denoise(model_wrap, sigmas):
|
||||||
|
noise = noise * torch.sqrt(1.0 + sigmas[0] ** 2.0)
|
||||||
|
else:
|
||||||
|
noise = noise * sigmas[0]
|
||||||
|
|
||||||
|
k_callback = None
|
||||||
|
total_steps = len(sigmas) - 1
|
||||||
|
if callback is not None:
|
||||||
|
k_callback = lambda x: callback(x["i"], x["denoised"], x["x"], total_steps)
|
||||||
|
|
||||||
|
if latent_image is not None:
|
||||||
|
noise += latent_image
|
||||||
|
|
||||||
|
samples = self.sampler_function(model_k, noise, sigmas, extra_args=extra_args, callback=k_callback, disable=disable_pbar, **self.extra_options)
|
||||||
|
return samples
|
||||||
|
|
||||||
|
|
||||||
def ksampler(sampler_name, extra_options={}, inpaint_options={}):
|
def ksampler(sampler_name, extra_options={}, inpaint_options={}):
|
||||||
class KSAMPLER(Sampler):
|
if sampler_name == "dpm_fast":
|
||||||
def sample(self, model_wrap, sigmas, extra_args, callback, noise, latent_image=None, denoise_mask=None, disable_pbar=False):
|
def dpm_fast_function(model, noise, sigmas, extra_args, callback, disable):
|
||||||
extra_args["denoise_mask"] = denoise_mask
|
|
||||||
model_k = KSamplerX0Inpaint(model_wrap)
|
|
||||||
model_k.latent_image = latent_image
|
|
||||||
if inpaint_options.get("random", False): #TODO: Should this be the default?
|
|
||||||
generator = torch.manual_seed(extra_args.get("seed", 41) + 1)
|
|
||||||
model_k.noise = torch.randn(noise.shape, generator=generator, device="cpu").to(noise.dtype).to(noise.device)
|
|
||||||
else:
|
|
||||||
model_k.noise = noise
|
|
||||||
|
|
||||||
if self.max_denoise(model_wrap, sigmas):
|
|
||||||
noise = noise * torch.sqrt(1.0 + sigmas[0] ** 2.0)
|
|
||||||
else:
|
|
||||||
noise = noise * sigmas[0]
|
|
||||||
|
|
||||||
k_callback = None
|
|
||||||
total_steps = len(sigmas) - 1
|
|
||||||
if callback is not None:
|
|
||||||
k_callback = lambda x: callback(x["i"], x["denoised"], x["x"], total_steps)
|
|
||||||
|
|
||||||
sigma_min = sigmas[-1]
|
sigma_min = sigmas[-1]
|
||||||
if sigma_min == 0:
|
if sigma_min == 0:
|
||||||
sigma_min = sigmas[-2]
|
sigma_min = sigmas[-2]
|
||||||
|
total_steps = len(sigmas) - 1
|
||||||
|
return k_diffusion_sampling.sample_dpm_fast(model, noise, sigma_min, sigmas[0], total_steps, extra_args=extra_args, callback=callback, disable=disable)
|
||||||
|
sampler_function = dpm_fast_function
|
||||||
|
elif sampler_name == "dpm_adaptive":
|
||||||
|
def dpm_adaptive_function(model, noise, sigmas, extra_args, callback, disable):
|
||||||
|
sigma_min = sigmas[-1]
|
||||||
|
if sigma_min == 0:
|
||||||
|
sigma_min = sigmas[-2]
|
||||||
|
return k_diffusion_sampling.sample_dpm_adaptive(model, noise, sigma_min, sigmas[0], extra_args=extra_args, callback=callback, disable=disable)
|
||||||
|
sampler_function = dpm_adaptive_function
|
||||||
|
else:
|
||||||
|
sampler_function = getattr(k_diffusion_sampling, "sample_{}".format(sampler_name))
|
||||||
|
|
||||||
if latent_image is not None:
|
return KSAMPLER(sampler_function, extra_options, inpaint_options)
|
||||||
noise += latent_image
|
|
||||||
if sampler_name == "dpm_fast":
|
|
||||||
samples = k_diffusion_sampling.sample_dpm_fast(model_k, noise, sigma_min, sigmas[0], total_steps, extra_args=extra_args, callback=k_callback, disable=disable_pbar)
|
|
||||||
elif sampler_name == "dpm_adaptive":
|
|
||||||
samples = k_diffusion_sampling.sample_dpm_adaptive(model_k, noise, sigma_min, sigmas[0], extra_args=extra_args, callback=k_callback, disable=disable_pbar)
|
|
||||||
else:
|
|
||||||
samples = getattr(k_diffusion_sampling, "sample_{}".format(sampler_name))(model_k, noise, sigmas, extra_args=extra_args, callback=k_callback, disable=disable_pbar, **extra_options)
|
|
||||||
return samples
|
|
||||||
return KSAMPLER
|
|
||||||
|
|
||||||
def wrap_model(model):
|
def wrap_model(model):
|
||||||
model_denoise = CFGNoisePredictor(model)
|
model_denoise = CFGNoisePredictor(model)
|
||||||
@ -574,6 +599,13 @@ def sample(model, noise, positive, negative, cfg, device, sampler, sigmas, model
|
|||||||
calculate_start_end_timesteps(model, negative)
|
calculate_start_end_timesteps(model, negative)
|
||||||
calculate_start_end_timesteps(model, positive)
|
calculate_start_end_timesteps(model, positive)
|
||||||
|
|
||||||
|
if latent_image is not None:
|
||||||
|
latent_image = model.process_latent_in(latent_image)
|
||||||
|
|
||||||
|
if hasattr(model, 'extra_conds'):
|
||||||
|
positive = encode_model_conds(model.extra_conds, positive, noise, device, "positive", latent_image=latent_image, denoise_mask=denoise_mask)
|
||||||
|
negative = encode_model_conds(model.extra_conds, negative, noise, device, "negative", latent_image=latent_image, denoise_mask=denoise_mask)
|
||||||
|
|
||||||
#make sure each cond area has an opposite one with the same area
|
#make sure each cond area has an opposite one with the same area
|
||||||
for c in positive:
|
for c in positive:
|
||||||
create_cond_with_same_area_if_none(negative, c)
|
create_cond_with_same_area_if_none(negative, c)
|
||||||
@ -585,13 +617,6 @@ def sample(model, noise, positive, negative, cfg, device, sampler, sigmas, model
|
|||||||
apply_empty_x_to_equal_area(list(filter(lambda c: c.get('control_apply_to_uncond', False) == True, positive)), negative, 'control', lambda cond_cnets, x: cond_cnets[x])
|
apply_empty_x_to_equal_area(list(filter(lambda c: c.get('control_apply_to_uncond', False) == True, positive)), negative, 'control', lambda cond_cnets, x: cond_cnets[x])
|
||||||
apply_empty_x_to_equal_area(positive, negative, 'gligen', lambda cond_cnets, x: cond_cnets[x])
|
apply_empty_x_to_equal_area(positive, negative, 'gligen', lambda cond_cnets, x: cond_cnets[x])
|
||||||
|
|
||||||
if latent_image is not None:
|
|
||||||
latent_image = model.process_latent_in(latent_image)
|
|
||||||
|
|
||||||
if hasattr(model, 'extra_conds'):
|
|
||||||
positive = encode_model_conds(model.extra_conds, positive, noise, device, "positive", latent_image=latent_image, denoise_mask=denoise_mask)
|
|
||||||
negative = encode_model_conds(model.extra_conds, negative, noise, device, "negative", latent_image=latent_image, denoise_mask=denoise_mask)
|
|
||||||
|
|
||||||
extra_args = {"cond":positive, "uncond":negative, "cond_scale": cfg, "model_options": model_options, "seed":seed}
|
extra_args = {"cond":positive, "uncond":negative, "cond_scale": cfg, "model_options": model_options, "seed":seed}
|
||||||
|
|
||||||
samples = sampler.sample(model_wrap, sigmas, extra_args, callback, noise, latent_image, denoise_mask, disable_pbar)
|
samples = sampler.sample(model_wrap, sigmas, extra_args, callback, noise, latent_image, denoise_mask, disable_pbar)
|
||||||
@ -617,11 +642,11 @@ def calculate_sigmas_scheduler(model, scheduler_name, steps):
|
|||||||
print("error invalid scheduler", self.scheduler)
|
print("error invalid scheduler", self.scheduler)
|
||||||
return sigmas
|
return sigmas
|
||||||
|
|
||||||
def sampler_class(name):
|
def sampler_object(name):
|
||||||
if name == "uni_pc":
|
if name == "uni_pc":
|
||||||
sampler = UNIPC
|
sampler = UNIPC()
|
||||||
elif name == "uni_pc_bh2":
|
elif name == "uni_pc_bh2":
|
||||||
sampler = UNIPCBH2
|
sampler = UNIPCBH2()
|
||||||
elif name == "ddim":
|
elif name == "ddim":
|
||||||
sampler = ksampler("euler", inpaint_options={"random": True})
|
sampler = ksampler("euler", inpaint_options={"random": True})
|
||||||
else:
|
else:
|
||||||
@ -686,6 +711,6 @@ class KSampler:
|
|||||||
else:
|
else:
|
||||||
return torch.zeros_like(noise)
|
return torch.zeros_like(noise)
|
||||||
|
|
||||||
sampler = sampler_class(self.sampler)
|
sampler = sampler_object(self.sampler)
|
||||||
|
|
||||||
return sample(self.model, noise, positive, negative, cfg, self.device, sampler(), sigmas, self.model_options, latent_image=latent_image, denoise_mask=denoise_mask, callback=callback, disable_pbar=disable_pbar, seed=seed)
|
return sample(self.model, noise, positive, negative, cfg, self.device, sampler, sigmas, self.model_options, latent_image=latent_image, denoise_mask=denoise_mask, callback=callback, disable_pbar=disable_pbar, seed=seed)
|
||||||
|
|||||||
104
comfy/sd.py
104
comfy/sd.py
@ -23,6 +23,7 @@ import comfy.model_patcher
|
|||||||
import comfy.lora
|
import comfy.lora
|
||||||
import comfy.t2i_adapter.adapter
|
import comfy.t2i_adapter.adapter
|
||||||
import comfy.supported_models_base
|
import comfy.supported_models_base
|
||||||
|
import comfy.taesd.taesd
|
||||||
|
|
||||||
def load_model_weights(model, sd):
|
def load_model_weights(model, sd):
|
||||||
m, u = model.load_state_dict(sd, strict=False)
|
m, u = model.load_state_dict(sd, strict=False)
|
||||||
@ -95,10 +96,7 @@ class CLIP:
|
|||||||
load_device = model_management.text_encoder_device()
|
load_device = model_management.text_encoder_device()
|
||||||
offload_device = model_management.text_encoder_offload_device()
|
offload_device = model_management.text_encoder_offload_device()
|
||||||
params['device'] = offload_device
|
params['device'] = offload_device
|
||||||
if model_management.should_use_fp16(load_device, prioritize_performance=False):
|
params['dtype'] = model_management.text_encoder_dtype(load_device)
|
||||||
params['dtype'] = torch.float16
|
|
||||||
else:
|
|
||||||
params['dtype'] = torch.float32
|
|
||||||
|
|
||||||
self.cond_stage_model = clip(**(params))
|
self.cond_stage_model = clip(**(params))
|
||||||
|
|
||||||
@ -153,14 +151,28 @@ class CLIP:
|
|||||||
return self.patcher.get_key_patches()
|
return self.patcher.get_key_patches()
|
||||||
|
|
||||||
class VAE:
|
class VAE:
|
||||||
def __init__(self, sd=None, device=None, config=None):
|
def __init__(self, sd=None, device=None, config=None, dtype=None):
|
||||||
if 'decoder.up_blocks.0.resnets.0.norm1.weight' in sd.keys(): #diffusers format
|
if 'decoder.up_blocks.0.resnets.0.norm1.weight' in sd.keys(): #diffusers format
|
||||||
sd = diffusers_convert.convert_vae_state_dict(sd)
|
sd = diffusers_convert.convert_vae_state_dict(sd)
|
||||||
|
|
||||||
|
self.memory_used_encode = lambda shape, dtype: (1767 * shape[2] * shape[3]) * model_management.dtype_size(dtype) #These are for AutoencoderKL and need tweaking (should be lower)
|
||||||
|
self.memory_used_decode = lambda shape, dtype: (2178 * shape[2] * shape[3] * 64) * model_management.dtype_size(dtype)
|
||||||
|
|
||||||
if config is None:
|
if config is None:
|
||||||
#default SD1.x/SD2.x VAE parameters
|
if "decoder.mid.block_1.mix_factor" in sd:
|
||||||
ddconfig = {'double_z': True, 'z_channels': 4, 'resolution': 256, 'in_channels': 3, 'out_ch': 3, 'ch': 128, 'ch_mult': [1, 2, 4, 4], 'num_res_blocks': 2, 'attn_resolutions': [], 'dropout': 0.0}
|
encoder_config = {'double_z': True, 'z_channels': 4, 'resolution': 256, 'in_channels': 3, 'out_ch': 3, 'ch': 128, 'ch_mult': [1, 2, 4, 4], 'num_res_blocks': 2, 'attn_resolutions': [], 'dropout': 0.0}
|
||||||
self.first_stage_model = AutoencoderKL(ddconfig=ddconfig, embed_dim=4)
|
decoder_config = encoder_config.copy()
|
||||||
|
decoder_config["video_kernel_size"] = [3, 1, 1]
|
||||||
|
decoder_config["alpha"] = 0.0
|
||||||
|
self.first_stage_model = AutoencodingEngine(regularizer_config={'target': "comfy.ldm.models.autoencoder.DiagonalGaussianRegularizer"},
|
||||||
|
encoder_config={'target': "comfy.ldm.modules.diffusionmodules.model.Encoder", 'params': encoder_config},
|
||||||
|
decoder_config={'target': "comfy.ldm.modules.temporal_ae.VideoDecoder", 'params': decoder_config})
|
||||||
|
elif "taesd_decoder.1.weight" in sd:
|
||||||
|
self.first_stage_model = comfy.taesd.taesd.TAESD()
|
||||||
|
else:
|
||||||
|
#default SD1.x/SD2.x VAE parameters
|
||||||
|
ddconfig = {'double_z': True, 'z_channels': 4, 'resolution': 256, 'in_channels': 3, 'out_ch': 3, 'ch': 128, 'ch_mult': [1, 2, 4, 4], 'num_res_blocks': 2, 'attn_resolutions': [], 'dropout': 0.0}
|
||||||
|
self.first_stage_model = AutoencoderKL(ddconfig=ddconfig, embed_dim=4)
|
||||||
else:
|
else:
|
||||||
self.first_stage_model = AutoencoderKL(**(config['params']))
|
self.first_stage_model = AutoencoderKL(**(config['params']))
|
||||||
self.first_stage_model = self.first_stage_model.eval()
|
self.first_stage_model = self.first_stage_model.eval()
|
||||||
@ -175,9 +187,14 @@ class VAE:
|
|||||||
if device is None:
|
if device is None:
|
||||||
device = model_management.vae_device()
|
device = model_management.vae_device()
|
||||||
self.device = device
|
self.device = device
|
||||||
self.offload_device = model_management.vae_offload_device()
|
offload_device = model_management.vae_offload_device()
|
||||||
self.vae_dtype = model_management.vae_dtype()
|
if dtype is None:
|
||||||
|
dtype = model_management.vae_dtype()
|
||||||
|
self.vae_dtype = dtype
|
||||||
self.first_stage_model.to(self.vae_dtype)
|
self.first_stage_model.to(self.vae_dtype)
|
||||||
|
self.output_device = model_management.intermediate_device()
|
||||||
|
|
||||||
|
self.patcher = comfy.model_patcher.ModelPatcher(self.first_stage_model, load_device=self.device, offload_device=offload_device)
|
||||||
|
|
||||||
def decode_tiled_(self, samples, tile_x=64, tile_y=64, overlap = 16):
|
def decode_tiled_(self, samples, tile_x=64, tile_y=64, overlap = 16):
|
||||||
steps = samples.shape[0] * comfy.utils.get_tiled_scale_steps(samples.shape[3], samples.shape[2], tile_x, tile_y, overlap)
|
steps = samples.shape[0] * comfy.utils.get_tiled_scale_steps(samples.shape[3], samples.shape[2], tile_x, tile_y, overlap)
|
||||||
@ -187,9 +204,9 @@ class VAE:
|
|||||||
|
|
||||||
decode_fn = lambda a: (self.first_stage_model.decode(a.to(self.vae_dtype).to(self.device)) + 1.0).float()
|
decode_fn = lambda a: (self.first_stage_model.decode(a.to(self.vae_dtype).to(self.device)) + 1.0).float()
|
||||||
output = torch.clamp((
|
output = torch.clamp((
|
||||||
(comfy.utils.tiled_scale(samples, decode_fn, tile_x // 2, tile_y * 2, overlap, upscale_amount = 8, pbar = pbar) +
|
(comfy.utils.tiled_scale(samples, decode_fn, tile_x // 2, tile_y * 2, overlap, upscale_amount = 8, output_device=self.output_device, pbar = pbar) +
|
||||||
comfy.utils.tiled_scale(samples, decode_fn, tile_x * 2, tile_y // 2, overlap, upscale_amount = 8, pbar = pbar) +
|
comfy.utils.tiled_scale(samples, decode_fn, tile_x * 2, tile_y // 2, overlap, upscale_amount = 8, output_device=self.output_device, pbar = pbar) +
|
||||||
comfy.utils.tiled_scale(samples, decode_fn, tile_x, tile_y, overlap, upscale_amount = 8, pbar = pbar))
|
comfy.utils.tiled_scale(samples, decode_fn, tile_x, tile_y, overlap, upscale_amount = 8, output_device=self.output_device, pbar = pbar))
|
||||||
/ 3.0) / 2.0, min=0.0, max=1.0)
|
/ 3.0) / 2.0, min=0.0, max=1.0)
|
||||||
return output
|
return output
|
||||||
|
|
||||||
@ -200,65 +217,59 @@ class VAE:
|
|||||||
pbar = comfy.utils.ProgressBar(steps)
|
pbar = comfy.utils.ProgressBar(steps)
|
||||||
|
|
||||||
encode_fn = lambda a: self.first_stage_model.encode((2. * a - 1.).to(self.vae_dtype).to(self.device)).float()
|
encode_fn = lambda a: self.first_stage_model.encode((2. * a - 1.).to(self.vae_dtype).to(self.device)).float()
|
||||||
samples = comfy.utils.tiled_scale(pixel_samples, encode_fn, tile_x, tile_y, overlap, upscale_amount = (1/8), out_channels=4, pbar=pbar)
|
samples = comfy.utils.tiled_scale(pixel_samples, encode_fn, tile_x, tile_y, overlap, upscale_amount = (1/8), out_channels=4, output_device=self.output_device, pbar=pbar)
|
||||||
samples += comfy.utils.tiled_scale(pixel_samples, encode_fn, tile_x * 2, tile_y // 2, overlap, upscale_amount = (1/8), out_channels=4, pbar=pbar)
|
samples += comfy.utils.tiled_scale(pixel_samples, encode_fn, tile_x * 2, tile_y // 2, overlap, upscale_amount = (1/8), out_channels=4, output_device=self.output_device, pbar=pbar)
|
||||||
samples += comfy.utils.tiled_scale(pixel_samples, encode_fn, tile_x // 2, tile_y * 2, overlap, upscale_amount = (1/8), out_channels=4, pbar=pbar)
|
samples += comfy.utils.tiled_scale(pixel_samples, encode_fn, tile_x // 2, tile_y * 2, overlap, upscale_amount = (1/8), out_channels=4, output_device=self.output_device, pbar=pbar)
|
||||||
samples /= 3.0
|
samples /= 3.0
|
||||||
return samples
|
return samples
|
||||||
|
|
||||||
def decode(self, samples_in):
|
def decode(self, samples_in):
|
||||||
self.first_stage_model = self.first_stage_model.to(self.device)
|
|
||||||
try:
|
try:
|
||||||
memory_used = (2562 * samples_in.shape[2] * samples_in.shape[3] * 64) * 1.7
|
memory_used = self.memory_used_decode(samples_in.shape, self.vae_dtype)
|
||||||
model_management.free_memory(memory_used, self.device)
|
model_management.load_models_gpu([self.patcher], memory_required=memory_used)
|
||||||
free_memory = model_management.get_free_memory(self.device)
|
free_memory = model_management.get_free_memory(self.device)
|
||||||
batch_number = int(free_memory / memory_used)
|
batch_number = int(free_memory / memory_used)
|
||||||
batch_number = max(1, batch_number)
|
batch_number = max(1, batch_number)
|
||||||
|
|
||||||
pixel_samples = torch.empty((samples_in.shape[0], 3, round(samples_in.shape[2] * 8), round(samples_in.shape[3] * 8)), device="cpu")
|
pixel_samples = torch.empty((samples_in.shape[0], 3, round(samples_in.shape[2] * 8), round(samples_in.shape[3] * 8)), device=self.output_device)
|
||||||
for x in range(0, samples_in.shape[0], batch_number):
|
for x in range(0, samples_in.shape[0], batch_number):
|
||||||
samples = samples_in[x:x+batch_number].to(self.vae_dtype).to(self.device)
|
samples = samples_in[x:x+batch_number].to(self.vae_dtype).to(self.device)
|
||||||
pixel_samples[x:x+batch_number] = torch.clamp((self.first_stage_model.decode(samples).cpu().float() + 1.0) / 2.0, min=0.0, max=1.0)
|
pixel_samples[x:x+batch_number] = torch.clamp((self.first_stage_model.decode(samples).to(self.output_device).float() + 1.0) / 2.0, min=0.0, max=1.0)
|
||||||
except model_management.OOM_EXCEPTION as e:
|
except model_management.OOM_EXCEPTION as e:
|
||||||
print("Warning: Ran out of memory when regular VAE decoding, retrying with tiled VAE decoding.")
|
print("Warning: Ran out of memory when regular VAE decoding, retrying with tiled VAE decoding.")
|
||||||
pixel_samples = self.decode_tiled_(samples_in)
|
pixel_samples = self.decode_tiled_(samples_in)
|
||||||
|
|
||||||
self.first_stage_model = self.first_stage_model.to(self.offload_device)
|
pixel_samples = pixel_samples.to(self.output_device).movedim(1,-1)
|
||||||
pixel_samples = pixel_samples.cpu().movedim(1,-1)
|
|
||||||
return pixel_samples
|
return pixel_samples
|
||||||
|
|
||||||
def decode_tiled(self, samples, tile_x=64, tile_y=64, overlap = 16):
|
def decode_tiled(self, samples, tile_x=64, tile_y=64, overlap = 16):
|
||||||
self.first_stage_model = self.first_stage_model.to(self.device)
|
model_management.load_model_gpu(self.patcher)
|
||||||
output = self.decode_tiled_(samples, tile_x, tile_y, overlap)
|
output = self.decode_tiled_(samples, tile_x, tile_y, overlap)
|
||||||
self.first_stage_model = self.first_stage_model.to(self.offload_device)
|
|
||||||
return output.movedim(1,-1)
|
return output.movedim(1,-1)
|
||||||
|
|
||||||
def encode(self, pixel_samples):
|
def encode(self, pixel_samples):
|
||||||
self.first_stage_model = self.first_stage_model.to(self.device)
|
|
||||||
pixel_samples = pixel_samples.movedim(-1,1)
|
pixel_samples = pixel_samples.movedim(-1,1)
|
||||||
try:
|
try:
|
||||||
memory_used = (2078 * pixel_samples.shape[2] * pixel_samples.shape[3]) * 1.7 #NOTE: this constant along with the one in the decode above are estimated from the mem usage for the VAE and could change.
|
memory_used = self.memory_used_encode(pixel_samples.shape, self.vae_dtype)
|
||||||
model_management.free_memory(memory_used, self.device)
|
model_management.load_models_gpu([self.patcher], memory_required=memory_used)
|
||||||
free_memory = model_management.get_free_memory(self.device)
|
free_memory = model_management.get_free_memory(self.device)
|
||||||
batch_number = int(free_memory / memory_used)
|
batch_number = int(free_memory / memory_used)
|
||||||
batch_number = max(1, batch_number)
|
batch_number = max(1, batch_number)
|
||||||
samples = torch.empty((pixel_samples.shape[0], 4, round(pixel_samples.shape[2] // 8), round(pixel_samples.shape[3] // 8)), device="cpu")
|
samples = torch.empty((pixel_samples.shape[0], 4, round(pixel_samples.shape[2] // 8), round(pixel_samples.shape[3] // 8)), device=self.output_device)
|
||||||
for x in range(0, pixel_samples.shape[0], batch_number):
|
for x in range(0, pixel_samples.shape[0], batch_number):
|
||||||
pixels_in = (2. * pixel_samples[x:x+batch_number] - 1.).to(self.vae_dtype).to(self.device)
|
pixels_in = (2. * pixel_samples[x:x+batch_number] - 1.).to(self.vae_dtype).to(self.device)
|
||||||
samples[x:x+batch_number] = self.first_stage_model.encode(pixels_in).cpu().float()
|
samples[x:x+batch_number] = self.first_stage_model.encode(pixels_in).to(self.output_device).float()
|
||||||
|
|
||||||
except model_management.OOM_EXCEPTION as e:
|
except model_management.OOM_EXCEPTION as e:
|
||||||
print("Warning: Ran out of memory when regular VAE encoding, retrying with tiled VAE encoding.")
|
print("Warning: Ran out of memory when regular VAE encoding, retrying with tiled VAE encoding.")
|
||||||
samples = self.encode_tiled_(pixel_samples)
|
samples = self.encode_tiled_(pixel_samples)
|
||||||
|
|
||||||
self.first_stage_model = self.first_stage_model.to(self.offload_device)
|
|
||||||
return samples
|
return samples
|
||||||
|
|
||||||
def encode_tiled(self, pixel_samples, tile_x=512, tile_y=512, overlap = 64):
|
def encode_tiled(self, pixel_samples, tile_x=512, tile_y=512, overlap = 64):
|
||||||
self.first_stage_model = self.first_stage_model.to(self.device)
|
model_management.load_model_gpu(self.patcher)
|
||||||
pixel_samples = pixel_samples.movedim(-1,1)
|
pixel_samples = pixel_samples.movedim(-1,1)
|
||||||
samples = self.encode_tiled_(pixel_samples, tile_x=tile_x, tile_y=tile_y, overlap=overlap)
|
samples = self.encode_tiled_(pixel_samples, tile_x=tile_x, tile_y=tile_y, overlap=overlap)
|
||||||
self.first_stage_model = self.first_stage_model.to(self.offload_device)
|
|
||||||
return samples
|
return samples
|
||||||
|
|
||||||
def get_sd(self):
|
def get_sd(self):
|
||||||
@ -424,11 +435,15 @@ def load_checkpoint_guess_config(ckpt_path, output_vae=True, output_clip=True, o
|
|||||||
|
|
||||||
parameters = comfy.utils.calculate_parameters(sd, "model.diffusion_model.")
|
parameters = comfy.utils.calculate_parameters(sd, "model.diffusion_model.")
|
||||||
unet_dtype = model_management.unet_dtype(model_params=parameters)
|
unet_dtype = model_management.unet_dtype(model_params=parameters)
|
||||||
|
load_device = model_management.get_torch_device()
|
||||||
|
manual_cast_dtype = model_management.unet_manual_cast(unet_dtype, load_device)
|
||||||
|
|
||||||
class WeightsLoader(torch.nn.Module):
|
class WeightsLoader(torch.nn.Module):
|
||||||
pass
|
pass
|
||||||
|
|
||||||
model_config = model_detection.model_config_from_unet(sd, "model.diffusion_model.", unet_dtype)
|
model_config = model_detection.model_config_from_unet(sd, "model.diffusion_model.", unet_dtype)
|
||||||
|
model_config.set_manual_cast(manual_cast_dtype)
|
||||||
|
|
||||||
if model_config is None:
|
if model_config is None:
|
||||||
raise RuntimeError("ERROR: Could not detect model type of: {}".format(ckpt_path))
|
raise RuntimeError("ERROR: Could not detect model type of: {}".format(ckpt_path))
|
||||||
|
|
||||||
@ -444,6 +459,7 @@ def load_checkpoint_guess_config(ckpt_path, output_vae=True, output_clip=True, o
|
|||||||
|
|
||||||
if output_vae:
|
if output_vae:
|
||||||
vae_sd = comfy.utils.state_dict_prefix_replace(sd, {"first_stage_model.": ""}, filter_keys=True)
|
vae_sd = comfy.utils.state_dict_prefix_replace(sd, {"first_stage_model.": ""}, filter_keys=True)
|
||||||
|
vae_sd = model_config.process_vae_state_dict(vae_sd)
|
||||||
vae = VAE(sd=vae_sd)
|
vae = VAE(sd=vae_sd)
|
||||||
|
|
||||||
if output_clip:
|
if output_clip:
|
||||||
@ -460,7 +476,7 @@ def load_checkpoint_guess_config(ckpt_path, output_vae=True, output_clip=True, o
|
|||||||
print("left over keys:", left_over)
|
print("left over keys:", left_over)
|
||||||
|
|
||||||
if output_model:
|
if output_model:
|
||||||
model_patcher = comfy.model_patcher.ModelPatcher(model, load_device=model_management.get_torch_device(), offload_device=model_management.unet_offload_device(), current_device=inital_load_device)
|
model_patcher = comfy.model_patcher.ModelPatcher(model, load_device=load_device, offload_device=model_management.unet_offload_device(), current_device=inital_load_device)
|
||||||
if inital_load_device != torch.device("cpu"):
|
if inital_load_device != torch.device("cpu"):
|
||||||
print("loaded straight to GPU")
|
print("loaded straight to GPU")
|
||||||
model_management.load_model_gpu(model_patcher)
|
model_management.load_model_gpu(model_patcher)
|
||||||
@ -468,20 +484,21 @@ def load_checkpoint_guess_config(ckpt_path, output_vae=True, output_clip=True, o
|
|||||||
return (model_patcher, clip, vae, clipvision)
|
return (model_patcher, clip, vae, clipvision)
|
||||||
|
|
||||||
|
|
||||||
def load_unet(unet_path): #load unet in diffusers format
|
def load_unet_state_dict(sd): #load unet in diffusers format
|
||||||
sd = comfy.utils.load_torch_file(unet_path)
|
|
||||||
parameters = comfy.utils.calculate_parameters(sd)
|
parameters = comfy.utils.calculate_parameters(sd)
|
||||||
unet_dtype = model_management.unet_dtype(model_params=parameters)
|
unet_dtype = model_management.unet_dtype(model_params=parameters)
|
||||||
|
load_device = model_management.get_torch_device()
|
||||||
|
manual_cast_dtype = model_management.unet_manual_cast(unet_dtype, load_device)
|
||||||
|
|
||||||
if "input_blocks.0.0.weight" in sd: #ldm
|
if "input_blocks.0.0.weight" in sd: #ldm
|
||||||
model_config = model_detection.model_config_from_unet(sd, "", unet_dtype)
|
model_config = model_detection.model_config_from_unet(sd, "", unet_dtype)
|
||||||
if model_config is None:
|
if model_config is None:
|
||||||
raise RuntimeError("ERROR: Could not detect model type of: {}".format(unet_path))
|
return None
|
||||||
new_sd = sd
|
new_sd = sd
|
||||||
|
|
||||||
else: #diffusers
|
else: #diffusers
|
||||||
model_config = model_detection.model_config_from_diffusers_unet(sd, unet_dtype)
|
model_config = model_detection.model_config_from_diffusers_unet(sd, unet_dtype)
|
||||||
if model_config is None:
|
if model_config is None:
|
||||||
print("ERROR UNSUPPORTED UNET", unet_path)
|
|
||||||
return None
|
return None
|
||||||
|
|
||||||
diffusers_keys = comfy.utils.unet_to_diffusers(model_config.unet_config)
|
diffusers_keys = comfy.utils.unet_to_diffusers(model_config.unet_config)
|
||||||
@ -493,13 +510,22 @@ def load_unet(unet_path): #load unet in diffusers format
|
|||||||
else:
|
else:
|
||||||
print(diffusers_keys[k], k)
|
print(diffusers_keys[k], k)
|
||||||
offload_device = model_management.unet_offload_device()
|
offload_device = model_management.unet_offload_device()
|
||||||
|
model_config.set_manual_cast(manual_cast_dtype)
|
||||||
model = model_config.get_model(new_sd, "")
|
model = model_config.get_model(new_sd, "")
|
||||||
model = model.to(offload_device)
|
model = model.to(offload_device)
|
||||||
model.load_model_weights(new_sd, "")
|
model.load_model_weights(new_sd, "")
|
||||||
left_over = sd.keys()
|
left_over = sd.keys()
|
||||||
if len(left_over) > 0:
|
if len(left_over) > 0:
|
||||||
print("left over keys in unet:", left_over)
|
print("left over keys in unet:", left_over)
|
||||||
return comfy.model_patcher.ModelPatcher(model, load_device=model_management.get_torch_device(), offload_device=offload_device)
|
return comfy.model_patcher.ModelPatcher(model, load_device=load_device, offload_device=offload_device)
|
||||||
|
|
||||||
|
def load_unet(unet_path):
|
||||||
|
sd = comfy.utils.load_torch_file(unet_path)
|
||||||
|
model = load_unet_state_dict(sd)
|
||||||
|
if model is None:
|
||||||
|
print("ERROR UNSUPPORTED UNET", unet_path)
|
||||||
|
raise RuntimeError("ERROR: Could not detect model type of: {}".format(unet_path))
|
||||||
|
return model
|
||||||
|
|
||||||
def save_checkpoint(output_path, model, clip, vae, metadata=None):
|
def save_checkpoint(output_path, model, clip, vae, metadata=None):
|
||||||
model_management.load_models_gpu([model, clip.load_model()])
|
model_management.load_models_gpu([model, clip.load_model()])
|
||||||
|
|||||||
@ -1,12 +1,14 @@
|
|||||||
import os
|
import os
|
||||||
|
|
||||||
from transformers import CLIPTokenizer, CLIPTextModel, CLIPTextConfig, modeling_utils
|
from transformers import CLIPTokenizer
|
||||||
import comfy.ops
|
import comfy.ops
|
||||||
import torch
|
import torch
|
||||||
import traceback
|
import traceback
|
||||||
import zipfile
|
import zipfile
|
||||||
from . import model_management
|
from . import model_management
|
||||||
import contextlib
|
import contextlib
|
||||||
|
import comfy.clip_model
|
||||||
|
import json
|
||||||
|
|
||||||
def gen_empty_tokens(special_tokens, length):
|
def gen_empty_tokens(special_tokens, length):
|
||||||
start_token = special_tokens.get("start", None)
|
start_token = special_tokens.get("start", None)
|
||||||
@ -37,7 +39,7 @@ class ClipTokenWeightEncoder:
|
|||||||
|
|
||||||
out, pooled = self.encode(to_encode)
|
out, pooled = self.encode(to_encode)
|
||||||
if pooled is not None:
|
if pooled is not None:
|
||||||
first_pooled = pooled[0:1].cpu()
|
first_pooled = pooled[0:1].to(model_management.intermediate_device())
|
||||||
else:
|
else:
|
||||||
first_pooled = pooled
|
first_pooled = pooled
|
||||||
|
|
||||||
@ -54,8 +56,8 @@ class ClipTokenWeightEncoder:
|
|||||||
output.append(z)
|
output.append(z)
|
||||||
|
|
||||||
if (len(output) == 0):
|
if (len(output) == 0):
|
||||||
return out[-1:].cpu(), first_pooled
|
return out[-1:].to(model_management.intermediate_device()), first_pooled
|
||||||
return torch.cat(output, dim=-2).cpu(), first_pooled
|
return torch.cat(output, dim=-2).to(model_management.intermediate_device()), first_pooled
|
||||||
|
|
||||||
class SDClipModel(torch.nn.Module, ClipTokenWeightEncoder):
|
class SDClipModel(torch.nn.Module, ClipTokenWeightEncoder):
|
||||||
"""Uses the CLIP transformer encoder for text (from huggingface)"""
|
"""Uses the CLIP transformer encoder for text (from huggingface)"""
|
||||||
@ -65,31 +67,19 @@ class SDClipModel(torch.nn.Module, ClipTokenWeightEncoder):
|
|||||||
"hidden"
|
"hidden"
|
||||||
]
|
]
|
||||||
def __init__(self, version="openai/clip-vit-large-patch14", device="cpu", max_length=77,
|
def __init__(self, version="openai/clip-vit-large-patch14", device="cpu", max_length=77,
|
||||||
freeze=True, layer="last", layer_idx=None, textmodel_json_config=None, textmodel_path=None, dtype=None,
|
freeze=True, layer="last", layer_idx=None, textmodel_json_config=None, dtype=None, model_class=comfy.clip_model.CLIPTextModel,
|
||||||
special_tokens={"start": 49406, "end": 49407, "pad": 49407},layer_norm_hidden_state=True, config_class=CLIPTextConfig,
|
special_tokens={"start": 49406, "end": 49407, "pad": 49407}, layer_norm_hidden_state=True): # clip-vit-base-patch32
|
||||||
model_class=CLIPTextModel, inner_name="text_model"): # clip-vit-base-patch32
|
|
||||||
super().__init__()
|
super().__init__()
|
||||||
assert layer in self.LAYERS
|
assert layer in self.LAYERS
|
||||||
self.num_layers = 12
|
|
||||||
if textmodel_path is not None:
|
|
||||||
self.transformer = model_class.from_pretrained(textmodel_path)
|
|
||||||
else:
|
|
||||||
if textmodel_json_config is None:
|
|
||||||
textmodel_json_config = os.path.join(os.path.dirname(os.path.realpath(__file__)), "sd1_clip_config.json")
|
|
||||||
config = config_class.from_json_file(textmodel_json_config)
|
|
||||||
self.num_layers = config.num_hidden_layers
|
|
||||||
with comfy.ops.use_comfy_ops(device, dtype):
|
|
||||||
with modeling_utils.no_init_weights():
|
|
||||||
self.transformer = model_class(config)
|
|
||||||
|
|
||||||
self.inner_name = inner_name
|
if textmodel_json_config is None:
|
||||||
if dtype is not None:
|
textmodel_json_config = os.path.join(os.path.dirname(os.path.realpath(__file__)), "sd1_clip_config.json")
|
||||||
self.transformer.to(dtype)
|
|
||||||
inner_model = getattr(self.transformer, self.inner_name)
|
with open(textmodel_json_config) as f:
|
||||||
if hasattr(inner_model, "embeddings"):
|
config = json.load(f)
|
||||||
inner_model.embeddings.to(torch.float32)
|
|
||||||
else:
|
self.transformer = model_class(config, dtype, device, comfy.ops.manual_cast)
|
||||||
self.transformer.set_input_embeddings(self.transformer.get_input_embeddings().to(torch.float32))
|
self.num_layers = self.transformer.num_layers
|
||||||
|
|
||||||
self.max_length = max_length
|
self.max_length = max_length
|
||||||
if freeze:
|
if freeze:
|
||||||
@ -104,7 +94,7 @@ class SDClipModel(torch.nn.Module, ClipTokenWeightEncoder):
|
|||||||
self.layer_norm_hidden_state = layer_norm_hidden_state
|
self.layer_norm_hidden_state = layer_norm_hidden_state
|
||||||
if layer == "hidden":
|
if layer == "hidden":
|
||||||
assert layer_idx is not None
|
assert layer_idx is not None
|
||||||
assert abs(layer_idx) <= self.num_layers
|
assert abs(layer_idx) < self.num_layers
|
||||||
self.clip_layer(layer_idx)
|
self.clip_layer(layer_idx)
|
||||||
self.layer_default = (self.layer, self.layer_idx)
|
self.layer_default = (self.layer, self.layer_idx)
|
||||||
|
|
||||||
@ -115,7 +105,7 @@ class SDClipModel(torch.nn.Module, ClipTokenWeightEncoder):
|
|||||||
param.requires_grad = False
|
param.requires_grad = False
|
||||||
|
|
||||||
def clip_layer(self, layer_idx):
|
def clip_layer(self, layer_idx):
|
||||||
if abs(layer_idx) >= self.num_layers:
|
if abs(layer_idx) > self.num_layers:
|
||||||
self.layer = "last"
|
self.layer = "last"
|
||||||
else:
|
else:
|
||||||
self.layer = "hidden"
|
self.layer = "hidden"
|
||||||
@ -170,41 +160,31 @@ class SDClipModel(torch.nn.Module, ClipTokenWeightEncoder):
|
|||||||
tokens = self.set_up_textual_embeddings(tokens, backup_embeds)
|
tokens = self.set_up_textual_embeddings(tokens, backup_embeds)
|
||||||
tokens = torch.LongTensor(tokens).to(device)
|
tokens = torch.LongTensor(tokens).to(device)
|
||||||
|
|
||||||
if getattr(self.transformer, self.inner_name).final_layer_norm.weight.dtype != torch.float32:
|
attention_mask = None
|
||||||
precision_scope = torch.autocast
|
if self.enable_attention_masks:
|
||||||
|
attention_mask = torch.zeros_like(tokens)
|
||||||
|
max_token = self.transformer.get_input_embeddings().weight.shape[0] - 1
|
||||||
|
for x in range(attention_mask.shape[0]):
|
||||||
|
for y in range(attention_mask.shape[1]):
|
||||||
|
attention_mask[x, y] = 1
|
||||||
|
if tokens[x, y] == max_token:
|
||||||
|
break
|
||||||
|
|
||||||
|
outputs = self.transformer(tokens, attention_mask, intermediate_output=self.layer_idx, final_layer_norm_intermediate=self.layer_norm_hidden_state)
|
||||||
|
self.transformer.set_input_embeddings(backup_embeds)
|
||||||
|
|
||||||
|
if self.layer == "last":
|
||||||
|
z = outputs[0]
|
||||||
else:
|
else:
|
||||||
precision_scope = lambda a, b: contextlib.nullcontext(a)
|
z = outputs[1]
|
||||||
|
|
||||||
with precision_scope(model_management.get_autocast_device(device), torch.float32):
|
if outputs[2] is not None:
|
||||||
attention_mask = None
|
pooled_output = outputs[2].float()
|
||||||
if self.enable_attention_masks:
|
else:
|
||||||
attention_mask = torch.zeros_like(tokens)
|
pooled_output = None
|
||||||
max_token = self.transformer.get_input_embeddings().weight.shape[0] - 1
|
|
||||||
for x in range(attention_mask.shape[0]):
|
|
||||||
for y in range(attention_mask.shape[1]):
|
|
||||||
attention_mask[x, y] = 1
|
|
||||||
if tokens[x, y] == max_token:
|
|
||||||
break
|
|
||||||
|
|
||||||
outputs = self.transformer(input_ids=tokens, attention_mask=attention_mask, output_hidden_states=self.layer=="hidden")
|
if self.text_projection is not None and pooled_output is not None:
|
||||||
self.transformer.set_input_embeddings(backup_embeds)
|
pooled_output = pooled_output.float().to(self.text_projection.device) @ self.text_projection.float()
|
||||||
|
|
||||||
if self.layer == "last":
|
|
||||||
z = outputs.last_hidden_state
|
|
||||||
elif self.layer == "pooled":
|
|
||||||
z = outputs.pooler_output[:, None, :]
|
|
||||||
else:
|
|
||||||
z = outputs.hidden_states[self.layer_idx]
|
|
||||||
if self.layer_norm_hidden_state:
|
|
||||||
z = getattr(self.transformer, self.inner_name).final_layer_norm(z)
|
|
||||||
|
|
||||||
if hasattr(outputs, "pooler_output"):
|
|
||||||
pooled_output = outputs.pooler_output.float()
|
|
||||||
else:
|
|
||||||
pooled_output = None
|
|
||||||
|
|
||||||
if self.text_projection is not None and pooled_output is not None:
|
|
||||||
pooled_output = pooled_output.float().to(self.text_projection.device) @ self.text_projection.float()
|
|
||||||
return z.float(), pooled_output
|
return z.float(), pooled_output
|
||||||
|
|
||||||
def encode(self, tokens):
|
def encode(self, tokens):
|
||||||
|
|||||||
@ -3,13 +3,13 @@ import torch
|
|||||||
import os
|
import os
|
||||||
|
|
||||||
class SD2ClipHModel(sd1_clip.SDClipModel):
|
class SD2ClipHModel(sd1_clip.SDClipModel):
|
||||||
def __init__(self, arch="ViT-H-14", device="cpu", max_length=77, freeze=True, layer="penultimate", layer_idx=None, textmodel_path=None, dtype=None):
|
def __init__(self, arch="ViT-H-14", device="cpu", max_length=77, freeze=True, layer="penultimate", layer_idx=None, dtype=None):
|
||||||
if layer == "penultimate":
|
if layer == "penultimate":
|
||||||
layer="hidden"
|
layer="hidden"
|
||||||
layer_idx=23
|
layer_idx=-2
|
||||||
|
|
||||||
textmodel_json_config = os.path.join(os.path.dirname(os.path.realpath(__file__)), "sd2_clip_config.json")
|
textmodel_json_config = os.path.join(os.path.dirname(os.path.realpath(__file__)), "sd2_clip_config.json")
|
||||||
super().__init__(device=device, freeze=freeze, layer=layer, layer_idx=layer_idx, textmodel_json_config=textmodel_json_config, textmodel_path=textmodel_path, dtype=dtype, special_tokens={"start": 49406, "end": 49407, "pad": 0})
|
super().__init__(device=device, freeze=freeze, layer=layer, layer_idx=layer_idx, textmodel_json_config=textmodel_json_config, dtype=dtype, special_tokens={"start": 49406, "end": 49407, "pad": 0})
|
||||||
|
|
||||||
class SD2ClipHTokenizer(sd1_clip.SDTokenizer):
|
class SD2ClipHTokenizer(sd1_clip.SDTokenizer):
|
||||||
def __init__(self, tokenizer_path=None, embedding_directory=None):
|
def __init__(self, tokenizer_path=None, embedding_directory=None):
|
||||||
|
|||||||
@ -3,13 +3,13 @@ import torch
|
|||||||
import os
|
import os
|
||||||
|
|
||||||
class SDXLClipG(sd1_clip.SDClipModel):
|
class SDXLClipG(sd1_clip.SDClipModel):
|
||||||
def __init__(self, device="cpu", max_length=77, freeze=True, layer="penultimate", layer_idx=None, textmodel_path=None, dtype=None):
|
def __init__(self, device="cpu", max_length=77, freeze=True, layer="penultimate", layer_idx=None, dtype=None):
|
||||||
if layer == "penultimate":
|
if layer == "penultimate":
|
||||||
layer="hidden"
|
layer="hidden"
|
||||||
layer_idx=-2
|
layer_idx=-2
|
||||||
|
|
||||||
textmodel_json_config = os.path.join(os.path.dirname(os.path.realpath(__file__)), "clip_config_bigg.json")
|
textmodel_json_config = os.path.join(os.path.dirname(os.path.realpath(__file__)), "clip_config_bigg.json")
|
||||||
super().__init__(device=device, freeze=freeze, layer=layer, layer_idx=layer_idx, textmodel_json_config=textmodel_json_config, textmodel_path=textmodel_path, dtype=dtype,
|
super().__init__(device=device, freeze=freeze, layer=layer, layer_idx=layer_idx, textmodel_json_config=textmodel_json_config, dtype=dtype,
|
||||||
special_tokens={"start": 49406, "end": 49407, "pad": 0}, layer_norm_hidden_state=False)
|
special_tokens={"start": 49406, "end": 49407, "pad": 0}, layer_norm_hidden_state=False)
|
||||||
|
|
||||||
def load_sd(self, sd):
|
def load_sd(self, sd):
|
||||||
@ -37,7 +37,7 @@ class SDXLTokenizer:
|
|||||||
class SDXLClipModel(torch.nn.Module):
|
class SDXLClipModel(torch.nn.Module):
|
||||||
def __init__(self, device="cpu", dtype=None):
|
def __init__(self, device="cpu", dtype=None):
|
||||||
super().__init__()
|
super().__init__()
|
||||||
self.clip_l = sd1_clip.SDClipModel(layer="hidden", layer_idx=11, device=device, dtype=dtype, layer_norm_hidden_state=False)
|
self.clip_l = sd1_clip.SDClipModel(layer="hidden", layer_idx=-2, device=device, dtype=dtype, layer_norm_hidden_state=False)
|
||||||
self.clip_g = SDXLClipG(device=device, dtype=dtype)
|
self.clip_g = SDXLClipG(device=device, dtype=dtype)
|
||||||
|
|
||||||
def clip_layer(self, layer_idx):
|
def clip_layer(self, layer_idx):
|
||||||
|
|||||||
@ -17,6 +17,7 @@ class SD15(supported_models_base.BASE):
|
|||||||
"model_channels": 320,
|
"model_channels": 320,
|
||||||
"use_linear_in_transformer": False,
|
"use_linear_in_transformer": False,
|
||||||
"adm_in_channels": None,
|
"adm_in_channels": None,
|
||||||
|
"use_temporal_attention": False,
|
||||||
}
|
}
|
||||||
|
|
||||||
unet_extra_config = {
|
unet_extra_config = {
|
||||||
@ -56,6 +57,7 @@ class SD20(supported_models_base.BASE):
|
|||||||
"model_channels": 320,
|
"model_channels": 320,
|
||||||
"use_linear_in_transformer": True,
|
"use_linear_in_transformer": True,
|
||||||
"adm_in_channels": None,
|
"adm_in_channels": None,
|
||||||
|
"use_temporal_attention": False,
|
||||||
}
|
}
|
||||||
|
|
||||||
latent_format = latent_formats.SD15
|
latent_format = latent_formats.SD15
|
||||||
@ -69,6 +71,10 @@ class SD20(supported_models_base.BASE):
|
|||||||
return model_base.ModelType.EPS
|
return model_base.ModelType.EPS
|
||||||
|
|
||||||
def process_clip_state_dict(self, state_dict):
|
def process_clip_state_dict(self, state_dict):
|
||||||
|
replace_prefix = {}
|
||||||
|
replace_prefix["conditioner.embedders.0.model."] = "cond_stage_model.model." #SD2 in sgm format
|
||||||
|
state_dict = utils.state_dict_prefix_replace(state_dict, replace_prefix)
|
||||||
|
|
||||||
state_dict = utils.transformers_convert(state_dict, "cond_stage_model.model.", "cond_stage_model.clip_h.transformer.text_model.", 24)
|
state_dict = utils.transformers_convert(state_dict, "cond_stage_model.model.", "cond_stage_model.clip_h.transformer.text_model.", 24)
|
||||||
return state_dict
|
return state_dict
|
||||||
|
|
||||||
@ -88,6 +94,7 @@ class SD21UnclipL(SD20):
|
|||||||
"model_channels": 320,
|
"model_channels": 320,
|
||||||
"use_linear_in_transformer": True,
|
"use_linear_in_transformer": True,
|
||||||
"adm_in_channels": 1536,
|
"adm_in_channels": 1536,
|
||||||
|
"use_temporal_attention": False,
|
||||||
}
|
}
|
||||||
|
|
||||||
clip_vision_prefix = "embedder.model.visual."
|
clip_vision_prefix = "embedder.model.visual."
|
||||||
@ -100,6 +107,7 @@ class SD21UnclipH(SD20):
|
|||||||
"model_channels": 320,
|
"model_channels": 320,
|
||||||
"use_linear_in_transformer": True,
|
"use_linear_in_transformer": True,
|
||||||
"adm_in_channels": 2048,
|
"adm_in_channels": 2048,
|
||||||
|
"use_temporal_attention": False,
|
||||||
}
|
}
|
||||||
|
|
||||||
clip_vision_prefix = "embedder.model.visual."
|
clip_vision_prefix = "embedder.model.visual."
|
||||||
@ -112,6 +120,7 @@ class SDXLRefiner(supported_models_base.BASE):
|
|||||||
"context_dim": 1280,
|
"context_dim": 1280,
|
||||||
"adm_in_channels": 2560,
|
"adm_in_channels": 2560,
|
||||||
"transformer_depth": [0, 0, 4, 4, 4, 4, 0, 0],
|
"transformer_depth": [0, 0, 4, 4, 4, 4, 0, 0],
|
||||||
|
"use_temporal_attention": False,
|
||||||
}
|
}
|
||||||
|
|
||||||
latent_format = latent_formats.SDXL
|
latent_format = latent_formats.SDXL
|
||||||
@ -148,7 +157,8 @@ class SDXL(supported_models_base.BASE):
|
|||||||
"use_linear_in_transformer": True,
|
"use_linear_in_transformer": True,
|
||||||
"transformer_depth": [0, 0, 2, 2, 10, 10],
|
"transformer_depth": [0, 0, 2, 2, 10, 10],
|
||||||
"context_dim": 2048,
|
"context_dim": 2048,
|
||||||
"adm_in_channels": 2816
|
"adm_in_channels": 2816,
|
||||||
|
"use_temporal_attention": False,
|
||||||
}
|
}
|
||||||
|
|
||||||
latent_format = latent_formats.SDXL
|
latent_format = latent_formats.SDXL
|
||||||
@ -203,8 +213,71 @@ class SSD1B(SDXL):
|
|||||||
"use_linear_in_transformer": True,
|
"use_linear_in_transformer": True,
|
||||||
"transformer_depth": [0, 0, 2, 2, 4, 4],
|
"transformer_depth": [0, 0, 2, 2, 4, 4],
|
||||||
"context_dim": 2048,
|
"context_dim": 2048,
|
||||||
"adm_in_channels": 2816
|
"adm_in_channels": 2816,
|
||||||
|
"use_temporal_attention": False,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
class Segmind_Vega(SDXL):
|
||||||
|
unet_config = {
|
||||||
|
"model_channels": 320,
|
||||||
|
"use_linear_in_transformer": True,
|
||||||
|
"transformer_depth": [0, 0, 1, 1, 2, 2],
|
||||||
|
"context_dim": 2048,
|
||||||
|
"adm_in_channels": 2816,
|
||||||
|
"use_temporal_attention": False,
|
||||||
|
}
|
||||||
|
|
||||||
models = [SD15, SD20, SD21UnclipL, SD21UnclipH, SDXLRefiner, SDXL, SSD1B]
|
class SVD_img2vid(supported_models_base.BASE):
|
||||||
|
unet_config = {
|
||||||
|
"model_channels": 320,
|
||||||
|
"in_channels": 8,
|
||||||
|
"use_linear_in_transformer": True,
|
||||||
|
"transformer_depth": [1, 1, 1, 1, 1, 1, 0, 0],
|
||||||
|
"context_dim": 1024,
|
||||||
|
"adm_in_channels": 768,
|
||||||
|
"use_temporal_attention": True,
|
||||||
|
"use_temporal_resblock": True
|
||||||
|
}
|
||||||
|
|
||||||
|
clip_vision_prefix = "conditioner.embedders.0.open_clip.model.visual."
|
||||||
|
|
||||||
|
latent_format = latent_formats.SD15
|
||||||
|
|
||||||
|
sampling_settings = {"sigma_max": 700.0, "sigma_min": 0.002}
|
||||||
|
|
||||||
|
def get_model(self, state_dict, prefix="", device=None):
|
||||||
|
out = model_base.SVD_img2vid(self, device=device)
|
||||||
|
return out
|
||||||
|
|
||||||
|
def clip_target(self):
|
||||||
|
return None
|
||||||
|
|
||||||
|
class Stable_Zero123(supported_models_base.BASE):
|
||||||
|
unet_config = {
|
||||||
|
"context_dim": 768,
|
||||||
|
"model_channels": 320,
|
||||||
|
"use_linear_in_transformer": False,
|
||||||
|
"adm_in_channels": None,
|
||||||
|
"use_temporal_attention": False,
|
||||||
|
"in_channels": 8,
|
||||||
|
}
|
||||||
|
|
||||||
|
unet_extra_config = {
|
||||||
|
"num_heads": 8,
|
||||||
|
"num_head_channels": -1,
|
||||||
|
}
|
||||||
|
|
||||||
|
clip_vision_prefix = "cond_stage_model.model.visual."
|
||||||
|
|
||||||
|
latent_format = latent_formats.SD15
|
||||||
|
|
||||||
|
def get_model(self, state_dict, prefix="", device=None):
|
||||||
|
out = model_base.Stable_Zero123(self, device=device, cc_projection_weight=state_dict["cc_projection.weight"], cc_projection_bias=state_dict["cc_projection.bias"])
|
||||||
|
return out
|
||||||
|
|
||||||
|
def clip_target(self):
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
models = [Stable_Zero123, SD15, SD20, SD21UnclipL, SD21UnclipH, SDXLRefiner, SDXL, SSD1B, Segmind_Vega]
|
||||||
|
models += [SVD_img2vid]
|
||||||
|
|||||||
@ -19,9 +19,11 @@ class BASE:
|
|||||||
clip_prefix = []
|
clip_prefix = []
|
||||||
clip_vision_prefix = None
|
clip_vision_prefix = None
|
||||||
noise_aug_config = None
|
noise_aug_config = None
|
||||||
beta_schedule = "linear"
|
sampling_settings = {}
|
||||||
latent_format = latent_formats.LatentFormat
|
latent_format = latent_formats.LatentFormat
|
||||||
|
|
||||||
|
manual_cast_dtype = None
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def matches(s, unet_config):
|
def matches(s, unet_config):
|
||||||
for k in s.unet_config:
|
for k in s.unet_config:
|
||||||
@ -53,6 +55,12 @@ class BASE:
|
|||||||
def process_clip_state_dict(self, state_dict):
|
def process_clip_state_dict(self, state_dict):
|
||||||
return state_dict
|
return state_dict
|
||||||
|
|
||||||
|
def process_unet_state_dict(self, state_dict):
|
||||||
|
return state_dict
|
||||||
|
|
||||||
|
def process_vae_state_dict(self, state_dict):
|
||||||
|
return state_dict
|
||||||
|
|
||||||
def process_clip_state_dict_for_saving(self, state_dict):
|
def process_clip_state_dict_for_saving(self, state_dict):
|
||||||
replace_prefix = {"": "cond_stage_model."}
|
replace_prefix = {"": "cond_stage_model."}
|
||||||
return utils.state_dict_prefix_replace(state_dict, replace_prefix)
|
return utils.state_dict_prefix_replace(state_dict, replace_prefix)
|
||||||
@ -65,3 +73,5 @@ class BASE:
|
|||||||
replace_prefix = {"": "first_stage_model."}
|
replace_prefix = {"": "first_stage_model."}
|
||||||
return utils.state_dict_prefix_replace(state_dict, replace_prefix)
|
return utils.state_dict_prefix_replace(state_dict, replace_prefix)
|
||||||
|
|
||||||
|
def set_manual_cast(self, manual_cast_dtype):
|
||||||
|
self.manual_cast_dtype = manual_cast_dtype
|
||||||
|
|||||||
@ -7,9 +7,10 @@ import torch
|
|||||||
import torch.nn as nn
|
import torch.nn as nn
|
||||||
|
|
||||||
import comfy.utils
|
import comfy.utils
|
||||||
|
import comfy.ops
|
||||||
|
|
||||||
def conv(n_in, n_out, **kwargs):
|
def conv(n_in, n_out, **kwargs):
|
||||||
return nn.Conv2d(n_in, n_out, 3, padding=1, **kwargs)
|
return comfy.ops.disable_weight_init.Conv2d(n_in, n_out, 3, padding=1, **kwargs)
|
||||||
|
|
||||||
class Clamp(nn.Module):
|
class Clamp(nn.Module):
|
||||||
def forward(self, x):
|
def forward(self, x):
|
||||||
@ -19,7 +20,7 @@ class Block(nn.Module):
|
|||||||
def __init__(self, n_in, n_out):
|
def __init__(self, n_in, n_out):
|
||||||
super().__init__()
|
super().__init__()
|
||||||
self.conv = nn.Sequential(conv(n_in, n_out), nn.ReLU(), conv(n_out, n_out), nn.ReLU(), conv(n_out, n_out))
|
self.conv = nn.Sequential(conv(n_in, n_out), nn.ReLU(), conv(n_out, n_out), nn.ReLU(), conv(n_out, n_out))
|
||||||
self.skip = nn.Conv2d(n_in, n_out, 1, bias=False) if n_in != n_out else nn.Identity()
|
self.skip = comfy.ops.disable_weight_init.Conv2d(n_in, n_out, 1, bias=False) if n_in != n_out else nn.Identity()
|
||||||
self.fuse = nn.ReLU()
|
self.fuse = nn.ReLU()
|
||||||
def forward(self, x):
|
def forward(self, x):
|
||||||
return self.fuse(self.conv(x) + self.skip(x))
|
return self.fuse(self.conv(x) + self.skip(x))
|
||||||
@ -46,15 +47,16 @@ class TAESD(nn.Module):
|
|||||||
latent_magnitude = 3
|
latent_magnitude = 3
|
||||||
latent_shift = 0.5
|
latent_shift = 0.5
|
||||||
|
|
||||||
def __init__(self, encoder_path="taesd_encoder.pth", decoder_path="taesd_decoder.pth"):
|
def __init__(self, encoder_path=None, decoder_path=None):
|
||||||
"""Initialize pretrained TAESD on the given device from the given checkpoints."""
|
"""Initialize pretrained TAESD on the given device from the given checkpoints."""
|
||||||
super().__init__()
|
super().__init__()
|
||||||
self.encoder = Encoder()
|
self.taesd_encoder = Encoder()
|
||||||
self.decoder = Decoder()
|
self.taesd_decoder = Decoder()
|
||||||
|
self.vae_scale = torch.nn.Parameter(torch.tensor(1.0))
|
||||||
if encoder_path is not None:
|
if encoder_path is not None:
|
||||||
self.encoder.load_state_dict(comfy.utils.load_torch_file(encoder_path, safe_load=True))
|
self.taesd_encoder.load_state_dict(comfy.utils.load_torch_file(encoder_path, safe_load=True))
|
||||||
if decoder_path is not None:
|
if decoder_path is not None:
|
||||||
self.decoder.load_state_dict(comfy.utils.load_torch_file(decoder_path, safe_load=True))
|
self.taesd_decoder.load_state_dict(comfy.utils.load_torch_file(decoder_path, safe_load=True))
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def scale_latents(x):
|
def scale_latents(x):
|
||||||
@ -65,3 +67,11 @@ class TAESD(nn.Module):
|
|||||||
def unscale_latents(x):
|
def unscale_latents(x):
|
||||||
"""[0, 1] -> raw latents"""
|
"""[0, 1] -> raw latents"""
|
||||||
return x.sub(TAESD.latent_shift).mul(2 * TAESD.latent_magnitude)
|
return x.sub(TAESD.latent_shift).mul(2 * TAESD.latent_magnitude)
|
||||||
|
|
||||||
|
def decode(self, x):
|
||||||
|
x_sample = self.taesd_decoder(x * self.vae_scale)
|
||||||
|
x_sample = x_sample.sub(0.5).mul(2)
|
||||||
|
return x_sample
|
||||||
|
|
||||||
|
def encode(self, x):
|
||||||
|
return self.taesd_encoder(x * 0.5 + 0.5) / self.vae_scale
|
||||||
|
|||||||
@ -239,6 +239,26 @@ def repeat_to_batch_size(tensor, batch_size):
|
|||||||
return tensor.repeat([math.ceil(batch_size / tensor.shape[0])] + [1] * (len(tensor.shape) - 1))[:batch_size]
|
return tensor.repeat([math.ceil(batch_size / tensor.shape[0])] + [1] * (len(tensor.shape) - 1))[:batch_size]
|
||||||
return tensor
|
return tensor
|
||||||
|
|
||||||
|
def resize_to_batch_size(tensor, batch_size):
|
||||||
|
in_batch_size = tensor.shape[0]
|
||||||
|
if in_batch_size == batch_size:
|
||||||
|
return tensor
|
||||||
|
|
||||||
|
if batch_size <= 1:
|
||||||
|
return tensor[:batch_size]
|
||||||
|
|
||||||
|
output = torch.empty([batch_size] + list(tensor.shape)[1:], dtype=tensor.dtype, device=tensor.device)
|
||||||
|
if batch_size < in_batch_size:
|
||||||
|
scale = (in_batch_size - 1) / (batch_size - 1)
|
||||||
|
for i in range(batch_size):
|
||||||
|
output[i] = tensor[min(round(i * scale), in_batch_size - 1)]
|
||||||
|
else:
|
||||||
|
scale = in_batch_size / batch_size
|
||||||
|
for i in range(batch_size):
|
||||||
|
output[i] = tensor[min(math.floor((i + 0.5) * scale), in_batch_size - 1)]
|
||||||
|
|
||||||
|
return output
|
||||||
|
|
||||||
def convert_sd_to(state_dict, dtype):
|
def convert_sd_to(state_dict, dtype):
|
||||||
keys = list(state_dict.keys())
|
keys = list(state_dict.keys())
|
||||||
for k in keys:
|
for k in keys:
|
||||||
@ -258,7 +278,7 @@ def set_attr(obj, attr, value):
|
|||||||
for name in attrs[:-1]:
|
for name in attrs[:-1]:
|
||||||
obj = getattr(obj, name)
|
obj = getattr(obj, name)
|
||||||
prev = getattr(obj, attrs[-1])
|
prev = getattr(obj, attrs[-1])
|
||||||
setattr(obj, attrs[-1], torch.nn.Parameter(value))
|
setattr(obj, attrs[-1], torch.nn.Parameter(value, requires_grad=False))
|
||||||
del prev
|
del prev
|
||||||
|
|
||||||
def copy_to_param(obj, attr, value):
|
def copy_to_param(obj, attr, value):
|
||||||
@ -307,23 +327,25 @@ def bislerp(samples, width, height):
|
|||||||
res[dot < 1e-5 - 1] = (b1 * (1.0-r) + b2 * r)[dot < 1e-5 - 1]
|
res[dot < 1e-5 - 1] = (b1 * (1.0-r) + b2 * r)[dot < 1e-5 - 1]
|
||||||
return res
|
return res
|
||||||
|
|
||||||
def generate_bilinear_data(length_old, length_new):
|
def generate_bilinear_data(length_old, length_new, device):
|
||||||
coords_1 = torch.arange(length_old).reshape((1,1,1,-1)).to(torch.float32)
|
coords_1 = torch.arange(length_old, dtype=torch.float32, device=device).reshape((1,1,1,-1))
|
||||||
coords_1 = torch.nn.functional.interpolate(coords_1, size=(1, length_new), mode="bilinear")
|
coords_1 = torch.nn.functional.interpolate(coords_1, size=(1, length_new), mode="bilinear")
|
||||||
ratios = coords_1 - coords_1.floor()
|
ratios = coords_1 - coords_1.floor()
|
||||||
coords_1 = coords_1.to(torch.int64)
|
coords_1 = coords_1.to(torch.int64)
|
||||||
|
|
||||||
coords_2 = torch.arange(length_old).reshape((1,1,1,-1)).to(torch.float32) + 1
|
coords_2 = torch.arange(length_old, dtype=torch.float32, device=device).reshape((1,1,1,-1)) + 1
|
||||||
coords_2[:,:,:,-1] -= 1
|
coords_2[:,:,:,-1] -= 1
|
||||||
coords_2 = torch.nn.functional.interpolate(coords_2, size=(1, length_new), mode="bilinear")
|
coords_2 = torch.nn.functional.interpolate(coords_2, size=(1, length_new), mode="bilinear")
|
||||||
coords_2 = coords_2.to(torch.int64)
|
coords_2 = coords_2.to(torch.int64)
|
||||||
return ratios, coords_1, coords_2
|
return ratios, coords_1, coords_2
|
||||||
|
|
||||||
|
orig_dtype = samples.dtype
|
||||||
|
samples = samples.float()
|
||||||
n,c,h,w = samples.shape
|
n,c,h,w = samples.shape
|
||||||
h_new, w_new = (height, width)
|
h_new, w_new = (height, width)
|
||||||
|
|
||||||
#linear w
|
#linear w
|
||||||
ratios, coords_1, coords_2 = generate_bilinear_data(w, w_new)
|
ratios, coords_1, coords_2 = generate_bilinear_data(w, w_new, samples.device)
|
||||||
coords_1 = coords_1.expand((n, c, h, -1))
|
coords_1 = coords_1.expand((n, c, h, -1))
|
||||||
coords_2 = coords_2.expand((n, c, h, -1))
|
coords_2 = coords_2.expand((n, c, h, -1))
|
||||||
ratios = ratios.expand((n, 1, h, -1))
|
ratios = ratios.expand((n, 1, h, -1))
|
||||||
@ -336,7 +358,7 @@ def bislerp(samples, width, height):
|
|||||||
result = result.reshape(n, h, w_new, c).movedim(-1, 1)
|
result = result.reshape(n, h, w_new, c).movedim(-1, 1)
|
||||||
|
|
||||||
#linear h
|
#linear h
|
||||||
ratios, coords_1, coords_2 = generate_bilinear_data(h, h_new)
|
ratios, coords_1, coords_2 = generate_bilinear_data(h, h_new, samples.device)
|
||||||
coords_1 = coords_1.reshape((1,1,-1,1)).expand((n, c, -1, w_new))
|
coords_1 = coords_1.reshape((1,1,-1,1)).expand((n, c, -1, w_new))
|
||||||
coords_2 = coords_2.reshape((1,1,-1,1)).expand((n, c, -1, w_new))
|
coords_2 = coords_2.reshape((1,1,-1,1)).expand((n, c, -1, w_new))
|
||||||
ratios = ratios.reshape((1,1,-1,1)).expand((n, 1, -1, w_new))
|
ratios = ratios.reshape((1,1,-1,1)).expand((n, 1, -1, w_new))
|
||||||
@ -347,14 +369,14 @@ def bislerp(samples, width, height):
|
|||||||
|
|
||||||
result = slerp(pass_1, pass_2, ratios)
|
result = slerp(pass_1, pass_2, ratios)
|
||||||
result = result.reshape(n, h_new, w_new, c).movedim(-1, 1)
|
result = result.reshape(n, h_new, w_new, c).movedim(-1, 1)
|
||||||
return result
|
return result.to(orig_dtype)
|
||||||
|
|
||||||
def lanczos(samples, width, height):
|
def lanczos(samples, width, height):
|
||||||
images = [Image.fromarray(np.clip(255. * image.movedim(0, -1).cpu().numpy(), 0, 255).astype(np.uint8)) for image in samples]
|
images = [Image.fromarray(np.clip(255. * image.movedim(0, -1).cpu().numpy(), 0, 255).astype(np.uint8)) for image in samples]
|
||||||
images = [image.resize((width, height), resample=Image.Resampling.LANCZOS) for image in images]
|
images = [image.resize((width, height), resample=Image.Resampling.LANCZOS) for image in images]
|
||||||
images = [torch.from_numpy(np.array(image).astype(np.float32) / 255.0).movedim(-1, 0) for image in images]
|
images = [torch.from_numpy(np.array(image).astype(np.float32) / 255.0).movedim(-1, 0) for image in images]
|
||||||
result = torch.stack(images)
|
result = torch.stack(images)
|
||||||
return result
|
return result.to(samples.device, samples.dtype)
|
||||||
|
|
||||||
def common_upscale(samples, width, height, upscale_method, crop):
|
def common_upscale(samples, width, height, upscale_method, crop):
|
||||||
if crop == "center":
|
if crop == "center":
|
||||||
@ -383,17 +405,17 @@ def get_tiled_scale_steps(width, height, tile_x, tile_y, overlap):
|
|||||||
return math.ceil((height / (tile_y - overlap))) * math.ceil((width / (tile_x - overlap)))
|
return math.ceil((height / (tile_y - overlap))) * math.ceil((width / (tile_x - overlap)))
|
||||||
|
|
||||||
@torch.inference_mode()
|
@torch.inference_mode()
|
||||||
def tiled_scale(samples, function, tile_x=64, tile_y=64, overlap = 8, upscale_amount = 4, out_channels = 3, pbar = None):
|
def tiled_scale(samples, function, tile_x=64, tile_y=64, overlap = 8, upscale_amount = 4, out_channels = 3, output_device="cpu", pbar = None):
|
||||||
output = torch.empty((samples.shape[0], out_channels, round(samples.shape[2] * upscale_amount), round(samples.shape[3] * upscale_amount)), device="cpu")
|
output = torch.empty((samples.shape[0], out_channels, round(samples.shape[2] * upscale_amount), round(samples.shape[3] * upscale_amount)), device=output_device)
|
||||||
for b in range(samples.shape[0]):
|
for b in range(samples.shape[0]):
|
||||||
s = samples[b:b+1]
|
s = samples[b:b+1]
|
||||||
out = torch.zeros((s.shape[0], out_channels, round(s.shape[2] * upscale_amount), round(s.shape[3] * upscale_amount)), device="cpu")
|
out = torch.zeros((s.shape[0], out_channels, round(s.shape[2] * upscale_amount), round(s.shape[3] * upscale_amount)), device=output_device)
|
||||||
out_div = torch.zeros((s.shape[0], out_channels, round(s.shape[2] * upscale_amount), round(s.shape[3] * upscale_amount)), device="cpu")
|
out_div = torch.zeros((s.shape[0], out_channels, round(s.shape[2] * upscale_amount), round(s.shape[3] * upscale_amount)), device=output_device)
|
||||||
for y in range(0, s.shape[2], tile_y - overlap):
|
for y in range(0, s.shape[2], tile_y - overlap):
|
||||||
for x in range(0, s.shape[3], tile_x - overlap):
|
for x in range(0, s.shape[3], tile_x - overlap):
|
||||||
s_in = s[:,:,y:y+tile_y,x:x+tile_x]
|
s_in = s[:,:,y:y+tile_y,x:x+tile_x]
|
||||||
|
|
||||||
ps = function(s_in).cpu()
|
ps = function(s_in).to(output_device)
|
||||||
mask = torch.ones_like(ps)
|
mask = torch.ones_like(ps)
|
||||||
feather = round(overlap * upscale_amount)
|
feather = round(overlap * upscale_amount)
|
||||||
for t in range(feather):
|
for t in range(feather):
|
||||||
|
|||||||
@ -291,7 +291,7 @@ class Canny:
|
|||||||
|
|
||||||
def detect_edge(self, image, low_threshold, high_threshold):
|
def detect_edge(self, image, low_threshold, high_threshold):
|
||||||
output = canny(image.to(comfy.model_management.get_torch_device()).movedim(-1, 1), low_threshold, high_threshold)
|
output = canny(image.to(comfy.model_management.get_torch_device()).movedim(-1, 1), low_threshold, high_threshold)
|
||||||
img_out = output[1].cpu().repeat(1, 3, 1, 1).movedim(1, -1)
|
img_out = output[1].to(comfy.model_management.intermediate_device()).repeat(1, 3, 1, 1).movedim(1, -1)
|
||||||
return (img_out,)
|
return (img_out,)
|
||||||
|
|
||||||
NODE_CLASS_MAPPINGS = {
|
NODE_CLASS_MAPPINGS = {
|
||||||
|
|||||||
@ -13,15 +13,21 @@ class BasicScheduler:
|
|||||||
{"model": ("MODEL",),
|
{"model": ("MODEL",),
|
||||||
"scheduler": (comfy.samplers.SCHEDULER_NAMES, ),
|
"scheduler": (comfy.samplers.SCHEDULER_NAMES, ),
|
||||||
"steps": ("INT", {"default": 20, "min": 1, "max": 10000}),
|
"steps": ("INT", {"default": 20, "min": 1, "max": 10000}),
|
||||||
|
"denoise": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 1.0, "step": 0.01}),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
RETURN_TYPES = ("SIGMAS",)
|
RETURN_TYPES = ("SIGMAS",)
|
||||||
CATEGORY = "sampling/custom_sampling"
|
CATEGORY = "sampling/custom_sampling/schedulers"
|
||||||
|
|
||||||
FUNCTION = "get_sigmas"
|
FUNCTION = "get_sigmas"
|
||||||
|
|
||||||
def get_sigmas(self, model, scheduler, steps):
|
def get_sigmas(self, model, scheduler, steps, denoise):
|
||||||
sigmas = comfy.samplers.calculate_sigmas_scheduler(model.model, scheduler, steps).cpu()
|
total_steps = steps
|
||||||
|
if denoise < 1.0:
|
||||||
|
total_steps = int(steps/denoise)
|
||||||
|
|
||||||
|
sigmas = comfy.samplers.calculate_sigmas_scheduler(model.model, scheduler, total_steps).cpu()
|
||||||
|
sigmas = sigmas[-(steps + 1):]
|
||||||
return (sigmas, )
|
return (sigmas, )
|
||||||
|
|
||||||
|
|
||||||
@ -36,7 +42,7 @@ class KarrasScheduler:
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
RETURN_TYPES = ("SIGMAS",)
|
RETURN_TYPES = ("SIGMAS",)
|
||||||
CATEGORY = "sampling/custom_sampling"
|
CATEGORY = "sampling/custom_sampling/schedulers"
|
||||||
|
|
||||||
FUNCTION = "get_sigmas"
|
FUNCTION = "get_sigmas"
|
||||||
|
|
||||||
@ -54,7 +60,7 @@ class ExponentialScheduler:
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
RETURN_TYPES = ("SIGMAS",)
|
RETURN_TYPES = ("SIGMAS",)
|
||||||
CATEGORY = "sampling/custom_sampling"
|
CATEGORY = "sampling/custom_sampling/schedulers"
|
||||||
|
|
||||||
FUNCTION = "get_sigmas"
|
FUNCTION = "get_sigmas"
|
||||||
|
|
||||||
@ -73,7 +79,7 @@ class PolyexponentialScheduler:
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
RETURN_TYPES = ("SIGMAS",)
|
RETURN_TYPES = ("SIGMAS",)
|
||||||
CATEGORY = "sampling/custom_sampling"
|
CATEGORY = "sampling/custom_sampling/schedulers"
|
||||||
|
|
||||||
FUNCTION = "get_sigmas"
|
FUNCTION = "get_sigmas"
|
||||||
|
|
||||||
@ -81,6 +87,27 @@ class PolyexponentialScheduler:
|
|||||||
sigmas = k_diffusion_sampling.get_sigmas_polyexponential(n=steps, sigma_min=sigma_min, sigma_max=sigma_max, rho=rho)
|
sigmas = k_diffusion_sampling.get_sigmas_polyexponential(n=steps, sigma_min=sigma_min, sigma_max=sigma_max, rho=rho)
|
||||||
return (sigmas, )
|
return (sigmas, )
|
||||||
|
|
||||||
|
class SDTurboScheduler:
|
||||||
|
@classmethod
|
||||||
|
def INPUT_TYPES(s):
|
||||||
|
return {"required":
|
||||||
|
{"model": ("MODEL",),
|
||||||
|
"steps": ("INT", {"default": 1, "min": 1, "max": 10}),
|
||||||
|
"denoise": ("FLOAT", {"default": 1.0, "min": 0, "max": 1.0, "step": 0.01}),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
RETURN_TYPES = ("SIGMAS",)
|
||||||
|
CATEGORY = "sampling/custom_sampling/schedulers"
|
||||||
|
|
||||||
|
FUNCTION = "get_sigmas"
|
||||||
|
|
||||||
|
def get_sigmas(self, model, steps, denoise):
|
||||||
|
start_step = 10 - int(10 * denoise)
|
||||||
|
timesteps = torch.flip(torch.arange(1, 11) * 100 - 1, (0,))[start_step:start_step + steps]
|
||||||
|
sigmas = model.model.model_sampling.sigma(timesteps)
|
||||||
|
sigmas = torch.cat([sigmas, sigmas.new_zeros([1])])
|
||||||
|
return (sigmas, )
|
||||||
|
|
||||||
class VPScheduler:
|
class VPScheduler:
|
||||||
@classmethod
|
@classmethod
|
||||||
def INPUT_TYPES(s):
|
def INPUT_TYPES(s):
|
||||||
@ -92,7 +119,7 @@ class VPScheduler:
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
RETURN_TYPES = ("SIGMAS",)
|
RETURN_TYPES = ("SIGMAS",)
|
||||||
CATEGORY = "sampling/custom_sampling"
|
CATEGORY = "sampling/custom_sampling/schedulers"
|
||||||
|
|
||||||
FUNCTION = "get_sigmas"
|
FUNCTION = "get_sigmas"
|
||||||
|
|
||||||
@ -109,7 +136,7 @@ class SplitSigmas:
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
RETURN_TYPES = ("SIGMAS","SIGMAS")
|
RETURN_TYPES = ("SIGMAS","SIGMAS")
|
||||||
CATEGORY = "sampling/custom_sampling"
|
CATEGORY = "sampling/custom_sampling/sigmas"
|
||||||
|
|
||||||
FUNCTION = "get_sigmas"
|
FUNCTION = "get_sigmas"
|
||||||
|
|
||||||
@ -118,6 +145,24 @@ class SplitSigmas:
|
|||||||
sigmas2 = sigmas[step:]
|
sigmas2 = sigmas[step:]
|
||||||
return (sigmas1, sigmas2)
|
return (sigmas1, sigmas2)
|
||||||
|
|
||||||
|
class FlipSigmas:
|
||||||
|
@classmethod
|
||||||
|
def INPUT_TYPES(s):
|
||||||
|
return {"required":
|
||||||
|
{"sigmas": ("SIGMAS", ),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
RETURN_TYPES = ("SIGMAS",)
|
||||||
|
CATEGORY = "sampling/custom_sampling/sigmas"
|
||||||
|
|
||||||
|
FUNCTION = "get_sigmas"
|
||||||
|
|
||||||
|
def get_sigmas(self, sigmas):
|
||||||
|
sigmas = sigmas.flip(0)
|
||||||
|
if sigmas[0] == 0:
|
||||||
|
sigmas[0] = 0.0001
|
||||||
|
return (sigmas,)
|
||||||
|
|
||||||
class KSamplerSelect:
|
class KSamplerSelect:
|
||||||
@classmethod
|
@classmethod
|
||||||
def INPUT_TYPES(s):
|
def INPUT_TYPES(s):
|
||||||
@ -126,12 +171,12 @@ class KSamplerSelect:
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
RETURN_TYPES = ("SAMPLER",)
|
RETURN_TYPES = ("SAMPLER",)
|
||||||
CATEGORY = "sampling/custom_sampling"
|
CATEGORY = "sampling/custom_sampling/samplers"
|
||||||
|
|
||||||
FUNCTION = "get_sampler"
|
FUNCTION = "get_sampler"
|
||||||
|
|
||||||
def get_sampler(self, sampler_name):
|
def get_sampler(self, sampler_name):
|
||||||
sampler = comfy.samplers.sampler_class(sampler_name)()
|
sampler = comfy.samplers.sampler_object(sampler_name)
|
||||||
return (sampler, )
|
return (sampler, )
|
||||||
|
|
||||||
class SamplerDPMPP_2M_SDE:
|
class SamplerDPMPP_2M_SDE:
|
||||||
@ -145,7 +190,7 @@ class SamplerDPMPP_2M_SDE:
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
RETURN_TYPES = ("SAMPLER",)
|
RETURN_TYPES = ("SAMPLER",)
|
||||||
CATEGORY = "sampling/custom_sampling"
|
CATEGORY = "sampling/custom_sampling/samplers"
|
||||||
|
|
||||||
FUNCTION = "get_sampler"
|
FUNCTION = "get_sampler"
|
||||||
|
|
||||||
@ -154,7 +199,7 @@ class SamplerDPMPP_2M_SDE:
|
|||||||
sampler_name = "dpmpp_2m_sde"
|
sampler_name = "dpmpp_2m_sde"
|
||||||
else:
|
else:
|
||||||
sampler_name = "dpmpp_2m_sde_gpu"
|
sampler_name = "dpmpp_2m_sde_gpu"
|
||||||
sampler = comfy.samplers.ksampler(sampler_name, {"eta": eta, "s_noise": s_noise, "solver_type": solver_type})()
|
sampler = comfy.samplers.ksampler(sampler_name, {"eta": eta, "s_noise": s_noise, "solver_type": solver_type})
|
||||||
return (sampler, )
|
return (sampler, )
|
||||||
|
|
||||||
|
|
||||||
@ -169,7 +214,7 @@ class SamplerDPMPP_SDE:
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
RETURN_TYPES = ("SAMPLER",)
|
RETURN_TYPES = ("SAMPLER",)
|
||||||
CATEGORY = "sampling/custom_sampling"
|
CATEGORY = "sampling/custom_sampling/samplers"
|
||||||
|
|
||||||
FUNCTION = "get_sampler"
|
FUNCTION = "get_sampler"
|
||||||
|
|
||||||
@ -178,7 +223,7 @@ class SamplerDPMPP_SDE:
|
|||||||
sampler_name = "dpmpp_sde"
|
sampler_name = "dpmpp_sde"
|
||||||
else:
|
else:
|
||||||
sampler_name = "dpmpp_sde_gpu"
|
sampler_name = "dpmpp_sde_gpu"
|
||||||
sampler = comfy.samplers.ksampler(sampler_name, {"eta": eta, "s_noise": s_noise, "r": r})()
|
sampler = comfy.samplers.ksampler(sampler_name, {"eta": eta, "s_noise": s_noise, "r": r})
|
||||||
return (sampler, )
|
return (sampler, )
|
||||||
|
|
||||||
class SamplerCustom:
|
class SamplerCustom:
|
||||||
@ -234,13 +279,15 @@ class SamplerCustom:
|
|||||||
|
|
||||||
NODE_CLASS_MAPPINGS = {
|
NODE_CLASS_MAPPINGS = {
|
||||||
"SamplerCustom": SamplerCustom,
|
"SamplerCustom": SamplerCustom,
|
||||||
|
"BasicScheduler": BasicScheduler,
|
||||||
"KarrasScheduler": KarrasScheduler,
|
"KarrasScheduler": KarrasScheduler,
|
||||||
"ExponentialScheduler": ExponentialScheduler,
|
"ExponentialScheduler": ExponentialScheduler,
|
||||||
"PolyexponentialScheduler": PolyexponentialScheduler,
|
"PolyexponentialScheduler": PolyexponentialScheduler,
|
||||||
"VPScheduler": VPScheduler,
|
"VPScheduler": VPScheduler,
|
||||||
|
"SDTurboScheduler": SDTurboScheduler,
|
||||||
"KSamplerSelect": KSamplerSelect,
|
"KSamplerSelect": KSamplerSelect,
|
||||||
"SamplerDPMPP_2M_SDE": SamplerDPMPP_2M_SDE,
|
"SamplerDPMPP_2M_SDE": SamplerDPMPP_2M_SDE,
|
||||||
"SamplerDPMPP_SDE": SamplerDPMPP_SDE,
|
"SamplerDPMPP_SDE": SamplerDPMPP_SDE,
|
||||||
"BasicScheduler": BasicScheduler,
|
|
||||||
"SplitSigmas": SplitSigmas,
|
"SplitSigmas": SplitSigmas,
|
||||||
|
"FlipSigmas": FlipSigmas,
|
||||||
}
|
}
|
||||||
|
|||||||
@ -2,9 +2,10 @@
|
|||||||
|
|
||||||
import math
|
import math
|
||||||
from einops import rearrange
|
from einops import rearrange
|
||||||
import random
|
# Use torch rng for consistency across generations
|
||||||
|
from torch import randint
|
||||||
|
|
||||||
def random_divisor(value: int, min_value: int, /, max_options: int = 1, counter = 0) -> int:
|
def random_divisor(value: int, min_value: int, /, max_options: int = 1) -> int:
|
||||||
min_value = min(min_value, value)
|
min_value = min(min_value, value)
|
||||||
|
|
||||||
# All big divisors of value (inclusive)
|
# All big divisors of value (inclusive)
|
||||||
@ -12,8 +13,10 @@ def random_divisor(value: int, min_value: int, /, max_options: int = 1, counter
|
|||||||
|
|
||||||
ns = [value // i for i in divisors[:max_options]] # has at least 1 element
|
ns = [value // i for i in divisors[:max_options]] # has at least 1 element
|
||||||
|
|
||||||
random.seed(counter)
|
if len(ns) - 1 > 0:
|
||||||
idx = random.randint(0, len(ns) - 1)
|
idx = randint(low=0, high=len(ns) - 1, size=(1,)).item()
|
||||||
|
else:
|
||||||
|
idx = 0
|
||||||
|
|
||||||
return ns[idx]
|
return ns[idx]
|
||||||
|
|
||||||
@ -42,7 +45,6 @@ class HyperTile:
|
|||||||
|
|
||||||
latent_tile_size = max(32, tile_size) // 8
|
latent_tile_size = max(32, tile_size) // 8
|
||||||
self.temp = None
|
self.temp = None
|
||||||
self.counter = 1
|
|
||||||
|
|
||||||
def hypertile_in(q, k, v, extra_options):
|
def hypertile_in(q, k, v, extra_options):
|
||||||
if q.shape[-1] in apply_to:
|
if q.shape[-1] in apply_to:
|
||||||
@ -53,10 +55,8 @@ class HyperTile:
|
|||||||
h, w = round(math.sqrt(hw * aspect_ratio)), round(math.sqrt(hw / aspect_ratio))
|
h, w = round(math.sqrt(hw * aspect_ratio)), round(math.sqrt(hw / aspect_ratio))
|
||||||
|
|
||||||
factor = 2**((q.shape[-1] // model_channels) - 1) if scale_depth else 1
|
factor = 2**((q.shape[-1] // model_channels) - 1) if scale_depth else 1
|
||||||
nh = random_divisor(h, latent_tile_size * factor, swap_size, self.counter)
|
nh = random_divisor(h, latent_tile_size * factor, swap_size)
|
||||||
self.counter += 1
|
nw = random_divisor(w, latent_tile_size * factor, swap_size)
|
||||||
nw = random_divisor(w, latent_tile_size * factor, swap_size, self.counter)
|
|
||||||
self.counter += 1
|
|
||||||
|
|
||||||
if nh * nw > 1:
|
if nh * nw > 1:
|
||||||
q = rearrange(q, "b (nh h nw w) c -> (b nh nw) (h w) c", h=h // nh, w=w // nw, nh=nh, nw=nw)
|
q = rearrange(q, "b (nh h nw w) c -> (b nh nw) (h w) c", h=h // nh, w=w // nw, nh=nh, nw=nw)
|
||||||
|
|||||||
175
comfy_extras/nodes_images.py
Normal file
175
comfy_extras/nodes_images.py
Normal file
@ -0,0 +1,175 @@
|
|||||||
|
import nodes
|
||||||
|
import folder_paths
|
||||||
|
from comfy.cli_args import args
|
||||||
|
|
||||||
|
from PIL import Image
|
||||||
|
from PIL.PngImagePlugin import PngInfo
|
||||||
|
|
||||||
|
import numpy as np
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
|
||||||
|
MAX_RESOLUTION = nodes.MAX_RESOLUTION
|
||||||
|
|
||||||
|
class ImageCrop:
|
||||||
|
@classmethod
|
||||||
|
def INPUT_TYPES(s):
|
||||||
|
return {"required": { "image": ("IMAGE",),
|
||||||
|
"width": ("INT", {"default": 512, "min": 1, "max": MAX_RESOLUTION, "step": 1}),
|
||||||
|
"height": ("INT", {"default": 512, "min": 1, "max": MAX_RESOLUTION, "step": 1}),
|
||||||
|
"x": ("INT", {"default": 0, "min": 0, "max": MAX_RESOLUTION, "step": 1}),
|
||||||
|
"y": ("INT", {"default": 0, "min": 0, "max": MAX_RESOLUTION, "step": 1}),
|
||||||
|
}}
|
||||||
|
RETURN_TYPES = ("IMAGE",)
|
||||||
|
FUNCTION = "crop"
|
||||||
|
|
||||||
|
CATEGORY = "image/transform"
|
||||||
|
|
||||||
|
def crop(self, image, width, height, x, y):
|
||||||
|
x = min(x, image.shape[2] - 1)
|
||||||
|
y = min(y, image.shape[1] - 1)
|
||||||
|
to_x = width + x
|
||||||
|
to_y = height + y
|
||||||
|
img = image[:,y:to_y, x:to_x, :]
|
||||||
|
return (img,)
|
||||||
|
|
||||||
|
class RepeatImageBatch:
|
||||||
|
@classmethod
|
||||||
|
def INPUT_TYPES(s):
|
||||||
|
return {"required": { "image": ("IMAGE",),
|
||||||
|
"amount": ("INT", {"default": 1, "min": 1, "max": 64}),
|
||||||
|
}}
|
||||||
|
RETURN_TYPES = ("IMAGE",)
|
||||||
|
FUNCTION = "repeat"
|
||||||
|
|
||||||
|
CATEGORY = "image/batch"
|
||||||
|
|
||||||
|
def repeat(self, image, amount):
|
||||||
|
s = image.repeat((amount, 1,1,1))
|
||||||
|
return (s,)
|
||||||
|
|
||||||
|
class SaveAnimatedWEBP:
|
||||||
|
def __init__(self):
|
||||||
|
self.output_dir = folder_paths.get_output_directory()
|
||||||
|
self.type = "output"
|
||||||
|
self.prefix_append = ""
|
||||||
|
|
||||||
|
methods = {"default": 4, "fastest": 0, "slowest": 6}
|
||||||
|
@classmethod
|
||||||
|
def INPUT_TYPES(s):
|
||||||
|
return {"required":
|
||||||
|
{"images": ("IMAGE", ),
|
||||||
|
"filename_prefix": ("STRING", {"default": "ComfyUI"}),
|
||||||
|
"fps": ("FLOAT", {"default": 6.0, "min": 0.01, "max": 1000.0, "step": 0.01}),
|
||||||
|
"lossless": ("BOOLEAN", {"default": True}),
|
||||||
|
"quality": ("INT", {"default": 80, "min": 0, "max": 100}),
|
||||||
|
"method": (list(s.methods.keys()),),
|
||||||
|
# "num_frames": ("INT", {"default": 0, "min": 0, "max": 8192}),
|
||||||
|
},
|
||||||
|
"hidden": {"prompt": "PROMPT", "extra_pnginfo": "EXTRA_PNGINFO"},
|
||||||
|
}
|
||||||
|
|
||||||
|
RETURN_TYPES = ()
|
||||||
|
FUNCTION = "save_images"
|
||||||
|
|
||||||
|
OUTPUT_NODE = True
|
||||||
|
|
||||||
|
CATEGORY = "image/animation"
|
||||||
|
|
||||||
|
def save_images(self, images, fps, filename_prefix, lossless, quality, method, num_frames=0, prompt=None, extra_pnginfo=None):
|
||||||
|
method = self.methods.get(method)
|
||||||
|
filename_prefix += self.prefix_append
|
||||||
|
full_output_folder, filename, counter, subfolder, filename_prefix = folder_paths.get_save_image_path(filename_prefix, self.output_dir, images[0].shape[1], images[0].shape[0])
|
||||||
|
results = list()
|
||||||
|
pil_images = []
|
||||||
|
for image in images:
|
||||||
|
i = 255. * image.cpu().numpy()
|
||||||
|
img = Image.fromarray(np.clip(i, 0, 255).astype(np.uint8))
|
||||||
|
pil_images.append(img)
|
||||||
|
|
||||||
|
metadata = pil_images[0].getexif()
|
||||||
|
if not args.disable_metadata:
|
||||||
|
if prompt is not None:
|
||||||
|
metadata[0x0110] = "prompt:{}".format(json.dumps(prompt))
|
||||||
|
if extra_pnginfo is not None:
|
||||||
|
inital_exif = 0x010f
|
||||||
|
for x in extra_pnginfo:
|
||||||
|
metadata[inital_exif] = "{}:{}".format(x, json.dumps(extra_pnginfo[x]))
|
||||||
|
inital_exif -= 1
|
||||||
|
|
||||||
|
if num_frames == 0:
|
||||||
|
num_frames = len(pil_images)
|
||||||
|
|
||||||
|
c = len(pil_images)
|
||||||
|
for i in range(0, c, num_frames):
|
||||||
|
file = f"{filename}_{counter:05}_.webp"
|
||||||
|
pil_images[i].save(os.path.join(full_output_folder, file), save_all=True, duration=int(1000.0/fps), append_images=pil_images[i + 1:i + num_frames], exif=metadata, lossless=lossless, quality=quality, method=method)
|
||||||
|
results.append({
|
||||||
|
"filename": file,
|
||||||
|
"subfolder": subfolder,
|
||||||
|
"type": self.type
|
||||||
|
})
|
||||||
|
counter += 1
|
||||||
|
|
||||||
|
animated = num_frames != 1
|
||||||
|
return { "ui": { "images": results, "animated": (animated,) } }
|
||||||
|
|
||||||
|
class SaveAnimatedPNG:
|
||||||
|
def __init__(self):
|
||||||
|
self.output_dir = folder_paths.get_output_directory()
|
||||||
|
self.type = "output"
|
||||||
|
self.prefix_append = ""
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def INPUT_TYPES(s):
|
||||||
|
return {"required":
|
||||||
|
{"images": ("IMAGE", ),
|
||||||
|
"filename_prefix": ("STRING", {"default": "ComfyUI"}),
|
||||||
|
"fps": ("FLOAT", {"default": 6.0, "min": 0.01, "max": 1000.0, "step": 0.01}),
|
||||||
|
"compress_level": ("INT", {"default": 4, "min": 0, "max": 9})
|
||||||
|
},
|
||||||
|
"hidden": {"prompt": "PROMPT", "extra_pnginfo": "EXTRA_PNGINFO"},
|
||||||
|
}
|
||||||
|
|
||||||
|
RETURN_TYPES = ()
|
||||||
|
FUNCTION = "save_images"
|
||||||
|
|
||||||
|
OUTPUT_NODE = True
|
||||||
|
|
||||||
|
CATEGORY = "image/animation"
|
||||||
|
|
||||||
|
def save_images(self, images, fps, compress_level, filename_prefix="ComfyUI", prompt=None, extra_pnginfo=None):
|
||||||
|
filename_prefix += self.prefix_append
|
||||||
|
full_output_folder, filename, counter, subfolder, filename_prefix = folder_paths.get_save_image_path(filename_prefix, self.output_dir, images[0].shape[1], images[0].shape[0])
|
||||||
|
results = list()
|
||||||
|
pil_images = []
|
||||||
|
for image in images:
|
||||||
|
i = 255. * image.cpu().numpy()
|
||||||
|
img = Image.fromarray(np.clip(i, 0, 255).astype(np.uint8))
|
||||||
|
pil_images.append(img)
|
||||||
|
|
||||||
|
metadata = None
|
||||||
|
if not args.disable_metadata:
|
||||||
|
metadata = PngInfo()
|
||||||
|
if prompt is not None:
|
||||||
|
metadata.add(b"comf", "prompt".encode("latin-1", "strict") + b"\0" + json.dumps(prompt).encode("latin-1", "strict"), after_idat=True)
|
||||||
|
if extra_pnginfo is not None:
|
||||||
|
for x in extra_pnginfo:
|
||||||
|
metadata.add(b"comf", x.encode("latin-1", "strict") + b"\0" + json.dumps(extra_pnginfo[x]).encode("latin-1", "strict"), after_idat=True)
|
||||||
|
|
||||||
|
file = f"{filename}_{counter:05}_.png"
|
||||||
|
pil_images[0].save(os.path.join(full_output_folder, file), pnginfo=metadata, compress_level=compress_level, save_all=True, duration=int(1000.0/fps), append_images=pil_images[1:])
|
||||||
|
results.append({
|
||||||
|
"filename": file,
|
||||||
|
"subfolder": subfolder,
|
||||||
|
"type": self.type
|
||||||
|
})
|
||||||
|
|
||||||
|
return { "ui": { "images": results, "animated": (True,)} }
|
||||||
|
|
||||||
|
NODE_CLASS_MAPPINGS = {
|
||||||
|
"ImageCrop": ImageCrop,
|
||||||
|
"RepeatImageBatch": RepeatImageBatch,
|
||||||
|
"SaveAnimatedWEBP": SaveAnimatedWEBP,
|
||||||
|
"SaveAnimatedPNG": SaveAnimatedPNG,
|
||||||
|
}
|
||||||
@ -1,10 +1,9 @@
|
|||||||
import comfy.utils
|
import comfy.utils
|
||||||
|
import torch
|
||||||
|
|
||||||
def reshape_latent_to(target_shape, latent):
|
def reshape_latent_to(target_shape, latent):
|
||||||
if latent.shape[1:] != target_shape[1:]:
|
if latent.shape[1:] != target_shape[1:]:
|
||||||
latent.movedim(1, -1)
|
|
||||||
latent = comfy.utils.common_upscale(latent, target_shape[3], target_shape[2], "bilinear", "center")
|
latent = comfy.utils.common_upscale(latent, target_shape[3], target_shape[2], "bilinear", "center")
|
||||||
latent.movedim(-1, 1)
|
|
||||||
return comfy.utils.repeat_to_batch_size(latent, target_shape[0])
|
return comfy.utils.repeat_to_batch_size(latent, target_shape[0])
|
||||||
|
|
||||||
|
|
||||||
@ -67,8 +66,66 @@ class LatentMultiply:
|
|||||||
samples_out["samples"] = s1 * multiplier
|
samples_out["samples"] = s1 * multiplier
|
||||||
return (samples_out,)
|
return (samples_out,)
|
||||||
|
|
||||||
|
class LatentInterpolate:
|
||||||
|
@classmethod
|
||||||
|
def INPUT_TYPES(s):
|
||||||
|
return {"required": { "samples1": ("LATENT",),
|
||||||
|
"samples2": ("LATENT",),
|
||||||
|
"ratio": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 1.0, "step": 0.01}),
|
||||||
|
}}
|
||||||
|
|
||||||
|
RETURN_TYPES = ("LATENT",)
|
||||||
|
FUNCTION = "op"
|
||||||
|
|
||||||
|
CATEGORY = "latent/advanced"
|
||||||
|
|
||||||
|
def op(self, samples1, samples2, ratio):
|
||||||
|
samples_out = samples1.copy()
|
||||||
|
|
||||||
|
s1 = samples1["samples"]
|
||||||
|
s2 = samples2["samples"]
|
||||||
|
|
||||||
|
s2 = reshape_latent_to(s1.shape, s2)
|
||||||
|
|
||||||
|
m1 = torch.linalg.vector_norm(s1, dim=(1))
|
||||||
|
m2 = torch.linalg.vector_norm(s2, dim=(1))
|
||||||
|
|
||||||
|
s1 = torch.nan_to_num(s1 / m1)
|
||||||
|
s2 = torch.nan_to_num(s2 / m2)
|
||||||
|
|
||||||
|
t = (s1 * ratio + s2 * (1.0 - ratio))
|
||||||
|
mt = torch.linalg.vector_norm(t, dim=(1))
|
||||||
|
st = torch.nan_to_num(t / mt)
|
||||||
|
|
||||||
|
samples_out["samples"] = st * (m1 * ratio + m2 * (1.0 - ratio))
|
||||||
|
return (samples_out,)
|
||||||
|
|
||||||
|
class LatentBatch:
|
||||||
|
@classmethod
|
||||||
|
def INPUT_TYPES(s):
|
||||||
|
return {"required": { "samples1": ("LATENT",), "samples2": ("LATENT",)}}
|
||||||
|
|
||||||
|
RETURN_TYPES = ("LATENT",)
|
||||||
|
FUNCTION = "batch"
|
||||||
|
|
||||||
|
CATEGORY = "latent/batch"
|
||||||
|
|
||||||
|
def batch(self, samples1, samples2):
|
||||||
|
samples_out = samples1.copy()
|
||||||
|
s1 = samples1["samples"]
|
||||||
|
s2 = samples2["samples"]
|
||||||
|
|
||||||
|
if s1.shape[1:] != s2.shape[1:]:
|
||||||
|
s2 = comfy.utils.common_upscale(s2, s1.shape[3], s1.shape[2], "bilinear", "center")
|
||||||
|
s = torch.cat((s1, s2), dim=0)
|
||||||
|
samples_out["samples"] = s
|
||||||
|
samples_out["batch_index"] = samples1.get("batch_index", [x for x in range(0, s1.shape[0])]) + samples2.get("batch_index", [x for x in range(0, s2.shape[0])])
|
||||||
|
return (samples_out,)
|
||||||
|
|
||||||
NODE_CLASS_MAPPINGS = {
|
NODE_CLASS_MAPPINGS = {
|
||||||
"LatentAdd": LatentAdd,
|
"LatentAdd": LatentAdd,
|
||||||
"LatentSubtract": LatentSubtract,
|
"LatentSubtract": LatentSubtract,
|
||||||
"LatentMultiply": LatentMultiply,
|
"LatentMultiply": LatentMultiply,
|
||||||
|
"LatentInterpolate": LatentInterpolate,
|
||||||
|
"LatentBatch": LatentBatch,
|
||||||
}
|
}
|
||||||
|
|||||||
@ -6,6 +6,7 @@ import comfy.utils
|
|||||||
from nodes import MAX_RESOLUTION
|
from nodes import MAX_RESOLUTION
|
||||||
|
|
||||||
def composite(destination, source, x, y, mask = None, multiplier = 8, resize_source = False):
|
def composite(destination, source, x, y, mask = None, multiplier = 8, resize_source = False):
|
||||||
|
source = source.to(destination.device)
|
||||||
if resize_source:
|
if resize_source:
|
||||||
source = torch.nn.functional.interpolate(source, size=(destination.shape[2], destination.shape[3]), mode="bilinear")
|
source = torch.nn.functional.interpolate(source, size=(destination.shape[2], destination.shape[3]), mode="bilinear")
|
||||||
|
|
||||||
@ -20,7 +21,7 @@ def composite(destination, source, x, y, mask = None, multiplier = 8, resize_sou
|
|||||||
if mask is None:
|
if mask is None:
|
||||||
mask = torch.ones_like(source)
|
mask = torch.ones_like(source)
|
||||||
else:
|
else:
|
||||||
mask = mask.clone()
|
mask = mask.to(destination.device, copy=True)
|
||||||
mask = torch.nn.functional.interpolate(mask.reshape((-1, 1, mask.shape[-2], mask.shape[-1])), size=(source.shape[2], source.shape[3]), mode="bilinear")
|
mask = torch.nn.functional.interpolate(mask.reshape((-1, 1, mask.shape[-2], mask.shape[-1])), size=(source.shape[2], source.shape[3]), mode="bilinear")
|
||||||
mask = comfy.utils.repeat_to_batch_size(mask, source.shape[0])
|
mask = comfy.utils.repeat_to_batch_size(mask, source.shape[0])
|
||||||
|
|
||||||
|
|||||||
@ -17,56 +17,32 @@ class LCM(comfy.model_sampling.EPS):
|
|||||||
|
|
||||||
return c_out * x0 + c_skip * model_input
|
return c_out * x0 + c_skip * model_input
|
||||||
|
|
||||||
class ModelSamplingDiscreteLCM(torch.nn.Module):
|
class ModelSamplingDiscreteDistilled(comfy.model_sampling.ModelSamplingDiscrete):
|
||||||
def __init__(self):
|
original_timesteps = 50
|
||||||
super().__init__()
|
|
||||||
self.sigma_data = 1.0
|
|
||||||
timesteps = 1000
|
|
||||||
beta_start = 0.00085
|
|
||||||
beta_end = 0.012
|
|
||||||
|
|
||||||
betas = torch.linspace(beta_start**0.5, beta_end**0.5, timesteps, dtype=torch.float32) ** 2
|
def __init__(self, model_config=None):
|
||||||
alphas = 1.0 - betas
|
super().__init__(model_config)
|
||||||
alphas_cumprod = torch.cumprod(alphas, dim=0)
|
|
||||||
|
|
||||||
original_timesteps = 50
|
self.skip_steps = self.num_timesteps // self.original_timesteps
|
||||||
self.skip_steps = timesteps // original_timesteps
|
|
||||||
|
|
||||||
|
sigmas_valid = torch.zeros((self.original_timesteps), dtype=torch.float32)
|
||||||
|
for x in range(self.original_timesteps):
|
||||||
|
sigmas_valid[self.original_timesteps - 1 - x] = self.sigmas[self.num_timesteps - 1 - x * self.skip_steps]
|
||||||
|
|
||||||
alphas_cumprod_valid = torch.zeros((original_timesteps), dtype=torch.float32)
|
self.set_sigmas(sigmas_valid)
|
||||||
for x in range(original_timesteps):
|
|
||||||
alphas_cumprod_valid[original_timesteps - 1 - x] = alphas_cumprod[timesteps - 1 - x * self.skip_steps]
|
|
||||||
|
|
||||||
sigmas = ((1 - alphas_cumprod_valid) / alphas_cumprod_valid) ** 0.5
|
|
||||||
self.set_sigmas(sigmas)
|
|
||||||
|
|
||||||
def set_sigmas(self, sigmas):
|
|
||||||
self.register_buffer('sigmas', sigmas)
|
|
||||||
self.register_buffer('log_sigmas', sigmas.log())
|
|
||||||
|
|
||||||
@property
|
|
||||||
def sigma_min(self):
|
|
||||||
return self.sigmas[0]
|
|
||||||
|
|
||||||
@property
|
|
||||||
def sigma_max(self):
|
|
||||||
return self.sigmas[-1]
|
|
||||||
|
|
||||||
def timestep(self, sigma):
|
def timestep(self, sigma):
|
||||||
log_sigma = sigma.log()
|
log_sigma = sigma.log()
|
||||||
dists = log_sigma.to(self.log_sigmas.device) - self.log_sigmas[:, None]
|
dists = log_sigma.to(self.log_sigmas.device) - self.log_sigmas[:, None]
|
||||||
return dists.abs().argmin(dim=0).view(sigma.shape) * self.skip_steps + (self.skip_steps - 1)
|
return (dists.abs().argmin(dim=0).view(sigma.shape) * self.skip_steps + (self.skip_steps - 1)).to(sigma.device)
|
||||||
|
|
||||||
def sigma(self, timestep):
|
def sigma(self, timestep):
|
||||||
t = torch.clamp(((timestep - (self.skip_steps - 1)) / self.skip_steps).float(), min=0, max=(len(self.sigmas) - 1))
|
t = torch.clamp(((timestep.float().to(self.log_sigmas.device) - (self.skip_steps - 1)) / self.skip_steps).float(), min=0, max=(len(self.sigmas) - 1))
|
||||||
low_idx = t.floor().long()
|
low_idx = t.floor().long()
|
||||||
high_idx = t.ceil().long()
|
high_idx = t.ceil().long()
|
||||||
w = t.frac()
|
w = t.frac()
|
||||||
log_sigma = (1 - w) * self.log_sigmas[low_idx] + w * self.log_sigmas[high_idx]
|
log_sigma = (1 - w) * self.log_sigmas[low_idx] + w * self.log_sigmas[high_idx]
|
||||||
return log_sigma.exp()
|
return log_sigma.exp().to(timestep.device)
|
||||||
|
|
||||||
def percent_to_sigma(self, percent):
|
|
||||||
return self.sigma(torch.tensor(percent * 999.0))
|
|
||||||
|
|
||||||
|
|
||||||
def rescale_zero_terminal_snr_sigmas(sigmas):
|
def rescale_zero_terminal_snr_sigmas(sigmas):
|
||||||
@ -111,18 +87,48 @@ class ModelSamplingDiscrete:
|
|||||||
sampling_type = comfy.model_sampling.V_PREDICTION
|
sampling_type = comfy.model_sampling.V_PREDICTION
|
||||||
elif sampling == "lcm":
|
elif sampling == "lcm":
|
||||||
sampling_type = LCM
|
sampling_type = LCM
|
||||||
sampling_base = ModelSamplingDiscreteLCM
|
sampling_base = ModelSamplingDiscreteDistilled
|
||||||
|
|
||||||
class ModelSamplingAdvanced(sampling_base, sampling_type):
|
class ModelSamplingAdvanced(sampling_base, sampling_type):
|
||||||
pass
|
pass
|
||||||
|
|
||||||
model_sampling = ModelSamplingAdvanced()
|
model_sampling = ModelSamplingAdvanced(model.model.model_config)
|
||||||
if zsnr:
|
if zsnr:
|
||||||
model_sampling.set_sigmas(rescale_zero_terminal_snr_sigmas(model_sampling.sigmas))
|
model_sampling.set_sigmas(rescale_zero_terminal_snr_sigmas(model_sampling.sigmas))
|
||||||
|
|
||||||
m.add_object_patch("model_sampling", model_sampling)
|
m.add_object_patch("model_sampling", model_sampling)
|
||||||
return (m, )
|
return (m, )
|
||||||
|
|
||||||
|
class ModelSamplingContinuousEDM:
|
||||||
|
@classmethod
|
||||||
|
def INPUT_TYPES(s):
|
||||||
|
return {"required": { "model": ("MODEL",),
|
||||||
|
"sampling": (["v_prediction", "eps"],),
|
||||||
|
"sigma_max": ("FLOAT", {"default": 120.0, "min": 0.0, "max": 1000.0, "step":0.001, "round": False}),
|
||||||
|
"sigma_min": ("FLOAT", {"default": 0.002, "min": 0.0, "max": 1000.0, "step":0.001, "round": False}),
|
||||||
|
}}
|
||||||
|
|
||||||
|
RETURN_TYPES = ("MODEL",)
|
||||||
|
FUNCTION = "patch"
|
||||||
|
|
||||||
|
CATEGORY = "advanced/model"
|
||||||
|
|
||||||
|
def patch(self, model, sampling, sigma_max, sigma_min):
|
||||||
|
m = model.clone()
|
||||||
|
|
||||||
|
if sampling == "eps":
|
||||||
|
sampling_type = comfy.model_sampling.EPS
|
||||||
|
elif sampling == "v_prediction":
|
||||||
|
sampling_type = comfy.model_sampling.V_PREDICTION
|
||||||
|
|
||||||
|
class ModelSamplingAdvanced(comfy.model_sampling.ModelSamplingContinuousEDM, sampling_type):
|
||||||
|
pass
|
||||||
|
|
||||||
|
model_sampling = ModelSamplingAdvanced(model.model.model_config)
|
||||||
|
model_sampling.set_sigma_range(sigma_min, sigma_max)
|
||||||
|
m.add_object_patch("model_sampling", model_sampling)
|
||||||
|
return (m, )
|
||||||
|
|
||||||
class RescaleCFG:
|
class RescaleCFG:
|
||||||
@classmethod
|
@classmethod
|
||||||
def INPUT_TYPES(s):
|
def INPUT_TYPES(s):
|
||||||
@ -164,5 +170,6 @@ class RescaleCFG:
|
|||||||
|
|
||||||
NODE_CLASS_MAPPINGS = {
|
NODE_CLASS_MAPPINGS = {
|
||||||
"ModelSamplingDiscrete": ModelSamplingDiscrete,
|
"ModelSamplingDiscrete": ModelSamplingDiscrete,
|
||||||
|
"ModelSamplingContinuousEDM": ModelSamplingContinuousEDM,
|
||||||
"RescaleCFG": RescaleCFG,
|
"RescaleCFG": RescaleCFG,
|
||||||
}
|
}
|
||||||
|
|||||||
53
comfy_extras/nodes_model_downscale.py
Normal file
53
comfy_extras/nodes_model_downscale.py
Normal file
@ -0,0 +1,53 @@
|
|||||||
|
import torch
|
||||||
|
import comfy.utils
|
||||||
|
|
||||||
|
class PatchModelAddDownscale:
|
||||||
|
upscale_methods = ["bicubic", "nearest-exact", "bilinear", "area", "bislerp"]
|
||||||
|
@classmethod
|
||||||
|
def INPUT_TYPES(s):
|
||||||
|
return {"required": { "model": ("MODEL",),
|
||||||
|
"block_number": ("INT", {"default": 3, "min": 1, "max": 32, "step": 1}),
|
||||||
|
"downscale_factor": ("FLOAT", {"default": 2.0, "min": 0.1, "max": 9.0, "step": 0.001}),
|
||||||
|
"start_percent": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 1.0, "step": 0.001}),
|
||||||
|
"end_percent": ("FLOAT", {"default": 0.35, "min": 0.0, "max": 1.0, "step": 0.001}),
|
||||||
|
"downscale_after_skip": ("BOOLEAN", {"default": True}),
|
||||||
|
"downscale_method": (s.upscale_methods,),
|
||||||
|
"upscale_method": (s.upscale_methods,),
|
||||||
|
}}
|
||||||
|
RETURN_TYPES = ("MODEL",)
|
||||||
|
FUNCTION = "patch"
|
||||||
|
|
||||||
|
CATEGORY = "_for_testing"
|
||||||
|
|
||||||
|
def patch(self, model, block_number, downscale_factor, start_percent, end_percent, downscale_after_skip, downscale_method, upscale_method):
|
||||||
|
sigma_start = model.model.model_sampling.percent_to_sigma(start_percent)
|
||||||
|
sigma_end = model.model.model_sampling.percent_to_sigma(end_percent)
|
||||||
|
|
||||||
|
def input_block_patch(h, transformer_options):
|
||||||
|
if transformer_options["block"][1] == block_number:
|
||||||
|
sigma = transformer_options["sigmas"][0].item()
|
||||||
|
if sigma <= sigma_start and sigma >= sigma_end:
|
||||||
|
h = comfy.utils.common_upscale(h, round(h.shape[-1] * (1.0 / downscale_factor)), round(h.shape[-2] * (1.0 / downscale_factor)), downscale_method, "disabled")
|
||||||
|
return h
|
||||||
|
|
||||||
|
def output_block_patch(h, hsp, transformer_options):
|
||||||
|
if h.shape[2] != hsp.shape[2]:
|
||||||
|
h = comfy.utils.common_upscale(h, hsp.shape[-1], hsp.shape[-2], upscale_method, "disabled")
|
||||||
|
return h, hsp
|
||||||
|
|
||||||
|
m = model.clone()
|
||||||
|
if downscale_after_skip:
|
||||||
|
m.set_model_input_block_patch_after_skip(input_block_patch)
|
||||||
|
else:
|
||||||
|
m.set_model_input_block_patch(input_block_patch)
|
||||||
|
m.set_model_output_block_patch(output_block_patch)
|
||||||
|
return (m, )
|
||||||
|
|
||||||
|
NODE_CLASS_MAPPINGS = {
|
||||||
|
"PatchModelAddDownscale": PatchModelAddDownscale,
|
||||||
|
}
|
||||||
|
|
||||||
|
NODE_DISPLAY_NAME_MAPPINGS = {
|
||||||
|
# Sampling
|
||||||
|
"PatchModelAddDownscale": "PatchModelAddDownscale (Kohya Deep Shrink)",
|
||||||
|
}
|
||||||
55
comfy_extras/nodes_perpneg.py
Normal file
55
comfy_extras/nodes_perpneg.py
Normal file
@ -0,0 +1,55 @@
|
|||||||
|
import torch
|
||||||
|
import comfy.model_management
|
||||||
|
import comfy.sample
|
||||||
|
import comfy.samplers
|
||||||
|
import comfy.utils
|
||||||
|
|
||||||
|
|
||||||
|
class PerpNeg:
|
||||||
|
@classmethod
|
||||||
|
def INPUT_TYPES(s):
|
||||||
|
return {"required": {"model": ("MODEL", ),
|
||||||
|
"empty_conditioning": ("CONDITIONING", ),
|
||||||
|
"neg_scale": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 100.0}),
|
||||||
|
}}
|
||||||
|
RETURN_TYPES = ("MODEL",)
|
||||||
|
FUNCTION = "patch"
|
||||||
|
|
||||||
|
CATEGORY = "_for_testing"
|
||||||
|
|
||||||
|
def patch(self, model, empty_conditioning, neg_scale):
|
||||||
|
m = model.clone()
|
||||||
|
nocond = comfy.sample.convert_cond(empty_conditioning)
|
||||||
|
|
||||||
|
def cfg_function(args):
|
||||||
|
model = args["model"]
|
||||||
|
noise_pred_pos = args["cond_denoised"]
|
||||||
|
noise_pred_neg = args["uncond_denoised"]
|
||||||
|
cond_scale = args["cond_scale"]
|
||||||
|
x = args["input"]
|
||||||
|
sigma = args["sigma"]
|
||||||
|
model_options = args["model_options"]
|
||||||
|
nocond_processed = comfy.samplers.encode_model_conds(model.extra_conds, nocond, x, x.device, "negative")
|
||||||
|
|
||||||
|
(noise_pred_nocond, _) = comfy.samplers.calc_cond_uncond_batch(model, nocond_processed, None, x, sigma, model_options)
|
||||||
|
|
||||||
|
pos = noise_pred_pos - noise_pred_nocond
|
||||||
|
neg = noise_pred_neg - noise_pred_nocond
|
||||||
|
perp = ((torch.mul(pos, neg).sum())/(torch.norm(neg)**2)) * neg
|
||||||
|
perp_neg = perp * neg_scale
|
||||||
|
cfg_result = noise_pred_nocond + cond_scale*(pos - perp_neg)
|
||||||
|
cfg_result = x - cfg_result
|
||||||
|
return cfg_result
|
||||||
|
|
||||||
|
m.set_model_sampler_cfg_function(cfg_function)
|
||||||
|
|
||||||
|
return (m, )
|
||||||
|
|
||||||
|
|
||||||
|
NODE_CLASS_MAPPINGS = {
|
||||||
|
"PerpNeg": PerpNeg,
|
||||||
|
}
|
||||||
|
|
||||||
|
NODE_DISPLAY_NAME_MAPPINGS = {
|
||||||
|
"PerpNeg": "Perp-Neg",
|
||||||
|
}
|
||||||
@ -226,7 +226,7 @@ class Sharpen:
|
|||||||
batch_size, height, width, channels = image.shape
|
batch_size, height, width, channels = image.shape
|
||||||
|
|
||||||
kernel_size = sharpen_radius * 2 + 1
|
kernel_size = sharpen_radius * 2 + 1
|
||||||
kernel = gaussian_kernel(kernel_size, sigma) * -(alpha*10)
|
kernel = gaussian_kernel(kernel_size, sigma, device=image.device) * -(alpha*10)
|
||||||
center = kernel_size // 2
|
center = kernel_size // 2
|
||||||
kernel[center, center] = kernel[center, center] - kernel.sum() + 1.0
|
kernel[center, center] = kernel[center, center] - kernel.sum() + 1.0
|
||||||
kernel = kernel.repeat(channels, 1, 1).unsqueeze(1)
|
kernel = kernel.repeat(channels, 1, 1).unsqueeze(1)
|
||||||
|
|||||||
@ -99,10 +99,40 @@ class LatentRebatch:
|
|||||||
|
|
||||||
return (output_list,)
|
return (output_list,)
|
||||||
|
|
||||||
|
class ImageRebatch:
|
||||||
|
@classmethod
|
||||||
|
def INPUT_TYPES(s):
|
||||||
|
return {"required": { "images": ("IMAGE",),
|
||||||
|
"batch_size": ("INT", {"default": 1, "min": 1, "max": 4096}),
|
||||||
|
}}
|
||||||
|
RETURN_TYPES = ("IMAGE",)
|
||||||
|
INPUT_IS_LIST = True
|
||||||
|
OUTPUT_IS_LIST = (True, )
|
||||||
|
|
||||||
|
FUNCTION = "rebatch"
|
||||||
|
|
||||||
|
CATEGORY = "image/batch"
|
||||||
|
|
||||||
|
def rebatch(self, images, batch_size):
|
||||||
|
batch_size = batch_size[0]
|
||||||
|
|
||||||
|
output_list = []
|
||||||
|
all_images = []
|
||||||
|
for img in images:
|
||||||
|
for i in range(img.shape[0]):
|
||||||
|
all_images.append(img[i:i+1])
|
||||||
|
|
||||||
|
for i in range(0, len(all_images), batch_size):
|
||||||
|
output_list.append(torch.cat(all_images[i:i+batch_size], dim=0))
|
||||||
|
|
||||||
|
return (output_list,)
|
||||||
|
|
||||||
NODE_CLASS_MAPPINGS = {
|
NODE_CLASS_MAPPINGS = {
|
||||||
"RebatchLatents": LatentRebatch,
|
"RebatchLatents": LatentRebatch,
|
||||||
|
"RebatchImages": ImageRebatch,
|
||||||
}
|
}
|
||||||
|
|
||||||
NODE_DISPLAY_NAME_MAPPINGS = {
|
NODE_DISPLAY_NAME_MAPPINGS = {
|
||||||
"RebatchLatents": "Rebatch Latents",
|
"RebatchLatents": "Rebatch Latents",
|
||||||
|
"RebatchImages": "Rebatch Images",
|
||||||
}
|
}
|
||||||
168
comfy_extras/nodes_sag.py
Normal file
168
comfy_extras/nodes_sag.py
Normal file
@ -0,0 +1,168 @@
|
|||||||
|
import torch
|
||||||
|
from torch import einsum
|
||||||
|
import torch.nn.functional as F
|
||||||
|
import math
|
||||||
|
|
||||||
|
from einops import rearrange, repeat
|
||||||
|
import os
|
||||||
|
from comfy.ldm.modules.attention import optimized_attention, _ATTN_PRECISION
|
||||||
|
import comfy.samplers
|
||||||
|
|
||||||
|
# from comfy/ldm/modules/attention.py
|
||||||
|
# but modified to return attention scores as well as output
|
||||||
|
def attention_basic_with_sim(q, k, v, heads, mask=None):
|
||||||
|
b, _, dim_head = q.shape
|
||||||
|
dim_head //= heads
|
||||||
|
scale = dim_head ** -0.5
|
||||||
|
|
||||||
|
h = heads
|
||||||
|
q, k, v = map(
|
||||||
|
lambda t: t.unsqueeze(3)
|
||||||
|
.reshape(b, -1, heads, dim_head)
|
||||||
|
.permute(0, 2, 1, 3)
|
||||||
|
.reshape(b * heads, -1, dim_head)
|
||||||
|
.contiguous(),
|
||||||
|
(q, k, v),
|
||||||
|
)
|
||||||
|
|
||||||
|
# force cast to fp32 to avoid overflowing
|
||||||
|
if _ATTN_PRECISION =="fp32":
|
||||||
|
sim = einsum('b i d, b j d -> b i j', q.float(), k.float()) * scale
|
||||||
|
else:
|
||||||
|
sim = einsum('b i d, b j d -> b i j', q, k) * scale
|
||||||
|
|
||||||
|
del q, k
|
||||||
|
|
||||||
|
if mask is not None:
|
||||||
|
mask = rearrange(mask, 'b ... -> b (...)')
|
||||||
|
max_neg_value = -torch.finfo(sim.dtype).max
|
||||||
|
mask = repeat(mask, 'b j -> (b h) () j', h=h)
|
||||||
|
sim.masked_fill_(~mask, max_neg_value)
|
||||||
|
|
||||||
|
# attention, what we cannot get enough of
|
||||||
|
sim = sim.softmax(dim=-1)
|
||||||
|
|
||||||
|
out = einsum('b i j, b j d -> b i d', sim.to(v.dtype), v)
|
||||||
|
out = (
|
||||||
|
out.unsqueeze(0)
|
||||||
|
.reshape(b, heads, -1, dim_head)
|
||||||
|
.permute(0, 2, 1, 3)
|
||||||
|
.reshape(b, -1, heads * dim_head)
|
||||||
|
)
|
||||||
|
return (out, sim)
|
||||||
|
|
||||||
|
def create_blur_map(x0, attn, sigma=3.0, threshold=1.0):
|
||||||
|
# reshape and GAP the attention map
|
||||||
|
_, hw1, hw2 = attn.shape
|
||||||
|
b, _, lh, lw = x0.shape
|
||||||
|
attn = attn.reshape(b, -1, hw1, hw2)
|
||||||
|
# Global Average Pool
|
||||||
|
mask = attn.mean(1, keepdim=False).sum(1, keepdim=False) > threshold
|
||||||
|
ratio = math.ceil(math.sqrt(lh * lw / hw1))
|
||||||
|
mid_shape = [math.ceil(lh / ratio), math.ceil(lw / ratio)]
|
||||||
|
|
||||||
|
# Reshape
|
||||||
|
mask = (
|
||||||
|
mask.reshape(b, *mid_shape)
|
||||||
|
.unsqueeze(1)
|
||||||
|
.type(attn.dtype)
|
||||||
|
)
|
||||||
|
# Upsample
|
||||||
|
mask = F.interpolate(mask, (lh, lw))
|
||||||
|
|
||||||
|
blurred = gaussian_blur_2d(x0, kernel_size=9, sigma=sigma)
|
||||||
|
blurred = blurred * mask + x0 * (1 - mask)
|
||||||
|
return blurred
|
||||||
|
|
||||||
|
def gaussian_blur_2d(img, kernel_size, sigma):
|
||||||
|
ksize_half = (kernel_size - 1) * 0.5
|
||||||
|
|
||||||
|
x = torch.linspace(-ksize_half, ksize_half, steps=kernel_size)
|
||||||
|
|
||||||
|
pdf = torch.exp(-0.5 * (x / sigma).pow(2))
|
||||||
|
|
||||||
|
x_kernel = pdf / pdf.sum()
|
||||||
|
x_kernel = x_kernel.to(device=img.device, dtype=img.dtype)
|
||||||
|
|
||||||
|
kernel2d = torch.mm(x_kernel[:, None], x_kernel[None, :])
|
||||||
|
kernel2d = kernel2d.expand(img.shape[-3], 1, kernel2d.shape[0], kernel2d.shape[1])
|
||||||
|
|
||||||
|
padding = [kernel_size // 2, kernel_size // 2, kernel_size // 2, kernel_size // 2]
|
||||||
|
|
||||||
|
img = F.pad(img, padding, mode="reflect")
|
||||||
|
img = F.conv2d(img, kernel2d, groups=img.shape[-3])
|
||||||
|
return img
|
||||||
|
|
||||||
|
class SelfAttentionGuidance:
|
||||||
|
@classmethod
|
||||||
|
def INPUT_TYPES(s):
|
||||||
|
return {"required": { "model": ("MODEL",),
|
||||||
|
"scale": ("FLOAT", {"default": 0.5, "min": -2.0, "max": 5.0, "step": 0.1}),
|
||||||
|
"blur_sigma": ("FLOAT", {"default": 2.0, "min": 0.0, "max": 10.0, "step": 0.1}),
|
||||||
|
}}
|
||||||
|
RETURN_TYPES = ("MODEL",)
|
||||||
|
FUNCTION = "patch"
|
||||||
|
|
||||||
|
CATEGORY = "_for_testing"
|
||||||
|
|
||||||
|
def patch(self, model, scale, blur_sigma):
|
||||||
|
m = model.clone()
|
||||||
|
|
||||||
|
attn_scores = None
|
||||||
|
|
||||||
|
# TODO: make this work properly with chunked batches
|
||||||
|
# currently, we can only save the attn from one UNet call
|
||||||
|
def attn_and_record(q, k, v, extra_options):
|
||||||
|
nonlocal attn_scores
|
||||||
|
# if uncond, save the attention scores
|
||||||
|
heads = extra_options["n_heads"]
|
||||||
|
cond_or_uncond = extra_options["cond_or_uncond"]
|
||||||
|
b = q.shape[0] // len(cond_or_uncond)
|
||||||
|
if 1 in cond_or_uncond:
|
||||||
|
uncond_index = cond_or_uncond.index(1)
|
||||||
|
# do the entire attention operation, but save the attention scores to attn_scores
|
||||||
|
(out, sim) = attention_basic_with_sim(q, k, v, heads=heads)
|
||||||
|
# when using a higher batch size, I BELIEVE the result batch dimension is [uc1, ... ucn, c1, ... cn]
|
||||||
|
n_slices = heads * b
|
||||||
|
attn_scores = sim[n_slices * uncond_index:n_slices * (uncond_index+1)]
|
||||||
|
return out
|
||||||
|
else:
|
||||||
|
return optimized_attention(q, k, v, heads=heads)
|
||||||
|
|
||||||
|
def post_cfg_function(args):
|
||||||
|
nonlocal attn_scores
|
||||||
|
uncond_attn = attn_scores
|
||||||
|
|
||||||
|
sag_scale = scale
|
||||||
|
sag_sigma = blur_sigma
|
||||||
|
sag_threshold = 1.0
|
||||||
|
model = args["model"]
|
||||||
|
uncond_pred = args["uncond_denoised"]
|
||||||
|
uncond = args["uncond"]
|
||||||
|
cfg_result = args["denoised"]
|
||||||
|
sigma = args["sigma"]
|
||||||
|
model_options = args["model_options"]
|
||||||
|
x = args["input"]
|
||||||
|
|
||||||
|
# create the adversarially blurred image
|
||||||
|
degraded = create_blur_map(uncond_pred, uncond_attn, sag_sigma, sag_threshold)
|
||||||
|
degraded_noised = degraded + x - uncond_pred
|
||||||
|
# call into the UNet
|
||||||
|
(sag, _) = comfy.samplers.calc_cond_uncond_batch(model, uncond, None, degraded_noised, sigma, model_options)
|
||||||
|
return cfg_result + (degraded - sag) * sag_scale
|
||||||
|
|
||||||
|
m.set_model_sampler_post_cfg_function(post_cfg_function, disable_cfg1_optimization=True)
|
||||||
|
|
||||||
|
# from diffusers:
|
||||||
|
# unet.mid_block.attentions[0].transformer_blocks[0].attn1.patch
|
||||||
|
m.set_model_attn1_replace(attn_and_record, "middle", 0, 0)
|
||||||
|
|
||||||
|
return (m, )
|
||||||
|
|
||||||
|
NODE_CLASS_MAPPINGS = {
|
||||||
|
"SelfAttentionGuidance": SelfAttentionGuidance,
|
||||||
|
}
|
||||||
|
|
||||||
|
NODE_DISPLAY_NAME_MAPPINGS = {
|
||||||
|
"SelfAttentionGuidance": "Self-Attention Guidance",
|
||||||
|
}
|
||||||
58
comfy_extras/nodes_stable3d.py
Normal file
58
comfy_extras/nodes_stable3d.py
Normal file
@ -0,0 +1,58 @@
|
|||||||
|
import torch
|
||||||
|
import nodes
|
||||||
|
import comfy.utils
|
||||||
|
|
||||||
|
def camera_embeddings(elevation, azimuth):
|
||||||
|
elevation = torch.as_tensor([elevation])
|
||||||
|
azimuth = torch.as_tensor([azimuth])
|
||||||
|
embeddings = torch.stack(
|
||||||
|
[
|
||||||
|
torch.deg2rad(
|
||||||
|
(90 - elevation) - (90)
|
||||||
|
), # Zero123 polar is 90-elevation
|
||||||
|
torch.sin(torch.deg2rad(azimuth)),
|
||||||
|
torch.cos(torch.deg2rad(azimuth)),
|
||||||
|
torch.deg2rad(
|
||||||
|
90 - torch.full_like(elevation, 0)
|
||||||
|
),
|
||||||
|
], dim=-1).unsqueeze(1)
|
||||||
|
|
||||||
|
return embeddings
|
||||||
|
|
||||||
|
|
||||||
|
class StableZero123_Conditioning:
|
||||||
|
@classmethod
|
||||||
|
def INPUT_TYPES(s):
|
||||||
|
return {"required": { "clip_vision": ("CLIP_VISION",),
|
||||||
|
"init_image": ("IMAGE",),
|
||||||
|
"vae": ("VAE",),
|
||||||
|
"width": ("INT", {"default": 256, "min": 16, "max": nodes.MAX_RESOLUTION, "step": 8}),
|
||||||
|
"height": ("INT", {"default": 256, "min": 16, "max": nodes.MAX_RESOLUTION, "step": 8}),
|
||||||
|
"batch_size": ("INT", {"default": 1, "min": 1, "max": 4096}),
|
||||||
|
"elevation": ("FLOAT", {"default": 0.0, "min": -180.0, "max": 180.0}),
|
||||||
|
"azimuth": ("FLOAT", {"default": 0.0, "min": -180.0, "max": 180.0}),
|
||||||
|
}}
|
||||||
|
RETURN_TYPES = ("CONDITIONING", "CONDITIONING", "LATENT")
|
||||||
|
RETURN_NAMES = ("positive", "negative", "latent")
|
||||||
|
|
||||||
|
FUNCTION = "encode"
|
||||||
|
|
||||||
|
CATEGORY = "conditioning/3d_models"
|
||||||
|
|
||||||
|
def encode(self, clip_vision, init_image, vae, width, height, batch_size, elevation, azimuth):
|
||||||
|
output = clip_vision.encode_image(init_image)
|
||||||
|
pooled = output.image_embeds.unsqueeze(0)
|
||||||
|
pixels = comfy.utils.common_upscale(init_image.movedim(-1,1), width, height, "bilinear", "center").movedim(1,-1)
|
||||||
|
encode_pixels = pixels[:,:,:,:3]
|
||||||
|
t = vae.encode(encode_pixels)
|
||||||
|
cam_embeds = camera_embeddings(elevation, azimuth)
|
||||||
|
cond = torch.cat([pooled, cam_embeds.repeat((pooled.shape[0], 1, 1))], dim=-1)
|
||||||
|
|
||||||
|
positive = [[cond, {"concat_latent_image": t}]]
|
||||||
|
negative = [[torch.zeros_like(pooled), {"concat_latent_image": torch.zeros_like(t)}]]
|
||||||
|
latent = torch.zeros([batch_size, 4, height // 8, width // 8])
|
||||||
|
return (positive, negative, {"samples":latent})
|
||||||
|
|
||||||
|
NODE_CLASS_MAPPINGS = {
|
||||||
|
"StableZero123_Conditioning": StableZero123_Conditioning,
|
||||||
|
}
|
||||||
89
comfy_extras/nodes_video_model.py
Normal file
89
comfy_extras/nodes_video_model.py
Normal file
@ -0,0 +1,89 @@
|
|||||||
|
import nodes
|
||||||
|
import torch
|
||||||
|
import comfy.utils
|
||||||
|
import comfy.sd
|
||||||
|
import folder_paths
|
||||||
|
|
||||||
|
|
||||||
|
class ImageOnlyCheckpointLoader:
|
||||||
|
@classmethod
|
||||||
|
def INPUT_TYPES(s):
|
||||||
|
return {"required": { "ckpt_name": (folder_paths.get_filename_list("checkpoints"), ),
|
||||||
|
}}
|
||||||
|
RETURN_TYPES = ("MODEL", "CLIP_VISION", "VAE")
|
||||||
|
FUNCTION = "load_checkpoint"
|
||||||
|
|
||||||
|
CATEGORY = "loaders/video_models"
|
||||||
|
|
||||||
|
def load_checkpoint(self, ckpt_name, output_vae=True, output_clip=True):
|
||||||
|
ckpt_path = folder_paths.get_full_path("checkpoints", ckpt_name)
|
||||||
|
out = comfy.sd.load_checkpoint_guess_config(ckpt_path, output_vae=True, output_clip=False, output_clipvision=True, embedding_directory=folder_paths.get_folder_paths("embeddings"))
|
||||||
|
return (out[0], out[3], out[2])
|
||||||
|
|
||||||
|
|
||||||
|
class SVD_img2vid_Conditioning:
|
||||||
|
@classmethod
|
||||||
|
def INPUT_TYPES(s):
|
||||||
|
return {"required": { "clip_vision": ("CLIP_VISION",),
|
||||||
|
"init_image": ("IMAGE",),
|
||||||
|
"vae": ("VAE",),
|
||||||
|
"width": ("INT", {"default": 1024, "min": 16, "max": nodes.MAX_RESOLUTION, "step": 8}),
|
||||||
|
"height": ("INT", {"default": 576, "min": 16, "max": nodes.MAX_RESOLUTION, "step": 8}),
|
||||||
|
"video_frames": ("INT", {"default": 14, "min": 1, "max": 4096}),
|
||||||
|
"motion_bucket_id": ("INT", {"default": 127, "min": 1, "max": 1023}),
|
||||||
|
"fps": ("INT", {"default": 6, "min": 1, "max": 1024}),
|
||||||
|
"augmentation_level": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 10.0, "step": 0.01})
|
||||||
|
}}
|
||||||
|
RETURN_TYPES = ("CONDITIONING", "CONDITIONING", "LATENT")
|
||||||
|
RETURN_NAMES = ("positive", "negative", "latent")
|
||||||
|
|
||||||
|
FUNCTION = "encode"
|
||||||
|
|
||||||
|
CATEGORY = "conditioning/video_models"
|
||||||
|
|
||||||
|
def encode(self, clip_vision, init_image, vae, width, height, video_frames, motion_bucket_id, fps, augmentation_level):
|
||||||
|
output = clip_vision.encode_image(init_image)
|
||||||
|
pooled = output.image_embeds.unsqueeze(0)
|
||||||
|
pixels = comfy.utils.common_upscale(init_image.movedim(-1,1), width, height, "bilinear", "center").movedim(1,-1)
|
||||||
|
encode_pixels = pixels[:,:,:,:3]
|
||||||
|
if augmentation_level > 0:
|
||||||
|
encode_pixels += torch.randn_like(pixels) * augmentation_level
|
||||||
|
t = vae.encode(encode_pixels)
|
||||||
|
positive = [[pooled, {"motion_bucket_id": motion_bucket_id, "fps": fps, "augmentation_level": augmentation_level, "concat_latent_image": t}]]
|
||||||
|
negative = [[torch.zeros_like(pooled), {"motion_bucket_id": motion_bucket_id, "fps": fps, "augmentation_level": augmentation_level, "concat_latent_image": torch.zeros_like(t)}]]
|
||||||
|
latent = torch.zeros([video_frames, 4, height // 8, width // 8])
|
||||||
|
return (positive, negative, {"samples":latent})
|
||||||
|
|
||||||
|
class VideoLinearCFGGuidance:
|
||||||
|
@classmethod
|
||||||
|
def INPUT_TYPES(s):
|
||||||
|
return {"required": { "model": ("MODEL",),
|
||||||
|
"min_cfg": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 100.0, "step":0.5, "round": 0.01}),
|
||||||
|
}}
|
||||||
|
RETURN_TYPES = ("MODEL",)
|
||||||
|
FUNCTION = "patch"
|
||||||
|
|
||||||
|
CATEGORY = "sampling/video_models"
|
||||||
|
|
||||||
|
def patch(self, model, min_cfg):
|
||||||
|
def linear_cfg(args):
|
||||||
|
cond = args["cond"]
|
||||||
|
uncond = args["uncond"]
|
||||||
|
cond_scale = args["cond_scale"]
|
||||||
|
|
||||||
|
scale = torch.linspace(min_cfg, cond_scale, cond.shape[0], device=cond.device).reshape((cond.shape[0], 1, 1, 1))
|
||||||
|
return uncond + scale * (cond - uncond)
|
||||||
|
|
||||||
|
m = model.clone()
|
||||||
|
m.set_model_sampler_cfg_function(linear_cfg)
|
||||||
|
return (m, )
|
||||||
|
|
||||||
|
NODE_CLASS_MAPPINGS = {
|
||||||
|
"ImageOnlyCheckpointLoader": ImageOnlyCheckpointLoader,
|
||||||
|
"SVD_img2vid_Conditioning": SVD_img2vid_Conditioning,
|
||||||
|
"VideoLinearCFGGuidance": VideoLinearCFGGuidance,
|
||||||
|
}
|
||||||
|
|
||||||
|
NODE_DISPLAY_NAME_MAPPINGS = {
|
||||||
|
"ImageOnlyCheckpointLoader": "Image Only Checkpoint Loader (img2vid model)",
|
||||||
|
}
|
||||||
83
execution.py
83
execution.py
@ -7,6 +7,7 @@ import threading
|
|||||||
import heapq
|
import heapq
|
||||||
import traceback
|
import traceback
|
||||||
import gc
|
import gc
|
||||||
|
import inspect
|
||||||
|
|
||||||
import torch
|
import torch
|
||||||
import nodes
|
import nodes
|
||||||
@ -382,6 +383,8 @@ class PromptExecutor:
|
|||||||
for x in executed:
|
for x in executed:
|
||||||
self.old_prompt[x] = copy.deepcopy(prompt[x])
|
self.old_prompt[x] = copy.deepcopy(prompt[x])
|
||||||
self.server.last_node_id = None
|
self.server.last_node_id = None
|
||||||
|
if comfy.model_management.DISABLE_SMART_MEMORY:
|
||||||
|
comfy.model_management.unload_all_models()
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
@ -400,6 +403,10 @@ def validate_inputs(prompt, item, validated):
|
|||||||
errors = []
|
errors = []
|
||||||
valid = True
|
valid = True
|
||||||
|
|
||||||
|
validate_function_inputs = []
|
||||||
|
if hasattr(obj_class, "VALIDATE_INPUTS"):
|
||||||
|
validate_function_inputs = inspect.getfullargspec(obj_class.VALIDATE_INPUTS).args
|
||||||
|
|
||||||
for x in required_inputs:
|
for x in required_inputs:
|
||||||
if x not in inputs:
|
if x not in inputs:
|
||||||
error = {
|
error = {
|
||||||
@ -529,29 +536,7 @@ def validate_inputs(prompt, item, validated):
|
|||||||
errors.append(error)
|
errors.append(error)
|
||||||
continue
|
continue
|
||||||
|
|
||||||
if hasattr(obj_class, "VALIDATE_INPUTS"):
|
if x not in validate_function_inputs:
|
||||||
input_data_all = get_input_data(inputs, obj_class, unique_id)
|
|
||||||
#ret = obj_class.VALIDATE_INPUTS(**input_data_all)
|
|
||||||
ret = map_node_over_list(obj_class, input_data_all, "VALIDATE_INPUTS")
|
|
||||||
for i, r in enumerate(ret):
|
|
||||||
if r is not True:
|
|
||||||
details = f"{x}"
|
|
||||||
if r is not False:
|
|
||||||
details += f" - {str(r)}"
|
|
||||||
|
|
||||||
error = {
|
|
||||||
"type": "custom_validation_failed",
|
|
||||||
"message": "Custom validation failed for node",
|
|
||||||
"details": details,
|
|
||||||
"extra_info": {
|
|
||||||
"input_name": x,
|
|
||||||
"input_config": info,
|
|
||||||
"received_value": val,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
errors.append(error)
|
|
||||||
continue
|
|
||||||
else:
|
|
||||||
if isinstance(type_input, list):
|
if isinstance(type_input, list):
|
||||||
if val not in type_input:
|
if val not in type_input:
|
||||||
input_config = info
|
input_config = info
|
||||||
@ -578,6 +563,35 @@ def validate_inputs(prompt, item, validated):
|
|||||||
errors.append(error)
|
errors.append(error)
|
||||||
continue
|
continue
|
||||||
|
|
||||||
|
if len(validate_function_inputs) > 0:
|
||||||
|
input_data_all = get_input_data(inputs, obj_class, unique_id)
|
||||||
|
input_filtered = {}
|
||||||
|
for x in input_data_all:
|
||||||
|
if x in validate_function_inputs:
|
||||||
|
input_filtered[x] = input_data_all[x]
|
||||||
|
|
||||||
|
#ret = obj_class.VALIDATE_INPUTS(**input_filtered)
|
||||||
|
ret = map_node_over_list(obj_class, input_filtered, "VALIDATE_INPUTS")
|
||||||
|
for x in input_filtered:
|
||||||
|
for i, r in enumerate(ret):
|
||||||
|
if r is not True:
|
||||||
|
details = f"{x}"
|
||||||
|
if r is not False:
|
||||||
|
details += f" - {str(r)}"
|
||||||
|
|
||||||
|
error = {
|
||||||
|
"type": "custom_validation_failed",
|
||||||
|
"message": "Custom validation failed for node",
|
||||||
|
"details": details,
|
||||||
|
"extra_info": {
|
||||||
|
"input_name": x,
|
||||||
|
"input_config": info,
|
||||||
|
"received_value": val,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
errors.append(error)
|
||||||
|
continue
|
||||||
|
|
||||||
if len(errors) > 0 or valid is not True:
|
if len(errors) > 0 or valid is not True:
|
||||||
ret = (False, errors, unique_id)
|
ret = (False, errors, unique_id)
|
||||||
else:
|
else:
|
||||||
@ -681,6 +695,7 @@ def validate_prompt(prompt):
|
|||||||
|
|
||||||
return (True, None, list(good_outputs), node_errors)
|
return (True, None, list(good_outputs), node_errors)
|
||||||
|
|
||||||
|
MAXIMUM_HISTORY_SIZE = 10000
|
||||||
|
|
||||||
class PromptQueue:
|
class PromptQueue:
|
||||||
def __init__(self, server):
|
def __init__(self, server):
|
||||||
@ -699,10 +714,12 @@ class PromptQueue:
|
|||||||
self.server.queue_updated()
|
self.server.queue_updated()
|
||||||
self.not_empty.notify()
|
self.not_empty.notify()
|
||||||
|
|
||||||
def get(self):
|
def get(self, timeout=None):
|
||||||
with self.not_empty:
|
with self.not_empty:
|
||||||
while len(self.queue) == 0:
|
while len(self.queue) == 0:
|
||||||
self.not_empty.wait()
|
self.not_empty.wait(timeout=timeout)
|
||||||
|
if timeout is not None and len(self.queue) == 0:
|
||||||
|
return None
|
||||||
item = heapq.heappop(self.queue)
|
item = heapq.heappop(self.queue)
|
||||||
i = self.task_counter
|
i = self.task_counter
|
||||||
self.currently_running[i] = copy.deepcopy(item)
|
self.currently_running[i] = copy.deepcopy(item)
|
||||||
@ -713,6 +730,8 @@ class PromptQueue:
|
|||||||
def task_done(self, item_id, outputs):
|
def task_done(self, item_id, outputs):
|
||||||
with self.mutex:
|
with self.mutex:
|
||||||
prompt = self.currently_running.pop(item_id)
|
prompt = self.currently_running.pop(item_id)
|
||||||
|
if len(self.history) > MAXIMUM_HISTORY_SIZE:
|
||||||
|
self.history.pop(next(iter(self.history)))
|
||||||
self.history[prompt[1]] = { "prompt": prompt, "outputs": {} }
|
self.history[prompt[1]] = { "prompt": prompt, "outputs": {} }
|
||||||
for o in outputs:
|
for o in outputs:
|
||||||
self.history[prompt[1]]["outputs"][o] = outputs[o]
|
self.history[prompt[1]]["outputs"][o] = outputs[o]
|
||||||
@ -747,10 +766,20 @@ class PromptQueue:
|
|||||||
return True
|
return True
|
||||||
return False
|
return False
|
||||||
|
|
||||||
def get_history(self, prompt_id=None):
|
def get_history(self, prompt_id=None, max_items=None, offset=-1):
|
||||||
with self.mutex:
|
with self.mutex:
|
||||||
if prompt_id is None:
|
if prompt_id is None:
|
||||||
return copy.deepcopy(self.history)
|
out = {}
|
||||||
|
i = 0
|
||||||
|
if offset < 0 and max_items is not None:
|
||||||
|
offset = len(self.history) - max_items
|
||||||
|
for k in self.history:
|
||||||
|
if i >= offset:
|
||||||
|
out[k] = self.history[k]
|
||||||
|
if max_items is not None and len(out) >= max_items:
|
||||||
|
break
|
||||||
|
i += 1
|
||||||
|
return out
|
||||||
elif prompt_id in self.history:
|
elif prompt_id in self.history:
|
||||||
return {prompt_id: copy.deepcopy(self.history[prompt_id])}
|
return {prompt_id: copy.deepcopy(self.history[prompt_id])}
|
||||||
else:
|
else:
|
||||||
|
|||||||
@ -38,7 +38,10 @@ input_directory = os.path.join(os.path.dirname(os.path.realpath(__file__)), "inp
|
|||||||
filename_list_cache = {}
|
filename_list_cache = {}
|
||||||
|
|
||||||
if not os.path.exists(input_directory):
|
if not os.path.exists(input_directory):
|
||||||
os.makedirs(input_directory)
|
try:
|
||||||
|
os.makedirs(input_directory)
|
||||||
|
except:
|
||||||
|
print("Failed to create input directory")
|
||||||
|
|
||||||
def set_output_directory(output_dir):
|
def set_output_directory(output_dir):
|
||||||
global output_directory
|
global output_directory
|
||||||
@ -181,8 +184,7 @@ def cached_filename_list_(folder_name):
|
|||||||
if folder_name not in filename_list_cache:
|
if folder_name not in filename_list_cache:
|
||||||
return None
|
return None
|
||||||
out = filename_list_cache[folder_name]
|
out = filename_list_cache[folder_name]
|
||||||
if time.perf_counter() < (out[2] + 0.5):
|
|
||||||
return out
|
|
||||||
for x in out[1]:
|
for x in out[1]:
|
||||||
time_modified = out[1][x]
|
time_modified = out[1][x]
|
||||||
folder = x
|
folder = x
|
||||||
@ -228,8 +230,12 @@ def get_save_image_path(filename_prefix, output_dir, image_width=0, image_height
|
|||||||
full_output_folder = os.path.join(output_dir, subfolder)
|
full_output_folder = os.path.join(output_dir, subfolder)
|
||||||
|
|
||||||
if os.path.commonpath((output_dir, os.path.abspath(full_output_folder))) != output_dir:
|
if os.path.commonpath((output_dir, os.path.abspath(full_output_folder))) != output_dir:
|
||||||
print("Saving image outside the output folder is not allowed.")
|
err = "**** ERROR: Saving image outside the output folder is not allowed." + \
|
||||||
return {}
|
"\n full_output_folder: " + os.path.abspath(full_output_folder) + \
|
||||||
|
"\n output_dir: " + output_dir + \
|
||||||
|
"\n commonpath: " + os.path.commonpath((output_dir, os.path.abspath(full_output_folder)))
|
||||||
|
print(err)
|
||||||
|
raise Exception(err)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
counter = max(filter(lambda a: a[1][:-1] == filename and a[1][-1] == "_", map(map_filename, os.listdir(full_output_folder))))[0] + 1
|
counter = max(filter(lambda a: a[1][:-1] == filename and a[1][-1] == "_", map(map_filename, os.listdir(full_output_folder))))[0] + 1
|
||||||
|
|||||||
@ -22,10 +22,7 @@ class TAESDPreviewerImpl(LatentPreviewer):
|
|||||||
self.taesd = taesd
|
self.taesd = taesd
|
||||||
|
|
||||||
def decode_latent_to_preview(self, x0):
|
def decode_latent_to_preview(self, x0):
|
||||||
x_sample = self.taesd.decoder(x0[:1])[0].detach()
|
x_sample = self.taesd.decode(x0[:1])[0].detach()
|
||||||
# x_sample = self.taesd.unscale_latents(x_sample).div(4).add(0.5) # returns value in [-2, 2]
|
|
||||||
x_sample = x_sample.sub(0.5).mul(2)
|
|
||||||
|
|
||||||
x_sample = torch.clamp((x_sample + 1.0) / 2.0, min=0.0, max=1.0)
|
x_sample = torch.clamp((x_sample + 1.0) / 2.0, min=0.0, max=1.0)
|
||||||
x_sample = 255. * np.moveaxis(x_sample.cpu().numpy(), 0, 2)
|
x_sample = 255. * np.moveaxis(x_sample.cpu().numpy(), 0, 2)
|
||||||
x_sample = x_sample.astype(np.uint8)
|
x_sample = x_sample.astype(np.uint8)
|
||||||
|
|||||||
51
main.py
51
main.py
@ -64,6 +64,10 @@ if __name__ == "__main__":
|
|||||||
os.environ['CUDA_VISIBLE_DEVICES'] = str(args.cuda_device)
|
os.environ['CUDA_VISIBLE_DEVICES'] = str(args.cuda_device)
|
||||||
print("Set cuda device to:", args.cuda_device)
|
print("Set cuda device to:", args.cuda_device)
|
||||||
|
|
||||||
|
if args.deterministic:
|
||||||
|
if 'CUBLAS_WORKSPACE_CONFIG' not in os.environ:
|
||||||
|
os.environ['CUBLAS_WORKSPACE_CONFIG'] = ":4096:8"
|
||||||
|
|
||||||
import cuda_malloc
|
import cuda_malloc
|
||||||
|
|
||||||
import comfy.utils
|
import comfy.utils
|
||||||
@ -88,18 +92,39 @@ def cuda_malloc_warning():
|
|||||||
|
|
||||||
def prompt_worker(q, server):
|
def prompt_worker(q, server):
|
||||||
e = execution.PromptExecutor(server)
|
e = execution.PromptExecutor(server)
|
||||||
while True:
|
last_gc_collect = 0
|
||||||
item, item_id = q.get()
|
need_gc = False
|
||||||
execution_start_time = time.perf_counter()
|
gc_collect_interval = 10.0
|
||||||
prompt_id = item[1]
|
|
||||||
e.execute(item[2], prompt_id, item[3], item[4])
|
|
||||||
q.task_done(item_id, e.outputs_ui)
|
|
||||||
if server.client_id is not None:
|
|
||||||
server.send_sync("executing", { "node": None, "prompt_id": prompt_id }, server.client_id)
|
|
||||||
|
|
||||||
print("Prompt executed in {:.2f} seconds".format(time.perf_counter() - execution_start_time))
|
while True:
|
||||||
gc.collect()
|
timeout = None
|
||||||
comfy.model_management.soft_empty_cache()
|
if need_gc:
|
||||||
|
timeout = max(gc_collect_interval - (current_time - last_gc_collect), 0.0)
|
||||||
|
|
||||||
|
queue_item = q.get(timeout=timeout)
|
||||||
|
if queue_item is not None:
|
||||||
|
item, item_id = queue_item
|
||||||
|
execution_start_time = time.perf_counter()
|
||||||
|
prompt_id = item[1]
|
||||||
|
server.last_prompt_id = prompt_id
|
||||||
|
|
||||||
|
e.execute(item[2], prompt_id, item[3], item[4])
|
||||||
|
need_gc = True
|
||||||
|
q.task_done(item_id, e.outputs_ui)
|
||||||
|
if server.client_id is not None:
|
||||||
|
server.send_sync("executing", { "node": None, "prompt_id": prompt_id }, server.client_id)
|
||||||
|
|
||||||
|
current_time = time.perf_counter()
|
||||||
|
execution_time = current_time - execution_start_time
|
||||||
|
print("Prompt executed in {:.2f} seconds".format(execution_time))
|
||||||
|
|
||||||
|
if need_gc:
|
||||||
|
current_time = time.perf_counter()
|
||||||
|
if (current_time - last_gc_collect) > gc_collect_interval:
|
||||||
|
gc.collect()
|
||||||
|
comfy.model_management.soft_empty_cache()
|
||||||
|
last_gc_collect = current_time
|
||||||
|
need_gc = False
|
||||||
|
|
||||||
async def run(server, address='', port=8188, verbose=True, call_on_start=None):
|
async def run(server, address='', port=8188, verbose=True, call_on_start=None):
|
||||||
await asyncio.gather(server.start(address, port, verbose, call_on_start), server.publish_loop())
|
await asyncio.gather(server.start(address, port, verbose, call_on_start), server.publish_loop())
|
||||||
@ -108,7 +133,9 @@ async def run(server, address='', port=8188, verbose=True, call_on_start=None):
|
|||||||
def hijack_progress(server):
|
def hijack_progress(server):
|
||||||
def hook(value, total, preview_image):
|
def hook(value, total, preview_image):
|
||||||
comfy.model_management.throw_exception_if_processing_interrupted()
|
comfy.model_management.throw_exception_if_processing_interrupted()
|
||||||
server.send_sync("progress", {"value": value, "max": total}, server.client_id)
|
progress = {"value": value, "max": total, "prompt_id": server.last_prompt_id, "node": server.last_node_id}
|
||||||
|
|
||||||
|
server.send_sync("progress", progress, server.client_id)
|
||||||
if preview_image is not None:
|
if preview_image is not None:
|
||||||
server.send_sync(BinaryEventTypes.UNENCODED_PREVIEW_IMAGE, preview_image, server.client_id)
|
server.send_sync(BinaryEventTypes.UNENCODED_PREVIEW_IMAGE, preview_image, server.client_id)
|
||||||
comfy.utils.set_progress_bar_global_hook(hook)
|
comfy.utils.set_progress_bar_global_hook(hook)
|
||||||
|
|||||||
135
nodes.py
135
nodes.py
@ -9,7 +9,7 @@ import math
|
|||||||
import time
|
import time
|
||||||
import random
|
import random
|
||||||
|
|
||||||
from PIL import Image, ImageOps
|
from PIL import Image, ImageOps, ImageSequence
|
||||||
from PIL.PngImagePlugin import PngInfo
|
from PIL.PngImagePlugin import PngInfo
|
||||||
import numpy as np
|
import numpy as np
|
||||||
import safetensors.torch
|
import safetensors.torch
|
||||||
@ -248,8 +248,8 @@ class ConditioningSetTimestepRange:
|
|||||||
c = []
|
c = []
|
||||||
for t in conditioning:
|
for t in conditioning:
|
||||||
d = t[1].copy()
|
d = t[1].copy()
|
||||||
d['start_percent'] = 1.0 - start
|
d['start_percent'] = start
|
||||||
d['end_percent'] = 1.0 - end
|
d['end_percent'] = end
|
||||||
n = [t[0], d]
|
n = [t[0], d]
|
||||||
c.append(n)
|
c.append(n)
|
||||||
return (c, )
|
return (c, )
|
||||||
@ -572,10 +572,69 @@ class LoraLoader:
|
|||||||
model_lora, clip_lora = comfy.sd.load_lora_for_models(model, clip, lora, strength_model, strength_clip)
|
model_lora, clip_lora = comfy.sd.load_lora_for_models(model, clip, lora, strength_model, strength_clip)
|
||||||
return (model_lora, clip_lora)
|
return (model_lora, clip_lora)
|
||||||
|
|
||||||
class VAELoader:
|
class LoraLoaderModelOnly(LoraLoader):
|
||||||
@classmethod
|
@classmethod
|
||||||
def INPUT_TYPES(s):
|
def INPUT_TYPES(s):
|
||||||
return {"required": { "vae_name": (folder_paths.get_filename_list("vae"), )}}
|
return {"required": { "model": ("MODEL",),
|
||||||
|
"lora_name": (folder_paths.get_filename_list("loras"), ),
|
||||||
|
"strength_model": ("FLOAT", {"default": 1.0, "min": -20.0, "max": 20.0, "step": 0.01}),
|
||||||
|
}}
|
||||||
|
RETURN_TYPES = ("MODEL",)
|
||||||
|
FUNCTION = "load_lora_model_only"
|
||||||
|
|
||||||
|
def load_lora_model_only(self, model, lora_name, strength_model):
|
||||||
|
return (self.load_lora(model, None, lora_name, strength_model, 0)[0],)
|
||||||
|
|
||||||
|
class VAELoader:
|
||||||
|
@staticmethod
|
||||||
|
def vae_list():
|
||||||
|
vaes = folder_paths.get_filename_list("vae")
|
||||||
|
approx_vaes = folder_paths.get_filename_list("vae_approx")
|
||||||
|
sdxl_taesd_enc = False
|
||||||
|
sdxl_taesd_dec = False
|
||||||
|
sd1_taesd_enc = False
|
||||||
|
sd1_taesd_dec = False
|
||||||
|
|
||||||
|
for v in approx_vaes:
|
||||||
|
if v.startswith("taesd_decoder."):
|
||||||
|
sd1_taesd_dec = True
|
||||||
|
elif v.startswith("taesd_encoder."):
|
||||||
|
sd1_taesd_enc = True
|
||||||
|
elif v.startswith("taesdxl_decoder."):
|
||||||
|
sdxl_taesd_dec = True
|
||||||
|
elif v.startswith("taesdxl_encoder."):
|
||||||
|
sdxl_taesd_enc = True
|
||||||
|
if sd1_taesd_dec and sd1_taesd_enc:
|
||||||
|
vaes.append("taesd")
|
||||||
|
if sdxl_taesd_dec and sdxl_taesd_enc:
|
||||||
|
vaes.append("taesdxl")
|
||||||
|
return vaes
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def load_taesd(name):
|
||||||
|
sd = {}
|
||||||
|
approx_vaes = folder_paths.get_filename_list("vae_approx")
|
||||||
|
|
||||||
|
encoder = next(filter(lambda a: a.startswith("{}_encoder.".format(name)), approx_vaes))
|
||||||
|
decoder = next(filter(lambda a: a.startswith("{}_decoder.".format(name)), approx_vaes))
|
||||||
|
|
||||||
|
enc = comfy.utils.load_torch_file(folder_paths.get_full_path("vae_approx", encoder))
|
||||||
|
for k in enc:
|
||||||
|
sd["taesd_encoder.{}".format(k)] = enc[k]
|
||||||
|
|
||||||
|
dec = comfy.utils.load_torch_file(folder_paths.get_full_path("vae_approx", decoder))
|
||||||
|
for k in dec:
|
||||||
|
sd["taesd_decoder.{}".format(k)] = dec[k]
|
||||||
|
|
||||||
|
if name == "taesd":
|
||||||
|
sd["vae_scale"] = torch.tensor(0.18215)
|
||||||
|
elif name == "taesdxl":
|
||||||
|
sd["vae_scale"] = torch.tensor(0.13025)
|
||||||
|
return sd
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def INPUT_TYPES(s):
|
||||||
|
return {"required": { "vae_name": (s.vae_list(), )}}
|
||||||
RETURN_TYPES = ("VAE",)
|
RETURN_TYPES = ("VAE",)
|
||||||
FUNCTION = "load_vae"
|
FUNCTION = "load_vae"
|
||||||
|
|
||||||
@ -583,8 +642,11 @@ class VAELoader:
|
|||||||
|
|
||||||
#TODO: scale factor?
|
#TODO: scale factor?
|
||||||
def load_vae(self, vae_name):
|
def load_vae(self, vae_name):
|
||||||
vae_path = folder_paths.get_full_path("vae", vae_name)
|
if vae_name in ["taesd", "taesdxl"]:
|
||||||
sd = comfy.utils.load_torch_file(vae_path)
|
sd = self.load_taesd(vae_name)
|
||||||
|
else:
|
||||||
|
vae_path = folder_paths.get_full_path("vae", vae_name)
|
||||||
|
sd = comfy.utils.load_torch_file(vae_path)
|
||||||
vae = comfy.sd.VAE(sd=sd)
|
vae = comfy.sd.VAE(sd=sd)
|
||||||
return (vae,)
|
return (vae,)
|
||||||
|
|
||||||
@ -685,7 +747,7 @@ class ControlNetApplyAdvanced:
|
|||||||
if prev_cnet in cnets:
|
if prev_cnet in cnets:
|
||||||
c_net = cnets[prev_cnet]
|
c_net = cnets[prev_cnet]
|
||||||
else:
|
else:
|
||||||
c_net = control_net.copy().set_cond_hint(control_hint, strength, (1.0 - start_percent, 1.0 - end_percent))
|
c_net = control_net.copy().set_cond_hint(control_hint, strength, (start_percent, end_percent))
|
||||||
c_net.set_previous_controlnet(prev_cnet)
|
c_net.set_previous_controlnet(prev_cnet)
|
||||||
cnets[prev_cnet] = c_net
|
cnets[prev_cnet] = c_net
|
||||||
|
|
||||||
@ -885,8 +947,8 @@ class GLIGENTextBoxApply:
|
|||||||
return (c, )
|
return (c, )
|
||||||
|
|
||||||
class EmptyLatentImage:
|
class EmptyLatentImage:
|
||||||
def __init__(self, device="cpu"):
|
def __init__(self):
|
||||||
self.device = device
|
self.device = comfy.model_management.intermediate_device()
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def INPUT_TYPES(s):
|
def INPUT_TYPES(s):
|
||||||
@ -899,7 +961,7 @@ class EmptyLatentImage:
|
|||||||
CATEGORY = "latent"
|
CATEGORY = "latent"
|
||||||
|
|
||||||
def generate(self, width, height, batch_size=1):
|
def generate(self, width, height, batch_size=1):
|
||||||
latent = torch.zeros([batch_size, 4, height // 8, width // 8])
|
latent = torch.zeros([batch_size, 4, height // 8, width // 8], device=self.device)
|
||||||
return ({"samples":latent}, )
|
return ({"samples":latent}, )
|
||||||
|
|
||||||
|
|
||||||
@ -1275,6 +1337,7 @@ class SaveImage:
|
|||||||
self.output_dir = folder_paths.get_output_directory()
|
self.output_dir = folder_paths.get_output_directory()
|
||||||
self.type = "output"
|
self.type = "output"
|
||||||
self.prefix_append = ""
|
self.prefix_append = ""
|
||||||
|
self.compress_level = 4
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def INPUT_TYPES(s):
|
def INPUT_TYPES(s):
|
||||||
@ -1308,7 +1371,7 @@ class SaveImage:
|
|||||||
metadata.add_text(x, json.dumps(extra_pnginfo[x]))
|
metadata.add_text(x, json.dumps(extra_pnginfo[x]))
|
||||||
|
|
||||||
file = f"{filename}_{counter:05}_.png"
|
file = f"{filename}_{counter:05}_.png"
|
||||||
img.save(os.path.join(full_output_folder, file), pnginfo=metadata, compress_level=4)
|
img.save(os.path.join(full_output_folder, file), pnginfo=metadata, compress_level=self.compress_level)
|
||||||
results.append({
|
results.append({
|
||||||
"filename": file,
|
"filename": file,
|
||||||
"subfolder": subfolder,
|
"subfolder": subfolder,
|
||||||
@ -1323,6 +1386,7 @@ class PreviewImage(SaveImage):
|
|||||||
self.output_dir = folder_paths.get_temp_directory()
|
self.output_dir = folder_paths.get_temp_directory()
|
||||||
self.type = "temp"
|
self.type = "temp"
|
||||||
self.prefix_append = "_temp_" + ''.join(random.choice("abcdefghijklmnopqrstupvxyz") for x in range(5))
|
self.prefix_append = "_temp_" + ''.join(random.choice("abcdefghijklmnopqrstupvxyz") for x in range(5))
|
||||||
|
self.compress_level = 1
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def INPUT_TYPES(s):
|
def INPUT_TYPES(s):
|
||||||
@ -1346,17 +1410,30 @@ class LoadImage:
|
|||||||
FUNCTION = "load_image"
|
FUNCTION = "load_image"
|
||||||
def load_image(self, image):
|
def load_image(self, image):
|
||||||
image_path = folder_paths.get_annotated_filepath(image)
|
image_path = folder_paths.get_annotated_filepath(image)
|
||||||
i = Image.open(image_path)
|
img = Image.open(image_path)
|
||||||
i = ImageOps.exif_transpose(i)
|
output_images = []
|
||||||
image = i.convert("RGB")
|
output_masks = []
|
||||||
image = np.array(image).astype(np.float32) / 255.0
|
for i in ImageSequence.Iterator(img):
|
||||||
image = torch.from_numpy(image)[None,]
|
i = ImageOps.exif_transpose(i)
|
||||||
if 'A' in i.getbands():
|
image = i.convert("RGB")
|
||||||
mask = np.array(i.getchannel('A')).astype(np.float32) / 255.0
|
image = np.array(image).astype(np.float32) / 255.0
|
||||||
mask = 1. - torch.from_numpy(mask)
|
image = torch.from_numpy(image)[None,]
|
||||||
|
if 'A' in i.getbands():
|
||||||
|
mask = np.array(i.getchannel('A')).astype(np.float32) / 255.0
|
||||||
|
mask = 1. - torch.from_numpy(mask)
|
||||||
|
else:
|
||||||
|
mask = torch.zeros((64,64), dtype=torch.float32, device="cpu")
|
||||||
|
output_images.append(image)
|
||||||
|
output_masks.append(mask.unsqueeze(0))
|
||||||
|
|
||||||
|
if len(output_images) > 1:
|
||||||
|
output_image = torch.cat(output_images, dim=0)
|
||||||
|
output_mask = torch.cat(output_masks, dim=0)
|
||||||
else:
|
else:
|
||||||
mask = torch.zeros((64,64), dtype=torch.float32, device="cpu")
|
output_image = output_images[0]
|
||||||
return (image, mask.unsqueeze(0))
|
output_mask = output_masks[0]
|
||||||
|
|
||||||
|
return (output_image, output_mask)
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def IS_CHANGED(s, image):
|
def IS_CHANGED(s, image):
|
||||||
@ -1414,13 +1491,10 @@ class LoadImageMask:
|
|||||||
return m.digest().hex()
|
return m.digest().hex()
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def VALIDATE_INPUTS(s, image, channel):
|
def VALIDATE_INPUTS(s, image):
|
||||||
if not folder_paths.exists_annotated_filepath(image):
|
if not folder_paths.exists_annotated_filepath(image):
|
||||||
return "Invalid image file: {}".format(image)
|
return "Invalid image file: {}".format(image)
|
||||||
|
|
||||||
if channel not in s._color_channels:
|
|
||||||
return "Invalid color channel: {}".format(channel)
|
|
||||||
|
|
||||||
return True
|
return True
|
||||||
|
|
||||||
class ImageScale:
|
class ImageScale:
|
||||||
@ -1654,6 +1728,7 @@ NODE_CLASS_MAPPINGS = {
|
|||||||
|
|
||||||
"ConditioningZeroOut": ConditioningZeroOut,
|
"ConditioningZeroOut": ConditioningZeroOut,
|
||||||
"ConditioningSetTimestepRange": ConditioningSetTimestepRange,
|
"ConditioningSetTimestepRange": ConditioningSetTimestepRange,
|
||||||
|
"LoraLoaderModelOnly": LoraLoaderModelOnly,
|
||||||
}
|
}
|
||||||
|
|
||||||
NODE_DISPLAY_NAME_MAPPINGS = {
|
NODE_DISPLAY_NAME_MAPPINGS = {
|
||||||
@ -1759,7 +1834,7 @@ def load_custom_nodes():
|
|||||||
node_paths = folder_paths.get_folder_paths("custom_nodes")
|
node_paths = folder_paths.get_folder_paths("custom_nodes")
|
||||||
node_import_times = []
|
node_import_times = []
|
||||||
for custom_node_path in node_paths:
|
for custom_node_path in node_paths:
|
||||||
possible_modules = os.listdir(custom_node_path)
|
possible_modules = os.listdir(os.path.realpath(custom_node_path))
|
||||||
if "__pycache__" in possible_modules:
|
if "__pycache__" in possible_modules:
|
||||||
possible_modules.remove("__pycache__")
|
possible_modules.remove("__pycache__")
|
||||||
|
|
||||||
@ -1799,6 +1874,12 @@ def init_custom_nodes():
|
|||||||
"nodes_custom_sampler.py",
|
"nodes_custom_sampler.py",
|
||||||
"nodes_hypertile.py",
|
"nodes_hypertile.py",
|
||||||
"nodes_model_advanced.py",
|
"nodes_model_advanced.py",
|
||||||
|
"nodes_model_downscale.py",
|
||||||
|
"nodes_images.py",
|
||||||
|
"nodes_video_model.py",
|
||||||
|
"nodes_sag.py",
|
||||||
|
"nodes_perpneg.py",
|
||||||
|
"nodes_stable3d.py",
|
||||||
]
|
]
|
||||||
|
|
||||||
for node_file in extras_files:
|
for node_file in extras_files:
|
||||||
|
|||||||
@ -1,10 +1,10 @@
|
|||||||
torch
|
torch
|
||||||
torchsde
|
torchsde
|
||||||
|
torchvision
|
||||||
einops
|
einops
|
||||||
transformers>=4.25.1
|
transformers>=4.25.1
|
||||||
safetensors>=0.3.0
|
safetensors>=0.3.0
|
||||||
aiohttp
|
aiohttp
|
||||||
accelerate
|
|
||||||
pyyaml
|
pyyaml
|
||||||
Pillow
|
Pillow
|
||||||
scipy
|
scipy
|
||||||
|
|||||||
@ -431,7 +431,10 @@ class PromptServer():
|
|||||||
|
|
||||||
@routes.get("/history")
|
@routes.get("/history")
|
||||||
async def get_history(request):
|
async def get_history(request):
|
||||||
return web.json_response(self.prompt_queue.get_history())
|
max_items = request.rel_url.query.get("max_items", None)
|
||||||
|
if max_items is not None:
|
||||||
|
max_items = int(max_items)
|
||||||
|
return web.json_response(self.prompt_queue.get_history(max_items=max_items))
|
||||||
|
|
||||||
@routes.get("/history/{prompt_id}")
|
@routes.get("/history/{prompt_id}")
|
||||||
async def get_history(request):
|
async def get_history(request):
|
||||||
@ -573,7 +576,7 @@ class PromptServer():
|
|||||||
bytesIO = BytesIO()
|
bytesIO = BytesIO()
|
||||||
header = struct.pack(">I", type_num)
|
header = struct.pack(">I", type_num)
|
||||||
bytesIO.write(header)
|
bytesIO.write(header)
|
||||||
image.save(bytesIO, format=image_type, quality=95, compress_level=4)
|
image.save(bytesIO, format=image_type, quality=95, compress_level=1)
|
||||||
preview_bytes = bytesIO.getvalue()
|
preview_bytes = bytesIO.getvalue()
|
||||||
await self.send_bytes(BinaryEventTypes.PREVIEW_IMAGE, preview_bytes, sid=sid)
|
await self.send_bytes(BinaryEventTypes.PREVIEW_IMAGE, preview_bytes, sid=sid)
|
||||||
|
|
||||||
|
|||||||
9
tests-ui/afterSetup.js
Normal file
9
tests-ui/afterSetup.js
Normal file
@ -0,0 +1,9 @@
|
|||||||
|
const { start } = require("./utils");
|
||||||
|
const lg = require("./utils/litegraph");
|
||||||
|
|
||||||
|
// Load things once per test file before to ensure its all warmed up for the tests
|
||||||
|
beforeAll(async () => {
|
||||||
|
lg.setup(global);
|
||||||
|
await start({ resetEnv: true });
|
||||||
|
lg.teardown(global);
|
||||||
|
});
|
||||||
@ -2,8 +2,10 @@
|
|||||||
const config = {
|
const config = {
|
||||||
testEnvironment: "jsdom",
|
testEnvironment: "jsdom",
|
||||||
setupFiles: ["./globalSetup.js"],
|
setupFiles: ["./globalSetup.js"],
|
||||||
|
setupFilesAfterEnv: ["./afterSetup.js"],
|
||||||
clearMocks: true,
|
clearMocks: true,
|
||||||
resetModules: true,
|
resetModules: true,
|
||||||
|
testTimeout: 10000
|
||||||
};
|
};
|
||||||
|
|
||||||
module.exports = config;
|
module.exports = config;
|
||||||
|
|||||||
@ -20,6 +20,7 @@ async function setup() {
|
|||||||
// Modify the response data to add some checkpoints
|
// Modify the response data to add some checkpoints
|
||||||
const objectInfo = JSON.parse(data);
|
const objectInfo = JSON.parse(data);
|
||||||
objectInfo.CheckpointLoaderSimple.input.required.ckpt_name[0] = ["model1.safetensors", "model2.ckpt"];
|
objectInfo.CheckpointLoaderSimple.input.required.ckpt_name[0] = ["model1.safetensors", "model2.ckpt"];
|
||||||
|
objectInfo.VAELoader.input.required.vae_name[0] = ["vae1.safetensors", "vae2.ckpt"];
|
||||||
|
|
||||||
data = JSON.stringify(objectInfo, undefined, "\t");
|
data = JSON.stringify(objectInfo, undefined, "\t");
|
||||||
|
|
||||||
|
|||||||
196
tests-ui/tests/extensions.test.js
Normal file
196
tests-ui/tests/extensions.test.js
Normal file
@ -0,0 +1,196 @@
|
|||||||
|
// @ts-check
|
||||||
|
/// <reference path="../node_modules/@types/jest/index.d.ts" />
|
||||||
|
const { start } = require("../utils");
|
||||||
|
const lg = require("../utils/litegraph");
|
||||||
|
|
||||||
|
describe("extensions", () => {
|
||||||
|
beforeEach(() => {
|
||||||
|
lg.setup(global);
|
||||||
|
});
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
lg.teardown(global);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("calls each extension hook", async () => {
|
||||||
|
const mockExtension = {
|
||||||
|
name: "TestExtension",
|
||||||
|
init: jest.fn(),
|
||||||
|
setup: jest.fn(),
|
||||||
|
addCustomNodeDefs: jest.fn(),
|
||||||
|
getCustomWidgets: jest.fn(),
|
||||||
|
beforeRegisterNodeDef: jest.fn(),
|
||||||
|
registerCustomNodes: jest.fn(),
|
||||||
|
loadedGraphNode: jest.fn(),
|
||||||
|
nodeCreated: jest.fn(),
|
||||||
|
beforeConfigureGraph: jest.fn(),
|
||||||
|
afterConfigureGraph: jest.fn(),
|
||||||
|
};
|
||||||
|
|
||||||
|
const { app, ez, graph } = await start({
|
||||||
|
async preSetup(app) {
|
||||||
|
app.registerExtension(mockExtension);
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
// Basic initialisation hooks should be called once, with app
|
||||||
|
expect(mockExtension.init).toHaveBeenCalledTimes(1);
|
||||||
|
expect(mockExtension.init).toHaveBeenCalledWith(app);
|
||||||
|
|
||||||
|
// Adding custom node defs should be passed the full list of nodes
|
||||||
|
expect(mockExtension.addCustomNodeDefs).toHaveBeenCalledTimes(1);
|
||||||
|
expect(mockExtension.addCustomNodeDefs.mock.calls[0][1]).toStrictEqual(app);
|
||||||
|
const defs = mockExtension.addCustomNodeDefs.mock.calls[0][0];
|
||||||
|
expect(defs).toHaveProperty("KSampler");
|
||||||
|
expect(defs).toHaveProperty("LoadImage");
|
||||||
|
|
||||||
|
// Get custom widgets is called once and should return new widget types
|
||||||
|
expect(mockExtension.getCustomWidgets).toHaveBeenCalledTimes(1);
|
||||||
|
expect(mockExtension.getCustomWidgets).toHaveBeenCalledWith(app);
|
||||||
|
|
||||||
|
// Before register node def will be called once per node type
|
||||||
|
const nodeNames = Object.keys(defs);
|
||||||
|
const nodeCount = nodeNames.length;
|
||||||
|
expect(mockExtension.beforeRegisterNodeDef).toHaveBeenCalledTimes(nodeCount);
|
||||||
|
for (let i = 0; i < 10; i++) {
|
||||||
|
// It should be send the JS class and the original JSON definition
|
||||||
|
const nodeClass = mockExtension.beforeRegisterNodeDef.mock.calls[i][0];
|
||||||
|
const nodeDef = mockExtension.beforeRegisterNodeDef.mock.calls[i][1];
|
||||||
|
|
||||||
|
expect(nodeClass.name).toBe("ComfyNode");
|
||||||
|
expect(nodeClass.comfyClass).toBe(nodeNames[i]);
|
||||||
|
expect(nodeDef.name).toBe(nodeNames[i]);
|
||||||
|
expect(nodeDef).toHaveProperty("input");
|
||||||
|
expect(nodeDef).toHaveProperty("output");
|
||||||
|
}
|
||||||
|
|
||||||
|
// Register custom nodes is called once after registerNode defs to allow adding other frontend nodes
|
||||||
|
expect(mockExtension.registerCustomNodes).toHaveBeenCalledTimes(1);
|
||||||
|
|
||||||
|
// Before configure graph will be called here as the default graph is being loaded
|
||||||
|
expect(mockExtension.beforeConfigureGraph).toHaveBeenCalledTimes(1);
|
||||||
|
// it gets sent the graph data that is going to be loaded
|
||||||
|
const graphData = mockExtension.beforeConfigureGraph.mock.calls[0][0];
|
||||||
|
|
||||||
|
// A node created is fired for each node constructor that is called
|
||||||
|
expect(mockExtension.nodeCreated).toHaveBeenCalledTimes(graphData.nodes.length);
|
||||||
|
for (let i = 0; i < graphData.nodes.length; i++) {
|
||||||
|
expect(mockExtension.nodeCreated.mock.calls[i][0].type).toBe(graphData.nodes[i].type);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Each node then calls loadedGraphNode to allow them to be updated
|
||||||
|
expect(mockExtension.loadedGraphNode).toHaveBeenCalledTimes(graphData.nodes.length);
|
||||||
|
for (let i = 0; i < graphData.nodes.length; i++) {
|
||||||
|
expect(mockExtension.loadedGraphNode.mock.calls[i][0].type).toBe(graphData.nodes[i].type);
|
||||||
|
}
|
||||||
|
|
||||||
|
// After configure is then called once all the setup is done
|
||||||
|
expect(mockExtension.afterConfigureGraph).toHaveBeenCalledTimes(1);
|
||||||
|
|
||||||
|
expect(mockExtension.setup).toHaveBeenCalledTimes(1);
|
||||||
|
expect(mockExtension.setup).toHaveBeenCalledWith(app);
|
||||||
|
|
||||||
|
// Ensure hooks are called in the correct order
|
||||||
|
const callOrder = [
|
||||||
|
"init",
|
||||||
|
"addCustomNodeDefs",
|
||||||
|
"getCustomWidgets",
|
||||||
|
"beforeRegisterNodeDef",
|
||||||
|
"registerCustomNodes",
|
||||||
|
"beforeConfigureGraph",
|
||||||
|
"nodeCreated",
|
||||||
|
"loadedGraphNode",
|
||||||
|
"afterConfigureGraph",
|
||||||
|
"setup",
|
||||||
|
];
|
||||||
|
for (let i = 1; i < callOrder.length; i++) {
|
||||||
|
const fn1 = mockExtension[callOrder[i - 1]];
|
||||||
|
const fn2 = mockExtension[callOrder[i]];
|
||||||
|
expect(fn1.mock.invocationCallOrder[0]).toBeLessThan(fn2.mock.invocationCallOrder[0]);
|
||||||
|
}
|
||||||
|
|
||||||
|
graph.clear();
|
||||||
|
|
||||||
|
// Ensure adding a new node calls the correct callback
|
||||||
|
ez.LoadImage();
|
||||||
|
expect(mockExtension.loadedGraphNode).toHaveBeenCalledTimes(graphData.nodes.length);
|
||||||
|
expect(mockExtension.nodeCreated).toHaveBeenCalledTimes(graphData.nodes.length + 1);
|
||||||
|
expect(mockExtension.nodeCreated.mock.lastCall[0].type).toBe("LoadImage");
|
||||||
|
|
||||||
|
// Reload the graph to ensure correct hooks are fired
|
||||||
|
await graph.reload();
|
||||||
|
|
||||||
|
// These hooks should not be fired again
|
||||||
|
expect(mockExtension.init).toHaveBeenCalledTimes(1);
|
||||||
|
expect(mockExtension.addCustomNodeDefs).toHaveBeenCalledTimes(1);
|
||||||
|
expect(mockExtension.getCustomWidgets).toHaveBeenCalledTimes(1);
|
||||||
|
expect(mockExtension.registerCustomNodes).toHaveBeenCalledTimes(1);
|
||||||
|
expect(mockExtension.beforeRegisterNodeDef).toHaveBeenCalledTimes(nodeCount);
|
||||||
|
expect(mockExtension.setup).toHaveBeenCalledTimes(1);
|
||||||
|
|
||||||
|
// These should be called again
|
||||||
|
expect(mockExtension.beforeConfigureGraph).toHaveBeenCalledTimes(2);
|
||||||
|
expect(mockExtension.nodeCreated).toHaveBeenCalledTimes(graphData.nodes.length + 2);
|
||||||
|
expect(mockExtension.loadedGraphNode).toHaveBeenCalledTimes(graphData.nodes.length + 1);
|
||||||
|
expect(mockExtension.afterConfigureGraph).toHaveBeenCalledTimes(2);
|
||||||
|
}, 15000);
|
||||||
|
|
||||||
|
it("allows custom nodeDefs and widgets to be registered", async () => {
|
||||||
|
const widgetMock = jest.fn((node, inputName, inputData, app) => {
|
||||||
|
expect(node.constructor.comfyClass).toBe("TestNode");
|
||||||
|
expect(inputName).toBe("test_input");
|
||||||
|
expect(inputData[0]).toBe("CUSTOMWIDGET");
|
||||||
|
expect(inputData[1]?.hello).toBe("world");
|
||||||
|
expect(app).toStrictEqual(app);
|
||||||
|
|
||||||
|
return {
|
||||||
|
widget: node.addWidget("button", inputName, "hello", () => {}),
|
||||||
|
};
|
||||||
|
});
|
||||||
|
|
||||||
|
// Register our extension that adds a custom node + widget type
|
||||||
|
const mockExtension = {
|
||||||
|
name: "TestExtension",
|
||||||
|
addCustomNodeDefs: (nodeDefs) => {
|
||||||
|
nodeDefs["TestNode"] = {
|
||||||
|
output: [],
|
||||||
|
output_name: [],
|
||||||
|
output_is_list: [],
|
||||||
|
name: "TestNode",
|
||||||
|
display_name: "TestNode",
|
||||||
|
category: "Test",
|
||||||
|
input: {
|
||||||
|
required: {
|
||||||
|
test_input: ["CUSTOMWIDGET", { hello: "world" }],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
};
|
||||||
|
},
|
||||||
|
getCustomWidgets: jest.fn(() => {
|
||||||
|
return {
|
||||||
|
CUSTOMWIDGET: widgetMock,
|
||||||
|
};
|
||||||
|
}),
|
||||||
|
};
|
||||||
|
|
||||||
|
const { graph, ez } = await start({
|
||||||
|
async preSetup(app) {
|
||||||
|
app.registerExtension(mockExtension);
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(mockExtension.getCustomWidgets).toBeCalledTimes(1);
|
||||||
|
|
||||||
|
graph.clear();
|
||||||
|
expect(widgetMock).toBeCalledTimes(0);
|
||||||
|
const node = ez.TestNode();
|
||||||
|
expect(widgetMock).toBeCalledTimes(1);
|
||||||
|
|
||||||
|
// Ensure our custom widget is created
|
||||||
|
expect(node.inputs.length).toBe(0);
|
||||||
|
expect(node.widgets.length).toBe(1);
|
||||||
|
const w = node.widgets[0].widget;
|
||||||
|
expect(w.name).toBe("test_input");
|
||||||
|
expect(w.type).toBe("button");
|
||||||
|
});
|
||||||
|
});
|
||||||
1005
tests-ui/tests/groupNode.test.js
Normal file
1005
tests-ui/tests/groupNode.test.js
Normal file
File diff suppressed because it is too large
Load Diff
@ -1,7 +1,13 @@
|
|||||||
// @ts-check
|
// @ts-check
|
||||||
/// <reference path="../node_modules/@types/jest/index.d.ts" />
|
/// <reference path="../node_modules/@types/jest/index.d.ts" />
|
||||||
|
|
||||||
const { start, makeNodeDef, checkBeforeAndAfterReload, assertNotNullOrUndefined } = require("../utils");
|
const {
|
||||||
|
start,
|
||||||
|
makeNodeDef,
|
||||||
|
checkBeforeAndAfterReload,
|
||||||
|
assertNotNullOrUndefined,
|
||||||
|
createDefaultWorkflow,
|
||||||
|
} = require("../utils");
|
||||||
const lg = require("../utils/litegraph");
|
const lg = require("../utils/litegraph");
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@ -14,10 +20,10 @@ const lg = require("../utils/litegraph");
|
|||||||
* @param { InstanceType<Ez["EzGraph"]> } graph
|
* @param { InstanceType<Ez["EzGraph"]> } graph
|
||||||
* @param { InstanceType<Ez["EzInput"]> } input
|
* @param { InstanceType<Ez["EzInput"]> } input
|
||||||
* @param { string } widgetType
|
* @param { string } widgetType
|
||||||
* @param { boolean } hasControlWidget
|
* @param { number } controlWidgetCount
|
||||||
* @returns
|
* @returns
|
||||||
*/
|
*/
|
||||||
async function connectPrimitiveAndReload(ez, graph, input, widgetType, hasControlWidget) {
|
async function connectPrimitiveAndReload(ez, graph, input, widgetType, controlWidgetCount = 0) {
|
||||||
// Connect to primitive and ensure its still connected after
|
// Connect to primitive and ensure its still connected after
|
||||||
let primitive = ez.PrimitiveNode();
|
let primitive = ez.PrimitiveNode();
|
||||||
primitive.outputs[0].connectTo(input);
|
primitive.outputs[0].connectTo(input);
|
||||||
@ -33,13 +39,17 @@ async function connectPrimitiveAndReload(ez, graph, input, widgetType, hasContro
|
|||||||
expect(valueWidget.widget.type).toBe(widgetType);
|
expect(valueWidget.widget.type).toBe(widgetType);
|
||||||
|
|
||||||
// Check if control_after_generate should be added
|
// Check if control_after_generate should be added
|
||||||
if (hasControlWidget) {
|
if (controlWidgetCount) {
|
||||||
const controlWidget = primitive.widgets.control_after_generate;
|
const controlWidget = primitive.widgets.control_after_generate;
|
||||||
expect(controlWidget.widget.type).toBe("combo");
|
expect(controlWidget.widget.type).toBe("combo");
|
||||||
|
if (widgetType === "combo") {
|
||||||
|
const filterWidget = primitive.widgets.control_filter_list;
|
||||||
|
expect(filterWidget.widget.type).toBe("string");
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Ensure we dont have other widgets
|
// Ensure we dont have other widgets
|
||||||
expect(primitive.node.widgets).toHaveLength(1 + +!!hasControlWidget);
|
expect(primitive.node.widgets).toHaveLength(1 + controlWidgetCount);
|
||||||
});
|
});
|
||||||
|
|
||||||
return primitive;
|
return primitive;
|
||||||
@ -55,8 +65,8 @@ describe("widget inputs", () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
[
|
[
|
||||||
{ name: "int", type: "INT", widget: "number", control: true },
|
{ name: "int", type: "INT", widget: "number", control: 1 },
|
||||||
{ name: "float", type: "FLOAT", widget: "number", control: true },
|
{ name: "float", type: "FLOAT", widget: "number", control: 1 },
|
||||||
{ name: "text", type: "STRING" },
|
{ name: "text", type: "STRING" },
|
||||||
{
|
{
|
||||||
name: "customtext",
|
name: "customtext",
|
||||||
@ -64,7 +74,7 @@ describe("widget inputs", () => {
|
|||||||
opt: { multiline: true },
|
opt: { multiline: true },
|
||||||
},
|
},
|
||||||
{ name: "toggle", type: "BOOLEAN" },
|
{ name: "toggle", type: "BOOLEAN" },
|
||||||
{ name: "combo", type: ["a", "b", "c"], control: true },
|
{ name: "combo", type: ["a", "b", "c"], control: 2 },
|
||||||
].forEach((c) => {
|
].forEach((c) => {
|
||||||
test(`widget conversion + primitive works on ${c.name}`, async () => {
|
test(`widget conversion + primitive works on ${c.name}`, async () => {
|
||||||
const { ez, graph } = await start({
|
const { ez, graph } = await start({
|
||||||
@ -106,7 +116,7 @@ describe("widget inputs", () => {
|
|||||||
n.widgets.ckpt_name.convertToInput();
|
n.widgets.ckpt_name.convertToInput();
|
||||||
expect(n.inputs.length).toEqual(inputCount + 1);
|
expect(n.inputs.length).toEqual(inputCount + 1);
|
||||||
|
|
||||||
const primitive = await connectPrimitiveAndReload(ez, graph, n.inputs.ckpt_name, "combo", true);
|
const primitive = await connectPrimitiveAndReload(ez, graph, n.inputs.ckpt_name, "combo", 2);
|
||||||
|
|
||||||
// Disconnect & reconnect
|
// Disconnect & reconnect
|
||||||
primitive.outputs[0].connections[0].disconnect();
|
primitive.outputs[0].connections[0].disconnect();
|
||||||
@ -198,8 +208,8 @@ describe("widget inputs", () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
expect(dialogShow).toBeCalledTimes(1);
|
expect(dialogShow).toBeCalledTimes(1);
|
||||||
expect(dialogShow.mock.calls[0][0]).toContain("the following node types were not found");
|
expect(dialogShow.mock.calls[0][0].innerHTML).toContain("the following node types were not found");
|
||||||
expect(dialogShow.mock.calls[0][0]).toContain("TestNode");
|
expect(dialogShow.mock.calls[0][0].innerHTML).toContain("TestNode");
|
||||||
});
|
});
|
||||||
|
|
||||||
test("defaultInput widgets can be converted back to inputs", async () => {
|
test("defaultInput widgets can be converted back to inputs", async () => {
|
||||||
@ -226,7 +236,7 @@ describe("widget inputs", () => {
|
|||||||
// Reload and ensure it still only has 1 converted widget
|
// Reload and ensure it still only has 1 converted widget
|
||||||
if (!assertNotNullOrUndefined(input)) return;
|
if (!assertNotNullOrUndefined(input)) return;
|
||||||
|
|
||||||
await connectPrimitiveAndReload(ez, graph, input, "number", true);
|
await connectPrimitiveAndReload(ez, graph, input, "number", 1);
|
||||||
n = graph.find(n);
|
n = graph.find(n);
|
||||||
expect(n.widgets).toHaveLength(1);
|
expect(n.widgets).toHaveLength(1);
|
||||||
w = n.widgets.example;
|
w = n.widgets.example;
|
||||||
@ -258,7 +268,7 @@ describe("widget inputs", () => {
|
|||||||
|
|
||||||
// Reload and ensure it still only has 1 converted widget
|
// Reload and ensure it still only has 1 converted widget
|
||||||
if (assertNotNullOrUndefined(input)) {
|
if (assertNotNullOrUndefined(input)) {
|
||||||
await connectPrimitiveAndReload(ez, graph, input, "number", true);
|
await connectPrimitiveAndReload(ez, graph, input, "number", 1);
|
||||||
n = graph.find(n);
|
n = graph.find(n);
|
||||||
expect(n.widgets).toHaveLength(1);
|
expect(n.widgets).toHaveLength(1);
|
||||||
expect(n.widgets.example.isConvertedToInput).toBeTruthy();
|
expect(n.widgets.example.isConvertedToInput).toBeTruthy();
|
||||||
@ -304,8 +314,8 @@ describe("widget inputs", () => {
|
|||||||
const { ez } = await start({
|
const { ez } = await start({
|
||||||
mockNodeDefs: {
|
mockNodeDefs: {
|
||||||
...makeNodeDef("TestNode1", {}, [["A", "B"]]),
|
...makeNodeDef("TestNode1", {}, [["A", "B"]]),
|
||||||
...makeNodeDef("TestNode2", { example: [["A", "B"], { forceInput: true}] }),
|
...makeNodeDef("TestNode2", { example: [["A", "B"], { forceInput: true }] }),
|
||||||
...makeNodeDef("TestNode3", { example: [["A", "B", "C"], { forceInput: true}] }),
|
...makeNodeDef("TestNode3", { example: [["A", "B", "C"], { forceInput: true }] }),
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
|
||||||
@ -316,4 +326,232 @@ describe("widget inputs", () => {
|
|||||||
n1.outputs[0].connectTo(n2.inputs[0]);
|
n1.outputs[0].connectTo(n2.inputs[0]);
|
||||||
expect(() => n1.outputs[0].connectTo(n3.inputs[0])).toThrow();
|
expect(() => n1.outputs[0].connectTo(n3.inputs[0])).toThrow();
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test("combo primitive can filter list when control_after_generate called", async () => {
|
||||||
|
const { ez } = await start({
|
||||||
|
mockNodeDefs: {
|
||||||
|
...makeNodeDef("TestNode1", { example: [["A", "B", "C", "D", "AA", "BB", "CC", "DD", "AAA", "BBB"], {}] }),
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
const n1 = ez.TestNode1();
|
||||||
|
n1.widgets.example.convertToInput();
|
||||||
|
const p = ez.PrimitiveNode();
|
||||||
|
p.outputs[0].connectTo(n1.inputs[0]);
|
||||||
|
|
||||||
|
const value = p.widgets.value;
|
||||||
|
const control = p.widgets.control_after_generate.widget;
|
||||||
|
const filter = p.widgets.control_filter_list;
|
||||||
|
|
||||||
|
expect(p.widgets.length).toBe(3);
|
||||||
|
control.value = "increment";
|
||||||
|
expect(value.value).toBe("A");
|
||||||
|
|
||||||
|
// Manually trigger after queue when set to increment
|
||||||
|
control["afterQueued"]();
|
||||||
|
expect(value.value).toBe("B");
|
||||||
|
|
||||||
|
// Filter to items containing D
|
||||||
|
filter.value = "D";
|
||||||
|
control["afterQueued"]();
|
||||||
|
expect(value.value).toBe("D");
|
||||||
|
control["afterQueued"]();
|
||||||
|
expect(value.value).toBe("DD");
|
||||||
|
|
||||||
|
// Check decrement
|
||||||
|
value.value = "BBB";
|
||||||
|
control.value = "decrement";
|
||||||
|
filter.value = "B";
|
||||||
|
control["afterQueued"]();
|
||||||
|
expect(value.value).toBe("BB");
|
||||||
|
control["afterQueued"]();
|
||||||
|
expect(value.value).toBe("B");
|
||||||
|
|
||||||
|
// Check regex works
|
||||||
|
value.value = "BBB";
|
||||||
|
filter.value = "/[AB]|^C$/";
|
||||||
|
control["afterQueued"]();
|
||||||
|
expect(value.value).toBe("AAA");
|
||||||
|
control["afterQueued"]();
|
||||||
|
expect(value.value).toBe("BB");
|
||||||
|
control["afterQueued"]();
|
||||||
|
expect(value.value).toBe("AA");
|
||||||
|
control["afterQueued"]();
|
||||||
|
expect(value.value).toBe("C");
|
||||||
|
control["afterQueued"]();
|
||||||
|
expect(value.value).toBe("B");
|
||||||
|
control["afterQueued"]();
|
||||||
|
expect(value.value).toBe("A");
|
||||||
|
|
||||||
|
// Check random
|
||||||
|
control.value = "randomize";
|
||||||
|
filter.value = "/D/";
|
||||||
|
for (let i = 0; i < 100; i++) {
|
||||||
|
control["afterQueued"]();
|
||||||
|
expect(value.value === "D" || value.value === "DD").toBeTruthy();
|
||||||
|
}
|
||||||
|
|
||||||
|
// Ensure it doesnt apply when fixed
|
||||||
|
control.value = "fixed";
|
||||||
|
value.value = "B";
|
||||||
|
filter.value = "C";
|
||||||
|
control["afterQueued"]();
|
||||||
|
expect(value.value).toBe("B");
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("reroutes", () => {
|
||||||
|
async function checkOutput(graph, values) {
|
||||||
|
expect((await graph.toPrompt()).output).toStrictEqual({
|
||||||
|
1: { inputs: { ckpt_name: "model1.safetensors" }, class_type: "CheckpointLoaderSimple" },
|
||||||
|
2: { inputs: { text: "positive", clip: ["1", 1] }, class_type: "CLIPTextEncode" },
|
||||||
|
3: { inputs: { text: "negative", clip: ["1", 1] }, class_type: "CLIPTextEncode" },
|
||||||
|
4: {
|
||||||
|
inputs: { width: values.width ?? 512, height: values.height ?? 512, batch_size: values?.batch_size ?? 1 },
|
||||||
|
class_type: "EmptyLatentImage",
|
||||||
|
},
|
||||||
|
5: {
|
||||||
|
inputs: {
|
||||||
|
seed: 0,
|
||||||
|
steps: 20,
|
||||||
|
cfg: 8,
|
||||||
|
sampler_name: "euler",
|
||||||
|
scheduler: values?.scheduler ?? "normal",
|
||||||
|
denoise: 1,
|
||||||
|
model: ["1", 0],
|
||||||
|
positive: ["2", 0],
|
||||||
|
negative: ["3", 0],
|
||||||
|
latent_image: ["4", 0],
|
||||||
|
},
|
||||||
|
class_type: "KSampler",
|
||||||
|
},
|
||||||
|
6: { inputs: { samples: ["5", 0], vae: ["1", 2] }, class_type: "VAEDecode" },
|
||||||
|
7: {
|
||||||
|
inputs: { filename_prefix: values.filename_prefix ?? "ComfyUI", images: ["6", 0] },
|
||||||
|
class_type: "SaveImage",
|
||||||
|
},
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
async function waitForWidget(node) {
|
||||||
|
// widgets are created slightly after the graph is ready
|
||||||
|
// hard to find an exact hook to get these so just wait for them to be ready
|
||||||
|
for (let i = 0; i < 10; i++) {
|
||||||
|
await new Promise((r) => setTimeout(r, 10));
|
||||||
|
if (node.widgets?.value) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
it("can connect primitive via a reroute path to a widget input", async () => {
|
||||||
|
const { ez, graph } = await start();
|
||||||
|
const nodes = createDefaultWorkflow(ez, graph);
|
||||||
|
|
||||||
|
nodes.empty.widgets.width.convertToInput();
|
||||||
|
nodes.sampler.widgets.scheduler.convertToInput();
|
||||||
|
nodes.save.widgets.filename_prefix.convertToInput();
|
||||||
|
|
||||||
|
let widthReroute = ez.Reroute();
|
||||||
|
let schedulerReroute = ez.Reroute();
|
||||||
|
let fileReroute = ez.Reroute();
|
||||||
|
|
||||||
|
let widthNext = widthReroute;
|
||||||
|
let schedulerNext = schedulerReroute;
|
||||||
|
let fileNext = fileReroute;
|
||||||
|
|
||||||
|
for (let i = 0; i < 5; i++) {
|
||||||
|
let next = ez.Reroute();
|
||||||
|
widthNext.outputs[0].connectTo(next.inputs[0]);
|
||||||
|
widthNext = next;
|
||||||
|
|
||||||
|
next = ez.Reroute();
|
||||||
|
schedulerNext.outputs[0].connectTo(next.inputs[0]);
|
||||||
|
schedulerNext = next;
|
||||||
|
|
||||||
|
next = ez.Reroute();
|
||||||
|
fileNext.outputs[0].connectTo(next.inputs[0]);
|
||||||
|
fileNext = next;
|
||||||
|
}
|
||||||
|
|
||||||
|
widthNext.outputs[0].connectTo(nodes.empty.inputs.width);
|
||||||
|
schedulerNext.outputs[0].connectTo(nodes.sampler.inputs.scheduler);
|
||||||
|
fileNext.outputs[0].connectTo(nodes.save.inputs.filename_prefix);
|
||||||
|
|
||||||
|
let widthPrimitive = ez.PrimitiveNode();
|
||||||
|
let schedulerPrimitive = ez.PrimitiveNode();
|
||||||
|
let filePrimitive = ez.PrimitiveNode();
|
||||||
|
|
||||||
|
widthPrimitive.outputs[0].connectTo(widthReroute.inputs[0]);
|
||||||
|
schedulerPrimitive.outputs[0].connectTo(schedulerReroute.inputs[0]);
|
||||||
|
filePrimitive.outputs[0].connectTo(fileReroute.inputs[0]);
|
||||||
|
expect(widthPrimitive.widgets.value.value).toBe(512);
|
||||||
|
widthPrimitive.widgets.value.value = 1024;
|
||||||
|
expect(schedulerPrimitive.widgets.value.value).toBe("normal");
|
||||||
|
schedulerPrimitive.widgets.value.value = "simple";
|
||||||
|
expect(filePrimitive.widgets.value.value).toBe("ComfyUI");
|
||||||
|
filePrimitive.widgets.value.value = "ComfyTest";
|
||||||
|
|
||||||
|
await checkBeforeAndAfterReload(graph, async () => {
|
||||||
|
widthPrimitive = graph.find(widthPrimitive);
|
||||||
|
schedulerPrimitive = graph.find(schedulerPrimitive);
|
||||||
|
filePrimitive = graph.find(filePrimitive);
|
||||||
|
await waitForWidget(filePrimitive);
|
||||||
|
expect(widthPrimitive.widgets.length).toBe(2);
|
||||||
|
expect(schedulerPrimitive.widgets.length).toBe(3);
|
||||||
|
expect(filePrimitive.widgets.length).toBe(1);
|
||||||
|
|
||||||
|
await checkOutput(graph, {
|
||||||
|
width: 1024,
|
||||||
|
scheduler: "simple",
|
||||||
|
filename_prefix: "ComfyTest",
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
|
it("can connect primitive via a reroute path to multiple widget inputs", async () => {
|
||||||
|
const { ez, graph } = await start();
|
||||||
|
const nodes = createDefaultWorkflow(ez, graph);
|
||||||
|
|
||||||
|
nodes.empty.widgets.width.convertToInput();
|
||||||
|
nodes.empty.widgets.height.convertToInput();
|
||||||
|
nodes.empty.widgets.batch_size.convertToInput();
|
||||||
|
|
||||||
|
let reroute = ez.Reroute();
|
||||||
|
let prevReroute = reroute;
|
||||||
|
for (let i = 0; i < 5; i++) {
|
||||||
|
const next = ez.Reroute();
|
||||||
|
prevReroute.outputs[0].connectTo(next.inputs[0]);
|
||||||
|
prevReroute = next;
|
||||||
|
}
|
||||||
|
|
||||||
|
const r1 = ez.Reroute(prevReroute.outputs[0]);
|
||||||
|
const r2 = ez.Reroute(prevReroute.outputs[0]);
|
||||||
|
const r3 = ez.Reroute(r2.outputs[0]);
|
||||||
|
const r4 = ez.Reroute(r2.outputs[0]);
|
||||||
|
|
||||||
|
r1.outputs[0].connectTo(nodes.empty.inputs.width);
|
||||||
|
r3.outputs[0].connectTo(nodes.empty.inputs.height);
|
||||||
|
r4.outputs[0].connectTo(nodes.empty.inputs.batch_size);
|
||||||
|
|
||||||
|
let primitive = ez.PrimitiveNode();
|
||||||
|
primitive.outputs[0].connectTo(reroute.inputs[0]);
|
||||||
|
expect(primitive.widgets.value.value).toBe(1);
|
||||||
|
primitive.widgets.value.value = 64;
|
||||||
|
|
||||||
|
await checkBeforeAndAfterReload(graph, async (r) => {
|
||||||
|
primitive = graph.find(primitive);
|
||||||
|
await waitForWidget(primitive);
|
||||||
|
|
||||||
|
// Ensure widget configs are merged
|
||||||
|
expect(primitive.widgets.value.widget.options?.min).toBe(16); // width/height min
|
||||||
|
expect(primitive.widgets.value.widget.options?.max).toBe(4096); // batch max
|
||||||
|
expect(primitive.widgets.value.widget.options?.step).toBe(80); // width/height step * 10
|
||||||
|
|
||||||
|
await checkOutput(graph, {
|
||||||
|
width: 64,
|
||||||
|
height: 64,
|
||||||
|
batch_size: 64,
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
@ -78,6 +78,14 @@ export class EzInput extends EzSlot {
|
|||||||
this.input = input;
|
this.input = input;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
get connection() {
|
||||||
|
const link = this.node.node.inputs?.[this.index]?.link;
|
||||||
|
if (link == null) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
return new EzConnection(this.node.app, this.node.app.graph.links[link]);
|
||||||
|
}
|
||||||
|
|
||||||
disconnect() {
|
disconnect() {
|
||||||
this.node.node.disconnectInput(this.index);
|
this.node.node.disconnectInput(this.index);
|
||||||
}
|
}
|
||||||
@ -117,7 +125,7 @@ export class EzOutput extends EzSlot {
|
|||||||
const inp = input.input;
|
const inp = input.input;
|
||||||
const inName = inp.name || inp.label || inp.type;
|
const inName = inp.name || inp.label || inp.type;
|
||||||
throw new Error(
|
throw new Error(
|
||||||
`Connecting from ${input.node.node.type}[${inName}#${input.index}] -> ${this.node.node.type}[${
|
`Connecting from ${input.node.node.type}#${input.node.id}[${inName}#${input.index}] -> ${this.node.node.type}#${this.node.id}[${
|
||||||
this.output.name ?? this.output.type
|
this.output.name ?? this.output.type
|
||||||
}#${this.index}] failed.`
|
}#${this.index}] failed.`
|
||||||
);
|
);
|
||||||
@ -150,7 +158,7 @@ export class EzNodeMenuItem {
|
|||||||
if (selectNode) {
|
if (selectNode) {
|
||||||
this.node.select();
|
this.node.select();
|
||||||
}
|
}
|
||||||
this.item.callback.call(this.node.node, undefined, undefined, undefined, undefined, this.node.node);
|
return this.item.callback.call(this.node.node, undefined, undefined, undefined, undefined, this.node.node);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -179,6 +187,7 @@ export class EzWidget {
|
|||||||
|
|
||||||
set value(v) {
|
set value(v) {
|
||||||
this.widget.value = v;
|
this.widget.value = v;
|
||||||
|
this.widget.callback?.call?.(this.widget, v)
|
||||||
}
|
}
|
||||||
|
|
||||||
get isConvertedToInput() {
|
get isConvertedToInput() {
|
||||||
@ -240,8 +249,12 @@ export class EzNode {
|
|||||||
return this.#makeLookupArray(() => this.app.canvas.getNodeMenuOptions(this.node), "content", EzNodeMenuItem);
|
return this.#makeLookupArray(() => this.app.canvas.getNodeMenuOptions(this.node), "content", EzNodeMenuItem);
|
||||||
}
|
}
|
||||||
|
|
||||||
select() {
|
get isRemoved() {
|
||||||
this.app.canvas.selectNode(this.node);
|
return !this.app.graph.getNodeById(this.id);
|
||||||
|
}
|
||||||
|
|
||||||
|
select(addToSelection = false) {
|
||||||
|
this.app.canvas.selectNode(this.node, addToSelection);
|
||||||
}
|
}
|
||||||
|
|
||||||
// /**
|
// /**
|
||||||
@ -275,12 +288,17 @@ export class EzNode {
|
|||||||
if (!s) return p;
|
if (!s) return p;
|
||||||
|
|
||||||
const name = s[nameProperty];
|
const name = s[nameProperty];
|
||||||
|
const item = new ctor(this, i, s);
|
||||||
// @ts-ignore
|
// @ts-ignore
|
||||||
if (!name || name in p) {
|
p.push(item);
|
||||||
throw new Error(`Unable to store ${nodeProperty} ${name} on array as name conflicts.`);
|
if (name) {
|
||||||
|
// @ts-ignore
|
||||||
|
if (name in p) {
|
||||||
|
throw new Error(`Unable to store ${nodeProperty} ${name} on array as name conflicts.`);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
// @ts-ignore
|
// @ts-ignore
|
||||||
p.push((p[name] = new ctor(this, i, s)));
|
p[name] = item;
|
||||||
return p;
|
return p;
|
||||||
}, Object.assign([], { $: this }));
|
}, Object.assign([], { $: this }));
|
||||||
}
|
}
|
||||||
@ -310,7 +328,7 @@ export class EzGraph {
|
|||||||
}
|
}
|
||||||
|
|
||||||
stringify() {
|
stringify() {
|
||||||
return JSON.stringify(this.app.graph.serialize(), undefined, "\t");
|
return JSON.stringify(this.app.graph.serialize(), undefined);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@ -348,6 +366,19 @@ export class EzGraph {
|
|||||||
}, 10);
|
}, 10);
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @returns { Promise<{
|
||||||
|
* workflow: {},
|
||||||
|
* output: Record<string, {
|
||||||
|
* class_name: string,
|
||||||
|
* inputs: Record<string, [string, number] | unknown>
|
||||||
|
* }>}> }
|
||||||
|
*/
|
||||||
|
toPrompt() {
|
||||||
|
// @ts-ignore
|
||||||
|
return this.app.graphToPrompt();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
export const Ez = {
|
export const Ez = {
|
||||||
@ -356,12 +387,12 @@ export const Ez = {
|
|||||||
* @example
|
* @example
|
||||||
* const { ez, graph } = Ez.graph(app);
|
* const { ez, graph } = Ez.graph(app);
|
||||||
* graph.clear();
|
* graph.clear();
|
||||||
* const [model, clip, vae] = ez.CheckpointLoaderSimple();
|
* const [model, clip, vae] = ez.CheckpointLoaderSimple().outputs;
|
||||||
* const [pos] = ez.CLIPTextEncode(clip, { text: "positive" });
|
* const [pos] = ez.CLIPTextEncode(clip, { text: "positive" }).outputs;
|
||||||
* const [neg] = ez.CLIPTextEncode(clip, { text: "negative" });
|
* const [neg] = ez.CLIPTextEncode(clip, { text: "negative" }).outputs;
|
||||||
* const [latent] = ez.KSampler(model, pos, neg, ...ez.EmptyLatentImage());
|
* const [latent] = ez.KSampler(model, pos, neg, ...ez.EmptyLatentImage().outputs).outputs;
|
||||||
* const [image] = ez.VAEDecode(latent, vae);
|
* const [image] = ez.VAEDecode(latent, vae).outputs;
|
||||||
* const saveNode = ez.SaveImage(image).node;
|
* const saveNode = ez.SaveImage(image);
|
||||||
* console.log(saveNode);
|
* console.log(saveNode);
|
||||||
* graph.arrange();
|
* graph.arrange();
|
||||||
* @param { app } app
|
* @param { app } app
|
||||||
|
|||||||
@ -1,16 +1,24 @@
|
|||||||
const { mockApi } = require("./setup");
|
const { mockApi } = require("./setup");
|
||||||
const { Ez } = require("./ezgraph");
|
const { Ez } = require("./ezgraph");
|
||||||
|
const lg = require("./litegraph");
|
||||||
|
|
||||||
/**
|
/**
|
||||||
*
|
*
|
||||||
* @param { Parameters<mockApi>[0] } config
|
* @param { Parameters<mockApi>[0] & { resetEnv?: boolean, preSetup?(app): Promise<void> } } config
|
||||||
* @returns
|
* @returns
|
||||||
*/
|
*/
|
||||||
export async function start(config = undefined) {
|
export async function start(config = {}) {
|
||||||
|
if(config.resetEnv) {
|
||||||
|
jest.resetModules();
|
||||||
|
jest.resetAllMocks();
|
||||||
|
lg.setup(global);
|
||||||
|
}
|
||||||
|
|
||||||
mockApi(config);
|
mockApi(config);
|
||||||
const { app } = require("../../web/scripts/app");
|
const { app } = require("../../web/scripts/app");
|
||||||
|
config.preSetup?.(app);
|
||||||
await app.setup();
|
await app.setup();
|
||||||
return Ez.graph(app, global["LiteGraph"], global["LGraphCanvas"]);
|
return { ...Ez.graph(app, global["LiteGraph"], global["LGraphCanvas"]), app };
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@ -37,19 +45,19 @@ export function makeNodeDef(name, input, output = {}) {
|
|||||||
output_name: [],
|
output_name: [],
|
||||||
output_is_list: [],
|
output_is_list: [],
|
||||||
input: {
|
input: {
|
||||||
required: {}
|
required: {},
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
for(const k in input) {
|
for (const k in input) {
|
||||||
nodeDef.input.required[k] = typeof input[k] === "string" ? [input[k], {}] : [...input[k]];
|
nodeDef.input.required[k] = typeof input[k] === "string" ? [input[k], {}] : [...input[k]];
|
||||||
}
|
}
|
||||||
if(output instanceof Array) {
|
if (output instanceof Array) {
|
||||||
output = output.reduce((p, c) => {
|
output = output.reduce((p, c) => {
|
||||||
p[c] = c;
|
p[c] = c;
|
||||||
return p;
|
return p;
|
||||||
}, {})
|
}, {});
|
||||||
}
|
}
|
||||||
for(const k in output) {
|
for (const k in output) {
|
||||||
nodeDef.output.push(output[k]);
|
nodeDef.output.push(output[k]);
|
||||||
nodeDef.output_name.push(k);
|
nodeDef.output_name.push(k);
|
||||||
nodeDef.output_is_list.push(false);
|
nodeDef.output_is_list.push(false);
|
||||||
@ -69,3 +77,39 @@ export function assertNotNullOrUndefined(x) {
|
|||||||
expect(x).not.toEqual(undefined);
|
expect(x).not.toEqual(undefined);
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
*
|
||||||
|
* @param { ReturnType<Ez["graph"]>["ez"] } ez
|
||||||
|
* @param { ReturnType<Ez["graph"]>["graph"] } graph
|
||||||
|
*/
|
||||||
|
export function createDefaultWorkflow(ez, graph) {
|
||||||
|
graph.clear();
|
||||||
|
const ckpt = ez.CheckpointLoaderSimple();
|
||||||
|
|
||||||
|
const pos = ez.CLIPTextEncode(ckpt.outputs.CLIP, { text: "positive" });
|
||||||
|
const neg = ez.CLIPTextEncode(ckpt.outputs.CLIP, { text: "negative" });
|
||||||
|
|
||||||
|
const empty = ez.EmptyLatentImage();
|
||||||
|
const sampler = ez.KSampler(
|
||||||
|
ckpt.outputs.MODEL,
|
||||||
|
pos.outputs.CONDITIONING,
|
||||||
|
neg.outputs.CONDITIONING,
|
||||||
|
empty.outputs.LATENT
|
||||||
|
);
|
||||||
|
|
||||||
|
const decode = ez.VAEDecode(sampler.outputs.LATENT, ckpt.outputs.VAE);
|
||||||
|
const save = ez.SaveImage(decode.outputs.IMAGE);
|
||||||
|
graph.arrange();
|
||||||
|
|
||||||
|
return { ckpt, pos, neg, empty, sampler, decode, save };
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function getNodeDefs() {
|
||||||
|
const { api } = require("../../web/scripts/api");
|
||||||
|
return api.getNodeDefs();
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function getNodeDef(nodeId) {
|
||||||
|
return (await getNodeDefs())[nodeId];
|
||||||
|
}
|
||||||
@ -30,16 +30,20 @@ export function mockApi({ mockExtensions, mockNodeDefs } = {}) {
|
|||||||
mockNodeDefs = JSON.parse(fs.readFileSync(path.resolve("./data/object_info.json")));
|
mockNodeDefs = JSON.parse(fs.readFileSync(path.resolve("./data/object_info.json")));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const events = new EventTarget();
|
||||||
|
const mockApi = {
|
||||||
|
addEventListener: events.addEventListener.bind(events),
|
||||||
|
removeEventListener: events.removeEventListener.bind(events),
|
||||||
|
dispatchEvent: events.dispatchEvent.bind(events),
|
||||||
|
getSystemStats: jest.fn(),
|
||||||
|
getExtensions: jest.fn(() => mockExtensions),
|
||||||
|
getNodeDefs: jest.fn(() => mockNodeDefs),
|
||||||
|
init: jest.fn(),
|
||||||
|
apiURL: jest.fn((x) => "../../web/" + x),
|
||||||
|
};
|
||||||
jest.mock("../../web/scripts/api", () => ({
|
jest.mock("../../web/scripts/api", () => ({
|
||||||
get api() {
|
get api() {
|
||||||
return {
|
return mockApi;
|
||||||
addEventListener: jest.fn(),
|
|
||||||
getSystemStats: jest.fn(),
|
|
||||||
getExtensions: jest.fn(() => mockExtensions),
|
|
||||||
getNodeDefs: jest.fn(() => mockNodeDefs),
|
|
||||||
init: jest.fn(),
|
|
||||||
apiURL: jest.fn((x) => "../../web/" + x),
|
|
||||||
};
|
|
||||||
},
|
},
|
||||||
}));
|
}));
|
||||||
}
|
}
|
||||||
|
|||||||
@ -174,6 +174,213 @@ const colorPalettes = {
|
|||||||
"tr-odd-bg-color": "#073642",
|
"tr-odd-bg-color": "#073642",
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
},
|
||||||
|
"arc": {
|
||||||
|
"id": "arc",
|
||||||
|
"name": "Arc",
|
||||||
|
"colors": {
|
||||||
|
"node_slot": {
|
||||||
|
"BOOLEAN": "",
|
||||||
|
"CLIP": "#eacb8b",
|
||||||
|
"CLIP_VISION": "#A8DADC",
|
||||||
|
"CLIP_VISION_OUTPUT": "#ad7452",
|
||||||
|
"CONDITIONING": "#cf876f",
|
||||||
|
"CONTROL_NET": "#00d78d",
|
||||||
|
"CONTROL_NET_WEIGHTS": "",
|
||||||
|
"FLOAT": "",
|
||||||
|
"GLIGEN": "",
|
||||||
|
"IMAGE": "#80a1c0",
|
||||||
|
"IMAGEUPLOAD": "",
|
||||||
|
"INT": "",
|
||||||
|
"LATENT": "#b38ead",
|
||||||
|
"LATENT_KEYFRAME": "",
|
||||||
|
"MASK": "#a3bd8d",
|
||||||
|
"MODEL": "#8978a7",
|
||||||
|
"SAMPLER": "",
|
||||||
|
"SIGMAS": "",
|
||||||
|
"STRING": "",
|
||||||
|
"STYLE_MODEL": "#C2FFAE",
|
||||||
|
"T2I_ADAPTER_WEIGHTS": "",
|
||||||
|
"TAESD": "#DCC274",
|
||||||
|
"TIMESTEP_KEYFRAME": "",
|
||||||
|
"UPSCALE_MODEL": "",
|
||||||
|
"VAE": "#be616b"
|
||||||
|
},
|
||||||
|
"litegraph_base": {
|
||||||
|
"BACKGROUND_IMAGE": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAGQAAABkCAYAAABw4pVUAAAACXBIWXMAAAsTAAALEwEAmpwYAAABcklEQVR4nO3YMUoDARgF4RfxBqZI6/0vZqFn0MYtrLIQMFN8U6V4LAtD+Jm9XG/v30OGl2e/AP7yevz4+vx45nvgF/+QGITEICQGITEIiUFIjNNC3q43u3/YnRJyPOzeQ+0e220nhRzReC8e7R7bbdvl+Jal1Bs46jEIiUFIDEJiEBKDkBhKPbZT6qHdptRTu02p53DUYxASg5AYhMQgJAYhMZR6bKfUQ7tNqad2m1LP4ajHICQGITEIiUFIDEJiKPXYTqmHdptST+02pZ7DUY9BSAxCYhASg5AYhMRQ6rGdUg/tNqWe2m1KPYejHoOQGITEICQGITEIiaHUYzulHtptSj2125R6Dkc9BiExCIlBSAxCYhASQ6nHdko9tNuUemq3KfUcjnoMQmIQEoOQGITEICSGUo/tlHpotyn11G5T6jkc9RiExCAkBiExCIlBSAylHtsp9dBuU+qp3abUczjqMQiJQUgMQmIQEoOQGITE+AHFISNQrFTGuwAAAABJRU5ErkJggg==",
|
||||||
|
"CLEAR_BACKGROUND_COLOR": "#2b2f38",
|
||||||
|
"NODE_TITLE_COLOR": "#b2b7bd",
|
||||||
|
"NODE_SELECTED_TITLE_COLOR": "#FFF",
|
||||||
|
"NODE_TEXT_SIZE": 14,
|
||||||
|
"NODE_TEXT_COLOR": "#AAA",
|
||||||
|
"NODE_SUBTEXT_SIZE": 12,
|
||||||
|
"NODE_DEFAULT_COLOR": "#2b2f38",
|
||||||
|
"NODE_DEFAULT_BGCOLOR": "#242730",
|
||||||
|
"NODE_DEFAULT_BOXCOLOR": "#6e7581",
|
||||||
|
"NODE_DEFAULT_SHAPE": "box",
|
||||||
|
"NODE_BOX_OUTLINE_COLOR": "#FFF",
|
||||||
|
"DEFAULT_SHADOW_COLOR": "rgba(0,0,0,0.5)",
|
||||||
|
"DEFAULT_GROUP_FONT": 22,
|
||||||
|
"WIDGET_BGCOLOR": "#2b2f38",
|
||||||
|
"WIDGET_OUTLINE_COLOR": "#6e7581",
|
||||||
|
"WIDGET_TEXT_COLOR": "#DDD",
|
||||||
|
"WIDGET_SECONDARY_TEXT_COLOR": "#b2b7bd",
|
||||||
|
"LINK_COLOR": "#9A9",
|
||||||
|
"EVENT_LINK_COLOR": "#A86",
|
||||||
|
"CONNECTING_LINK_COLOR": "#AFA"
|
||||||
|
},
|
||||||
|
"comfy_base": {
|
||||||
|
"fg-color": "#fff",
|
||||||
|
"bg-color": "#2b2f38",
|
||||||
|
"comfy-menu-bg": "#242730",
|
||||||
|
"comfy-input-bg": "#2b2f38",
|
||||||
|
"input-text": "#ddd",
|
||||||
|
"descrip-text": "#b2b7bd",
|
||||||
|
"drag-text": "#ccc",
|
||||||
|
"error-text": "#ff4444",
|
||||||
|
"border-color": "#6e7581",
|
||||||
|
"tr-even-bg-color": "#2b2f38",
|
||||||
|
"tr-odd-bg-color": "#242730"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
},
|
||||||
|
"nord": {
|
||||||
|
"id": "nord",
|
||||||
|
"name": "Nord",
|
||||||
|
"colors": {
|
||||||
|
"node_slot": {
|
||||||
|
"BOOLEAN": "",
|
||||||
|
"CLIP": "#eacb8b",
|
||||||
|
"CLIP_VISION": "#A8DADC",
|
||||||
|
"CLIP_VISION_OUTPUT": "#ad7452",
|
||||||
|
"CONDITIONING": "#cf876f",
|
||||||
|
"CONTROL_NET": "#00d78d",
|
||||||
|
"CONTROL_NET_WEIGHTS": "",
|
||||||
|
"FLOAT": "",
|
||||||
|
"GLIGEN": "",
|
||||||
|
"IMAGE": "#80a1c0",
|
||||||
|
"IMAGEUPLOAD": "",
|
||||||
|
"INT": "",
|
||||||
|
"LATENT": "#b38ead",
|
||||||
|
"LATENT_KEYFRAME": "",
|
||||||
|
"MASK": "#a3bd8d",
|
||||||
|
"MODEL": "#8978a7",
|
||||||
|
"SAMPLER": "",
|
||||||
|
"SIGMAS": "",
|
||||||
|
"STRING": "",
|
||||||
|
"STYLE_MODEL": "#C2FFAE",
|
||||||
|
"T2I_ADAPTER_WEIGHTS": "",
|
||||||
|
"TAESD": "#DCC274",
|
||||||
|
"TIMESTEP_KEYFRAME": "",
|
||||||
|
"UPSCALE_MODEL": "",
|
||||||
|
"VAE": "#be616b"
|
||||||
|
},
|
||||||
|
"litegraph_base": {
|
||||||
|
"BACKGROUND_IMAGE": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAGQAAABkCAIAAAD/gAIDAAAACXBIWXMAAAsTAAALEwEAmpwYAAAFu2lUWHRYTUw6Y29tLmFkb2JlLnhtcAAAAAAAPD94cGFja2V0IGJlZ2luPSLvu78iIGlkPSJXNU0wTXBDZWhpSHpyZVN6TlRjemtjOWQiPz4gPHg6eG1wbWV0YSB4bWxuczp4PSJhZG9iZTpuczptZXRhLyIgeDp4bXB0az0iQWRvYmUgWE1QIENvcmUgOS4xLWMwMDEgNzkuMTQ2Mjg5OSwgMjAyMy8wNi8yNS0yMDowMTo1NSAgICAgICAgIj4gPHJkZjpSREYgeG1sbnM6cmRmPSJodHRwOi8vd3d3LnczLm9yZy8xOTk5LzAyLzIyLXJkZi1zeW50YXgtbnMjIj4gPHJkZjpEZXNjcmlwdGlvbiByZGY6YWJvdXQ9IiIgeG1sbnM6eG1wPSJodHRwOi8vbnMuYWRvYmUuY29tL3hhcC8xLjAvIiB4bWxuczpkYz0iaHR0cDovL3B1cmwub3JnL2RjL2VsZW1lbnRzLzEuMS8iIHhtbG5zOnBob3Rvc2hvcD0iaHR0cDovL25zLmFkb2JlLmNvbS9waG90b3Nob3AvMS4wLyIgeG1sbnM6eG1wTU09Imh0dHA6Ly9ucy5hZG9iZS5jb20veGFwLzEuMC9tbS8iIHhtbG5zOnN0RXZ0PSJodHRwOi8vbnMuYWRvYmUuY29tL3hhcC8xLjAvc1R5cGUvUmVzb3VyY2VFdmVudCMiIHhtcDpDcmVhdG9yVG9vbD0iQWRvYmUgUGhvdG9zaG9wIDI1LjEgKFdpbmRvd3MpIiB4bXA6Q3JlYXRlRGF0ZT0iMjAyMy0xMS0xM1QwMDoxODowMiswMTowMCIgeG1wOk1vZGlmeURhdGU9IjIwMjMtMTEtMTVUMDE6MjA6NDUrMDE6MDAiIHhtcDpNZXRhZGF0YURhdGU9IjIwMjMtMTEtMTVUMDE6MjA6NDUrMDE6MDAiIGRjOmZvcm1hdD0iaW1hZ2UvcG5nIiBwaG90b3Nob3A6Q29sb3JNb2RlPSIzIiB4bXBNTTpJbnN0YW5jZUlEPSJ4bXAuaWlkOjUwNDFhMmZjLTEzNzQtMTk0ZC1hZWY4LTYxMzM1MTVmNjUwMCIgeG1wTU06RG9jdW1lbnRJRD0ieG1wLmRpZDoyMzFiMTBiMC1iNGZiLTAyNGUtYjEyZS0zMDUzMDNjZDA3YzgiIHhtcE1NOk9yaWdpbmFsRG9jdW1lbnRJRD0ieG1wLmRpZDoyMzFiMTBiMC1iNGZiLTAyNGUtYjEyZS0zMDUzMDNjZDA3YzgiPiA8eG1wTU06SGlzdG9yeT4gPHJkZjpTZXE+IDxyZGY6bGkgc3RFdnQ6YWN0aW9uPSJjcmVhdGVkIiBzdEV2dDppbnN0YW5jZUlEPSJ4bXAuaWlkOjIzMWIxMGIwLWI0ZmItMDI0ZS1iMTJlLTMwNTMwM2NkMDdjOCIgc3RFdnQ6d2hlbj0iMjAyMy0xMS0xM1QwMDoxODowMiswMTowMCIgc3RFdnQ6c29mdHdhcmVBZ2VudD0iQWRvYmUgUGhvdG9zaG9wIDI1LjEgKFdpbmRvd3MpIi8+IDxyZGY6bGkgc3RFdnQ6YWN0aW9uPSJzYXZlZCIgc3RFdnQ6aW5zdGFuY2VJRD0ieG1wLmlpZDo1MDQxYTJmYy0xMzc0LTE5NGQtYWVmOC02MTMzNTE1ZjY1MDAiIHN0RXZ0OndoZW49IjIwMjMtMTEtMTVUMDE6MjA6NDUrMDE6MDAiIHN0RXZ0OnNvZnR3YXJlQWdlbnQ9IkFkb2JlIFBob3Rvc2hvcCAyNS4xIChXaW5kb3dzKSIgc3RFdnQ6Y2hhbmdlZD0iLyIvPiA8L3JkZjpTZXE+IDwveG1wTU06SGlzdG9yeT4gPC9yZGY6RGVzY3JpcHRpb24+IDwvcmRmOlJERj4gPC94OnhtcG1ldGE+IDw/eHBhY2tldCBlbmQ9InIiPz73jWg/AAAAyUlEQVR42u3WKwoAIBRFQRdiMb1idv9Lsxn9gEFw4Dbb8JCTojbbXEJwjJVL2HKwYMGCBQuWLbDmjr+9zrBGjHl1WVcvy2DBggULFizTWQpewSt4HzwsgwULFiwFr7MUvMtS8D54WLBgGSxYCl7BK3iXZbBgwYIFC5bpLAWv4BW8Dx6WwYIFC5aC11kK3mUpeB88LFiwDBYsBa/gFbzLMliwYMGCBct0loJX8AreBw/LYMGCBUvB6ywF77IUvA8eFixYBgsWrNfWAZPltufdad+1AAAAAElFTkSuQmCC",
|
||||||
|
"CLEAR_BACKGROUND_COLOR": "#212732",
|
||||||
|
"NODE_TITLE_COLOR": "#999",
|
||||||
|
"NODE_SELECTED_TITLE_COLOR": "#e5eaf0",
|
||||||
|
"NODE_TEXT_SIZE": 14,
|
||||||
|
"NODE_TEXT_COLOR": "#bcc2c8",
|
||||||
|
"NODE_SUBTEXT_SIZE": 12,
|
||||||
|
"NODE_DEFAULT_COLOR": "#2e3440",
|
||||||
|
"NODE_DEFAULT_BGCOLOR": "#161b22",
|
||||||
|
"NODE_DEFAULT_BOXCOLOR": "#545d70",
|
||||||
|
"NODE_DEFAULT_SHAPE": "box",
|
||||||
|
"NODE_BOX_OUTLINE_COLOR": "#e5eaf0",
|
||||||
|
"DEFAULT_SHADOW_COLOR": "rgba(0,0,0,0.5)",
|
||||||
|
"DEFAULT_GROUP_FONT": 24,
|
||||||
|
"WIDGET_BGCOLOR": "#2e3440",
|
||||||
|
"WIDGET_OUTLINE_COLOR": "#545d70",
|
||||||
|
"WIDGET_TEXT_COLOR": "#bcc2c8",
|
||||||
|
"WIDGET_SECONDARY_TEXT_COLOR": "#999",
|
||||||
|
"LINK_COLOR": "#9A9",
|
||||||
|
"EVENT_LINK_COLOR": "#A86",
|
||||||
|
"CONNECTING_LINK_COLOR": "#AFA"
|
||||||
|
},
|
||||||
|
"comfy_base": {
|
||||||
|
"fg-color": "#e5eaf0",
|
||||||
|
"bg-color": "#2e3440",
|
||||||
|
"comfy-menu-bg": "#161b22",
|
||||||
|
"comfy-input-bg": "#2e3440",
|
||||||
|
"input-text": "#bcc2c8",
|
||||||
|
"descrip-text": "#999",
|
||||||
|
"drag-text": "#ccc",
|
||||||
|
"error-text": "#ff4444",
|
||||||
|
"border-color": "#545d70",
|
||||||
|
"tr-even-bg-color": "#2e3440",
|
||||||
|
"tr-odd-bg-color": "#161b22"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
},
|
||||||
|
"github": {
|
||||||
|
"id": "github",
|
||||||
|
"name": "Github",
|
||||||
|
"colors": {
|
||||||
|
"node_slot": {
|
||||||
|
"BOOLEAN": "",
|
||||||
|
"CLIP": "#eacb8b",
|
||||||
|
"CLIP_VISION": "#A8DADC",
|
||||||
|
"CLIP_VISION_OUTPUT": "#ad7452",
|
||||||
|
"CONDITIONING": "#cf876f",
|
||||||
|
"CONTROL_NET": "#00d78d",
|
||||||
|
"CONTROL_NET_WEIGHTS": "",
|
||||||
|
"FLOAT": "",
|
||||||
|
"GLIGEN": "",
|
||||||
|
"IMAGE": "#80a1c0",
|
||||||
|
"IMAGEUPLOAD": "",
|
||||||
|
"INT": "",
|
||||||
|
"LATENT": "#b38ead",
|
||||||
|
"LATENT_KEYFRAME": "",
|
||||||
|
"MASK": "#a3bd8d",
|
||||||
|
"MODEL": "#8978a7",
|
||||||
|
"SAMPLER": "",
|
||||||
|
"SIGMAS": "",
|
||||||
|
"STRING": "",
|
||||||
|
"STYLE_MODEL": "#C2FFAE",
|
||||||
|
"T2I_ADAPTER_WEIGHTS": "",
|
||||||
|
"TAESD": "#DCC274",
|
||||||
|
"TIMESTEP_KEYFRAME": "",
|
||||||
|
"UPSCALE_MODEL": "",
|
||||||
|
"VAE": "#be616b"
|
||||||
|
},
|
||||||
|
"litegraph_base": {
|
||||||
|
"BACKGROUND_IMAGE": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAGQAAABkCAIAAAD/gAIDAAAACXBIWXMAAAsTAAALEwEAmpwYAAAGlmlUWHRYTUw6Y29tLmFkb2JlLnhtcAAAAAAAPD94cGFja2V0IGJlZ2luPSLvu78iIGlkPSJXNU0wTXBDZWhpSHpyZVN6TlRjemtjOWQiPz4gPHg6eG1wbWV0YSB4bWxuczp4PSJhZG9iZTpuczptZXRhLyIgeDp4bXB0az0iQWRvYmUgWE1QIENvcmUgOS4xLWMwMDEgNzkuMTQ2Mjg5OSwgMjAyMy8wNi8yNS0yMDowMTo1NSAgICAgICAgIj4gPHJkZjpSREYgeG1sbnM6cmRmPSJodHRwOi8vd3d3LnczLm9yZy8xOTk5LzAyLzIyLXJkZi1zeW50YXgtbnMjIj4gPHJkZjpEZXNjcmlwdGlvbiByZGY6YWJvdXQ9IiIgeG1sbnM6eG1wPSJodHRwOi8vbnMuYWRvYmUuY29tL3hhcC8xLjAvIiB4bWxuczpkYz0iaHR0cDovL3B1cmwub3JnL2RjL2VsZW1lbnRzLzEuMS8iIHhtbG5zOnBob3Rvc2hvcD0iaHR0cDovL25zLmFkb2JlLmNvbS9waG90b3Nob3AvMS4wLyIgeG1sbnM6eG1wTU09Imh0dHA6Ly9ucy5hZG9iZS5jb20veGFwLzEuMC9tbS8iIHhtbG5zOnN0RXZ0PSJodHRwOi8vbnMuYWRvYmUuY29tL3hhcC8xLjAvc1R5cGUvUmVzb3VyY2VFdmVudCMiIHhtcDpDcmVhdG9yVG9vbD0iQWRvYmUgUGhvdG9zaG9wIDI1LjEgKFdpbmRvd3MpIiB4bXA6Q3JlYXRlRGF0ZT0iMjAyMy0xMS0xM1QwMDoxODowMiswMTowMCIgeG1wOk1vZGlmeURhdGU9IjIwMjMtMTEtMTVUMDI6MDQ6NTkrMDE6MDAiIHhtcDpNZXRhZGF0YURhdGU9IjIwMjMtMTEtMTVUMDI6MDQ6NTkrMDE6MDAiIGRjOmZvcm1hdD0iaW1hZ2UvcG5nIiBwaG90b3Nob3A6Q29sb3JNb2RlPSIzIiB4bXBNTTpJbnN0YW5jZUlEPSJ4bXAuaWlkOmIyYzRhNjA5LWJmYTctYTg0MC1iOGFlLTk3MzE2ZjM1ZGIyNyIgeG1wTU06RG9jdW1lbnRJRD0iYWRvYmU6ZG9jaWQ6cGhvdG9zaG9wOjk0ZmNlZGU4LTE1MTctZmQ0MC04ZGU3LWYzOTgxM2E3ODk5ZiIgeG1wTU06T3JpZ2luYWxEb2N1bWVudElEPSJ4bXAuZGlkOjIzMWIxMGIwLWI0ZmItMDI0ZS1iMTJlLTMwNTMwM2NkMDdjOCI+IDx4bXBNTTpIaXN0b3J5PiA8cmRmOlNlcT4gPHJkZjpsaSBzdEV2dDphY3Rpb249ImNyZWF0ZWQiIHN0RXZ0Omluc3RhbmNlSUQ9InhtcC5paWQ6MjMxYjEwYjAtYjRmYi0wMjRlLWIxMmUtMzA1MzAzY2QwN2M4IiBzdEV2dDp3aGVuPSIyMDIzLTExLTEzVDAwOjE4OjAyKzAxOjAwIiBzdEV2dDpzb2Z0d2FyZUFnZW50PSJBZG9iZSBQaG90b3Nob3AgMjUuMSAoV2luZG93cykiLz4gPHJkZjpsaSBzdEV2dDphY3Rpb249InNhdmVkIiBzdEV2dDppbnN0YW5jZUlEPSJ4bXAuaWlkOjQ4OWY1NzlmLTJkNjUtZWQ0Zi04OTg0LTA4NGE2MGE1ZTMzNSIgc3RFdnQ6d2hlbj0iMjAyMy0xMS0xNVQwMjowNDo1OSswMTowMCIgc3RFdnQ6c29mdHdhcmVBZ2VudD0iQWRvYmUgUGhvdG9zaG9wIDI1LjEgKFdpbmRvd3MpIiBzdEV2dDpjaGFuZ2VkPSIvIi8+IDxyZGY6bGkgc3RFdnQ6YWN0aW9uPSJzYXZlZCIgc3RFdnQ6aW5zdGFuY2VJRD0ieG1wLmlpZDpiMmM0YTYwOS1iZmE3LWE4NDAtYjhhZS05NzMxNmYzNWRiMjciIHN0RXZ0OndoZW49IjIwMjMtMTEtMTVUMDI6MDQ6NTkrMDE6MDAiIHN0RXZ0OnNvZnR3YXJlQWdlbnQ9IkFkb2JlIFBob3Rvc2hvcCAyNS4xIChXaW5kb3dzKSIgc3RFdnQ6Y2hhbmdlZD0iLyIvPiA8L3JkZjpTZXE+IDwveG1wTU06SGlzdG9yeT4gPC9yZGY6RGVzY3JpcHRpb24+IDwvcmRmOlJERj4gPC94OnhtcG1ldGE+IDw/eHBhY2tldCBlbmQ9InIiPz4OTe6GAAAAx0lEQVR42u3WMQoAIQxFwRzJys77X8vSLiRgITif7bYbgrwYc/mKXyBoY4VVBgsWLFiwYFmOlTv+9jfDOjHmr8u6eVkGCxYsWLBgmc5S8ApewXvgYRksWLBgKXidpeBdloL3wMOCBctgwVLwCl7BuyyDBQsWLFiwTGcpeAWv4D3wsAwWLFiwFLzOUvAuS8F74GHBgmWwYCl4Ba/gXZbBggULFixYprMUvIJX8B54WAYLFixYCl5nKXiXpeA98LBgwTJYsGC9tg1o8f4TTtqzNQAAAABJRU5ErkJggg==",
|
||||||
|
"CLEAR_BACKGROUND_COLOR": "#040506",
|
||||||
|
"NODE_TITLE_COLOR": "#999",
|
||||||
|
"NODE_SELECTED_TITLE_COLOR": "#e5eaf0",
|
||||||
|
"NODE_TEXT_SIZE": 14,
|
||||||
|
"NODE_TEXT_COLOR": "#bcc2c8",
|
||||||
|
"NODE_SUBTEXT_SIZE": 12,
|
||||||
|
"NODE_DEFAULT_COLOR": "#161b22",
|
||||||
|
"NODE_DEFAULT_BGCOLOR": "#13171d",
|
||||||
|
"NODE_DEFAULT_BOXCOLOR": "#30363d",
|
||||||
|
"NODE_DEFAULT_SHAPE": "box",
|
||||||
|
"NODE_BOX_OUTLINE_COLOR": "#e5eaf0",
|
||||||
|
"DEFAULT_SHADOW_COLOR": "rgba(0,0,0,0.5)",
|
||||||
|
"DEFAULT_GROUP_FONT": 24,
|
||||||
|
"WIDGET_BGCOLOR": "#161b22",
|
||||||
|
"WIDGET_OUTLINE_COLOR": "#30363d",
|
||||||
|
"WIDGET_TEXT_COLOR": "#bcc2c8",
|
||||||
|
"WIDGET_SECONDARY_TEXT_COLOR": "#999",
|
||||||
|
"LINK_COLOR": "#9A9",
|
||||||
|
"EVENT_LINK_COLOR": "#A86",
|
||||||
|
"CONNECTING_LINK_COLOR": "#AFA"
|
||||||
|
},
|
||||||
|
"comfy_base": {
|
||||||
|
"fg-color": "#e5eaf0",
|
||||||
|
"bg-color": "#161b22",
|
||||||
|
"comfy-menu-bg": "#13171d",
|
||||||
|
"comfy-input-bg": "#161b22",
|
||||||
|
"input-text": "#bcc2c8",
|
||||||
|
"descrip-text": "#999",
|
||||||
|
"drag-text": "#ccc",
|
||||||
|
"error-text": "#ff4444",
|
||||||
|
"border-color": "#30363d",
|
||||||
|
"tr-even-bg-color": "#161b22",
|
||||||
|
"tr-odd-bg-color": "#13171d"
|
||||||
|
}
|
||||||
|
},
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|||||||
1172
web/extensions/core/groupNode.js
Normal file
1172
web/extensions/core/groupNode.js
Normal file
File diff suppressed because it is too large
Load Diff
@ -33,6 +33,18 @@ function loadedImageToBlob(image) {
|
|||||||
return blob;
|
return blob;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function loadImage(imagePath) {
|
||||||
|
return new Promise((resolve, reject) => {
|
||||||
|
const image = new Image();
|
||||||
|
|
||||||
|
image.onload = function() {
|
||||||
|
resolve(image);
|
||||||
|
};
|
||||||
|
|
||||||
|
image.src = imagePath;
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
async function uploadMask(filepath, formData) {
|
async function uploadMask(filepath, formData) {
|
||||||
await api.fetchApi('/upload/mask', {
|
await api.fetchApi('/upload/mask', {
|
||||||
method: 'POST',
|
method: 'POST',
|
||||||
@ -42,7 +54,7 @@ async function uploadMask(filepath, formData) {
|
|||||||
});
|
});
|
||||||
|
|
||||||
ComfyApp.clipspace.imgs[ComfyApp.clipspace['selectedIndex']] = new Image();
|
ComfyApp.clipspace.imgs[ComfyApp.clipspace['selectedIndex']] = new Image();
|
||||||
ComfyApp.clipspace.imgs[ComfyApp.clipspace['selectedIndex']].src = api.apiURL("/view?" + new URLSearchParams(filepath).toString() + app.getPreviewFormatParam());
|
ComfyApp.clipspace.imgs[ComfyApp.clipspace['selectedIndex']].src = api.apiURL("/view?" + new URLSearchParams(filepath).toString() + app.getPreviewFormatParam() + app.getRandParam());
|
||||||
|
|
||||||
if(ComfyApp.clipspace.images)
|
if(ComfyApp.clipspace.images)
|
||||||
ComfyApp.clipspace.images[ComfyApp.clipspace['selectedIndex']] = filepath;
|
ComfyApp.clipspace.images[ComfyApp.clipspace['selectedIndex']] = filepath;
|
||||||
@ -50,25 +62,25 @@ async function uploadMask(filepath, formData) {
|
|||||||
ClipspaceDialog.invalidatePreview();
|
ClipspaceDialog.invalidatePreview();
|
||||||
}
|
}
|
||||||
|
|
||||||
function prepareRGB(image, backupCanvas, backupCtx) {
|
function prepare_mask(image, maskCanvas, maskCtx) {
|
||||||
// paste mask data into alpha channel
|
// paste mask data into alpha channel
|
||||||
backupCtx.drawImage(image, 0, 0, backupCanvas.width, backupCanvas.height);
|
maskCtx.drawImage(image, 0, 0, maskCanvas.width, maskCanvas.height);
|
||||||
const backupData = backupCtx.getImageData(0, 0, backupCanvas.width, backupCanvas.height);
|
const maskData = maskCtx.getImageData(0, 0, maskCanvas.width, maskCanvas.height);
|
||||||
|
|
||||||
// refine mask image
|
// invert mask
|
||||||
for (let i = 0; i < backupData.data.length; i += 4) {
|
for (let i = 0; i < maskData.data.length; i += 4) {
|
||||||
if(backupData.data[i+3] == 255)
|
if(maskData.data[i+3] == 255)
|
||||||
backupData.data[i+3] = 0;
|
maskData.data[i+3] = 0;
|
||||||
else
|
else
|
||||||
backupData.data[i+3] = 255;
|
maskData.data[i+3] = 255;
|
||||||
|
|
||||||
backupData.data[i] = 0;
|
maskData.data[i] = 0;
|
||||||
backupData.data[i+1] = 0;
|
maskData.data[i+1] = 0;
|
||||||
backupData.data[i+2] = 0;
|
maskData.data[i+2] = 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
backupCtx.globalCompositeOperation = 'source-over';
|
maskCtx.globalCompositeOperation = 'source-over';
|
||||||
backupCtx.putImageData(backupData, 0, 0);
|
maskCtx.putImageData(maskData, 0, 0);
|
||||||
}
|
}
|
||||||
|
|
||||||
class MaskEditorDialog extends ComfyDialog {
|
class MaskEditorDialog extends ComfyDialog {
|
||||||
@ -155,10 +167,6 @@ class MaskEditorDialog extends ComfyDialog {
|
|||||||
|
|
||||||
// If it is specified as relative, using it only as a hidden placeholder for padding is recommended
|
// If it is specified as relative, using it only as a hidden placeholder for padding is recommended
|
||||||
// to prevent anomalies where it exceeds a certain size and goes outside of the window.
|
// to prevent anomalies where it exceeds a certain size and goes outside of the window.
|
||||||
var placeholder = document.createElement("div");
|
|
||||||
placeholder.style.position = "relative";
|
|
||||||
placeholder.style.height = "50px";
|
|
||||||
|
|
||||||
var bottom_panel = document.createElement("div");
|
var bottom_panel = document.createElement("div");
|
||||||
bottom_panel.style.position = "absolute";
|
bottom_panel.style.position = "absolute";
|
||||||
bottom_panel.style.bottom = "0px";
|
bottom_panel.style.bottom = "0px";
|
||||||
@ -180,18 +188,16 @@ class MaskEditorDialog extends ComfyDialog {
|
|||||||
this.brush = brush;
|
this.brush = brush;
|
||||||
this.element.appendChild(imgCanvas);
|
this.element.appendChild(imgCanvas);
|
||||||
this.element.appendChild(maskCanvas);
|
this.element.appendChild(maskCanvas);
|
||||||
this.element.appendChild(placeholder); // must below z-index than bottom_panel to avoid covering button
|
|
||||||
this.element.appendChild(bottom_panel);
|
this.element.appendChild(bottom_panel);
|
||||||
document.body.appendChild(brush);
|
document.body.appendChild(brush);
|
||||||
|
|
||||||
var brush_size_slider = this.createLeftSlider(self, "Thickness", (event) => {
|
this.brush_size_slider = this.createLeftSlider(self, "Thickness", (event) => {
|
||||||
self.brush_size = event.target.value;
|
self.brush_size = event.target.value;
|
||||||
self.updateBrushPreview(self, null, null);
|
self.updateBrushPreview(self, null, null);
|
||||||
});
|
});
|
||||||
var clearButton = this.createLeftButton("Clear",
|
var clearButton = this.createLeftButton("Clear",
|
||||||
() => {
|
() => {
|
||||||
self.maskCtx.clearRect(0, 0, self.maskCanvas.width, self.maskCanvas.height);
|
self.maskCtx.clearRect(0, 0, self.maskCanvas.width, self.maskCanvas.height);
|
||||||
self.backupCtx.clearRect(0, 0, self.backupCanvas.width, self.backupCanvas.height);
|
|
||||||
});
|
});
|
||||||
var cancelButton = this.createRightButton("Cancel", () => {
|
var cancelButton = this.createRightButton("Cancel", () => {
|
||||||
document.removeEventListener("mouseup", MaskEditorDialog.handleMouseUp);
|
document.removeEventListener("mouseup", MaskEditorDialog.handleMouseUp);
|
||||||
@ -207,40 +213,42 @@ class MaskEditorDialog extends ComfyDialog {
|
|||||||
|
|
||||||
this.element.appendChild(imgCanvas);
|
this.element.appendChild(imgCanvas);
|
||||||
this.element.appendChild(maskCanvas);
|
this.element.appendChild(maskCanvas);
|
||||||
this.element.appendChild(placeholder); // must below z-index than bottom_panel to avoid covering button
|
|
||||||
this.element.appendChild(bottom_panel);
|
this.element.appendChild(bottom_panel);
|
||||||
|
|
||||||
bottom_panel.appendChild(clearButton);
|
bottom_panel.appendChild(clearButton);
|
||||||
bottom_panel.appendChild(this.saveButton);
|
bottom_panel.appendChild(this.saveButton);
|
||||||
bottom_panel.appendChild(cancelButton);
|
bottom_panel.appendChild(cancelButton);
|
||||||
bottom_panel.appendChild(brush_size_slider);
|
bottom_panel.appendChild(this.brush_size_slider);
|
||||||
|
|
||||||
|
imgCanvas.style.position = "absolute";
|
||||||
|
maskCanvas.style.position = "absolute";
|
||||||
|
|
||||||
imgCanvas.style.position = "relative";
|
|
||||||
imgCanvas.style.top = "200";
|
imgCanvas.style.top = "200";
|
||||||
imgCanvas.style.left = "0";
|
imgCanvas.style.left = "0";
|
||||||
|
|
||||||
maskCanvas.style.position = "absolute";
|
maskCanvas.style.top = imgCanvas.style.top;
|
||||||
|
maskCanvas.style.left = imgCanvas.style.left;
|
||||||
}
|
}
|
||||||
|
|
||||||
show() {
|
async show() {
|
||||||
|
this.zoom_ratio = 1.0;
|
||||||
|
this.pan_x = 0;
|
||||||
|
this.pan_y = 0;
|
||||||
|
|
||||||
if(!this.is_layout_created) {
|
if(!this.is_layout_created) {
|
||||||
// layout
|
// layout
|
||||||
const imgCanvas = document.createElement('canvas');
|
const imgCanvas = document.createElement('canvas');
|
||||||
const maskCanvas = document.createElement('canvas');
|
const maskCanvas = document.createElement('canvas');
|
||||||
const backupCanvas = document.createElement('canvas');
|
|
||||||
|
|
||||||
imgCanvas.id = "imageCanvas";
|
imgCanvas.id = "imageCanvas";
|
||||||
maskCanvas.id = "maskCanvas";
|
maskCanvas.id = "maskCanvas";
|
||||||
backupCanvas.id = "backupCanvas";
|
|
||||||
|
|
||||||
this.setlayout(imgCanvas, maskCanvas);
|
this.setlayout(imgCanvas, maskCanvas);
|
||||||
|
|
||||||
// prepare content
|
// prepare content
|
||||||
this.imgCanvas = imgCanvas;
|
this.imgCanvas = imgCanvas;
|
||||||
this.maskCanvas = maskCanvas;
|
this.maskCanvas = maskCanvas;
|
||||||
this.backupCanvas = backupCanvas;
|
this.maskCtx = maskCanvas.getContext('2d', {willReadFrequently: true });
|
||||||
this.maskCtx = maskCanvas.getContext('2d');
|
|
||||||
this.backupCtx = backupCanvas.getContext('2d');
|
|
||||||
|
|
||||||
this.setEventHandler(maskCanvas);
|
this.setEventHandler(maskCanvas);
|
||||||
|
|
||||||
@ -252,6 +260,8 @@ class MaskEditorDialog extends ComfyDialog {
|
|||||||
mutations.forEach(function(mutation) {
|
mutations.forEach(function(mutation) {
|
||||||
if (mutation.type === 'attributes' && mutation.attributeName === 'style') {
|
if (mutation.type === 'attributes' && mutation.attributeName === 'style') {
|
||||||
if(self.last_display_style && self.last_display_style != 'none' && self.element.style.display == 'none') {
|
if(self.last_display_style && self.last_display_style != 'none' && self.element.style.display == 'none') {
|
||||||
|
document.removeEventListener("mouseup", MaskEditorDialog.handleMouseUp);
|
||||||
|
self.brush.style.display = "none";
|
||||||
ComfyApp.onClipspaceEditorClosed();
|
ComfyApp.onClipspaceEditorClosed();
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -264,7 +274,8 @@ class MaskEditorDialog extends ComfyDialog {
|
|||||||
observer.observe(this.element, config);
|
observer.observe(this.element, config);
|
||||||
}
|
}
|
||||||
|
|
||||||
this.setImages(this.imgCanvas, this.backupCanvas);
|
// The keydown event needs to be reconfigured when closing the dialog as it gets removed.
|
||||||
|
document.addEventListener('keydown', MaskEditorDialog.handleKeyDown);
|
||||||
|
|
||||||
if(ComfyApp.clipspace_return_node) {
|
if(ComfyApp.clipspace_return_node) {
|
||||||
this.saveButton.innerText = "Save to node";
|
this.saveButton.innerText = "Save to node";
|
||||||
@ -275,97 +286,157 @@ class MaskEditorDialog extends ComfyDialog {
|
|||||||
this.saveButton.disabled = false;
|
this.saveButton.disabled = false;
|
||||||
|
|
||||||
this.element.style.display = "block";
|
this.element.style.display = "block";
|
||||||
|
this.element.style.width = "85%";
|
||||||
|
this.element.style.margin = "0 7.5%";
|
||||||
|
this.element.style.height = "100vh";
|
||||||
|
this.element.style.top = "50%";
|
||||||
|
this.element.style.left = "42%";
|
||||||
this.element.style.zIndex = 8888; // NOTE: alert dialog must be high priority.
|
this.element.style.zIndex = 8888; // NOTE: alert dialog must be high priority.
|
||||||
|
|
||||||
|
await this.setImages(this.imgCanvas);
|
||||||
|
|
||||||
|
this.is_visible = true;
|
||||||
}
|
}
|
||||||
|
|
||||||
isOpened() {
|
isOpened() {
|
||||||
return this.element.style.display == "block";
|
return this.element.style.display == "block";
|
||||||
}
|
}
|
||||||
|
|
||||||
setImages(imgCanvas, backupCanvas) {
|
invalidateCanvas(orig_image, mask_image) {
|
||||||
const imgCtx = imgCanvas.getContext('2d');
|
this.imgCanvas.width = orig_image.width;
|
||||||
const backupCtx = backupCanvas.getContext('2d');
|
this.imgCanvas.height = orig_image.height;
|
||||||
|
|
||||||
|
this.maskCanvas.width = orig_image.width;
|
||||||
|
this.maskCanvas.height = orig_image.height;
|
||||||
|
|
||||||
|
let imgCtx = this.imgCanvas.getContext('2d', {willReadFrequently: true });
|
||||||
|
let maskCtx = this.maskCanvas.getContext('2d', {willReadFrequently: true });
|
||||||
|
|
||||||
|
imgCtx.drawImage(orig_image, 0, 0, orig_image.width, orig_image.height);
|
||||||
|
prepare_mask(mask_image, this.maskCanvas, maskCtx);
|
||||||
|
}
|
||||||
|
|
||||||
|
async setImages(imgCanvas) {
|
||||||
|
let self = this;
|
||||||
|
|
||||||
|
const imgCtx = imgCanvas.getContext('2d', {willReadFrequently: true });
|
||||||
const maskCtx = this.maskCtx;
|
const maskCtx = this.maskCtx;
|
||||||
const maskCanvas = this.maskCanvas;
|
const maskCanvas = this.maskCanvas;
|
||||||
|
|
||||||
backupCtx.clearRect(0,0,this.backupCanvas.width,this.backupCanvas.height);
|
|
||||||
imgCtx.clearRect(0,0,this.imgCanvas.width,this.imgCanvas.height);
|
imgCtx.clearRect(0,0,this.imgCanvas.width,this.imgCanvas.height);
|
||||||
maskCtx.clearRect(0,0,this.maskCanvas.width,this.maskCanvas.height);
|
maskCtx.clearRect(0,0,this.maskCanvas.width,this.maskCanvas.height);
|
||||||
|
|
||||||
// image load
|
// image load
|
||||||
const orig_image = new Image();
|
|
||||||
window.addEventListener("resize", () => {
|
|
||||||
// repositioning
|
|
||||||
imgCanvas.width = window.innerWidth - 250;
|
|
||||||
imgCanvas.height = window.innerHeight - 200;
|
|
||||||
|
|
||||||
// redraw image
|
|
||||||
let drawWidth = orig_image.width;
|
|
||||||
let drawHeight = orig_image.height;
|
|
||||||
if (orig_image.width > imgCanvas.width) {
|
|
||||||
drawWidth = imgCanvas.width;
|
|
||||||
drawHeight = (drawWidth / orig_image.width) * orig_image.height;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (drawHeight > imgCanvas.height) {
|
|
||||||
drawHeight = imgCanvas.height;
|
|
||||||
drawWidth = (drawHeight / orig_image.height) * orig_image.width;
|
|
||||||
}
|
|
||||||
|
|
||||||
imgCtx.drawImage(orig_image, 0, 0, drawWidth, drawHeight);
|
|
||||||
|
|
||||||
// update mask
|
|
||||||
maskCanvas.width = drawWidth;
|
|
||||||
maskCanvas.height = drawHeight;
|
|
||||||
maskCanvas.style.top = imgCanvas.offsetTop + "px";
|
|
||||||
maskCanvas.style.left = imgCanvas.offsetLeft + "px";
|
|
||||||
backupCtx.drawImage(maskCanvas, 0, 0, maskCanvas.width, maskCanvas.height, 0, 0, backupCanvas.width, backupCanvas.height);
|
|
||||||
maskCtx.drawImage(backupCanvas, 0, 0, backupCanvas.width, backupCanvas.height, 0, 0, maskCanvas.width, maskCanvas.height);
|
|
||||||
});
|
|
||||||
|
|
||||||
const filepath = ComfyApp.clipspace.images;
|
const filepath = ComfyApp.clipspace.images;
|
||||||
|
|
||||||
const touched_image = new Image();
|
|
||||||
|
|
||||||
touched_image.onload = function() {
|
|
||||||
backupCanvas.width = touched_image.width;
|
|
||||||
backupCanvas.height = touched_image.height;
|
|
||||||
|
|
||||||
prepareRGB(touched_image, backupCanvas, backupCtx);
|
|
||||||
};
|
|
||||||
|
|
||||||
const alpha_url = new URL(ComfyApp.clipspace.imgs[ComfyApp.clipspace['selectedIndex']].src)
|
const alpha_url = new URL(ComfyApp.clipspace.imgs[ComfyApp.clipspace['selectedIndex']].src)
|
||||||
alpha_url.searchParams.delete('channel');
|
alpha_url.searchParams.delete('channel');
|
||||||
alpha_url.searchParams.delete('preview');
|
alpha_url.searchParams.delete('preview');
|
||||||
alpha_url.searchParams.set('channel', 'a');
|
alpha_url.searchParams.set('channel', 'a');
|
||||||
touched_image.src = alpha_url;
|
let mask_image = await loadImage(alpha_url);
|
||||||
|
|
||||||
// original image load
|
// original image load
|
||||||
orig_image.onload = function() {
|
|
||||||
window.dispatchEvent(new Event('resize'));
|
|
||||||
};
|
|
||||||
|
|
||||||
const rgb_url = new URL(ComfyApp.clipspace.imgs[ComfyApp.clipspace['selectedIndex']].src);
|
const rgb_url = new URL(ComfyApp.clipspace.imgs[ComfyApp.clipspace['selectedIndex']].src);
|
||||||
rgb_url.searchParams.delete('channel');
|
rgb_url.searchParams.delete('channel');
|
||||||
rgb_url.searchParams.set('channel', 'rgb');
|
rgb_url.searchParams.set('channel', 'rgb');
|
||||||
orig_image.src = rgb_url;
|
this.image = new Image();
|
||||||
this.image = orig_image;
|
this.image.onload = function() {
|
||||||
|
maskCanvas.width = self.image.width;
|
||||||
|
maskCanvas.height = self.image.height;
|
||||||
|
|
||||||
|
self.invalidateCanvas(self.image, mask_image);
|
||||||
|
self.initializeCanvasPanZoom();
|
||||||
|
};
|
||||||
|
this.image.src = rgb_url;
|
||||||
}
|
}
|
||||||
|
|
||||||
setEventHandler(maskCanvas) {
|
initializeCanvasPanZoom() {
|
||||||
maskCanvas.addEventListener("contextmenu", (event) => {
|
// set initialize
|
||||||
event.preventDefault();
|
let drawWidth = this.image.width;
|
||||||
});
|
let drawHeight = this.image.height;
|
||||||
|
|
||||||
|
let width = this.element.clientWidth;
|
||||||
|
let height = this.element.clientHeight;
|
||||||
|
|
||||||
|
if (this.image.width > width) {
|
||||||
|
drawWidth = width;
|
||||||
|
drawHeight = (drawWidth / this.image.width) * this.image.height;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (drawHeight > height) {
|
||||||
|
drawHeight = height;
|
||||||
|
drawWidth = (drawHeight / this.image.height) * this.image.width;
|
||||||
|
}
|
||||||
|
|
||||||
|
this.zoom_ratio = drawWidth/this.image.width;
|
||||||
|
|
||||||
|
const canvasX = (width - drawWidth) / 2;
|
||||||
|
const canvasY = (height - drawHeight) / 2;
|
||||||
|
this.pan_x = canvasX;
|
||||||
|
this.pan_y = canvasY;
|
||||||
|
|
||||||
|
this.invalidatePanZoom();
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
invalidatePanZoom() {
|
||||||
|
let raw_width = this.image.width * this.zoom_ratio;
|
||||||
|
let raw_height = this.image.height * this.zoom_ratio;
|
||||||
|
|
||||||
|
if(this.pan_x + raw_width < 10) {
|
||||||
|
this.pan_x = 10 - raw_width;
|
||||||
|
}
|
||||||
|
|
||||||
|
if(this.pan_y + raw_height < 10) {
|
||||||
|
this.pan_y = 10 - raw_height;
|
||||||
|
}
|
||||||
|
|
||||||
|
let width = `${raw_width}px`;
|
||||||
|
let height = `${raw_height}px`;
|
||||||
|
|
||||||
|
let left = `${this.pan_x}px`;
|
||||||
|
let top = `${this.pan_y}px`;
|
||||||
|
|
||||||
|
this.maskCanvas.style.width = width;
|
||||||
|
this.maskCanvas.style.height = height;
|
||||||
|
this.maskCanvas.style.left = left;
|
||||||
|
this.maskCanvas.style.top = top;
|
||||||
|
|
||||||
|
this.imgCanvas.style.width = width;
|
||||||
|
this.imgCanvas.style.height = height;
|
||||||
|
this.imgCanvas.style.left = left;
|
||||||
|
this.imgCanvas.style.top = top;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
setEventHandler(maskCanvas) {
|
||||||
const self = this;
|
const self = this;
|
||||||
maskCanvas.addEventListener('wheel', (event) => this.handleWheelEvent(self,event));
|
|
||||||
maskCanvas.addEventListener('pointerdown', (event) => this.handlePointerDown(self,event));
|
if(!this.handler_registered) {
|
||||||
document.addEventListener('pointerup', MaskEditorDialog.handlePointerUp);
|
maskCanvas.addEventListener("contextmenu", (event) => {
|
||||||
maskCanvas.addEventListener('pointermove', (event) => this.draw_move(self,event));
|
event.preventDefault();
|
||||||
maskCanvas.addEventListener('touchmove', (event) => this.draw_move(self,event));
|
});
|
||||||
maskCanvas.addEventListener('pointerover', (event) => { this.brush.style.display = "block"; });
|
|
||||||
maskCanvas.addEventListener('pointerleave', (event) => { this.brush.style.display = "none"; });
|
this.element.addEventListener('wheel', (event) => this.handleWheelEvent(self,event));
|
||||||
document.addEventListener('keydown', MaskEditorDialog.handleKeyDown);
|
this.element.addEventListener('pointermove', (event) => this.pointMoveEvent(self,event));
|
||||||
|
this.element.addEventListener('touchmove', (event) => this.pointMoveEvent(self,event));
|
||||||
|
|
||||||
|
this.element.addEventListener('dragstart', (event) => {
|
||||||
|
if(event.ctrlKey) {
|
||||||
|
event.preventDefault();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
maskCanvas.addEventListener('pointerdown', (event) => this.handlePointerDown(self,event));
|
||||||
|
maskCanvas.addEventListener('pointermove', (event) => this.draw_move(self,event));
|
||||||
|
maskCanvas.addEventListener('touchmove', (event) => this.draw_move(self,event));
|
||||||
|
maskCanvas.addEventListener('pointerover', (event) => { this.brush.style.display = "block"; });
|
||||||
|
maskCanvas.addEventListener('pointerleave', (event) => { this.brush.style.display = "none"; });
|
||||||
|
|
||||||
|
document.addEventListener('pointerup', MaskEditorDialog.handlePointerUp);
|
||||||
|
|
||||||
|
this.handler_registered = true;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
brush_size = 10;
|
brush_size = 10;
|
||||||
@ -378,8 +449,10 @@ class MaskEditorDialog extends ComfyDialog {
|
|||||||
const self = MaskEditorDialog.instance;
|
const self = MaskEditorDialog.instance;
|
||||||
if (event.key === ']') {
|
if (event.key === ']') {
|
||||||
self.brush_size = Math.min(self.brush_size+2, 100);
|
self.brush_size = Math.min(self.brush_size+2, 100);
|
||||||
|
self.brush_slider_input.value = self.brush_size;
|
||||||
} else if (event.key === '[') {
|
} else if (event.key === '[') {
|
||||||
self.brush_size = Math.max(self.brush_size-2, 1);
|
self.brush_size = Math.max(self.brush_size-2, 1);
|
||||||
|
self.brush_slider_input.value = self.brush_size;
|
||||||
} else if(event.key === 'Enter') {
|
} else if(event.key === 'Enter') {
|
||||||
self.save();
|
self.save();
|
||||||
}
|
}
|
||||||
@ -389,6 +462,10 @@ class MaskEditorDialog extends ComfyDialog {
|
|||||||
|
|
||||||
static handlePointerUp(event) {
|
static handlePointerUp(event) {
|
||||||
event.preventDefault();
|
event.preventDefault();
|
||||||
|
|
||||||
|
this.mousedown_x = null;
|
||||||
|
this.mousedown_y = null;
|
||||||
|
|
||||||
MaskEditorDialog.instance.drawing_mode = false;
|
MaskEditorDialog.instance.drawing_mode = false;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -398,24 +475,70 @@ class MaskEditorDialog extends ComfyDialog {
|
|||||||
var centerX = self.cursorX;
|
var centerX = self.cursorX;
|
||||||
var centerY = self.cursorY;
|
var centerY = self.cursorY;
|
||||||
|
|
||||||
brush.style.width = self.brush_size * 2 + "px";
|
brush.style.width = self.brush_size * 2 * this.zoom_ratio + "px";
|
||||||
brush.style.height = self.brush_size * 2 + "px";
|
brush.style.height = self.brush_size * 2 * this.zoom_ratio + "px";
|
||||||
brush.style.left = (centerX - self.brush_size) + "px";
|
brush.style.left = (centerX - self.brush_size * this.zoom_ratio) + "px";
|
||||||
brush.style.top = (centerY - self.brush_size) + "px";
|
brush.style.top = (centerY - self.brush_size * this.zoom_ratio) + "px";
|
||||||
}
|
}
|
||||||
|
|
||||||
handleWheelEvent(self, event) {
|
handleWheelEvent(self, event) {
|
||||||
if(event.deltaY < 0)
|
event.preventDefault();
|
||||||
self.brush_size = Math.min(self.brush_size+2, 100);
|
|
||||||
else
|
|
||||||
self.brush_size = Math.max(self.brush_size-2, 1);
|
|
||||||
|
|
||||||
self.brush_slider_input.value = self.brush_size;
|
if(event.ctrlKey) {
|
||||||
|
// zoom canvas
|
||||||
|
if(event.deltaY < 0) {
|
||||||
|
this.zoom_ratio = Math.min(10.0, this.zoom_ratio+0.2);
|
||||||
|
}
|
||||||
|
else {
|
||||||
|
this.zoom_ratio = Math.max(0.2, this.zoom_ratio-0.2);
|
||||||
|
}
|
||||||
|
|
||||||
|
this.invalidatePanZoom();
|
||||||
|
}
|
||||||
|
else {
|
||||||
|
// adjust brush size
|
||||||
|
if(event.deltaY < 0)
|
||||||
|
this.brush_size = Math.min(this.brush_size+2, 100);
|
||||||
|
else
|
||||||
|
this.brush_size = Math.max(this.brush_size-2, 1);
|
||||||
|
|
||||||
|
this.brush_slider_input.value = this.brush_size;
|
||||||
|
|
||||||
|
this.updateBrushPreview(this);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pointMoveEvent(self, event) {
|
||||||
|
this.cursorX = event.pageX;
|
||||||
|
this.cursorY = event.pageY;
|
||||||
|
|
||||||
self.updateBrushPreview(self);
|
self.updateBrushPreview(self);
|
||||||
|
|
||||||
|
if(event.ctrlKey) {
|
||||||
|
event.preventDefault();
|
||||||
|
self.pan_move(self, event);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pan_move(self, event) {
|
||||||
|
if(event.buttons == 1) {
|
||||||
|
if(this.mousedown_x) {
|
||||||
|
let deltaX = this.mousedown_x - event.clientX;
|
||||||
|
let deltaY = this.mousedown_y - event.clientY;
|
||||||
|
|
||||||
|
self.pan_x = this.mousedown_pan_x - deltaX;
|
||||||
|
self.pan_y = this.mousedown_pan_y - deltaY;
|
||||||
|
|
||||||
|
self.invalidatePanZoom();
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
draw_move(self, event) {
|
draw_move(self, event) {
|
||||||
|
if(event.ctrlKey) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
event.preventDefault();
|
event.preventDefault();
|
||||||
|
|
||||||
this.cursorX = event.pageX;
|
this.cursorX = event.pageX;
|
||||||
@ -439,6 +562,9 @@ class MaskEditorDialog extends ComfyDialog {
|
|||||||
y = event.targetTouches[0].clientY - maskRect.top;
|
y = event.targetTouches[0].clientY - maskRect.top;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
x /= self.zoom_ratio;
|
||||||
|
y /= self.zoom_ratio;
|
||||||
|
|
||||||
var brush_size = this.brush_size;
|
var brush_size = this.brush_size;
|
||||||
if(event instanceof PointerEvent && event.pointerType == 'pen') {
|
if(event instanceof PointerEvent && event.pointerType == 'pen') {
|
||||||
brush_size *= event.pressure;
|
brush_size *= event.pressure;
|
||||||
@ -489,8 +615,8 @@ class MaskEditorDialog extends ComfyDialog {
|
|||||||
}
|
}
|
||||||
else if(event.buttons == 2 || event.buttons == 5 || event.buttons == 32) {
|
else if(event.buttons == 2 || event.buttons == 5 || event.buttons == 32) {
|
||||||
const maskRect = self.maskCanvas.getBoundingClientRect();
|
const maskRect = self.maskCanvas.getBoundingClientRect();
|
||||||
const x = event.offsetX || event.targetTouches[0].clientX - maskRect.left;
|
const x = (event.offsetX || event.targetTouches[0].clientX - maskRect.left) / self.zoom_ratio;
|
||||||
const y = event.offsetY || event.targetTouches[0].clientY - maskRect.top;
|
const y = (event.offsetY || event.targetTouches[0].clientY - maskRect.top) / self.zoom_ratio;
|
||||||
|
|
||||||
var brush_size = this.brush_size;
|
var brush_size = this.brush_size;
|
||||||
if(event instanceof PointerEvent && event.pointerType == 'pen') {
|
if(event instanceof PointerEvent && event.pointerType == 'pen') {
|
||||||
@ -540,6 +666,17 @@ class MaskEditorDialog extends ComfyDialog {
|
|||||||
}
|
}
|
||||||
|
|
||||||
handlePointerDown(self, event) {
|
handlePointerDown(self, event) {
|
||||||
|
if(event.ctrlKey) {
|
||||||
|
if (event.buttons == 1) {
|
||||||
|
this.mousedown_x = event.clientX;
|
||||||
|
this.mousedown_y = event.clientY;
|
||||||
|
|
||||||
|
this.mousedown_pan_x = this.pan_x;
|
||||||
|
this.mousedown_pan_y = this.pan_y;
|
||||||
|
}
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
var brush_size = this.brush_size;
|
var brush_size = this.brush_size;
|
||||||
if(event instanceof PointerEvent && event.pointerType == 'pen') {
|
if(event instanceof PointerEvent && event.pointerType == 'pen') {
|
||||||
brush_size *= event.pressure;
|
brush_size *= event.pressure;
|
||||||
@ -551,8 +688,8 @@ class MaskEditorDialog extends ComfyDialog {
|
|||||||
|
|
||||||
event.preventDefault();
|
event.preventDefault();
|
||||||
const maskRect = self.maskCanvas.getBoundingClientRect();
|
const maskRect = self.maskCanvas.getBoundingClientRect();
|
||||||
const x = event.offsetX || event.targetTouches[0].clientX - maskRect.left;
|
const x = (event.offsetX || event.targetTouches[0].clientX - maskRect.left) / self.zoom_ratio;
|
||||||
const y = event.offsetY || event.targetTouches[0].clientY - maskRect.top;
|
const y = (event.offsetY || event.targetTouches[0].clientY - maskRect.top) / self.zoom_ratio;
|
||||||
|
|
||||||
self.maskCtx.beginPath();
|
self.maskCtx.beginPath();
|
||||||
if (event.button == 0) {
|
if (event.button == 0) {
|
||||||
@ -570,15 +707,18 @@ class MaskEditorDialog extends ComfyDialog {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async save() {
|
async save() {
|
||||||
const backupCtx = this.backupCanvas.getContext('2d', {willReadFrequently:true});
|
const backupCanvas = document.createElement('canvas');
|
||||||
|
const backupCtx = backupCanvas.getContext('2d', {willReadFrequently:true});
|
||||||
|
backupCanvas.width = this.image.width;
|
||||||
|
backupCanvas.height = this.image.height;
|
||||||
|
|
||||||
backupCtx.clearRect(0,0,this.backupCanvas.width,this.backupCanvas.height);
|
backupCtx.clearRect(0,0, backupCanvas.width, backupCanvas.height);
|
||||||
backupCtx.drawImage(this.maskCanvas,
|
backupCtx.drawImage(this.maskCanvas,
|
||||||
0, 0, this.maskCanvas.width, this.maskCanvas.height,
|
0, 0, this.maskCanvas.width, this.maskCanvas.height,
|
||||||
0, 0, this.backupCanvas.width, this.backupCanvas.height);
|
0, 0, backupCanvas.width, backupCanvas.height);
|
||||||
|
|
||||||
// paste mask data into alpha channel
|
// paste mask data into alpha channel
|
||||||
const backupData = backupCtx.getImageData(0, 0, this.backupCanvas.width, this.backupCanvas.height);
|
const backupData = backupCtx.getImageData(0, 0, backupCanvas.width, backupCanvas.height);
|
||||||
|
|
||||||
// refine mask image
|
// refine mask image
|
||||||
for (let i = 0; i < backupData.data.length; i += 4) {
|
for (let i = 0; i < backupData.data.length; i += 4) {
|
||||||
@ -615,7 +755,7 @@ class MaskEditorDialog extends ComfyDialog {
|
|||||||
ComfyApp.clipspace.widgets[index].value = item;
|
ComfyApp.clipspace.widgets[index].value = item;
|
||||||
}
|
}
|
||||||
|
|
||||||
const dataURL = this.backupCanvas.toDataURL();
|
const dataURL = backupCanvas.toDataURL();
|
||||||
const blob = dataURLToBlob(dataURL);
|
const blob = dataURLToBlob(dataURL);
|
||||||
|
|
||||||
let original_url = new URL(this.image.src);
|
let original_url = new URL(this.image.src);
|
||||||
|
|||||||
@ -1,5 +1,6 @@
|
|||||||
import { app } from "../../scripts/app.js";
|
import { app } from "../../scripts/app.js";
|
||||||
import { ComfyDialog, $el } from "../../scripts/ui.js";
|
import { ComfyDialog, $el } from "../../scripts/ui.js";
|
||||||
|
import { GroupNodeConfig, GroupNodeHandler } from "./groupNode.js";
|
||||||
|
|
||||||
// Adds the ability to save and add multiple nodes as a template
|
// Adds the ability to save and add multiple nodes as a template
|
||||||
// To save:
|
// To save:
|
||||||
@ -34,7 +35,7 @@ class ManageTemplates extends ComfyDialog {
|
|||||||
type: "file",
|
type: "file",
|
||||||
accept: ".json",
|
accept: ".json",
|
||||||
multiple: true,
|
multiple: true,
|
||||||
style: {display: "none"},
|
style: { display: "none" },
|
||||||
parent: document.body,
|
parent: document.body,
|
||||||
onchange: () => this.importAll(),
|
onchange: () => this.importAll(),
|
||||||
});
|
});
|
||||||
@ -109,13 +110,13 @@ class ManageTemplates extends ComfyDialog {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
const json = JSON.stringify({templates: this.templates}, null, 2); // convert the data to a JSON string
|
const json = JSON.stringify({ templates: this.templates }, null, 2); // convert the data to a JSON string
|
||||||
const blob = new Blob([json], {type: "application/json"});
|
const blob = new Blob([json], { type: "application/json" });
|
||||||
const url = URL.createObjectURL(blob);
|
const url = URL.createObjectURL(blob);
|
||||||
const a = $el("a", {
|
const a = $el("a", {
|
||||||
href: url,
|
href: url,
|
||||||
download: "node_templates.json",
|
download: "node_templates.json",
|
||||||
style: {display: "none"},
|
style: { display: "none" },
|
||||||
parent: document.body,
|
parent: document.body,
|
||||||
});
|
});
|
||||||
a.click();
|
a.click();
|
||||||
@ -163,7 +164,7 @@ class ManageTemplates extends ComfyDialog {
|
|||||||
var prev_i = el.dataset.id;
|
var prev_i = el.dataset.id;
|
||||||
|
|
||||||
if ( el == this.draggedEl && prev_i != i ) {
|
if ( el == this.draggedEl && prev_i != i ) {
|
||||||
[this.templates[i], this.templates[prev_i]] = [this.templates[prev_i], this.templates[i]];
|
this.templates.splice(i, 0, this.templates.splice(prev_i, 1)[0]);
|
||||||
}
|
}
|
||||||
el.dataset.id = i;
|
el.dataset.id = i;
|
||||||
});
|
});
|
||||||
@ -291,11 +292,11 @@ app.registerExtension({
|
|||||||
setup() {
|
setup() {
|
||||||
const manage = new ManageTemplates();
|
const manage = new ManageTemplates();
|
||||||
|
|
||||||
const clipboardAction = (cb) => {
|
const clipboardAction = async (cb) => {
|
||||||
// We use the clipboard functions but dont want to overwrite the current user clipboard
|
// We use the clipboard functions but dont want to overwrite the current user clipboard
|
||||||
// Restore it after we've run our callback
|
// Restore it after we've run our callback
|
||||||
const old = localStorage.getItem("litegrapheditor_clipboard");
|
const old = localStorage.getItem("litegrapheditor_clipboard");
|
||||||
cb();
|
await cb();
|
||||||
localStorage.setItem("litegrapheditor_clipboard", old);
|
localStorage.setItem("litegrapheditor_clipboard", old);
|
||||||
};
|
};
|
||||||
|
|
||||||
@ -309,13 +310,31 @@ app.registerExtension({
|
|||||||
disabled: !Object.keys(app.canvas.selected_nodes || {}).length,
|
disabled: !Object.keys(app.canvas.selected_nodes || {}).length,
|
||||||
callback: () => {
|
callback: () => {
|
||||||
const name = prompt("Enter name");
|
const name = prompt("Enter name");
|
||||||
if (!name || !name.trim()) return;
|
if (!name?.trim()) return;
|
||||||
|
|
||||||
clipboardAction(() => {
|
clipboardAction(() => {
|
||||||
app.canvas.copyToClipboard();
|
app.canvas.copyToClipboard();
|
||||||
|
let data = localStorage.getItem("litegrapheditor_clipboard");
|
||||||
|
data = JSON.parse(data);
|
||||||
|
const nodeIds = Object.keys(app.canvas.selected_nodes);
|
||||||
|
for (let i = 0; i < nodeIds.length; i++) {
|
||||||
|
const node = app.graph.getNodeById(nodeIds[i]);
|
||||||
|
const nodeData = node?.constructor.nodeData;
|
||||||
|
|
||||||
|
let groupData = GroupNodeHandler.getGroupData(node);
|
||||||
|
if (groupData) {
|
||||||
|
groupData = groupData.nodeData;
|
||||||
|
if (!data.groupNodes) {
|
||||||
|
data.groupNodes = {};
|
||||||
|
}
|
||||||
|
data.groupNodes[nodeData.name] = groupData;
|
||||||
|
data.nodes[i].type = nodeData.name;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
manage.templates.push({
|
manage.templates.push({
|
||||||
name,
|
name,
|
||||||
data: localStorage.getItem("litegrapheditor_clipboard"),
|
data: JSON.stringify(data),
|
||||||
});
|
});
|
||||||
manage.store();
|
manage.store();
|
||||||
});
|
});
|
||||||
@ -323,15 +342,19 @@ app.registerExtension({
|
|||||||
});
|
});
|
||||||
|
|
||||||
// Map each template to a menu item
|
// Map each template to a menu item
|
||||||
const subItems = manage.templates.map((t) => ({
|
const subItems = manage.templates.map((t) => {
|
||||||
content: t.name,
|
return {
|
||||||
callback: () => {
|
content: t.name,
|
||||||
clipboardAction(() => {
|
callback: () => {
|
||||||
localStorage.setItem("litegrapheditor_clipboard", t.data);
|
clipboardAction(async () => {
|
||||||
app.canvas.pasteFromClipboard();
|
const data = JSON.parse(t.data);
|
||||||
});
|
await GroupNodeConfig.registerFromWorkflow(data.groupNodes, {});
|
||||||
},
|
localStorage.setItem("litegrapheditor_clipboard", t.data);
|
||||||
}));
|
app.canvas.pasteFromClipboard();
|
||||||
|
});
|
||||||
|
},
|
||||||
|
};
|
||||||
|
});
|
||||||
|
|
||||||
subItems.push(null, {
|
subItems.push(null, {
|
||||||
content: "Manage",
|
content: "Manage",
|
||||||
|
|||||||
@ -1,10 +1,11 @@
|
|||||||
import { app } from "../../scripts/app.js";
|
import { app } from "../../scripts/app.js";
|
||||||
|
import { mergeIfValid, getWidgetConfig, setWidgetConfig } from "./widgetInputs.js";
|
||||||
|
|
||||||
// Node that allows you to redirect connections for cleaner graphs
|
// Node that allows you to redirect connections for cleaner graphs
|
||||||
|
|
||||||
app.registerExtension({
|
app.registerExtension({
|
||||||
name: "Comfy.RerouteNode",
|
name: "Comfy.RerouteNode",
|
||||||
registerCustomNodes() {
|
registerCustomNodes(app) {
|
||||||
class RerouteNode {
|
class RerouteNode {
|
||||||
constructor() {
|
constructor() {
|
||||||
if (!this.properties) {
|
if (!this.properties) {
|
||||||
@ -16,6 +17,12 @@ app.registerExtension({
|
|||||||
this.addInput("", "*");
|
this.addInput("", "*");
|
||||||
this.addOutput(this.properties.showOutputText ? "*" : "", "*");
|
this.addOutput(this.properties.showOutputText ? "*" : "", "*");
|
||||||
|
|
||||||
|
this.onAfterGraphConfigured = function () {
|
||||||
|
requestAnimationFrame(() => {
|
||||||
|
this.onConnectionsChange(LiteGraph.INPUT, null, true, null);
|
||||||
|
});
|
||||||
|
};
|
||||||
|
|
||||||
this.onConnectionsChange = function (type, index, connected, link_info) {
|
this.onConnectionsChange = function (type, index, connected, link_info) {
|
||||||
this.applyOrientation();
|
this.applyOrientation();
|
||||||
|
|
||||||
@ -47,6 +54,7 @@ app.registerExtension({
|
|||||||
const linkId = currentNode.inputs[0].link;
|
const linkId = currentNode.inputs[0].link;
|
||||||
if (linkId !== null) {
|
if (linkId !== null) {
|
||||||
const link = app.graph.links[linkId];
|
const link = app.graph.links[linkId];
|
||||||
|
if (!link) return;
|
||||||
const node = app.graph.getNodeById(link.origin_id);
|
const node = app.graph.getNodeById(link.origin_id);
|
||||||
const type = node.constructor.type;
|
const type = node.constructor.type;
|
||||||
if (type === "Reroute") {
|
if (type === "Reroute") {
|
||||||
@ -54,8 +62,7 @@ app.registerExtension({
|
|||||||
// We've found a circle
|
// We've found a circle
|
||||||
currentNode.disconnectInput(link.target_slot);
|
currentNode.disconnectInput(link.target_slot);
|
||||||
currentNode = null;
|
currentNode = null;
|
||||||
}
|
} else {
|
||||||
else {
|
|
||||||
// Move the previous node
|
// Move the previous node
|
||||||
currentNode = node;
|
currentNode = node;
|
||||||
}
|
}
|
||||||
@ -94,8 +101,11 @@ app.registerExtension({
|
|||||||
updateNodes.push(node);
|
updateNodes.push(node);
|
||||||
} else {
|
} else {
|
||||||
// We've found an output
|
// We've found an output
|
||||||
const nodeOutType = node.inputs && node.inputs[link?.target_slot] && node.inputs[link.target_slot].type ? node.inputs[link.target_slot].type : null;
|
const nodeOutType =
|
||||||
if (inputType && nodeOutType !== inputType) {
|
node.inputs && node.inputs[link?.target_slot] && node.inputs[link.target_slot].type
|
||||||
|
? node.inputs[link.target_slot].type
|
||||||
|
: null;
|
||||||
|
if (inputType && inputType !== "*" && nodeOutType !== inputType) {
|
||||||
// The output doesnt match our input so disconnect it
|
// The output doesnt match our input so disconnect it
|
||||||
node.disconnectInput(link.target_slot);
|
node.disconnectInput(link.target_slot);
|
||||||
} else {
|
} else {
|
||||||
@ -111,6 +121,9 @@ app.registerExtension({
|
|||||||
const displayType = inputType || outputType || "*";
|
const displayType = inputType || outputType || "*";
|
||||||
const color = LGraphCanvas.link_type_colors[displayType];
|
const color = LGraphCanvas.link_type_colors[displayType];
|
||||||
|
|
||||||
|
let widgetConfig;
|
||||||
|
let targetWidget;
|
||||||
|
let widgetType;
|
||||||
// Update the types of each node
|
// Update the types of each node
|
||||||
for (const node of updateNodes) {
|
for (const node of updateNodes) {
|
||||||
// If we dont have an input type we are always wildcard but we'll show the output type
|
// If we dont have an input type we are always wildcard but we'll show the output type
|
||||||
@ -125,10 +138,38 @@ app.registerExtension({
|
|||||||
const link = app.graph.links[l];
|
const link = app.graph.links[l];
|
||||||
if (link) {
|
if (link) {
|
||||||
link.color = color;
|
link.color = color;
|
||||||
|
|
||||||
|
if (app.configuringGraph) continue;
|
||||||
|
const targetNode = app.graph.getNodeById(link.target_id);
|
||||||
|
const targetInput = targetNode.inputs?.[link.target_slot];
|
||||||
|
if (targetInput?.widget) {
|
||||||
|
const config = getWidgetConfig(targetInput);
|
||||||
|
if (!widgetConfig) {
|
||||||
|
widgetConfig = config[1] ?? {};
|
||||||
|
widgetType = config[0];
|
||||||
|
}
|
||||||
|
if (!targetWidget) {
|
||||||
|
targetWidget = targetNode.widgets?.find((w) => w.name === targetInput.widget.name);
|
||||||
|
}
|
||||||
|
|
||||||
|
const merged = mergeIfValid(targetInput, [config[0], widgetConfig]);
|
||||||
|
if (merged.customConfig) {
|
||||||
|
widgetConfig = merged.customConfig;
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
for (const node of updateNodes) {
|
||||||
|
if (widgetConfig && outputType) {
|
||||||
|
node.inputs[0].widget = { name: "value" };
|
||||||
|
setWidgetConfig(node.inputs[0], [widgetType ?? displayType, widgetConfig], targetWidget);
|
||||||
|
} else {
|
||||||
|
setWidgetConfig(node.inputs[0], null);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
if (inputNode) {
|
if (inputNode) {
|
||||||
const link = app.graph.links[inputNode.inputs[0].link];
|
const link = app.graph.links[inputNode.inputs[0].link];
|
||||||
if (link) {
|
if (link) {
|
||||||
|
|||||||
@ -1,5 +1,5 @@
|
|||||||
import { app } from "../../scripts/app.js";
|
import { app } from "../../scripts/app.js";
|
||||||
|
import { applyTextReplacements } from "../../scripts/utils.js";
|
||||||
// Use widget values and dates in output filenames
|
// Use widget values and dates in output filenames
|
||||||
|
|
||||||
app.registerExtension({
|
app.registerExtension({
|
||||||
@ -7,84 +7,19 @@ app.registerExtension({
|
|||||||
async beforeRegisterNodeDef(nodeType, nodeData, app) {
|
async beforeRegisterNodeDef(nodeType, nodeData, app) {
|
||||||
if (nodeData.name === "SaveImage") {
|
if (nodeData.name === "SaveImage") {
|
||||||
const onNodeCreated = nodeType.prototype.onNodeCreated;
|
const onNodeCreated = nodeType.prototype.onNodeCreated;
|
||||||
|
// When the SaveImage node is created we want to override the serialization of the output name widget to run our S&R
|
||||||
// Simple date formatter
|
|
||||||
const parts = {
|
|
||||||
d: (d) => d.getDate(),
|
|
||||||
M: (d) => d.getMonth() + 1,
|
|
||||||
h: (d) => d.getHours(),
|
|
||||||
m: (d) => d.getMinutes(),
|
|
||||||
s: (d) => d.getSeconds(),
|
|
||||||
};
|
|
||||||
const format =
|
|
||||||
Object.keys(parts)
|
|
||||||
.map((k) => k + k + "?")
|
|
||||||
.join("|") + "|yyy?y?";
|
|
||||||
|
|
||||||
function formatDate(text, date) {
|
|
||||||
return text.replace(new RegExp(format, "g"), function (text) {
|
|
||||||
if (text === "yy") return (date.getFullYear() + "").substring(2);
|
|
||||||
if (text === "yyyy") return date.getFullYear();
|
|
||||||
if (text[0] in parts) {
|
|
||||||
const p = parts[text[0]](date);
|
|
||||||
return (p + "").padStart(text.length, "0");
|
|
||||||
}
|
|
||||||
return text;
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
// When the SaveImage node is created we want to override the serialization of the output name widget to run our S&R
|
|
||||||
nodeType.prototype.onNodeCreated = function () {
|
nodeType.prototype.onNodeCreated = function () {
|
||||||
const r = onNodeCreated ? onNodeCreated.apply(this, arguments) : undefined;
|
const r = onNodeCreated ? onNodeCreated.apply(this, arguments) : undefined;
|
||||||
|
|
||||||
const widget = this.widgets.find((w) => w.name === "filename_prefix");
|
const widget = this.widgets.find((w) => w.name === "filename_prefix");
|
||||||
widget.serializeValue = () => {
|
widget.serializeValue = () => {
|
||||||
return widget.value.replace(/%([^%]+)%/g, function (match, text) {
|
return applyTextReplacements(app, widget.value);
|
||||||
const split = text.split(".");
|
|
||||||
if (split.length !== 2) {
|
|
||||||
// Special handling for dates
|
|
||||||
if (split[0].startsWith("date:")) {
|
|
||||||
return formatDate(split[0].substring(5), new Date());
|
|
||||||
}
|
|
||||||
|
|
||||||
if (text !== "width" && text !== "height") {
|
|
||||||
// Dont warn on standard replacements
|
|
||||||
console.warn("Invalid replacement pattern", text);
|
|
||||||
}
|
|
||||||
return match;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Find node with matching S&R property name
|
|
||||||
let nodes = app.graph._nodes.filter((n) => n.properties?.["Node name for S&R"] === split[0]);
|
|
||||||
// If we cant, see if there is a node with that title
|
|
||||||
if (!nodes.length) {
|
|
||||||
nodes = app.graph._nodes.filter((n) => n.title === split[0]);
|
|
||||||
}
|
|
||||||
if (!nodes.length) {
|
|
||||||
console.warn("Unable to find node", split[0]);
|
|
||||||
return match;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (nodes.length > 1) {
|
|
||||||
console.warn("Multiple nodes matched", split[0], "using first match");
|
|
||||||
}
|
|
||||||
|
|
||||||
const node = nodes[0];
|
|
||||||
|
|
||||||
const widget = node.widgets?.find((w) => w.name === split[1]);
|
|
||||||
if (!widget) {
|
|
||||||
console.warn("Unable to find widget", split[1], "on node", split[0], node);
|
|
||||||
return match;
|
|
||||||
}
|
|
||||||
|
|
||||||
return ((widget.value ?? "") + "").replaceAll(/\/|\\/g, "_");
|
|
||||||
});
|
|
||||||
};
|
};
|
||||||
|
|
||||||
return r;
|
return r;
|
||||||
};
|
};
|
||||||
} else {
|
} else {
|
||||||
// When any other node is created add a property to alias the node
|
// When any other node is created add a property to alias the node
|
||||||
const onNodeCreated = nodeType.prototype.onNodeCreated;
|
const onNodeCreated = nodeType.prototype.onNodeCreated;
|
||||||
nodeType.prototype.onNodeCreated = function () {
|
nodeType.prototype.onNodeCreated = function () {
|
||||||
const r = onNodeCreated ? onNodeCreated.apply(this, arguments) : undefined;
|
const r = onNodeCreated ? onNodeCreated.apply(this, arguments) : undefined;
|
||||||
|
|||||||
147
web/extensions/core/undoRedo.js
Normal file
147
web/extensions/core/undoRedo.js
Normal file
@ -0,0 +1,147 @@
|
|||||||
|
import { app } from "../../scripts/app.js";
|
||||||
|
|
||||||
|
const MAX_HISTORY = 50;
|
||||||
|
|
||||||
|
let undo = [];
|
||||||
|
let redo = [];
|
||||||
|
let activeState = null;
|
||||||
|
let isOurLoad = false;
|
||||||
|
function checkState() {
|
||||||
|
const currentState = app.graph.serialize();
|
||||||
|
if (!graphEqual(activeState, currentState)) {
|
||||||
|
undo.push(activeState);
|
||||||
|
if (undo.length > MAX_HISTORY) {
|
||||||
|
undo.shift();
|
||||||
|
}
|
||||||
|
activeState = clone(currentState);
|
||||||
|
redo.length = 0;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const loadGraphData = app.loadGraphData;
|
||||||
|
app.loadGraphData = async function () {
|
||||||
|
const v = await loadGraphData.apply(this, arguments);
|
||||||
|
if (isOurLoad) {
|
||||||
|
isOurLoad = false;
|
||||||
|
} else {
|
||||||
|
checkState();
|
||||||
|
}
|
||||||
|
return v;
|
||||||
|
};
|
||||||
|
|
||||||
|
function clone(obj) {
|
||||||
|
try {
|
||||||
|
if (typeof structuredClone !== "undefined") {
|
||||||
|
return structuredClone(obj);
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
// structuredClone is stricter than using JSON.parse/stringify so fallback to that
|
||||||
|
}
|
||||||
|
|
||||||
|
return JSON.parse(JSON.stringify(obj));
|
||||||
|
}
|
||||||
|
|
||||||
|
function graphEqual(a, b, root = true) {
|
||||||
|
if (a === b) return true;
|
||||||
|
|
||||||
|
if (typeof a == "object" && a && typeof b == "object" && b) {
|
||||||
|
const keys = Object.getOwnPropertyNames(a);
|
||||||
|
|
||||||
|
if (keys.length != Object.getOwnPropertyNames(b).length) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
for (const key of keys) {
|
||||||
|
let av = a[key];
|
||||||
|
let bv = b[key];
|
||||||
|
if (root && key === "nodes") {
|
||||||
|
// Nodes need to be sorted as the order changes when selecting nodes
|
||||||
|
av = [...av].sort((a, b) => a.id - b.id);
|
||||||
|
bv = [...bv].sort((a, b) => a.id - b.id);
|
||||||
|
}
|
||||||
|
if (!graphEqual(av, bv, false)) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
const undoRedo = async (e) => {
|
||||||
|
const updateState = async (source, target) => {
|
||||||
|
const prevState = source.pop();
|
||||||
|
if (prevState) {
|
||||||
|
target.push(activeState);
|
||||||
|
isOurLoad = true;
|
||||||
|
await app.loadGraphData(prevState, false);
|
||||||
|
activeState = prevState;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (e.ctrlKey || e.metaKey) {
|
||||||
|
if (e.key === "y") {
|
||||||
|
updateState(redo, undo);
|
||||||
|
return true;
|
||||||
|
} else if (e.key === "z") {
|
||||||
|
updateState(undo, redo);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const bindInput = (activeEl) => {
|
||||||
|
if (activeEl?.tagName !== "CANVAS" && activeEl?.tagName !== "BODY") {
|
||||||
|
for (const evt of ["change", "input", "blur"]) {
|
||||||
|
if (`on${evt}` in activeEl) {
|
||||||
|
const listener = () => {
|
||||||
|
checkState();
|
||||||
|
activeEl.removeEventListener(evt, listener);
|
||||||
|
};
|
||||||
|
activeEl.addEventListener(evt, listener);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
window.addEventListener(
|
||||||
|
"keydown",
|
||||||
|
(e) => {
|
||||||
|
requestAnimationFrame(async () => {
|
||||||
|
const activeEl = document.activeElement;
|
||||||
|
if (activeEl?.tagName === "INPUT" || activeEl?.type === "textarea") {
|
||||||
|
// Ignore events on inputs, they have their native history
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check if this is a ctrl+z ctrl+y
|
||||||
|
if (await undoRedo(e)) return;
|
||||||
|
|
||||||
|
// If our active element is some type of input then handle changes after they're done
|
||||||
|
if (bindInput(activeEl)) return;
|
||||||
|
checkState();
|
||||||
|
});
|
||||||
|
},
|
||||||
|
true
|
||||||
|
);
|
||||||
|
|
||||||
|
// Handle clicking DOM elements (e.g. widgets)
|
||||||
|
window.addEventListener("mouseup", () => {
|
||||||
|
checkState();
|
||||||
|
});
|
||||||
|
|
||||||
|
// Handle litegraph clicks
|
||||||
|
const processMouseUp = LGraphCanvas.prototype.processMouseUp;
|
||||||
|
LGraphCanvas.prototype.processMouseUp = function (e) {
|
||||||
|
const v = processMouseUp.apply(this, arguments);
|
||||||
|
checkState();
|
||||||
|
return v;
|
||||||
|
};
|
||||||
|
const processMouseDown = LGraphCanvas.prototype.processMouseDown;
|
||||||
|
LGraphCanvas.prototype.processMouseDown = function (e) {
|
||||||
|
const v = processMouseDown.apply(this, arguments);
|
||||||
|
checkState();
|
||||||
|
return v;
|
||||||
|
};
|
||||||
@ -1,10 +1,16 @@
|
|||||||
import { ComfyWidgets, addValueControlWidget } from "../../scripts/widgets.js";
|
import { ComfyWidgets, addValueControlWidgets } from "../../scripts/widgets.js";
|
||||||
import { app } from "../../scripts/app.js";
|
import { app } from "../../scripts/app.js";
|
||||||
|
import { applyTextReplacements } from "../../scripts/utils.js";
|
||||||
|
|
||||||
const CONVERTED_TYPE = "converted-widget";
|
const CONVERTED_TYPE = "converted-widget";
|
||||||
const VALID_TYPES = ["STRING", "combo", "number", "BOOLEAN"];
|
const VALID_TYPES = ["STRING", "combo", "number", "BOOLEAN"];
|
||||||
const CONFIG = Symbol();
|
const CONFIG = Symbol();
|
||||||
const GET_CONFIG = Symbol();
|
const GET_CONFIG = Symbol();
|
||||||
|
const TARGET = Symbol(); // Used for reroutes to specify the real target widget
|
||||||
|
|
||||||
|
export function getWidgetConfig(slot) {
|
||||||
|
return slot.widget[CONFIG] ?? slot.widget[GET_CONFIG]();
|
||||||
|
}
|
||||||
|
|
||||||
function getConfig(widgetName) {
|
function getConfig(widgetName) {
|
||||||
const { nodeData } = this.constructor;
|
const { nodeData } = this.constructor;
|
||||||
@ -100,7 +106,6 @@ function getWidgetType(config) {
|
|||||||
return { type };
|
return { type };
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
function isValidCombo(combo, obj) {
|
function isValidCombo(combo, obj) {
|
||||||
// New input isnt a combo
|
// New input isnt a combo
|
||||||
if (!(obj instanceof Array)) {
|
if (!(obj instanceof Array)) {
|
||||||
@ -121,6 +126,135 @@ function isValidCombo(combo, obj) {
|
|||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export function setWidgetConfig(slot, config, target) {
|
||||||
|
if (!slot.widget) return;
|
||||||
|
if (config) {
|
||||||
|
slot.widget[GET_CONFIG] = () => config;
|
||||||
|
slot.widget[TARGET] = target;
|
||||||
|
} else {
|
||||||
|
delete slot.widget;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (slot.link) {
|
||||||
|
const link = app.graph.links[slot.link];
|
||||||
|
if (link) {
|
||||||
|
const originNode = app.graph.getNodeById(link.origin_id);
|
||||||
|
if (originNode.type === "PrimitiveNode") {
|
||||||
|
if (config) {
|
||||||
|
originNode.recreateWidget();
|
||||||
|
} else if(!app.configuringGraph) {
|
||||||
|
originNode.disconnectOutput(0);
|
||||||
|
originNode.onLastDisconnect();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export function mergeIfValid(output, config2, forceUpdate, recreateWidget, config1) {
|
||||||
|
if (!config1) {
|
||||||
|
config1 = output.widget[CONFIG] ?? output.widget[GET_CONFIG]();
|
||||||
|
}
|
||||||
|
|
||||||
|
if (config1[0] instanceof Array) {
|
||||||
|
if (!isValidCombo(config1[0], config2[0])) return false;
|
||||||
|
} else if (config1[0] !== config2[0]) {
|
||||||
|
// Types dont match
|
||||||
|
console.log(`connection rejected: types dont match`, config1[0], config2[0]);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
const keys = new Set([...Object.keys(config1[1] ?? {}), ...Object.keys(config2[1] ?? {})]);
|
||||||
|
|
||||||
|
let customConfig;
|
||||||
|
const getCustomConfig = () => {
|
||||||
|
if (!customConfig) {
|
||||||
|
if (typeof structuredClone === "undefined") {
|
||||||
|
customConfig = JSON.parse(JSON.stringify(config1[1] ?? {}));
|
||||||
|
} else {
|
||||||
|
customConfig = structuredClone(config1[1] ?? {});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return customConfig;
|
||||||
|
};
|
||||||
|
|
||||||
|
const isNumber = config1[0] === "INT" || config1[0] === "FLOAT";
|
||||||
|
for (const k of keys.values()) {
|
||||||
|
if (k !== "default" && k !== "forceInput" && k !== "defaultInput" && k !== "control_after_generate" && k !== "multiline") {
|
||||||
|
let v1 = config1[1][k];
|
||||||
|
let v2 = config2[1]?.[k];
|
||||||
|
|
||||||
|
if (v1 === v2 || (!v1 && !v2)) continue;
|
||||||
|
|
||||||
|
if (isNumber) {
|
||||||
|
if (k === "min") {
|
||||||
|
const theirMax = config2[1]?.["max"];
|
||||||
|
if (theirMax != null && v1 > theirMax) {
|
||||||
|
console.log("connection rejected: min > max", v1, theirMax);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
getCustomConfig()[k] = v1 == null ? v2 : v2 == null ? v1 : Math.max(v1, v2);
|
||||||
|
continue;
|
||||||
|
} else if (k === "max") {
|
||||||
|
const theirMin = config2[1]?.["min"];
|
||||||
|
if (theirMin != null && v1 < theirMin) {
|
||||||
|
console.log("connection rejected: max < min", v1, theirMin);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
getCustomConfig()[k] = v1 == null ? v2 : v2 == null ? v1 : Math.min(v1, v2);
|
||||||
|
continue;
|
||||||
|
} else if (k === "step") {
|
||||||
|
let step;
|
||||||
|
if (v1 == null) {
|
||||||
|
// No current step
|
||||||
|
step = v2;
|
||||||
|
} else if (v2 == null) {
|
||||||
|
// No new step
|
||||||
|
step = v1;
|
||||||
|
} else {
|
||||||
|
if (v1 < v2) {
|
||||||
|
// Ensure v1 is larger for the mod
|
||||||
|
const a = v2;
|
||||||
|
v2 = v1;
|
||||||
|
v1 = a;
|
||||||
|
}
|
||||||
|
if (v1 % v2) {
|
||||||
|
console.log("connection rejected: steps not divisible", "current:", v1, "new:", v2);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
step = v1;
|
||||||
|
}
|
||||||
|
|
||||||
|
getCustomConfig()[k] = step;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log(`connection rejected: config ${k} values dont match`, v1, v2);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (customConfig || forceUpdate) {
|
||||||
|
if (customConfig) {
|
||||||
|
output.widget[CONFIG] = [config1[0], customConfig];
|
||||||
|
}
|
||||||
|
|
||||||
|
const widget = recreateWidget?.call(this);
|
||||||
|
// When deleting a node this can be null
|
||||||
|
if (widget) {
|
||||||
|
const min = widget.options.min;
|
||||||
|
const max = widget.options.max;
|
||||||
|
if (min != null && widget.value < min) widget.value = min;
|
||||||
|
if (max != null && widget.value > max) widget.value = max;
|
||||||
|
widget.callback(widget.value);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return { customConfig };
|
||||||
|
}
|
||||||
|
|
||||||
app.registerExtension({
|
app.registerExtension({
|
||||||
name: "Comfy.WidgetInputs",
|
name: "Comfy.WidgetInputs",
|
||||||
async beforeRegisterNodeDef(nodeType, nodeData, app) {
|
async beforeRegisterNodeDef(nodeType, nodeData, app) {
|
||||||
@ -301,14 +435,19 @@ app.registerExtension({
|
|||||||
};
|
};
|
||||||
},
|
},
|
||||||
registerCustomNodes() {
|
registerCustomNodes() {
|
||||||
|
const replacePropertyName = "Run widget replace on values";
|
||||||
class PrimitiveNode {
|
class PrimitiveNode {
|
||||||
constructor() {
|
constructor() {
|
||||||
this.addOutput("connect to widget input", "*");
|
this.addOutput("connect to widget input", "*");
|
||||||
this.serialize_widgets = true;
|
this.serialize_widgets = true;
|
||||||
this.isVirtualNode = true;
|
this.isVirtualNode = true;
|
||||||
|
|
||||||
|
if (!this.properties || !(replacePropertyName in this.properties)) {
|
||||||
|
this.addProperty(replacePropertyName, false, "boolean");
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
applyToGraph() {
|
applyToGraph(extraLinks = []) {
|
||||||
if (!this.outputs[0].links?.length) return;
|
if (!this.outputs[0].links?.length) return;
|
||||||
|
|
||||||
function get_links(node) {
|
function get_links(node) {
|
||||||
@ -325,20 +464,30 @@ app.registerExtension({
|
|||||||
return links;
|
return links;
|
||||||
}
|
}
|
||||||
|
|
||||||
let links = get_links(this);
|
let links = [...get_links(this).map((l) => app.graph.links[l]), ...extraLinks];
|
||||||
|
let v = this.widgets?.[0].value;
|
||||||
|
if(v && this.properties[replacePropertyName]) {
|
||||||
|
v = applyTextReplacements(app, v);
|
||||||
|
}
|
||||||
|
|
||||||
// For each output link copy our value over the original widget value
|
// For each output link copy our value over the original widget value
|
||||||
for (const l of links) {
|
for (const linkInfo of links) {
|
||||||
const linkInfo = app.graph.links[l];
|
|
||||||
const node = this.graph.getNodeById(linkInfo.target_id);
|
const node = this.graph.getNodeById(linkInfo.target_id);
|
||||||
const input = node.inputs[linkInfo.target_slot];
|
const input = node.inputs[linkInfo.target_slot];
|
||||||
const widgetName = input.widget.name;
|
let widget;
|
||||||
if (widgetName) {
|
if (input.widget[TARGET]) {
|
||||||
const widget = node.widgets.find((w) => w.name === widgetName);
|
widget = input.widget[TARGET];
|
||||||
if (widget) {
|
} else {
|
||||||
widget.value = this.widgets[0].value;
|
const widgetName = input.widget.name;
|
||||||
if (widget.callback) {
|
if (widgetName) {
|
||||||
widget.callback(widget.value, app.canvas, node, app.canvas.graph_mouse, {});
|
widget = node.widgets.find((w) => w.name === widgetName);
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (widget) {
|
||||||
|
widget.value = v;
|
||||||
|
if (widget.callback) {
|
||||||
|
widget.callback(widget.value, app.canvas, node, app.canvas.graph_mouse, {});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -391,26 +540,34 @@ app.registerExtension({
|
|||||||
this.#mergeWidgetConfig();
|
this.#mergeWidgetConfig();
|
||||||
|
|
||||||
if (!links?.length) {
|
if (!links?.length) {
|
||||||
this.#onLastDisconnect();
|
this.onLastDisconnect();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
onConnectOutput(slot, type, input, target_node, target_slot) {
|
onConnectOutput(slot, type, input, target_node, target_slot) {
|
||||||
// Fires before the link is made allowing us to reject it if it isn't valid
|
// Fires before the link is made allowing us to reject it if it isn't valid
|
||||||
|
|
||||||
// No widget, we cant connect
|
// No widget, we cant connect
|
||||||
if (!input.widget) {
|
if (!input.widget) {
|
||||||
if (!(input.type in ComfyWidgets)) return false;
|
if (!(input.type in ComfyWidgets)) return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (this.outputs[slot].links?.length) {
|
if (this.outputs[slot].links?.length) {
|
||||||
return this.#isValidConnection(input);
|
const valid = this.#isValidConnection(input);
|
||||||
|
if (valid) {
|
||||||
|
// On connect of additional outputs, copy our value to their widget
|
||||||
|
this.applyToGraph([{ target_id: target_node.id, target_slot }]);
|
||||||
|
}
|
||||||
|
return valid;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#onFirstConnection(recreating) {
|
#onFirstConnection(recreating) {
|
||||||
// First connection can fire before the graph is ready on initial load so random things can be missing
|
// First connection can fire before the graph is ready on initial load so random things can be missing
|
||||||
|
if (!this.outputs[0].links) {
|
||||||
|
this.onLastDisconnect();
|
||||||
|
return;
|
||||||
|
}
|
||||||
const linkId = this.outputs[0].links[0];
|
const linkId = this.outputs[0].links[0];
|
||||||
const link = this.graph.links[linkId];
|
const link = this.graph.links[linkId];
|
||||||
if (!link) return;
|
if (!link) return;
|
||||||
@ -438,10 +595,10 @@ app.registerExtension({
|
|||||||
this.outputs[0].name = type;
|
this.outputs[0].name = type;
|
||||||
this.outputs[0].widget = widget;
|
this.outputs[0].widget = widget;
|
||||||
|
|
||||||
this.#createWidget(widget[CONFIG] ?? config, theirNode, widget.name, recreating);
|
this.#createWidget(widget[CONFIG] ?? config, theirNode, widget.name, recreating, widget[TARGET]);
|
||||||
}
|
}
|
||||||
|
|
||||||
#createWidget(inputData, node, widgetName, recreating) {
|
#createWidget(inputData, node, widgetName, recreating, targetWidget) {
|
||||||
let type = inputData[0];
|
let type = inputData[0];
|
||||||
|
|
||||||
if (type instanceof Array) {
|
if (type instanceof Array) {
|
||||||
@ -455,19 +612,33 @@ app.registerExtension({
|
|||||||
widget = this.addWidget(type, "value", null, () => {}, {});
|
widget = this.addWidget(type, "value", null, () => {}, {});
|
||||||
}
|
}
|
||||||
|
|
||||||
if (node?.widgets && widget) {
|
if (targetWidget) {
|
||||||
|
widget.value = targetWidget.value;
|
||||||
|
} else if (node?.widgets && widget) {
|
||||||
const theirWidget = node.widgets.find((w) => w.name === widgetName);
|
const theirWidget = node.widgets.find((w) => w.name === widgetName);
|
||||||
if (theirWidget) {
|
if (theirWidget) {
|
||||||
widget.value = theirWidget.value;
|
widget.value = theirWidget.value;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (widget.type === "number" || widget.type === "combo") {
|
if (!inputData?.[1]?.control_after_generate && (widget.type === "number" || widget.type === "combo")) {
|
||||||
let control_value = this.widgets_values?.[1];
|
let control_value = this.widgets_values?.[1];
|
||||||
if (!control_value) {
|
if (!control_value) {
|
||||||
control_value = "fixed";
|
control_value = "fixed";
|
||||||
}
|
}
|
||||||
addValueControlWidget(this, widget, control_value);
|
addValueControlWidgets(this, widget, control_value, undefined, inputData);
|
||||||
|
let filter = this.widgets_values?.[2];
|
||||||
|
if (filter && this.widgets.length === 3) {
|
||||||
|
this.widgets[2].value = filter;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Restore any saved control values
|
||||||
|
const controlValues = this.controlValues;
|
||||||
|
if(this.lastType === this.widgets[0].type && controlValues?.length === this.widgets.length - 1) {
|
||||||
|
for(let i = 0; i < controlValues.length; i++) {
|
||||||
|
this.widgets[i + 1].value = controlValues[i];
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// When our value changes, update other widgets to reflect our changes
|
// When our value changes, update other widgets to reflect our changes
|
||||||
@ -498,11 +669,14 @@ app.registerExtension({
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#recreateWidget() {
|
recreateWidget() {
|
||||||
const values = this.widgets.map((w) => w.value);
|
const values = this.widgets?.map((w) => w.value);
|
||||||
this.#removeWidgets();
|
this.#removeWidgets();
|
||||||
this.#onFirstConnection(true);
|
this.#onFirstConnection(true);
|
||||||
for (let i = 0; i < this.widgets?.length; i++) this.widgets[i].value = values[i];
|
if (values?.length) {
|
||||||
|
for (let i = 0; i < this.widgets?.length; i++) this.widgets[i].value = values[i];
|
||||||
|
}
|
||||||
|
return this.widgets?.[0];
|
||||||
}
|
}
|
||||||
|
|
||||||
#mergeWidgetConfig() {
|
#mergeWidgetConfig() {
|
||||||
@ -518,7 +692,7 @@ app.registerExtension({
|
|||||||
if (links?.length < 2 && hasConfig) {
|
if (links?.length < 2 && hasConfig) {
|
||||||
// Copy the widget options from the source
|
// Copy the widget options from the source
|
||||||
if (links.length) {
|
if (links.length) {
|
||||||
this.#recreateWidget();
|
this.recreateWidget();
|
||||||
}
|
}
|
||||||
|
|
||||||
return;
|
return;
|
||||||
@ -543,108 +717,8 @@ app.registerExtension({
|
|||||||
#isValidConnection(input, forceUpdate) {
|
#isValidConnection(input, forceUpdate) {
|
||||||
// Only allow connections where the configs match
|
// Only allow connections where the configs match
|
||||||
const output = this.outputs[0];
|
const output = this.outputs[0];
|
||||||
const config1 = output.widget[CONFIG] ?? output.widget[GET_CONFIG]();
|
|
||||||
const config2 = input.widget[GET_CONFIG]();
|
const config2 = input.widget[GET_CONFIG]();
|
||||||
|
return !!mergeIfValid.call(this, output, config2, forceUpdate, this.recreateWidget);
|
||||||
if (config1[0] instanceof Array) {
|
|
||||||
if (!isValidCombo(config1[0], config2[0])) return false;
|
|
||||||
} else if (config1[0] !== config2[0]) {
|
|
||||||
// Types dont match
|
|
||||||
console.log(`connection rejected: types dont match`, config1[0], config2[0]);
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
const keys = new Set([...Object.keys(config1[1] ?? {}), ...Object.keys(config2[1] ?? {})]);
|
|
||||||
|
|
||||||
let customConfig;
|
|
||||||
const getCustomConfig = () => {
|
|
||||||
if (!customConfig) {
|
|
||||||
if (typeof structuredClone === "undefined") {
|
|
||||||
customConfig = JSON.parse(JSON.stringify(config1[1] ?? {}));
|
|
||||||
} else {
|
|
||||||
customConfig = structuredClone(config1[1] ?? {});
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return customConfig;
|
|
||||||
};
|
|
||||||
|
|
||||||
const isNumber = config1[0] === "INT" || config1[0] === "FLOAT";
|
|
||||||
for (const k of keys.values()) {
|
|
||||||
if (k !== "default" && k !== "forceInput" && k !== "defaultInput") {
|
|
||||||
let v1 = config1[1][k];
|
|
||||||
let v2 = config2[1][k];
|
|
||||||
|
|
||||||
if (v1 === v2 || (!v1 && !v2)) continue;
|
|
||||||
|
|
||||||
if (isNumber) {
|
|
||||||
if (k === "min") {
|
|
||||||
const theirMax = config2[1]["max"];
|
|
||||||
if (theirMax != null && v1 > theirMax) {
|
|
||||||
console.log("connection rejected: min > max", v1, theirMax);
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
getCustomConfig()[k] = v1 == null ? v2 : v2 == null ? v1 : Math.max(v1, v2);
|
|
||||||
continue;
|
|
||||||
} else if (k === "max") {
|
|
||||||
const theirMin = config2[1]["min"];
|
|
||||||
if (theirMin != null && v1 < theirMin) {
|
|
||||||
console.log("connection rejected: max < min", v1, theirMin);
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
getCustomConfig()[k] = v1 == null ? v2 : v2 == null ? v1 : Math.min(v1, v2);
|
|
||||||
continue;
|
|
||||||
} else if (k === "step") {
|
|
||||||
let step;
|
|
||||||
if (v1 == null) {
|
|
||||||
// No current step
|
|
||||||
step = v2;
|
|
||||||
} else if (v2 == null) {
|
|
||||||
// No new step
|
|
||||||
step = v1;
|
|
||||||
} else {
|
|
||||||
if (v1 < v2) {
|
|
||||||
// Ensure v1 is larger for the mod
|
|
||||||
const a = v2;
|
|
||||||
v2 = v1;
|
|
||||||
v1 = a;
|
|
||||||
}
|
|
||||||
if (v1 % v2) {
|
|
||||||
console.log("connection rejected: steps not divisible", "current:", v1, "new:", v2);
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
step = v1;
|
|
||||||
}
|
|
||||||
|
|
||||||
getCustomConfig()[k] = step;
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
console.log(`connection rejected: config ${k} values dont match`, v1, v2);
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (customConfig || forceUpdate) {
|
|
||||||
if (customConfig) {
|
|
||||||
output.widget[CONFIG] = [config1[0], customConfig];
|
|
||||||
}
|
|
||||||
|
|
||||||
this.#recreateWidget();
|
|
||||||
|
|
||||||
const widget = this.widgets[0];
|
|
||||||
// When deleting a node this can be null
|
|
||||||
if (widget) {
|
|
||||||
const min = widget.options.min;
|
|
||||||
const max = widget.options.max;
|
|
||||||
if (min != null && widget.value < min) widget.value = min;
|
|
||||||
if (max != null && widget.value > max) widget.value = max;
|
|
||||||
widget.callback(widget.value);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return true;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#removeWidgets() {
|
#removeWidgets() {
|
||||||
@ -655,11 +729,20 @@ app.registerExtension({
|
|||||||
w.onRemove();
|
w.onRemove();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Temporarily store the current values in case the node is being recreated
|
||||||
|
// e.g. by group node conversion
|
||||||
|
this.controlValues = [];
|
||||||
|
this.lastType = this.widgets[0]?.type;
|
||||||
|
for(let i = 1; i < this.widgets.length; i++) {
|
||||||
|
this.controlValues.push(this.widgets[i].value);
|
||||||
|
}
|
||||||
|
setTimeout(() => { delete this.lastType; delete this.controlValues }, 15);
|
||||||
this.widgets.length = 0;
|
this.widgets.length = 0;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#onLastDisconnect() {
|
onLastDisconnect() {
|
||||||
// We cant remove + re-add the output here as if you drag a link over the same link
|
// We cant remove + re-add the output here as if you drag a link over the same link
|
||||||
// it removes, then re-adds, causing it to break
|
// it removes, then re-adds, causing it to break
|
||||||
this.outputs[0].type = "*";
|
this.outputs[0].type = "*";
|
||||||
|
|||||||
@ -48,7 +48,7 @@
|
|||||||
EVENT_LINK_COLOR: "#A86",
|
EVENT_LINK_COLOR: "#A86",
|
||||||
CONNECTING_LINK_COLOR: "#AFA",
|
CONNECTING_LINK_COLOR: "#AFA",
|
||||||
|
|
||||||
MAX_NUMBER_OF_NODES: 1000, //avoid infinite loops
|
MAX_NUMBER_OF_NODES: 10000, //avoid infinite loops
|
||||||
DEFAULT_POSITION: [100, 100], //default node position
|
DEFAULT_POSITION: [100, 100], //default node position
|
||||||
VALID_SHAPES: ["default", "box", "round", "card"], //,"circle"
|
VALID_SHAPES: ["default", "box", "round", "card"], //,"circle"
|
||||||
|
|
||||||
@ -2533,7 +2533,7 @@
|
|||||||
var w = this.widgets[i];
|
var w = this.widgets[i];
|
||||||
if(!w)
|
if(!w)
|
||||||
continue;
|
continue;
|
||||||
if(w.options && w.options.property && this.properties[ w.options.property ])
|
if(w.options && w.options.property && (this.properties[ w.options.property ] != undefined))
|
||||||
w.value = JSON.parse( JSON.stringify( this.properties[ w.options.property ] ) );
|
w.value = JSON.parse( JSON.stringify( this.properties[ w.options.property ] ) );
|
||||||
}
|
}
|
||||||
if (info.widgets_values) {
|
if (info.widgets_values) {
|
||||||
@ -3788,16 +3788,42 @@
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* returns the bounding of the object, used for rendering purposes
|
* returns the bounding of the object, used for rendering purposes
|
||||||
* bounding is: [topleft_cornerx, topleft_cornery, width, height]
|
|
||||||
* @method getBounding
|
* @method getBounding
|
||||||
* @return {Float32Array[4]} the total size
|
* @param out {Float32Array[4]?} [optional] a place to store the output, to free garbage
|
||||||
|
* @param compute_outer {boolean?} [optional] set to true to include the shadow and connection points in the bounding calculation
|
||||||
|
* @return {Float32Array[4]} the bounding box in format of [topleft_cornerx, topleft_cornery, width, height]
|
||||||
*/
|
*/
|
||||||
LGraphNode.prototype.getBounding = function(out) {
|
LGraphNode.prototype.getBounding = function(out, compute_outer) {
|
||||||
out = out || new Float32Array(4);
|
out = out || new Float32Array(4);
|
||||||
out[0] = this.pos[0] - 4;
|
const nodePos = this.pos;
|
||||||
out[1] = this.pos[1] - LiteGraph.NODE_TITLE_HEIGHT;
|
const isCollapsed = this.flags.collapsed;
|
||||||
out[2] = this.flags.collapsed ? (this._collapsed_width || LiteGraph.NODE_COLLAPSED_WIDTH) : this.size[0] + 4;
|
const nodeSize = this.size;
|
||||||
out[3] = this.flags.collapsed ? LiteGraph.NODE_TITLE_HEIGHT : this.size[1] + LiteGraph.NODE_TITLE_HEIGHT;
|
|
||||||
|
let left_offset = 0;
|
||||||
|
// 1 offset due to how nodes are rendered
|
||||||
|
let right_offset = 1 ;
|
||||||
|
let top_offset = 0;
|
||||||
|
let bottom_offset = 0;
|
||||||
|
|
||||||
|
if (compute_outer) {
|
||||||
|
// 4 offset for collapsed node connection points
|
||||||
|
left_offset = 4;
|
||||||
|
// 6 offset for right shadow and collapsed node connection points
|
||||||
|
right_offset = 6 + left_offset;
|
||||||
|
// 4 offset for collapsed nodes top connection points
|
||||||
|
top_offset = 4;
|
||||||
|
// 5 offset for bottom shadow and collapsed node connection points
|
||||||
|
bottom_offset = 5 + top_offset;
|
||||||
|
}
|
||||||
|
|
||||||
|
out[0] = nodePos[0] - left_offset;
|
||||||
|
out[1] = nodePos[1] - LiteGraph.NODE_TITLE_HEIGHT - top_offset;
|
||||||
|
out[2] = isCollapsed ?
|
||||||
|
(this._collapsed_width || LiteGraph.NODE_COLLAPSED_WIDTH) + right_offset :
|
||||||
|
nodeSize[0] + right_offset;
|
||||||
|
out[3] = isCollapsed ?
|
||||||
|
LiteGraph.NODE_TITLE_HEIGHT + bottom_offset :
|
||||||
|
nodeSize[1] + LiteGraph.NODE_TITLE_HEIGHT + bottom_offset;
|
||||||
|
|
||||||
if (this.onBounding) {
|
if (this.onBounding) {
|
||||||
this.onBounding(out);
|
this.onBounding(out);
|
||||||
@ -5714,10 +5740,10 @@ LGraphNode.prototype.executeAction = function(action)
|
|||||||
* @method enableWebGL
|
* @method enableWebGL
|
||||||
**/
|
**/
|
||||||
LGraphCanvas.prototype.enableWebGL = function() {
|
LGraphCanvas.prototype.enableWebGL = function() {
|
||||||
if (typeof GL === undefined) {
|
if (typeof GL === "undefined") {
|
||||||
throw "litegl.js must be included to use a WebGL canvas";
|
throw "litegl.js must be included to use a WebGL canvas";
|
||||||
}
|
}
|
||||||
if (typeof enableWebGLCanvas === undefined) {
|
if (typeof enableWebGLCanvas === "undefined") {
|
||||||
throw "webglCanvas.js must be included to use this feature";
|
throw "webglCanvas.js must be included to use this feature";
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -7110,15 +7136,16 @@ LGraphNode.prototype.executeAction = function(action)
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
LGraphCanvas.prototype.copyToClipboard = function() {
|
LGraphCanvas.prototype.copyToClipboard = function(nodes) {
|
||||||
var clipboard_info = {
|
var clipboard_info = {
|
||||||
nodes: [],
|
nodes: [],
|
||||||
links: []
|
links: []
|
||||||
};
|
};
|
||||||
var index = 0;
|
var index = 0;
|
||||||
var selected_nodes_array = [];
|
var selected_nodes_array = [];
|
||||||
for (var i in this.selected_nodes) {
|
if (!nodes) nodes = this.selected_nodes;
|
||||||
var node = this.selected_nodes[i];
|
for (var i in nodes) {
|
||||||
|
var node = nodes[i];
|
||||||
if (node.clonable === false)
|
if (node.clonable === false)
|
||||||
continue;
|
continue;
|
||||||
node._relative_id = index;
|
node._relative_id = index;
|
||||||
@ -7673,7 +7700,7 @@ LGraphNode.prototype.executeAction = function(action)
|
|||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!overlapBounding(this.visible_area, n.getBounding(temp))) {
|
if (!overlapBounding(this.visible_area, n.getBounding(temp, true))) {
|
||||||
continue;
|
continue;
|
||||||
} //out of the visible area
|
} //out of the visible area
|
||||||
|
|
||||||
@ -11335,6 +11362,7 @@ LGraphNode.prototype.executeAction = function(action)
|
|||||||
name_element.innerText = title;
|
name_element.innerText = title;
|
||||||
var value_element = dialog.querySelector(".value");
|
var value_element = dialog.querySelector(".value");
|
||||||
value_element.value = value;
|
value_element.value = value;
|
||||||
|
value_element.select();
|
||||||
|
|
||||||
var input = value_element;
|
var input = value_element;
|
||||||
input.addEventListener("keydown", function(e) {
|
input.addEventListener("keydown", function(e) {
|
||||||
@ -11702,7 +11730,7 @@ LGraphNode.prototype.executeAction = function(action)
|
|||||||
default:
|
default:
|
||||||
iS = 0; // try with first if no name set
|
iS = 0; // try with first if no name set
|
||||||
}
|
}
|
||||||
if (typeof options.node_from.outputs[iS] !== undefined){
|
if (typeof options.node_from.outputs[iS] !== "undefined"){
|
||||||
if (iS!==false && iS>-1){
|
if (iS!==false && iS>-1){
|
||||||
options.node_from.connectByType( iS, node, options.node_from.outputs[iS].type );
|
options.node_from.connectByType( iS, node, options.node_from.outputs[iS].type );
|
||||||
}
|
}
|
||||||
@ -11730,7 +11758,7 @@ LGraphNode.prototype.executeAction = function(action)
|
|||||||
default:
|
default:
|
||||||
iS = 0; // try with first if no name set
|
iS = 0; // try with first if no name set
|
||||||
}
|
}
|
||||||
if (typeof options.node_to.inputs[iS] !== undefined){
|
if (typeof options.node_to.inputs[iS] !== "undefined"){
|
||||||
if (iS!==false && iS>-1){
|
if (iS!==false && iS>-1){
|
||||||
// try connection
|
// try connection
|
||||||
options.node_to.connectByTypeOutput(iS,node,options.node_to.inputs[iS].type);
|
options.node_to.connectByTypeOutput(iS,node,options.node_to.inputs[iS].type);
|
||||||
|
|||||||
@ -254,9 +254,9 @@ class ComfyApi extends EventTarget {
|
|||||||
* Gets the prompt execution history
|
* Gets the prompt execution history
|
||||||
* @returns Prompt history including node outputs
|
* @returns Prompt history including node outputs
|
||||||
*/
|
*/
|
||||||
async getHistory() {
|
async getHistory(max_items=200) {
|
||||||
try {
|
try {
|
||||||
const res = await this.fetchApi("/history");
|
const res = await this.fetchApi(`/history?max_items=${max_items}`);
|
||||||
return { History: Object.values(await res.json()) };
|
return { History: Object.values(await res.json()) };
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error(error);
|
console.error(error);
|
||||||
|
|||||||
@ -4,7 +4,10 @@ import { ComfyUI, $el } from "./ui.js";
|
|||||||
import { api } from "./api.js";
|
import { api } from "./api.js";
|
||||||
import { defaultGraph } from "./defaultGraph.js";
|
import { defaultGraph } from "./defaultGraph.js";
|
||||||
import { getPngMetadata, getWebpMetadata, importA1111, getLatentMetadata } from "./pnginfo.js";
|
import { getPngMetadata, getWebpMetadata, importA1111, getLatentMetadata } from "./pnginfo.js";
|
||||||
|
import { addDomClippingSetting } from "./domWidget.js";
|
||||||
|
import { createImageHost, calculateImageGrid } from "./ui/imagePreview.js"
|
||||||
|
|
||||||
|
export const ANIM_PREVIEW_WIDGET = "$$comfy_animation_preview"
|
||||||
|
|
||||||
function sanitizeNodeName(string) {
|
function sanitizeNodeName(string) {
|
||||||
let entityMap = {
|
let entityMap = {
|
||||||
@ -83,6 +86,10 @@ export class ComfyApp {
|
|||||||
return "";
|
return "";
|
||||||
}
|
}
|
||||||
|
|
||||||
|
getRandParam() {
|
||||||
|
return "&rand=" + Math.random();
|
||||||
|
}
|
||||||
|
|
||||||
static isImageNode(node) {
|
static isImageNode(node) {
|
||||||
return node.imgs || (node && node.widgets && node.widgets.findIndex(obj => obj.name === 'image') >= 0);
|
return node.imgs || (node && node.widgets && node.widgets.findIndex(obj => obj.name === 'image') >= 0);
|
||||||
}
|
}
|
||||||
@ -405,8 +412,10 @@ export class ComfyApp {
|
|||||||
return shiftY;
|
return shiftY;
|
||||||
}
|
}
|
||||||
|
|
||||||
node.prototype.setSizeForImage = function () {
|
node.prototype.setSizeForImage = function (force) {
|
||||||
if (this.inputHeight) {
|
if(!force && this.animatedImages) return;
|
||||||
|
|
||||||
|
if (this.inputHeight || this.freeWidgetSpace > 210) {
|
||||||
this.setSize(this.size);
|
this.setSize(this.size);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
@ -422,13 +431,20 @@ export class ComfyApp {
|
|||||||
let imagesChanged = false
|
let imagesChanged = false
|
||||||
|
|
||||||
const output = app.nodeOutputs[this.id + ""];
|
const output = app.nodeOutputs[this.id + ""];
|
||||||
if (output && output.images) {
|
if (output?.images) {
|
||||||
|
this.animatedImages = output?.animated?.find(Boolean);
|
||||||
if (this.images !== output.images) {
|
if (this.images !== output.images) {
|
||||||
this.images = output.images;
|
this.images = output.images;
|
||||||
imagesChanged = true;
|
imagesChanged = true;
|
||||||
imgURLs = imgURLs.concat(output.images.map(params => {
|
imgURLs = imgURLs.concat(
|
||||||
return api.apiURL("/view?" + new URLSearchParams(params).toString() + app.getPreviewFormatParam());
|
output.images.map((params) => {
|
||||||
}))
|
return api.apiURL(
|
||||||
|
"/view?" +
|
||||||
|
new URLSearchParams(params).toString() +
|
||||||
|
(this.animatedImages ? "" : app.getPreviewFormatParam()) + app.getRandParam()
|
||||||
|
);
|
||||||
|
})
|
||||||
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -507,7 +523,35 @@ export class ComfyApp {
|
|||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (this.imgs && this.imgs.length) {
|
if (this.imgs?.length) {
|
||||||
|
const widgetIdx = this.widgets?.findIndex((w) => w.name === ANIM_PREVIEW_WIDGET);
|
||||||
|
|
||||||
|
if(this.animatedImages) {
|
||||||
|
// Instead of using the canvas we'll use a IMG
|
||||||
|
if(widgetIdx > -1) {
|
||||||
|
// Replace content
|
||||||
|
const widget = this.widgets[widgetIdx];
|
||||||
|
widget.options.host.updateImages(this.imgs);
|
||||||
|
} else {
|
||||||
|
const host = createImageHost(this);
|
||||||
|
this.setSizeForImage(true);
|
||||||
|
const widget = this.addDOMWidget(ANIM_PREVIEW_WIDGET, "img", host.el, {
|
||||||
|
host,
|
||||||
|
getHeight: host.getHeight,
|
||||||
|
onDraw: host.onDraw,
|
||||||
|
hideOnZoom: false
|
||||||
|
});
|
||||||
|
widget.serializeValue = () => undefined;
|
||||||
|
widget.options.host.updateImages(this.imgs);
|
||||||
|
}
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (widgetIdx > -1) {
|
||||||
|
this.widgets[widgetIdx].onRemove?.();
|
||||||
|
this.widgets.splice(widgetIdx, 1);
|
||||||
|
}
|
||||||
|
|
||||||
const canvas = app.graph.list_of_graphcanvas[0];
|
const canvas = app.graph.list_of_graphcanvas[0];
|
||||||
const mouse = canvas.graph_mouse;
|
const mouse = canvas.graph_mouse;
|
||||||
if (!canvas.pointer_is_down && this.pointerDown) {
|
if (!canvas.pointer_is_down && this.pointerDown) {
|
||||||
@ -547,31 +591,7 @@ export class ComfyApp {
|
|||||||
}
|
}
|
||||||
else {
|
else {
|
||||||
cell_padding = 0;
|
cell_padding = 0;
|
||||||
let best = 0;
|
({ cellWidth, cellHeight, cols, shiftX } = calculateImageGrid(this.imgs, dw, dh));
|
||||||
let w = this.imgs[0].naturalWidth;
|
|
||||||
let h = this.imgs[0].naturalHeight;
|
|
||||||
|
|
||||||
// compact style
|
|
||||||
for (let c = 1; c <= numImages; c++) {
|
|
||||||
const rows = Math.ceil(numImages / c);
|
|
||||||
const cW = dw / c;
|
|
||||||
const cH = dh / rows;
|
|
||||||
const scaleX = cW / w;
|
|
||||||
const scaleY = cH / h;
|
|
||||||
|
|
||||||
const scale = Math.min(scaleX, scaleY, 1);
|
|
||||||
const imageW = w * scale;
|
|
||||||
const imageH = h * scale;
|
|
||||||
const area = imageW * imageH * numImages;
|
|
||||||
|
|
||||||
if (area > best) {
|
|
||||||
best = area;
|
|
||||||
cellWidth = imageW;
|
|
||||||
cellHeight = imageH;
|
|
||||||
cols = c;
|
|
||||||
shiftX = c * ((cW - imageW) / 2);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
let anyHovered = false;
|
let anyHovered = false;
|
||||||
@ -763,7 +783,7 @@ export class ComfyApp {
|
|||||||
* Adds a handler on paste that extracts and loads images or workflows from pasted JSON data
|
* Adds a handler on paste that extracts and loads images or workflows from pasted JSON data
|
||||||
*/
|
*/
|
||||||
#addPasteHandler() {
|
#addPasteHandler() {
|
||||||
document.addEventListener("paste", (e) => {
|
document.addEventListener("paste", async (e) => {
|
||||||
// ctrl+shift+v is used to paste nodes with connections
|
// ctrl+shift+v is used to paste nodes with connections
|
||||||
// this is handled by litegraph
|
// this is handled by litegraph
|
||||||
if(this.shiftDown) return;
|
if(this.shiftDown) return;
|
||||||
@ -811,7 +831,7 @@ export class ComfyApp {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if (workflow && workflow.version && workflow.nodes && workflow.extra) {
|
if (workflow && workflow.version && workflow.nodes && workflow.extra) {
|
||||||
this.loadGraphData(workflow);
|
await this.loadGraphData(workflow);
|
||||||
}
|
}
|
||||||
else {
|
else {
|
||||||
if (e.target.type === "text" || e.target.type === "textarea") {
|
if (e.target.type === "text" || e.target.type === "textarea") {
|
||||||
@ -1161,7 +1181,19 @@ export class ComfyApp {
|
|||||||
});
|
});
|
||||||
|
|
||||||
api.addEventListener("executed", ({ detail }) => {
|
api.addEventListener("executed", ({ detail }) => {
|
||||||
this.nodeOutputs[detail.node] = detail.output;
|
const output = this.nodeOutputs[detail.node];
|
||||||
|
if (detail.merge && output) {
|
||||||
|
for (const k in detail.output ?? {}) {
|
||||||
|
const v = output[k];
|
||||||
|
if (v instanceof Array) {
|
||||||
|
output[k] = v.concat(detail.output[k]);
|
||||||
|
} else {
|
||||||
|
output[k] = detail.output[k];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
this.nodeOutputs[detail.node] = detail.output;
|
||||||
|
}
|
||||||
const node = this.graph.getNodeById(detail.node);
|
const node = this.graph.getNodeById(detail.node);
|
||||||
if (node) {
|
if (node) {
|
||||||
if (node.onExecuted)
|
if (node.onExecuted)
|
||||||
@ -1272,9 +1304,11 @@ export class ComfyApp {
|
|||||||
canvasEl.tabIndex = "1";
|
canvasEl.tabIndex = "1";
|
||||||
document.body.prepend(canvasEl);
|
document.body.prepend(canvasEl);
|
||||||
|
|
||||||
|
addDomClippingSetting();
|
||||||
this.#addProcessMouseHandler();
|
this.#addProcessMouseHandler();
|
||||||
this.#addProcessKeyHandler();
|
this.#addProcessKeyHandler();
|
||||||
this.#addConfigureHandler();
|
this.#addConfigureHandler();
|
||||||
|
this.#addApiUpdateHandlers();
|
||||||
|
|
||||||
this.graph = new LGraph();
|
this.graph = new LGraph();
|
||||||
|
|
||||||
@ -1311,7 +1345,7 @@ export class ComfyApp {
|
|||||||
const json = localStorage.getItem("workflow");
|
const json = localStorage.getItem("workflow");
|
||||||
if (json) {
|
if (json) {
|
||||||
const workflow = JSON.parse(json);
|
const workflow = JSON.parse(json);
|
||||||
this.loadGraphData(workflow);
|
await this.loadGraphData(workflow);
|
||||||
restored = true;
|
restored = true;
|
||||||
}
|
}
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
@ -1320,7 +1354,7 @@ export class ComfyApp {
|
|||||||
|
|
||||||
// We failed to restore a workflow so load the default
|
// We failed to restore a workflow so load the default
|
||||||
if (!restored) {
|
if (!restored) {
|
||||||
this.loadGraphData();
|
await this.loadGraphData();
|
||||||
}
|
}
|
||||||
|
|
||||||
// Save current workflow automatically
|
// Save current workflow automatically
|
||||||
@ -1328,7 +1362,6 @@ export class ComfyApp {
|
|||||||
|
|
||||||
this.#addDrawNodeHandler();
|
this.#addDrawNodeHandler();
|
||||||
this.#addDrawGroupsHandler();
|
this.#addDrawGroupsHandler();
|
||||||
this.#addApiUpdateHandlers();
|
|
||||||
this.#addDropHandler();
|
this.#addDropHandler();
|
||||||
this.#addCopyHandler();
|
this.#addCopyHandler();
|
||||||
this.#addPasteHandler();
|
this.#addPasteHandler();
|
||||||
@ -1348,11 +1381,95 @@ export class ComfyApp {
|
|||||||
await this.#invokeExtensionsAsync("registerCustomNodes");
|
await this.#invokeExtensionsAsync("registerCustomNodes");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
getWidgetType(inputData, inputName) {
|
||||||
|
const type = inputData[0];
|
||||||
|
|
||||||
|
if (Array.isArray(type)) {
|
||||||
|
return "COMBO";
|
||||||
|
} else if (`${type}:${inputName}` in this.widgets) {
|
||||||
|
return `${type}:${inputName}`;
|
||||||
|
} else if (type in this.widgets) {
|
||||||
|
return type;
|
||||||
|
} else {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async registerNodeDef(nodeId, nodeData) {
|
||||||
|
const self = this;
|
||||||
|
const node = Object.assign(
|
||||||
|
function ComfyNode() {
|
||||||
|
var inputs = nodeData["input"]["required"];
|
||||||
|
if (nodeData["input"]["optional"] != undefined) {
|
||||||
|
inputs = Object.assign({}, nodeData["input"]["required"], nodeData["input"]["optional"]);
|
||||||
|
}
|
||||||
|
const config = { minWidth: 1, minHeight: 1 };
|
||||||
|
for (const inputName in inputs) {
|
||||||
|
const inputData = inputs[inputName];
|
||||||
|
const type = inputData[0];
|
||||||
|
|
||||||
|
let widgetCreated = true;
|
||||||
|
const widgetType = self.getWidgetType(inputData, inputName);
|
||||||
|
if(widgetType) {
|
||||||
|
if(widgetType === "COMBO") {
|
||||||
|
Object.assign(config, self.widgets.COMBO(this, inputName, inputData, app) || {});
|
||||||
|
} else {
|
||||||
|
Object.assign(config, self.widgets[widgetType](this, inputName, inputData, app) || {});
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
// Node connection inputs
|
||||||
|
this.addInput(inputName, type);
|
||||||
|
widgetCreated = false;
|
||||||
|
}
|
||||||
|
|
||||||
|
if(widgetCreated && inputData[1]?.forceInput && config?.widget) {
|
||||||
|
if (!config.widget.options) config.widget.options = {};
|
||||||
|
config.widget.options.forceInput = inputData[1].forceInput;
|
||||||
|
}
|
||||||
|
if(widgetCreated && inputData[1]?.defaultInput && config?.widget) {
|
||||||
|
if (!config.widget.options) config.widget.options = {};
|
||||||
|
config.widget.options.defaultInput = inputData[1].defaultInput;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for (const o in nodeData["output"]) {
|
||||||
|
let output = nodeData["output"][o];
|
||||||
|
if(output instanceof Array) output = "COMBO";
|
||||||
|
const outputName = nodeData["output_name"][o] || output;
|
||||||
|
const outputShape = nodeData["output_is_list"][o] ? LiteGraph.GRID_SHAPE : LiteGraph.CIRCLE_SHAPE ;
|
||||||
|
this.addOutput(outputName, output, { shape: outputShape });
|
||||||
|
}
|
||||||
|
|
||||||
|
const s = this.computeSize();
|
||||||
|
s[0] = Math.max(config.minWidth, s[0] * 1.5);
|
||||||
|
s[1] = Math.max(config.minHeight, s[1]);
|
||||||
|
this.size = s;
|
||||||
|
this.serialize_widgets = true;
|
||||||
|
|
||||||
|
app.#invokeExtensionsAsync("nodeCreated", this);
|
||||||
|
},
|
||||||
|
{
|
||||||
|
title: nodeData.display_name || nodeData.name,
|
||||||
|
comfyClass: nodeData.name,
|
||||||
|
nodeData
|
||||||
|
}
|
||||||
|
);
|
||||||
|
node.prototype.comfyClass = nodeData.name;
|
||||||
|
|
||||||
|
this.#addNodeContextMenuHandler(node);
|
||||||
|
this.#addDrawBackgroundHandler(node, app);
|
||||||
|
this.#addNodeKeyHandler(node);
|
||||||
|
|
||||||
|
await this.#invokeExtensionsAsync("beforeRegisterNodeDef", node, nodeData);
|
||||||
|
LiteGraph.registerNodeType(nodeId, node);
|
||||||
|
node.category = nodeData.category;
|
||||||
|
}
|
||||||
|
|
||||||
async registerNodesFromDefs(defs) {
|
async registerNodesFromDefs(defs) {
|
||||||
await this.#invokeExtensionsAsync("addCustomNodeDefs", defs);
|
await this.#invokeExtensionsAsync("addCustomNodeDefs", defs);
|
||||||
|
|
||||||
// Generate list of known widgets
|
// Generate list of known widgets
|
||||||
const widgets = Object.assign(
|
this.widgets = Object.assign(
|
||||||
{},
|
{},
|
||||||
ComfyWidgets,
|
ComfyWidgets,
|
||||||
...(await this.#invokeExtensionsAsync("getCustomWidgets")).filter(Boolean)
|
...(await this.#invokeExtensionsAsync("getCustomWidgets")).filter(Boolean)
|
||||||
@ -1360,75 +1477,7 @@ export class ComfyApp {
|
|||||||
|
|
||||||
// Register a node for each definition
|
// Register a node for each definition
|
||||||
for (const nodeId in defs) {
|
for (const nodeId in defs) {
|
||||||
const nodeData = defs[nodeId];
|
this.registerNodeDef(nodeId, defs[nodeId]);
|
||||||
const node = Object.assign(
|
|
||||||
function ComfyNode() {
|
|
||||||
var inputs = nodeData["input"]["required"];
|
|
||||||
if (nodeData["input"]["optional"] != undefined){
|
|
||||||
inputs = Object.assign({}, nodeData["input"]["required"], nodeData["input"]["optional"])
|
|
||||||
}
|
|
||||||
const config = { minWidth: 1, minHeight: 1 };
|
|
||||||
for (const inputName in inputs) {
|
|
||||||
const inputData = inputs[inputName];
|
|
||||||
const type = inputData[0];
|
|
||||||
|
|
||||||
let widgetCreated = true;
|
|
||||||
if (Array.isArray(type)) {
|
|
||||||
// Enums
|
|
||||||
Object.assign(config, widgets.COMBO(this, inputName, inputData, app) || {});
|
|
||||||
} else if (`${type}:${inputName}` in widgets) {
|
|
||||||
// Support custom widgets by Type:Name
|
|
||||||
Object.assign(config, widgets[`${type}:${inputName}`](this, inputName, inputData, app) || {});
|
|
||||||
} else if (type in widgets) {
|
|
||||||
// Standard type widgets
|
|
||||||
Object.assign(config, widgets[type](this, inputName, inputData, app) || {});
|
|
||||||
} else {
|
|
||||||
// Node connection inputs
|
|
||||||
this.addInput(inputName, type);
|
|
||||||
widgetCreated = false;
|
|
||||||
}
|
|
||||||
|
|
||||||
if(widgetCreated && inputData[1]?.forceInput && config?.widget) {
|
|
||||||
if (!config.widget.options) config.widget.options = {};
|
|
||||||
config.widget.options.forceInput = inputData[1].forceInput;
|
|
||||||
}
|
|
||||||
if(widgetCreated && inputData[1]?.defaultInput && config?.widget) {
|
|
||||||
if (!config.widget.options) config.widget.options = {};
|
|
||||||
config.widget.options.defaultInput = inputData[1].defaultInput;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
for (const o in nodeData["output"]) {
|
|
||||||
let output = nodeData["output"][o];
|
|
||||||
if(output instanceof Array) output = "COMBO";
|
|
||||||
const outputName = nodeData["output_name"][o] || output;
|
|
||||||
const outputShape = nodeData["output_is_list"][o] ? LiteGraph.GRID_SHAPE : LiteGraph.CIRCLE_SHAPE ;
|
|
||||||
this.addOutput(outputName, output, { shape: outputShape });
|
|
||||||
}
|
|
||||||
|
|
||||||
const s = this.computeSize();
|
|
||||||
s[0] = Math.max(config.minWidth, s[0] * 1.5);
|
|
||||||
s[1] = Math.max(config.minHeight, s[1]);
|
|
||||||
this.size = s;
|
|
||||||
this.serialize_widgets = true;
|
|
||||||
|
|
||||||
app.#invokeExtensionsAsync("nodeCreated", this);
|
|
||||||
},
|
|
||||||
{
|
|
||||||
title: nodeData.display_name || nodeData.name,
|
|
||||||
comfyClass: nodeData.name,
|
|
||||||
nodeData
|
|
||||||
}
|
|
||||||
);
|
|
||||||
node.prototype.comfyClass = nodeData.name;
|
|
||||||
|
|
||||||
this.#addNodeContextMenuHandler(node);
|
|
||||||
this.#addDrawBackgroundHandler(node, app);
|
|
||||||
this.#addNodeKeyHandler(node);
|
|
||||||
|
|
||||||
await this.#invokeExtensionsAsync("beforeRegisterNodeDef", node, nodeData);
|
|
||||||
LiteGraph.registerNodeType(nodeId, node);
|
|
||||||
node.category = nodeData.category;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -1470,10 +1519,37 @@ export class ComfyApp {
|
|||||||
}
|
}
|
||||||
|
|
||||||
showMissingNodesError(missingNodeTypes, hasAddedNodes = true) {
|
showMissingNodesError(missingNodeTypes, hasAddedNodes = true) {
|
||||||
|
let seenTypes = new Set();
|
||||||
|
|
||||||
this.ui.dialog.show(
|
this.ui.dialog.show(
|
||||||
`When loading the graph, the following node types were not found: <ul>${Array.from(new Set(missingNodeTypes)).map(
|
$el("div.comfy-missing-nodes", [
|
||||||
(t) => `<li>${t}</li>`
|
$el("span", { textContent: "When loading the graph, the following node types were not found: " }),
|
||||||
).join("")}</ul>${hasAddedNodes ? "Nodes that have failed to load will show as red on the graph." : ""}`
|
$el(
|
||||||
|
"ul",
|
||||||
|
Array.from(new Set(missingNodeTypes)).map((t) => {
|
||||||
|
let children = [];
|
||||||
|
if (typeof t === "object") {
|
||||||
|
if(seenTypes.has(t.type)) return null;
|
||||||
|
seenTypes.add(t.type);
|
||||||
|
children.push($el("span", { textContent: t.type }));
|
||||||
|
if (t.hint) {
|
||||||
|
children.push($el("span", { textContent: t.hint }));
|
||||||
|
}
|
||||||
|
if (t.action) {
|
||||||
|
children.push($el("button", { onclick: t.action.callback, textContent: t.action.text }));
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
if(seenTypes.has(t)) return null;
|
||||||
|
seenTypes.add(t);
|
||||||
|
children.push($el("span", { textContent: t }));
|
||||||
|
}
|
||||||
|
return $el("li", children);
|
||||||
|
}).filter(Boolean)
|
||||||
|
),
|
||||||
|
...(hasAddedNodes
|
||||||
|
? [$el("span", { textContent: "Nodes that have failed to load will show as red on the graph." })]
|
||||||
|
: []),
|
||||||
|
])
|
||||||
);
|
);
|
||||||
this.logging.addEntry("Comfy.App", "warn", {
|
this.logging.addEntry("Comfy.App", "warn", {
|
||||||
MissingNodes: missingNodeTypes,
|
MissingNodes: missingNodeTypes,
|
||||||
@ -1483,32 +1559,39 @@ export class ComfyApp {
|
|||||||
/**
|
/**
|
||||||
* Populates the graph with the specified workflow data
|
* Populates the graph with the specified workflow data
|
||||||
* @param {*} graphData A serialized graph object
|
* @param {*} graphData A serialized graph object
|
||||||
|
* @param { boolean } clean If the graph state, e.g. images, should be cleared
|
||||||
*/
|
*/
|
||||||
loadGraphData(graphData) {
|
async loadGraphData(graphData, clean = true) {
|
||||||
this.clean();
|
if (clean !== false) {
|
||||||
|
this.clean();
|
||||||
|
}
|
||||||
|
|
||||||
let reset_invalid_values = false;
|
let reset_invalid_values = false;
|
||||||
if (!graphData) {
|
if (!graphData) {
|
||||||
if (typeof structuredClone === "undefined")
|
graphData = defaultGraph;
|
||||||
{
|
|
||||||
graphData = JSON.parse(JSON.stringify(defaultGraph));
|
|
||||||
}else
|
|
||||||
{
|
|
||||||
graphData = structuredClone(defaultGraph);
|
|
||||||
}
|
|
||||||
reset_invalid_values = true;
|
reset_invalid_values = true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (typeof structuredClone === "undefined")
|
||||||
|
{
|
||||||
|
graphData = JSON.parse(JSON.stringify(graphData));
|
||||||
|
}else
|
||||||
|
{
|
||||||
|
graphData = structuredClone(graphData);
|
||||||
|
}
|
||||||
|
|
||||||
const missingNodeTypes = [];
|
const missingNodeTypes = [];
|
||||||
|
await this.#invokeExtensionsAsync("beforeConfigureGraph", graphData, missingNodeTypes);
|
||||||
for (let n of graphData.nodes) {
|
for (let n of graphData.nodes) {
|
||||||
// Patch T2IAdapterLoader to ControlNetLoader since they are the same node now
|
// Patch T2IAdapterLoader to ControlNetLoader since they are the same node now
|
||||||
if (n.type == "T2IAdapterLoader") n.type = "ControlNetLoader";
|
if (n.type == "T2IAdapterLoader") n.type = "ControlNetLoader";
|
||||||
if (n.type == "ConditioningAverage ") n.type = "ConditioningAverage"; //typo fix
|
if (n.type == "ConditioningAverage ") n.type = "ConditioningAverage"; //typo fix
|
||||||
|
if (n.type == "SDV_img2vid_Conditioning") n.type = "SVD_img2vid_Conditioning"; //typo fix
|
||||||
|
|
||||||
// Find missing node types
|
// Find missing node types
|
||||||
if (!(n.type in LiteGraph.registered_node_types)) {
|
if (!(n.type in LiteGraph.registered_node_types)) {
|
||||||
n.type = sanitizeNodeName(n.type);
|
|
||||||
missingNodeTypes.push(n.type);
|
missingNodeTypes.push(n.type);
|
||||||
|
n.type = sanitizeNodeName(n.type);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -1600,6 +1683,7 @@ export class ComfyApp {
|
|||||||
if (missingNodeTypes.length) {
|
if (missingNodeTypes.length) {
|
||||||
this.showMissingNodesError(missingNodeTypes);
|
this.showMissingNodesError(missingNodeTypes);
|
||||||
}
|
}
|
||||||
|
await this.#invokeExtensionsAsync("afterConfigureGraph", missingNodeTypes);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@ -1607,92 +1691,110 @@ export class ComfyApp {
|
|||||||
* @returns The workflow and node links
|
* @returns The workflow and node links
|
||||||
*/
|
*/
|
||||||
async graphToPrompt() {
|
async graphToPrompt() {
|
||||||
for (const node of this.graph.computeExecutionOrder(false)) {
|
for (const outerNode of this.graph.computeExecutionOrder(false)) {
|
||||||
if (node.isVirtualNode) {
|
const innerNodes = outerNode.getInnerNodes ? outerNode.getInnerNodes() : [outerNode];
|
||||||
// Don't serialize frontend only nodes but let them make changes
|
for (const node of innerNodes) {
|
||||||
if (node.applyToGraph) {
|
if (node.isVirtualNode) {
|
||||||
node.applyToGraph();
|
// Don't serialize frontend only nodes but let them make changes
|
||||||
|
if (node.applyToGraph) {
|
||||||
|
node.applyToGraph();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
continue;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
const workflow = this.graph.serialize();
|
const workflow = this.graph.serialize();
|
||||||
const output = {};
|
const output = {};
|
||||||
// Process nodes in order of execution
|
// Process nodes in order of execution
|
||||||
for (const node of this.graph.computeExecutionOrder(false)) {
|
for (const outerNode of this.graph.computeExecutionOrder(false)) {
|
||||||
const n = workflow.nodes.find((n) => n.id === node.id);
|
const skipNode = outerNode.mode === 2 || outerNode.mode === 4;
|
||||||
|
const innerNodes = (!skipNode && outerNode.getInnerNodes) ? outerNode.getInnerNodes() : [outerNode];
|
||||||
|
for (const node of innerNodes) {
|
||||||
|
if (node.isVirtualNode) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
if (node.isVirtualNode) {
|
if (node.mode === 2 || node.mode === 4) {
|
||||||
continue;
|
// Don't serialize muted nodes
|
||||||
}
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
if (node.mode === 2 || node.mode === 4) {
|
const inputs = {};
|
||||||
// Don't serialize muted nodes
|
const widgets = node.widgets;
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
const inputs = {};
|
// Store all widget values
|
||||||
const widgets = node.widgets;
|
if (widgets) {
|
||||||
|
for (const i in widgets) {
|
||||||
// Store all widget values
|
const widget = widgets[i];
|
||||||
if (widgets) {
|
if (!widget.options || widget.options.serialize !== false) {
|
||||||
for (const i in widgets) {
|
inputs[widget.name] = widget.serializeValue ? await widget.serializeValue(node, i) : widget.value;
|
||||||
const widget = widgets[i];
|
}
|
||||||
if (!widget.options || widget.options.serialize !== false) {
|
|
||||||
inputs[widget.name] = widget.serializeValue ? await widget.serializeValue(n, i) : widget.value;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
// Store all node links
|
// Store all node links
|
||||||
for (let i in node.inputs) {
|
for (let i in node.inputs) {
|
||||||
let parent = node.getInputNode(i);
|
let parent = node.getInputNode(i);
|
||||||
if (parent) {
|
if (parent) {
|
||||||
let link = node.getInputLink(i);
|
let link = node.getInputLink(i);
|
||||||
while (parent.mode === 4 || parent.isVirtualNode) {
|
while (parent.mode === 4 || parent.isVirtualNode) {
|
||||||
let found = false;
|
let found = false;
|
||||||
if (parent.isVirtualNode) {
|
if (parent.isVirtualNode) {
|
||||||
link = parent.getInputLink(link.origin_slot);
|
link = parent.getInputLink(link.origin_slot);
|
||||||
if (link) {
|
if (link) {
|
||||||
parent = parent.getInputNode(link.target_slot);
|
parent = parent.getInputNode(link.target_slot);
|
||||||
if (parent) {
|
if (parent) {
|
||||||
found = true;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} else if (link && parent.mode === 4) {
|
|
||||||
let all_inputs = [link.origin_slot];
|
|
||||||
if (parent.inputs) {
|
|
||||||
all_inputs = all_inputs.concat(Object.keys(parent.inputs))
|
|
||||||
for (let parent_input in all_inputs) {
|
|
||||||
parent_input = all_inputs[parent_input];
|
|
||||||
if (parent.inputs[parent_input]?.type === node.inputs[i].type) {
|
|
||||||
link = parent.getInputLink(parent_input);
|
|
||||||
if (link) {
|
|
||||||
parent = parent.getInputNode(parent_input);
|
|
||||||
}
|
|
||||||
found = true;
|
found = true;
|
||||||
break;
|
}
|
||||||
|
}
|
||||||
|
} else if (link && parent.mode === 4) {
|
||||||
|
let all_inputs = [link.origin_slot];
|
||||||
|
if (parent.inputs) {
|
||||||
|
all_inputs = all_inputs.concat(Object.keys(parent.inputs))
|
||||||
|
for (let parent_input in all_inputs) {
|
||||||
|
parent_input = all_inputs[parent_input];
|
||||||
|
if (parent.inputs[parent_input]?.type === node.inputs[i].type) {
|
||||||
|
link = parent.getInputLink(parent_input);
|
||||||
|
if (link) {
|
||||||
|
parent = parent.getInputNode(parent_input);
|
||||||
|
}
|
||||||
|
found = true;
|
||||||
|
break;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (!found) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!found) {
|
if (link) {
|
||||||
break;
|
if (parent?.updateLink) {
|
||||||
|
link = parent.updateLink(link);
|
||||||
|
}
|
||||||
|
if (link) {
|
||||||
|
inputs[node.inputs[i].name] = [String(link.origin_id), parseInt(link.origin_slot)];
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (link) {
|
|
||||||
inputs[node.inputs[i].name] = [String(link.origin_id), parseInt(link.origin_slot)];
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
output[String(node.id)] = {
|
let node_data = {
|
||||||
inputs,
|
inputs,
|
||||||
class_type: node.comfyClass,
|
class_type: node.comfyClass,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
if (this.ui.settings.getSettingValue("Comfy.DevMode")) {
|
||||||
|
// Ignored by the backend.
|
||||||
|
node_data["_meta"] = {
|
||||||
|
title: node.title,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
output[String(node.id)] = node_data;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Remove inputs connected to removed nodes
|
// Remove inputs connected to removed nodes
|
||||||
@ -1812,7 +1914,9 @@ export class ComfyApp {
|
|||||||
const pngInfo = await getPngMetadata(file);
|
const pngInfo = await getPngMetadata(file);
|
||||||
if (pngInfo) {
|
if (pngInfo) {
|
||||||
if (pngInfo.workflow) {
|
if (pngInfo.workflow) {
|
||||||
this.loadGraphData(JSON.parse(pngInfo.workflow));
|
await this.loadGraphData(JSON.parse(pngInfo.workflow));
|
||||||
|
} else if (pngInfo.prompt) {
|
||||||
|
this.loadApiJson(JSON.parse(pngInfo.prompt));
|
||||||
} else if (pngInfo.parameters) {
|
} else if (pngInfo.parameters) {
|
||||||
importA1111(this.graph, pngInfo.parameters);
|
importA1111(this.graph, pngInfo.parameters);
|
||||||
}
|
}
|
||||||
@ -1824,25 +1928,29 @@ export class ComfyApp {
|
|||||||
this.loadGraphData(JSON.parse(pngInfo.workflow));
|
this.loadGraphData(JSON.parse(pngInfo.workflow));
|
||||||
} else if (pngInfo.Workflow) {
|
} else if (pngInfo.Workflow) {
|
||||||
this.loadGraphData(JSON.parse(pngInfo.Workflow)); // Support loading workflows from that webp custom node.
|
this.loadGraphData(JSON.parse(pngInfo.Workflow)); // Support loading workflows from that webp custom node.
|
||||||
|
} else if (pngInfo.prompt) {
|
||||||
|
this.loadApiJson(JSON.parse(pngInfo.prompt));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
} else if (file.type === "application/json" || file.name?.endsWith(".json")) {
|
} else if (file.type === "application/json" || file.name?.endsWith(".json")) {
|
||||||
const reader = new FileReader();
|
const reader = new FileReader();
|
||||||
reader.onload = () => {
|
reader.onload = async () => {
|
||||||
const jsonContent = JSON.parse(reader.result);
|
const jsonContent = JSON.parse(reader.result);
|
||||||
if (jsonContent?.templates) {
|
if (jsonContent?.templates) {
|
||||||
this.loadTemplateData(jsonContent);
|
this.loadTemplateData(jsonContent);
|
||||||
} else if(this.isApiJson(jsonContent)) {
|
} else if(this.isApiJson(jsonContent)) {
|
||||||
this.loadApiJson(jsonContent);
|
this.loadApiJson(jsonContent);
|
||||||
} else {
|
} else {
|
||||||
this.loadGraphData(jsonContent);
|
await this.loadGraphData(jsonContent);
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
reader.readAsText(file);
|
reader.readAsText(file);
|
||||||
} else if (file.name?.endsWith(".latent") || file.name?.endsWith(".safetensors")) {
|
} else if (file.name?.endsWith(".latent") || file.name?.endsWith(".safetensors")) {
|
||||||
const info = await getLatentMetadata(file);
|
const info = await getLatentMetadata(file);
|
||||||
if (info.workflow) {
|
if (info.workflow) {
|
||||||
this.loadGraphData(JSON.parse(info.workflow));
|
await this.loadGraphData(JSON.parse(info.workflow));
|
||||||
|
} else if (info.prompt) {
|
||||||
|
this.loadApiJson(JSON.parse(info.prompt));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -1863,7 +1971,7 @@ export class ComfyApp {
|
|||||||
for (const id of ids) {
|
for (const id of ids) {
|
||||||
const data = apiData[id];
|
const data = apiData[id];
|
||||||
const node = LiteGraph.createNode(data.class_type);
|
const node = LiteGraph.createNode(data.class_type);
|
||||||
node.id = id;
|
node.id = isNaN(+id) ? id : +id;
|
||||||
graph.add(node);
|
graph.add(node);
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -1912,12 +2020,8 @@ export class ComfyApp {
|
|||||||
async refreshComboInNodes() {
|
async refreshComboInNodes() {
|
||||||
const defs = await api.getNodeDefs();
|
const defs = await api.getNodeDefs();
|
||||||
|
|
||||||
for(const nodeId in LiteGraph.registered_node_types) {
|
for (const nodeId in defs) {
|
||||||
const node = LiteGraph.registered_node_types[nodeId];
|
this.registerNodeDef(nodeId, defs[nodeId]);
|
||||||
const nodeDef = defs[nodeId];
|
|
||||||
if(!nodeDef) continue;
|
|
||||||
|
|
||||||
node.nodeData = nodeDef;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
for(let nodeNum in this.graph._nodes) {
|
for(let nodeNum in this.graph._nodes) {
|
||||||
|
|||||||
325
web/scripts/domWidget.js
Normal file
325
web/scripts/domWidget.js
Normal file
@ -0,0 +1,325 @@
|
|||||||
|
import { app, ANIM_PREVIEW_WIDGET } from "./app.js";
|
||||||
|
|
||||||
|
const SIZE = Symbol();
|
||||||
|
|
||||||
|
function intersect(a, b) {
|
||||||
|
const x = Math.max(a.x, b.x);
|
||||||
|
const num1 = Math.min(a.x + a.width, b.x + b.width);
|
||||||
|
const y = Math.max(a.y, b.y);
|
||||||
|
const num2 = Math.min(a.y + a.height, b.y + b.height);
|
||||||
|
if (num1 >= x && num2 >= y) return [x, y, num1 - x, num2 - y];
|
||||||
|
else return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
function getClipPath(node, element, elRect) {
|
||||||
|
const selectedNode = Object.values(app.canvas.selected_nodes)[0];
|
||||||
|
if (selectedNode && selectedNode !== node) {
|
||||||
|
const MARGIN = 7;
|
||||||
|
const scale = app.canvas.ds.scale;
|
||||||
|
|
||||||
|
const bounding = selectedNode.getBounding();
|
||||||
|
const intersection = intersect(
|
||||||
|
{ x: elRect.x / scale, y: elRect.y / scale, width: elRect.width / scale, height: elRect.height / scale },
|
||||||
|
{
|
||||||
|
x: selectedNode.pos[0] + app.canvas.ds.offset[0] - MARGIN,
|
||||||
|
y: selectedNode.pos[1] + app.canvas.ds.offset[1] - LiteGraph.NODE_TITLE_HEIGHT - MARGIN,
|
||||||
|
width: bounding[2] + MARGIN + MARGIN,
|
||||||
|
height: bounding[3] + MARGIN + MARGIN,
|
||||||
|
}
|
||||||
|
);
|
||||||
|
|
||||||
|
if (!intersection) {
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
|
||||||
|
const widgetRect = element.getBoundingClientRect();
|
||||||
|
const clipX = intersection[0] - widgetRect.x / scale + "px";
|
||||||
|
const clipY = intersection[1] - widgetRect.y / scale + "px";
|
||||||
|
const clipWidth = intersection[2] + "px";
|
||||||
|
const clipHeight = intersection[3] + "px";
|
||||||
|
const path = `polygon(0% 0%, 0% 100%, ${clipX} 100%, ${clipX} ${clipY}, calc(${clipX} + ${clipWidth}) ${clipY}, calc(${clipX} + ${clipWidth}) calc(${clipY} + ${clipHeight}), ${clipX} calc(${clipY} + ${clipHeight}), ${clipX} 100%, 100% 100%, 100% 0%)`;
|
||||||
|
return path;
|
||||||
|
}
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
|
||||||
|
function computeSize(size) {
|
||||||
|
if (this.widgets?.[0]?.last_y == null) return;
|
||||||
|
|
||||||
|
let y = this.widgets[0].last_y;
|
||||||
|
let freeSpace = size[1] - y;
|
||||||
|
|
||||||
|
let widgetHeight = 0;
|
||||||
|
let dom = [];
|
||||||
|
for (const w of this.widgets) {
|
||||||
|
if (w.type === "converted-widget") {
|
||||||
|
// Ignore
|
||||||
|
delete w.computedHeight;
|
||||||
|
} else if (w.computeSize) {
|
||||||
|
widgetHeight += w.computeSize()[1] + 4;
|
||||||
|
} else if (w.element) {
|
||||||
|
// Extract DOM widget size info
|
||||||
|
const styles = getComputedStyle(w.element);
|
||||||
|
let minHeight = w.options.getMinHeight?.() ?? parseInt(styles.getPropertyValue("--comfy-widget-min-height"));
|
||||||
|
let maxHeight = w.options.getMaxHeight?.() ?? parseInt(styles.getPropertyValue("--comfy-widget-max-height"));
|
||||||
|
|
||||||
|
let prefHeight = w.options.getHeight?.() ?? styles.getPropertyValue("--comfy-widget-height");
|
||||||
|
if (prefHeight.endsWith?.("%")) {
|
||||||
|
prefHeight = size[1] * (parseFloat(prefHeight.substring(0, prefHeight.length - 1)) / 100);
|
||||||
|
} else {
|
||||||
|
prefHeight = parseInt(prefHeight);
|
||||||
|
if (isNaN(minHeight)) {
|
||||||
|
minHeight = prefHeight;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (isNaN(minHeight)) {
|
||||||
|
minHeight = 50;
|
||||||
|
}
|
||||||
|
if (!isNaN(maxHeight)) {
|
||||||
|
if (!isNaN(prefHeight)) {
|
||||||
|
prefHeight = Math.min(prefHeight, maxHeight);
|
||||||
|
} else {
|
||||||
|
prefHeight = maxHeight;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
dom.push({
|
||||||
|
minHeight,
|
||||||
|
prefHeight,
|
||||||
|
w,
|
||||||
|
});
|
||||||
|
} else {
|
||||||
|
widgetHeight += LiteGraph.NODE_WIDGET_HEIGHT + 4;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
freeSpace -= widgetHeight;
|
||||||
|
|
||||||
|
// Calculate sizes with all widgets at their min height
|
||||||
|
const prefGrow = []; // Nodes that want to grow to their prefd size
|
||||||
|
const canGrow = []; // Nodes that can grow to auto size
|
||||||
|
let growBy = 0;
|
||||||
|
for (const d of dom) {
|
||||||
|
freeSpace -= d.minHeight;
|
||||||
|
if (isNaN(d.prefHeight)) {
|
||||||
|
canGrow.push(d);
|
||||||
|
d.w.computedHeight = d.minHeight;
|
||||||
|
} else {
|
||||||
|
const diff = d.prefHeight - d.minHeight;
|
||||||
|
if (diff > 0) {
|
||||||
|
prefGrow.push(d);
|
||||||
|
growBy += diff;
|
||||||
|
d.diff = diff;
|
||||||
|
} else {
|
||||||
|
d.w.computedHeight = d.minHeight;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (this.imgs && !this.widgets.find((w) => w.name === ANIM_PREVIEW_WIDGET)) {
|
||||||
|
// Allocate space for image
|
||||||
|
freeSpace -= 220;
|
||||||
|
}
|
||||||
|
|
||||||
|
this.freeWidgetSpace = freeSpace;
|
||||||
|
|
||||||
|
if (freeSpace < 0) {
|
||||||
|
// Not enough space for all widgets so we need to grow
|
||||||
|
size[1] -= freeSpace;
|
||||||
|
this.graph.setDirtyCanvas(true);
|
||||||
|
} else {
|
||||||
|
// Share the space between each
|
||||||
|
const growDiff = freeSpace - growBy;
|
||||||
|
if (growDiff > 0) {
|
||||||
|
// All pref sizes can be fulfilled
|
||||||
|
freeSpace = growDiff;
|
||||||
|
for (const d of prefGrow) {
|
||||||
|
d.w.computedHeight = d.prefHeight;
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
// We need to grow evenly
|
||||||
|
const shared = -growDiff / prefGrow.length;
|
||||||
|
for (const d of prefGrow) {
|
||||||
|
d.w.computedHeight = d.prefHeight - shared;
|
||||||
|
}
|
||||||
|
freeSpace = 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (freeSpace > 0 && canGrow.length) {
|
||||||
|
// Grow any that are auto height
|
||||||
|
const shared = freeSpace / canGrow.length;
|
||||||
|
for (const d of canGrow) {
|
||||||
|
d.w.computedHeight += shared;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Position each of the widgets
|
||||||
|
for (const w of this.widgets) {
|
||||||
|
w.y = y;
|
||||||
|
if (w.computedHeight) {
|
||||||
|
y += w.computedHeight;
|
||||||
|
} else if (w.computeSize) {
|
||||||
|
y += w.computeSize()[1] + 4;
|
||||||
|
} else {
|
||||||
|
y += LiteGraph.NODE_WIDGET_HEIGHT + 4;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Override the compute visible nodes function to allow us to hide/show DOM elements when the node goes offscreen
|
||||||
|
const elementWidgets = new Set();
|
||||||
|
const computeVisibleNodes = LGraphCanvas.prototype.computeVisibleNodes;
|
||||||
|
LGraphCanvas.prototype.computeVisibleNodes = function () {
|
||||||
|
const visibleNodes = computeVisibleNodes.apply(this, arguments);
|
||||||
|
for (const node of app.graph._nodes) {
|
||||||
|
if (elementWidgets.has(node)) {
|
||||||
|
const hidden = visibleNodes.indexOf(node) === -1;
|
||||||
|
for (const w of node.widgets) {
|
||||||
|
if (w.element) {
|
||||||
|
w.element.hidden = hidden;
|
||||||
|
w.element.style.display = hidden ? "none" : undefined;
|
||||||
|
if (hidden) {
|
||||||
|
w.options.onHide?.(w);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return visibleNodes;
|
||||||
|
};
|
||||||
|
|
||||||
|
let enableDomClipping = true;
|
||||||
|
|
||||||
|
export function addDomClippingSetting() {
|
||||||
|
app.ui.settings.addSetting({
|
||||||
|
id: "Comfy.DOMClippingEnabled",
|
||||||
|
name: "Enable DOM element clipping (enabling may reduce performance)",
|
||||||
|
type: "boolean",
|
||||||
|
defaultValue: enableDomClipping,
|
||||||
|
onChange(value) {
|
||||||
|
enableDomClipping = !!value;
|
||||||
|
},
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
LGraphNode.prototype.addDOMWidget = function (name, type, element, options) {
|
||||||
|
options = { hideOnZoom: true, selectOn: ["focus", "click"], ...options };
|
||||||
|
|
||||||
|
if (!element.parentElement) {
|
||||||
|
document.body.append(element);
|
||||||
|
}
|
||||||
|
|
||||||
|
let mouseDownHandler;
|
||||||
|
if (element.blur) {
|
||||||
|
mouseDownHandler = (event) => {
|
||||||
|
if (!element.contains(event.target)) {
|
||||||
|
element.blur();
|
||||||
|
}
|
||||||
|
};
|
||||||
|
document.addEventListener("mousedown", mouseDownHandler);
|
||||||
|
}
|
||||||
|
|
||||||
|
const widget = {
|
||||||
|
type,
|
||||||
|
name,
|
||||||
|
get value() {
|
||||||
|
return options.getValue?.() ?? undefined;
|
||||||
|
},
|
||||||
|
set value(v) {
|
||||||
|
options.setValue?.(v);
|
||||||
|
widget.callback?.(widget.value);
|
||||||
|
},
|
||||||
|
draw: function (ctx, node, widgetWidth, y, widgetHeight) {
|
||||||
|
if (widget.computedHeight == null) {
|
||||||
|
computeSize.call(node, node.size);
|
||||||
|
}
|
||||||
|
|
||||||
|
const hidden =
|
||||||
|
node.flags?.collapsed ||
|
||||||
|
(!!options.hideOnZoom && app.canvas.ds.scale < 0.5) ||
|
||||||
|
widget.computedHeight <= 0 ||
|
||||||
|
widget.type === "converted-widget";
|
||||||
|
element.hidden = hidden;
|
||||||
|
element.style.display = hidden ? "none" : null;
|
||||||
|
if (hidden) {
|
||||||
|
widget.options.onHide?.(widget);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const margin = 10;
|
||||||
|
const elRect = ctx.canvas.getBoundingClientRect();
|
||||||
|
const transform = new DOMMatrix()
|
||||||
|
.scaleSelf(elRect.width / ctx.canvas.width, elRect.height / ctx.canvas.height)
|
||||||
|
.multiplySelf(ctx.getTransform())
|
||||||
|
.translateSelf(margin, margin + y);
|
||||||
|
|
||||||
|
const scale = new DOMMatrix().scaleSelf(transform.a, transform.d);
|
||||||
|
|
||||||
|
Object.assign(element.style, {
|
||||||
|
transformOrigin: "0 0",
|
||||||
|
transform: scale,
|
||||||
|
left: `${transform.a + transform.e}px`,
|
||||||
|
top: `${transform.d + transform.f}px`,
|
||||||
|
width: `${widgetWidth - margin * 2}px`,
|
||||||
|
height: `${(widget.computedHeight ?? 50) - margin * 2}px`,
|
||||||
|
position: "absolute",
|
||||||
|
zIndex: app.graph._nodes.indexOf(node),
|
||||||
|
});
|
||||||
|
|
||||||
|
if (enableDomClipping) {
|
||||||
|
element.style.clipPath = getClipPath(node, element, elRect);
|
||||||
|
element.style.willChange = "clip-path";
|
||||||
|
}
|
||||||
|
|
||||||
|
this.options.onDraw?.(widget);
|
||||||
|
},
|
||||||
|
element,
|
||||||
|
options,
|
||||||
|
onRemove() {
|
||||||
|
if (mouseDownHandler) {
|
||||||
|
document.removeEventListener("mousedown", mouseDownHandler);
|
||||||
|
}
|
||||||
|
element.remove();
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
for (const evt of options.selectOn) {
|
||||||
|
element.addEventListener(evt, () => {
|
||||||
|
app.canvas.selectNode(this);
|
||||||
|
app.canvas.bringToFront(this);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
this.addCustomWidget(widget);
|
||||||
|
elementWidgets.add(this);
|
||||||
|
|
||||||
|
const collapse = this.collapse;
|
||||||
|
this.collapse = function() {
|
||||||
|
collapse.apply(this, arguments);
|
||||||
|
if(this.flags?.collapsed) {
|
||||||
|
element.hidden = true;
|
||||||
|
element.style.display = "none";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const onRemoved = this.onRemoved;
|
||||||
|
this.onRemoved = function () {
|
||||||
|
element.remove();
|
||||||
|
elementWidgets.delete(this);
|
||||||
|
onRemoved?.apply(this, arguments);
|
||||||
|
};
|
||||||
|
|
||||||
|
if (!this[SIZE]) {
|
||||||
|
this[SIZE] = true;
|
||||||
|
const onResize = this.onResize;
|
||||||
|
this.onResize = function (size) {
|
||||||
|
options.beforeResize?.call(widget, this);
|
||||||
|
computeSize.call(this, size);
|
||||||
|
onResize?.apply(this, arguments);
|
||||||
|
options.afterResize?.call(widget, this);
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
return widget;
|
||||||
|
};
|
||||||
@ -24,7 +24,7 @@ export function getPngMetadata(file) {
|
|||||||
const length = dataView.getUint32(offset);
|
const length = dataView.getUint32(offset);
|
||||||
// Get the chunk type
|
// Get the chunk type
|
||||||
const type = String.fromCharCode(...pngData.slice(offset + 4, offset + 8));
|
const type = String.fromCharCode(...pngData.slice(offset + 4, offset + 8));
|
||||||
if (type === "tEXt") {
|
if (type === "tEXt" || type == "comf") {
|
||||||
// Get the keyword
|
// Get the keyword
|
||||||
let keyword_end = offset + 8;
|
let keyword_end = offset + 8;
|
||||||
while (pngData[keyword_end] !== 0) {
|
while (pngData[keyword_end] !== 0) {
|
||||||
@ -50,7 +50,6 @@ export function getPngMetadata(file) {
|
|||||||
function parseExifData(exifData) {
|
function parseExifData(exifData) {
|
||||||
// Check for the correct TIFF header (0x4949 for little-endian or 0x4D4D for big-endian)
|
// Check for the correct TIFF header (0x4949 for little-endian or 0x4D4D for big-endian)
|
||||||
const isLittleEndian = new Uint16Array(exifData.slice(0, 2))[0] === 0x4949;
|
const isLittleEndian = new Uint16Array(exifData.slice(0, 2))[0] === 0x4949;
|
||||||
console.log(exifData);
|
|
||||||
|
|
||||||
// Function to read 16-bit and 32-bit integers from binary data
|
// Function to read 16-bit and 32-bit integers from binary data
|
||||||
function readInt(offset, isLittleEndian, length) {
|
function readInt(offset, isLittleEndian, length) {
|
||||||
@ -126,6 +125,9 @@ export function getWebpMetadata(file) {
|
|||||||
const chunk_length = dataView.getUint32(offset + 4, true);
|
const chunk_length = dataView.getUint32(offset + 4, true);
|
||||||
const chunk_type = String.fromCharCode(...webp.slice(offset, offset + 4));
|
const chunk_type = String.fromCharCode(...webp.slice(offset, offset + 4));
|
||||||
if (chunk_type === "EXIF") {
|
if (chunk_type === "EXIF") {
|
||||||
|
if (String.fromCharCode(...webp.slice(offset + 8, offset + 8 + 6)) == "Exif\0\0") {
|
||||||
|
offset += 6;
|
||||||
|
}
|
||||||
let data = parseExifData(webp.slice(offset + 8, offset + 8 + chunk_length));
|
let data = parseExifData(webp.slice(offset + 8, offset + 8 + chunk_length));
|
||||||
for (var key in data) {
|
for (var key in data) {
|
||||||
var value = data[key];
|
var value = data[key];
|
||||||
|
|||||||
@ -462,8 +462,8 @@ class ComfyList {
|
|||||||
return $el("div", {textContent: item.prompt[0] + ": "}, [
|
return $el("div", {textContent: item.prompt[0] + ": "}, [
|
||||||
$el("button", {
|
$el("button", {
|
||||||
textContent: "Load",
|
textContent: "Load",
|
||||||
onclick: () => {
|
onclick: async () => {
|
||||||
app.loadGraphData(item.prompt[3].extra_pnginfo.workflow);
|
await app.loadGraphData(item.prompt[3].extra_pnginfo.workflow);
|
||||||
if (item.outputs) {
|
if (item.outputs) {
|
||||||
app.nodeOutputs = item.outputs;
|
app.nodeOutputs = item.outputs;
|
||||||
}
|
}
|
||||||
@ -599,7 +599,7 @@ export class ComfyUI {
|
|||||||
const fileInput = $el("input", {
|
const fileInput = $el("input", {
|
||||||
id: "comfy-file-input",
|
id: "comfy-file-input",
|
||||||
type: "file",
|
type: "file",
|
||||||
accept: ".json,image/png,.latent,.safetensors",
|
accept: ".json,image/png,.latent,.safetensors,image/webp",
|
||||||
style: {display: "none"},
|
style: {display: "none"},
|
||||||
parent: document.body,
|
parent: document.body,
|
||||||
onchange: () => {
|
onchange: () => {
|
||||||
@ -784,9 +784,9 @@ export class ComfyUI {
|
|||||||
}
|
}
|
||||||
}),
|
}),
|
||||||
$el("button", {
|
$el("button", {
|
||||||
id: "comfy-load-default-button", textContent: "Load Default", onclick: () => {
|
id: "comfy-load-default-button", textContent: "Load Default", onclick: async () => {
|
||||||
if (!confirmClear.value || confirm("Load default workflow?")) {
|
if (!confirmClear.value || confirm("Load default workflow?")) {
|
||||||
app.loadGraphData()
|
await app.loadGraphData()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}),
|
}),
|
||||||
|
|||||||
97
web/scripts/ui/imagePreview.js
Normal file
97
web/scripts/ui/imagePreview.js
Normal file
@ -0,0 +1,97 @@
|
|||||||
|
import { $el } from "../ui.js";
|
||||||
|
|
||||||
|
export function calculateImageGrid(imgs, dw, dh) {
|
||||||
|
let best = 0;
|
||||||
|
let w = imgs[0].naturalWidth;
|
||||||
|
let h = imgs[0].naturalHeight;
|
||||||
|
const numImages = imgs.length;
|
||||||
|
|
||||||
|
let cellWidth, cellHeight, cols, rows, shiftX;
|
||||||
|
// compact style
|
||||||
|
for (let c = 1; c <= numImages; c++) {
|
||||||
|
const r = Math.ceil(numImages / c);
|
||||||
|
const cW = dw / c;
|
||||||
|
const cH = dh / r;
|
||||||
|
const scaleX = cW / w;
|
||||||
|
const scaleY = cH / h;
|
||||||
|
|
||||||
|
const scale = Math.min(scaleX, scaleY, 1);
|
||||||
|
const imageW = w * scale;
|
||||||
|
const imageH = h * scale;
|
||||||
|
const area = imageW * imageH * numImages;
|
||||||
|
|
||||||
|
if (area > best) {
|
||||||
|
best = area;
|
||||||
|
cellWidth = imageW;
|
||||||
|
cellHeight = imageH;
|
||||||
|
cols = c;
|
||||||
|
rows = r;
|
||||||
|
shiftX = c * ((cW - imageW) / 2);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return { cellWidth, cellHeight, cols, rows, shiftX };
|
||||||
|
}
|
||||||
|
|
||||||
|
export function createImageHost(node) {
|
||||||
|
const el = $el("div.comfy-img-preview");
|
||||||
|
let currentImgs;
|
||||||
|
let first = true;
|
||||||
|
|
||||||
|
function updateSize() {
|
||||||
|
let w = null;
|
||||||
|
let h = null;
|
||||||
|
|
||||||
|
if (currentImgs) {
|
||||||
|
let elH = el.clientHeight;
|
||||||
|
if (first) {
|
||||||
|
first = false;
|
||||||
|
// On first run, if we are small then grow a bit
|
||||||
|
if (elH < 190) {
|
||||||
|
elH = 190;
|
||||||
|
}
|
||||||
|
el.style.setProperty("--comfy-widget-min-height", elH);
|
||||||
|
} else {
|
||||||
|
el.style.setProperty("--comfy-widget-min-height", null);
|
||||||
|
}
|
||||||
|
|
||||||
|
const nw = node.size[0];
|
||||||
|
({ cellWidth: w, cellHeight: h } = calculateImageGrid(currentImgs, nw - 20, elH));
|
||||||
|
w += "px";
|
||||||
|
h += "px";
|
||||||
|
|
||||||
|
el.style.setProperty("--comfy-img-preview-width", w);
|
||||||
|
el.style.setProperty("--comfy-img-preview-height", h);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
el,
|
||||||
|
updateImages(imgs) {
|
||||||
|
if (imgs !== currentImgs) {
|
||||||
|
if (currentImgs == null) {
|
||||||
|
requestAnimationFrame(() => {
|
||||||
|
updateSize();
|
||||||
|
});
|
||||||
|
}
|
||||||
|
el.replaceChildren(...imgs);
|
||||||
|
currentImgs = imgs;
|
||||||
|
node.onResize(node.size);
|
||||||
|
node.graph.setDirtyCanvas(true, true);
|
||||||
|
}
|
||||||
|
},
|
||||||
|
getHeight() {
|
||||||
|
updateSize();
|
||||||
|
},
|
||||||
|
onDraw() {
|
||||||
|
// Element from point uses a hittest find elements so we need to toggle pointer events
|
||||||
|
el.style.pointerEvents = "all";
|
||||||
|
const over = document.elementFromPoint(app.canvas.mouse[0], app.canvas.mouse[1]);
|
||||||
|
el.style.pointerEvents = "none";
|
||||||
|
|
||||||
|
if(!over) return;
|
||||||
|
// Set the overIndex so Open Image etc work
|
||||||
|
const idx = currentImgs.indexOf(over);
|
||||||
|
node.overIndex = idx;
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
67
web/scripts/utils.js
Normal file
67
web/scripts/utils.js
Normal file
@ -0,0 +1,67 @@
|
|||||||
|
// Simple date formatter
|
||||||
|
const parts = {
|
||||||
|
d: (d) => d.getDate(),
|
||||||
|
M: (d) => d.getMonth() + 1,
|
||||||
|
h: (d) => d.getHours(),
|
||||||
|
m: (d) => d.getMinutes(),
|
||||||
|
s: (d) => d.getSeconds(),
|
||||||
|
};
|
||||||
|
const format =
|
||||||
|
Object.keys(parts)
|
||||||
|
.map((k) => k + k + "?")
|
||||||
|
.join("|") + "|yyy?y?";
|
||||||
|
|
||||||
|
function formatDate(text, date) {
|
||||||
|
return text.replace(new RegExp(format, "g"), function (text) {
|
||||||
|
if (text === "yy") return (date.getFullYear() + "").substring(2);
|
||||||
|
if (text === "yyyy") return date.getFullYear();
|
||||||
|
if (text[0] in parts) {
|
||||||
|
const p = parts[text[0]](date);
|
||||||
|
return (p + "").padStart(text.length, "0");
|
||||||
|
}
|
||||||
|
return text;
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
export function applyTextReplacements(app, value) {
|
||||||
|
return value.replace(/%([^%]+)%/g, function (match, text) {
|
||||||
|
const split = text.split(".");
|
||||||
|
if (split.length !== 2) {
|
||||||
|
// Special handling for dates
|
||||||
|
if (split[0].startsWith("date:")) {
|
||||||
|
return formatDate(split[0].substring(5), new Date());
|
||||||
|
}
|
||||||
|
|
||||||
|
if (text !== "width" && text !== "height") {
|
||||||
|
// Dont warn on standard replacements
|
||||||
|
console.warn("Invalid replacement pattern", text);
|
||||||
|
}
|
||||||
|
return match;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Find node with matching S&R property name
|
||||||
|
let nodes = app.graph._nodes.filter((n) => n.properties?.["Node name for S&R"] === split[0]);
|
||||||
|
// If we cant, see if there is a node with that title
|
||||||
|
if (!nodes.length) {
|
||||||
|
nodes = app.graph._nodes.filter((n) => n.title === split[0]);
|
||||||
|
}
|
||||||
|
if (!nodes.length) {
|
||||||
|
console.warn("Unable to find node", split[0]);
|
||||||
|
return match;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (nodes.length > 1) {
|
||||||
|
console.warn("Multiple nodes matched", split[0], "using first match");
|
||||||
|
}
|
||||||
|
|
||||||
|
const node = nodes[0];
|
||||||
|
|
||||||
|
const widget = node.widgets?.find((w) => w.name === split[1]);
|
||||||
|
if (!widget) {
|
||||||
|
console.warn("Unable to find widget", split[1], "on node", split[0], node);
|
||||||
|
return match;
|
||||||
|
}
|
||||||
|
|
||||||
|
return ((widget.value ?? "") + "").replaceAll(/\/|\\/g, "_");
|
||||||
|
});
|
||||||
|
}
|
||||||
@ -1,4 +1,5 @@
|
|||||||
import { api } from "./api.js"
|
import { api } from "./api.js"
|
||||||
|
import "./domWidget.js";
|
||||||
|
|
||||||
function getNumberDefaults(inputData, defaultStep, precision, enable_rounding) {
|
function getNumberDefaults(inputData, defaultStep, precision, enable_rounding) {
|
||||||
let defaultVal = inputData[1]["default"];
|
let defaultVal = inputData[1]["default"];
|
||||||
@ -22,18 +23,89 @@ function getNumberDefaults(inputData, defaultStep, precision, enable_rounding) {
|
|||||||
return { val: defaultVal, config: { min, max, step: 10.0 * step, round, precision } };
|
return { val: defaultVal, config: { min, max, step: 10.0 * step, round, precision } };
|
||||||
}
|
}
|
||||||
|
|
||||||
export function addValueControlWidget(node, targetWidget, defaultValue = "randomize", values) {
|
export function addValueControlWidget(node, targetWidget, defaultValue = "randomize", values, widgetName, inputData) {
|
||||||
const valueControl = node.addWidget("combo", "control_after_generate", defaultValue, function (v) { }, {
|
let name = inputData[1]?.control_after_generate;
|
||||||
values: ["fixed", "increment", "decrement", "randomize"],
|
if(typeof name !== "string") {
|
||||||
serialize: false, // Don't include this in prompt.
|
name = widgetName;
|
||||||
});
|
}
|
||||||
valueControl.afterQueued = () => {
|
const widgets = addValueControlWidgets(node, targetWidget, defaultValue, {
|
||||||
|
addFilterList: false,
|
||||||
|
controlAfterGenerateName: name
|
||||||
|
}, inputData);
|
||||||
|
return widgets[0];
|
||||||
|
}
|
||||||
|
|
||||||
|
export function addValueControlWidgets(node, targetWidget, defaultValue = "randomize", options, inputData) {
|
||||||
|
if (!defaultValue) defaultValue = "randomize";
|
||||||
|
if (!options) options = {};
|
||||||
|
|
||||||
|
const getName = (defaultName, optionName) => {
|
||||||
|
let name = defaultName;
|
||||||
|
if (options[optionName]) {
|
||||||
|
name = options[optionName];
|
||||||
|
} else if (typeof inputData?.[1]?.[defaultName] === "string") {
|
||||||
|
name = inputData?.[1]?.[defaultName];
|
||||||
|
} else if (inputData?.[1]?.control_prefix) {
|
||||||
|
name = inputData?.[1]?.control_prefix + " " + name
|
||||||
|
}
|
||||||
|
return name;
|
||||||
|
}
|
||||||
|
|
||||||
|
const widgets = [];
|
||||||
|
const valueControl = node.addWidget(
|
||||||
|
"combo",
|
||||||
|
getName("control_after_generate", "controlAfterGenerateName"),
|
||||||
|
defaultValue,
|
||||||
|
function () {},
|
||||||
|
{
|
||||||
|
values: ["fixed", "increment", "decrement", "randomize"],
|
||||||
|
serialize: false, // Don't include this in prompt.
|
||||||
|
}
|
||||||
|
);
|
||||||
|
widgets.push(valueControl);
|
||||||
|
|
||||||
|
const isCombo = targetWidget.type === "combo";
|
||||||
|
let comboFilter;
|
||||||
|
if (isCombo && options.addFilterList !== false) {
|
||||||
|
comboFilter = node.addWidget(
|
||||||
|
"string",
|
||||||
|
getName("control_filter_list", "controlFilterListName"),
|
||||||
|
"",
|
||||||
|
function () {},
|
||||||
|
{
|
||||||
|
serialize: false, // Don't include this in prompt.
|
||||||
|
}
|
||||||
|
);
|
||||||
|
widgets.push(comboFilter);
|
||||||
|
}
|
||||||
|
|
||||||
|
valueControl.afterQueued = () => {
|
||||||
var v = valueControl.value;
|
var v = valueControl.value;
|
||||||
|
|
||||||
if (targetWidget.type == "combo" && v !== "fixed") {
|
if (isCombo && v !== "fixed") {
|
||||||
let current_index = targetWidget.options.values.indexOf(targetWidget.value);
|
let values = targetWidget.options.values;
|
||||||
let current_length = targetWidget.options.values.length;
|
const filter = comboFilter?.value;
|
||||||
|
if (filter) {
|
||||||
|
let check;
|
||||||
|
if (filter.startsWith("/") && filter.endsWith("/")) {
|
||||||
|
try {
|
||||||
|
const regex = new RegExp(filter.substring(1, filter.length - 1));
|
||||||
|
check = (item) => regex.test(item);
|
||||||
|
} catch (error) {
|
||||||
|
console.error("Error constructing RegExp filter for node " + node.id, filter, error);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (!check) {
|
||||||
|
const lower = filter.toLocaleLowerCase();
|
||||||
|
check = (item) => item.toLocaleLowerCase().includes(lower);
|
||||||
|
}
|
||||||
|
values = values.filter(item => check(item));
|
||||||
|
if (!values.length && targetWidget.options.values.length) {
|
||||||
|
console.warn("Filter for node " + node.id + " has filtered out all items", filter);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let current_index = values.indexOf(targetWidget.value);
|
||||||
|
let current_length = values.length;
|
||||||
|
|
||||||
switch (v) {
|
switch (v) {
|
||||||
case "increment":
|
case "increment":
|
||||||
@ -50,11 +122,12 @@ export function addValueControlWidget(node, targetWidget, defaultValue = "random
|
|||||||
current_index = Math.max(0, current_index);
|
current_index = Math.max(0, current_index);
|
||||||
current_index = Math.min(current_length - 1, current_index);
|
current_index = Math.min(current_length - 1, current_index);
|
||||||
if (current_index >= 0) {
|
if (current_index >= 0) {
|
||||||
let value = targetWidget.options.values[current_index];
|
let value = values[current_index];
|
||||||
targetWidget.value = value;
|
targetWidget.value = value;
|
||||||
targetWidget.callback(value);
|
targetWidget.callback(value);
|
||||||
}
|
}
|
||||||
} else { //number
|
} else {
|
||||||
|
//number
|
||||||
let min = targetWidget.options.min;
|
let min = targetWidget.options.min;
|
||||||
let max = targetWidget.options.max;
|
let max = targetWidget.options.max;
|
||||||
// limit to something that javascript can handle
|
// limit to something that javascript can handle
|
||||||
@ -77,186 +150,68 @@ export function addValueControlWidget(node, targetWidget, defaultValue = "random
|
|||||||
default:
|
default:
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
/*check if values are over or under their respective
|
/*check if values are over or under their respective
|
||||||
* ranges and set them to min or max.*/
|
* ranges and set them to min or max.*/
|
||||||
if (targetWidget.value < min)
|
if (targetWidget.value < min) targetWidget.value = min;
|
||||||
targetWidget.value = min;
|
|
||||||
|
|
||||||
if (targetWidget.value > max)
|
if (targetWidget.value > max)
|
||||||
targetWidget.value = max;
|
targetWidget.value = max;
|
||||||
targetWidget.callback(targetWidget.value);
|
targetWidget.callback(targetWidget.value);
|
||||||
}
|
}
|
||||||
}
|
};
|
||||||
return valueControl;
|
return widgets;
|
||||||
};
|
};
|
||||||
|
|
||||||
function seedWidget(node, inputName, inputData, app) {
|
function seedWidget(node, inputName, inputData, app, widgetName) {
|
||||||
const seed = ComfyWidgets.INT(node, inputName, inputData, app);
|
const seed = createIntWidget(node, inputName, inputData, app, true);
|
||||||
const seedControl = addValueControlWidget(node, seed.widget, "randomize");
|
const seedControl = addValueControlWidget(node, seed.widget, "randomize", undefined, widgetName, inputData);
|
||||||
|
|
||||||
seed.widget.linkedWidgets = [seedControl];
|
seed.widget.linkedWidgets = [seedControl];
|
||||||
return seed;
|
return seed;
|
||||||
}
|
}
|
||||||
|
|
||||||
const MultilineSymbol = Symbol();
|
function createIntWidget(node, inputName, inputData, app, isSeedInput) {
|
||||||
const MultilineResizeSymbol = Symbol();
|
const control = inputData[1]?.control_after_generate;
|
||||||
|
if (!isSeedInput && control) {
|
||||||
|
return seedWidget(node, inputName, inputData, app, typeof control === "string" ? control : undefined);
|
||||||
|
}
|
||||||
|
|
||||||
|
let widgetType = isSlider(inputData[1]["display"], app);
|
||||||
|
const { val, config } = getNumberDefaults(inputData, 1, 0, true);
|
||||||
|
Object.assign(config, { precision: 0 });
|
||||||
|
return {
|
||||||
|
widget: node.addWidget(
|
||||||
|
widgetType,
|
||||||
|
inputName,
|
||||||
|
val,
|
||||||
|
function (v) {
|
||||||
|
const s = this.options.step / 10;
|
||||||
|
this.value = Math.round(v / s) * s;
|
||||||
|
},
|
||||||
|
config
|
||||||
|
),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
function addMultilineWidget(node, name, opts, app) {
|
function addMultilineWidget(node, name, opts, app) {
|
||||||
const MIN_SIZE = 50;
|
const inputEl = document.createElement("textarea");
|
||||||
|
inputEl.className = "comfy-multiline-input";
|
||||||
|
inputEl.value = opts.defaultVal;
|
||||||
|
inputEl.placeholder = opts.placeholder || name;
|
||||||
|
|
||||||
function computeSize(size) {
|
const widget = node.addDOMWidget(name, "customtext", inputEl, {
|
||||||
if (node.widgets[0].last_y == null) return;
|
getValue() {
|
||||||
|
return inputEl.value;
|
||||||
let y = node.widgets[0].last_y;
|
|
||||||
let freeSpace = size[1] - y;
|
|
||||||
|
|
||||||
// Compute the height of all non customtext widgets
|
|
||||||
let widgetHeight = 0;
|
|
||||||
const multi = [];
|
|
||||||
for (let i = 0; i < node.widgets.length; i++) {
|
|
||||||
const w = node.widgets[i];
|
|
||||||
if (w.type === "customtext") {
|
|
||||||
multi.push(w);
|
|
||||||
} else {
|
|
||||||
if (w.computeSize) {
|
|
||||||
widgetHeight += w.computeSize()[1] + 4;
|
|
||||||
} else {
|
|
||||||
widgetHeight += LiteGraph.NODE_WIDGET_HEIGHT + 4;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// See how large each text input can be
|
|
||||||
freeSpace -= widgetHeight;
|
|
||||||
freeSpace /= multi.length + (!!node.imgs?.length);
|
|
||||||
|
|
||||||
if (freeSpace < MIN_SIZE) {
|
|
||||||
// There isnt enough space for all the widgets, increase the size of the node
|
|
||||||
freeSpace = MIN_SIZE;
|
|
||||||
node.size[1] = y + widgetHeight + freeSpace * (multi.length + (!!node.imgs?.length));
|
|
||||||
node.graph.setDirtyCanvas(true);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Position each of the widgets
|
|
||||||
for (const w of node.widgets) {
|
|
||||||
w.y = y;
|
|
||||||
if (w.type === "customtext") {
|
|
||||||
y += freeSpace;
|
|
||||||
w.computedHeight = freeSpace - multi.length*4;
|
|
||||||
} else if (w.computeSize) {
|
|
||||||
y += w.computeSize()[1] + 4;
|
|
||||||
} else {
|
|
||||||
y += LiteGraph.NODE_WIDGET_HEIGHT + 4;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
node.inputHeight = freeSpace;
|
|
||||||
}
|
|
||||||
|
|
||||||
const widget = {
|
|
||||||
type: "customtext",
|
|
||||||
name,
|
|
||||||
get value() {
|
|
||||||
return this.inputEl.value;
|
|
||||||
},
|
},
|
||||||
set value(x) {
|
setValue(v) {
|
||||||
this.inputEl.value = x;
|
inputEl.value = v;
|
||||||
},
|
},
|
||||||
draw: function (ctx, _, widgetWidth, y, widgetHeight) {
|
|
||||||
if (!this.parent.inputHeight) {
|
|
||||||
// If we are initially offscreen when created we wont have received a resize event
|
|
||||||
// Calculate it here instead
|
|
||||||
computeSize(node.size);
|
|
||||||
}
|
|
||||||
const visible = app.canvas.ds.scale > 0.5 && this.type === "customtext";
|
|
||||||
const margin = 10;
|
|
||||||
const elRect = ctx.canvas.getBoundingClientRect();
|
|
||||||
const transform = new DOMMatrix()
|
|
||||||
.scaleSelf(elRect.width / ctx.canvas.width, elRect.height / ctx.canvas.height)
|
|
||||||
.multiplySelf(ctx.getTransform())
|
|
||||||
.translateSelf(margin, margin + y);
|
|
||||||
|
|
||||||
const scale = new DOMMatrix().scaleSelf(transform.a, transform.d)
|
|
||||||
Object.assign(this.inputEl.style, {
|
|
||||||
transformOrigin: "0 0",
|
|
||||||
transform: scale,
|
|
||||||
left: `${transform.a + transform.e}px`,
|
|
||||||
top: `${transform.d + transform.f}px`,
|
|
||||||
width: `${widgetWidth - (margin * 2)}px`,
|
|
||||||
height: `${this.parent.inputHeight - (margin * 2)}px`,
|
|
||||||
position: "absolute",
|
|
||||||
background: (!node.color)?'':node.color,
|
|
||||||
color: (!node.color)?'':'white',
|
|
||||||
zIndex: app.graph._nodes.indexOf(node),
|
|
||||||
});
|
|
||||||
this.inputEl.hidden = !visible;
|
|
||||||
},
|
|
||||||
};
|
|
||||||
widget.inputEl = document.createElement("textarea");
|
|
||||||
widget.inputEl.className = "comfy-multiline-input";
|
|
||||||
widget.inputEl.value = opts.defaultVal;
|
|
||||||
widget.inputEl.placeholder = opts.placeholder || "";
|
|
||||||
document.addEventListener("mousedown", function (event) {
|
|
||||||
if (!widget.inputEl.contains(event.target)) {
|
|
||||||
widget.inputEl.blur();
|
|
||||||
}
|
|
||||||
});
|
});
|
||||||
widget.parent = node;
|
widget.inputEl = inputEl;
|
||||||
document.body.appendChild(widget.inputEl);
|
|
||||||
|
|
||||||
node.addCustomWidget(widget);
|
inputEl.addEventListener("input", () => {
|
||||||
|
widget.callback?.(widget.value);
|
||||||
app.canvas.onDrawBackground = function () {
|
});
|
||||||
// Draw node isnt fired once the node is off the screen
|
|
||||||
// if it goes off screen quickly, the input may not be removed
|
|
||||||
// this shifts it off screen so it can be moved back if the node is visible.
|
|
||||||
for (let n in app.graph._nodes) {
|
|
||||||
n = graph._nodes[n];
|
|
||||||
for (let w in n.widgets) {
|
|
||||||
let wid = n.widgets[w];
|
|
||||||
if (Object.hasOwn(wid, "inputEl")) {
|
|
||||||
wid.inputEl.style.left = -8000 + "px";
|
|
||||||
wid.inputEl.style.position = "absolute";
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
node.onRemoved = function () {
|
|
||||||
// When removing this node we need to remove the input from the DOM
|
|
||||||
for (let y in this.widgets) {
|
|
||||||
if (this.widgets[y].inputEl) {
|
|
||||||
this.widgets[y].inputEl.remove();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
widget.onRemove = () => {
|
|
||||||
widget.inputEl?.remove();
|
|
||||||
|
|
||||||
// Restore original size handler if we are the last
|
|
||||||
if (!--node[MultilineSymbol]) {
|
|
||||||
node.onResize = node[MultilineResizeSymbol];
|
|
||||||
delete node[MultilineSymbol];
|
|
||||||
delete node[MultilineResizeSymbol];
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
if (node[MultilineSymbol]) {
|
|
||||||
node[MultilineSymbol]++;
|
|
||||||
} else {
|
|
||||||
node[MultilineSymbol] = 1;
|
|
||||||
const onResize = (node[MultilineResizeSymbol] = node.onResize);
|
|
||||||
|
|
||||||
node.onResize = function (size) {
|
|
||||||
computeSize(size);
|
|
||||||
|
|
||||||
// Call original resizer handler
|
|
||||||
if (onResize) {
|
|
||||||
onResize.apply(this, arguments);
|
|
||||||
}
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
return { minWidth: 400, minHeight: 200, widget };
|
return { minWidth: 400, minHeight: 200, widget };
|
||||||
}
|
}
|
||||||
@ -288,31 +243,26 @@ export const ComfyWidgets = {
|
|||||||
}, config) };
|
}, config) };
|
||||||
},
|
},
|
||||||
INT(node, inputName, inputData, app) {
|
INT(node, inputName, inputData, app) {
|
||||||
let widgetType = isSlider(inputData[1]["display"], app);
|
return createIntWidget(node, inputName, inputData, app);
|
||||||
const { val, config } = getNumberDefaults(inputData, 1, 0, true);
|
|
||||||
Object.assign(config, { precision: 0 });
|
|
||||||
return {
|
|
||||||
widget: node.addWidget(
|
|
||||||
widgetType,
|
|
||||||
inputName,
|
|
||||||
val,
|
|
||||||
function (v) {
|
|
||||||
const s = this.options.step / 10;
|
|
||||||
this.value = Math.round(v / s) * s;
|
|
||||||
},
|
|
||||||
config
|
|
||||||
),
|
|
||||||
};
|
|
||||||
},
|
},
|
||||||
BOOLEAN(node, inputName, inputData) {
|
BOOLEAN(node, inputName, inputData) {
|
||||||
let defaultVal = inputData[1]["default"];
|
let defaultVal = false;
|
||||||
|
let options = {};
|
||||||
|
if (inputData[1]) {
|
||||||
|
if (inputData[1].default)
|
||||||
|
defaultVal = inputData[1].default;
|
||||||
|
if (inputData[1].label_on)
|
||||||
|
options["on"] = inputData[1].label_on;
|
||||||
|
if (inputData[1].label_off)
|
||||||
|
options["off"] = inputData[1].label_off;
|
||||||
|
}
|
||||||
return {
|
return {
|
||||||
widget: node.addWidget(
|
widget: node.addWidget(
|
||||||
"toggle",
|
"toggle",
|
||||||
inputName,
|
inputName,
|
||||||
defaultVal,
|
defaultVal,
|
||||||
() => {},
|
() => {},
|
||||||
{"on": inputData[1].label_on, "off": inputData[1].label_off}
|
options,
|
||||||
)
|
)
|
||||||
};
|
};
|
||||||
},
|
},
|
||||||
@ -338,10 +288,14 @@ export const ComfyWidgets = {
|
|||||||
if (inputData[1] && inputData[1].default) {
|
if (inputData[1] && inputData[1].default) {
|
||||||
defaultValue = inputData[1].default;
|
defaultValue = inputData[1].default;
|
||||||
}
|
}
|
||||||
return { widget: node.addWidget("combo", inputName, defaultValue, () => {}, { values: type }) };
|
const res = { widget: node.addWidget("combo", inputName, defaultValue, () => {}, { values: type }) };
|
||||||
|
if (inputData[1]?.control_after_generate) {
|
||||||
|
res.widget.linkedWidgets = addValueControlWidgets(node, res.widget, undefined, undefined, inputData);
|
||||||
|
}
|
||||||
|
return res;
|
||||||
},
|
},
|
||||||
IMAGEUPLOAD(node, inputName, inputData, app) {
|
IMAGEUPLOAD(node, inputName, inputData, app) {
|
||||||
const imageWidget = node.widgets.find((w) => w.name === "image");
|
const imageWidget = node.widgets.find((w) => w.name === (inputData[1]?.widget ?? "image"));
|
||||||
let uploadWidget;
|
let uploadWidget;
|
||||||
|
|
||||||
function showImage(name) {
|
function showImage(name) {
|
||||||
@ -356,7 +310,7 @@ export const ComfyWidgets = {
|
|||||||
subfolder = name.substring(0, folder_separator);
|
subfolder = name.substring(0, folder_separator);
|
||||||
name = name.substring(folder_separator + 1);
|
name = name.substring(folder_separator + 1);
|
||||||
}
|
}
|
||||||
img.src = api.apiURL(`/view?filename=${encodeURIComponent(name)}&type=input&subfolder=${subfolder}${app.getPreviewFormatParam()}`);
|
img.src = api.apiURL(`/view?filename=${encodeURIComponent(name)}&type=input&subfolder=${subfolder}${app.getPreviewFormatParam()}${app.getRandParam()}`);
|
||||||
node.setSizeForImage?.();
|
node.setSizeForImage?.();
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -455,9 +409,10 @@ export const ComfyWidgets = {
|
|||||||
document.body.append(fileInput);
|
document.body.append(fileInput);
|
||||||
|
|
||||||
// Create the button widget for selecting the files
|
// Create the button widget for selecting the files
|
||||||
uploadWidget = node.addWidget("button", "choose file to upload", "image", () => {
|
uploadWidget = node.addWidget("button", inputName, "image", () => {
|
||||||
fileInput.click();
|
fileInput.click();
|
||||||
});
|
});
|
||||||
|
uploadWidget.label = "choose file to upload";
|
||||||
uploadWidget.serialize = false;
|
uploadWidget.serialize = false;
|
||||||
|
|
||||||
// Add handler to check if an image is being dragged over our node
|
// Add handler to check if an image is being dragged over our node
|
||||||
|
|||||||
@ -409,6 +409,26 @@ dialog::backdrop {
|
|||||||
width: calc(100% - 10px);
|
width: calc(100% - 10px);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
.comfy-img-preview {
|
||||||
|
pointer-events: none;
|
||||||
|
overflow: hidden;
|
||||||
|
display: flex;
|
||||||
|
flex-wrap: wrap;
|
||||||
|
align-content: flex-start;
|
||||||
|
justify-content: center;
|
||||||
|
}
|
||||||
|
|
||||||
|
.comfy-img-preview img {
|
||||||
|
object-fit: contain;
|
||||||
|
width: var(--comfy-img-preview-width);
|
||||||
|
height: var(--comfy-img-preview-height);
|
||||||
|
}
|
||||||
|
|
||||||
|
.comfy-missing-nodes li button {
|
||||||
|
font-size: 12px;
|
||||||
|
margin-left: 5px;
|
||||||
|
}
|
||||||
|
|
||||||
/* Search box */
|
/* Search box */
|
||||||
|
|
||||||
.litegraph.litesearchbox {
|
.litegraph.litesearchbox {
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user