diff --git a/.gitattributes b/.gitattributes
index 4391de678..5b3c15bb4 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -1,2 +1,3 @@
/web/assets/** linguist-generated
/web/** linguist-vendored
+comfy_api_nodes/apis/__init__.py linguist-generated
diff --git a/.github/ISSUE_TEMPLATE/bug-report.yml b/.github/ISSUE_TEMPLATE/bug-report.yml
index 69ce998eb..3cf2717b7 100644
--- a/.github/ISSUE_TEMPLATE/bug-report.yml
+++ b/.github/ISSUE_TEMPLATE/bug-report.yml
@@ -22,7 +22,7 @@ body:
description: Please confirm you have tried to reproduce the issue with all custom nodes disabled.
options:
- label: I have tried disabling custom nodes and the issue persists (see [how to disable custom nodes](https://docs.comfy.org/troubleshooting/custom-node-issues#step-1%3A-test-with-all-custom-nodes-disabled) if you need help)
- required: true
+ required: false
- type: textarea
attributes:
label: Expected Behavior
diff --git a/.github/ISSUE_TEMPLATE/user-support.yml b/.github/ISSUE_TEMPLATE/user-support.yml
index 50657d493..281661f92 100644
--- a/.github/ISSUE_TEMPLATE/user-support.yml
+++ b/.github/ISSUE_TEMPLATE/user-support.yml
@@ -18,7 +18,7 @@ body:
description: Please confirm you have tried to reproduce the issue with all custom nodes disabled.
options:
- label: I have tried disabling custom nodes and the issue persists (see [how to disable custom nodes](https://docs.comfy.org/troubleshooting/custom-node-issues#step-1%3A-test-with-all-custom-nodes-disabled) if you need help)
- required: true
+ required: false
- type: textarea
attributes:
label: Your question
diff --git a/.github/workflows/stable-release.yml b/.github/workflows/stable-release.yml
index 61105abe4..2bc8e5905 100644
--- a/.github/workflows/stable-release.yml
+++ b/.github/workflows/stable-release.yml
@@ -12,17 +12,17 @@ on:
description: 'CUDA version'
required: true
type: string
- default: "128"
+ default: "129"
python_minor:
description: 'Python minor version'
required: true
type: string
- default: "12"
+ default: "13"
python_patch:
description: 'Python patch version'
required: true
type: string
- default: "10"
+ default: "6"
jobs:
@@ -66,8 +66,13 @@ jobs:
curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py
./python.exe get-pip.py
./python.exe -s -m pip install ../cu${{ inputs.cu }}_python_deps/*
- sed -i '1i../ComfyUI' ./python3${{ inputs.python_minor }}._pth
- cd ..
+ sed -i '1i../ComfyUI' ./python3${{ inputs.python_minor }}._pth
+
+ rm ./Lib/site-packages/torch/lib/dnnl.lib #I don't think this is actually used and I need the space
+ rm ./Lib/site-packages/torch/lib/libprotoc.lib
+ rm ./Lib/site-packages/torch/lib/libprotobuf.lib
+
+ cd ..
git clone --depth 1 https://github.com/comfyanonymous/taesd
cp taesd/*.safetensors ./ComfyUI_copy/models/vae_approx/
@@ -85,7 +90,7 @@ jobs:
cd ..
- "C:\Program Files\7-Zip\7z.exe" a -t7z -m0=lzma2 -mx=9 -mfb=128 -md=512m -ms=on -mf=BCJ2 ComfyUI_windows_portable.7z ComfyUI_windows_portable
+ "C:\Program Files\7-Zip\7z.exe" a -t7z -m0=lzma2 -mx=9 -mfb=128 -md=768m -ms=on -mf=BCJ2 ComfyUI_windows_portable.7z ComfyUI_windows_portable
mv ComfyUI_windows_portable.7z ComfyUI/ComfyUI_windows_portable_nvidia.7z
cd ComfyUI_windows_portable
diff --git a/.github/workflows/windows_release_dependencies.yml b/.github/workflows/windows_release_dependencies.yml
index dfdb96d50..7761cc1ed 100644
--- a/.github/workflows/windows_release_dependencies.yml
+++ b/.github/workflows/windows_release_dependencies.yml
@@ -17,19 +17,19 @@ on:
description: 'cuda version'
required: true
type: string
- default: "128"
+ default: "129"
python_minor:
description: 'python minor version'
required: true
type: string
- default: "12"
+ default: "13"
python_patch:
description: 'python patch version'
required: true
type: string
- default: "10"
+ default: "6"
# push:
# branches:
# - master
diff --git a/.github/workflows/windows_release_package.yml b/.github/workflows/windows_release_package.yml
index 3926a65f3..46375698e 100644
--- a/.github/workflows/windows_release_package.yml
+++ b/.github/workflows/windows_release_package.yml
@@ -7,19 +7,19 @@ on:
description: 'cuda version'
required: true
type: string
- default: "128"
+ default: "129"
python_minor:
description: 'python minor version'
required: true
type: string
- default: "12"
+ default: "13"
python_patch:
description: 'python patch version'
required: true
type: string
- default: "10"
+ default: "6"
# push:
# branches:
# - master
@@ -64,6 +64,10 @@ jobs:
./python.exe get-pip.py
./python.exe -s -m pip install ../cu${{ inputs.cu }}_python_deps/*
sed -i '1i../ComfyUI' ./python3${{ inputs.python_minor }}._pth
+
+ rm ./Lib/site-packages/torch/lib/dnnl.lib #I don't think this is actually used and I need the space
+ rm ./Lib/site-packages/torch/lib/libprotoc.lib
+ rm ./Lib/site-packages/torch/lib/libprotobuf.lib
cd ..
git clone --depth 1 https://github.com/comfyanonymous/taesd
@@ -82,7 +86,7 @@ jobs:
cd ..
- "C:\Program Files\7-Zip\7z.exe" a -t7z -m0=lzma2 -mx=9 -mfb=128 -md=512m -ms=on -mf=BCJ2 ComfyUI_windows_portable.7z ComfyUI_windows_portable
+ "C:\Program Files\7-Zip\7z.exe" a -t7z -m0=lzma2 -mx=9 -mfb=128 -md=768m -ms=on -mf=BCJ2 ComfyUI_windows_portable.7z ComfyUI_windows_portable
mv ComfyUI_windows_portable.7z ComfyUI/new_ComfyUI_windows_portable_nvidia_cu${{ inputs.cu }}_or_cpu.7z
cd ComfyUI_windows_portable
diff --git a/CODEOWNERS b/CODEOWNERS
index c4acbf06e..c8acd66d5 100644
--- a/CODEOWNERS
+++ b/CODEOWNERS
@@ -5,20 +5,21 @@
# Inlined the team members for now.
# Maintainers
-*.md @yoland68 @robinjhuang @webfiltered @pythongosssss @ltdrdata @Kosinkadink @christian-byrne
-/tests/ @yoland68 @robinjhuang @webfiltered @pythongosssss @ltdrdata @Kosinkadink @christian-byrne
-/tests-unit/ @yoland68 @robinjhuang @webfiltered @pythongosssss @ltdrdata @Kosinkadink @christian-byrne
-/notebooks/ @yoland68 @robinjhuang @webfiltered @pythongosssss @ltdrdata @Kosinkadink @christian-byrne
-/script_examples/ @yoland68 @robinjhuang @webfiltered @pythongosssss @ltdrdata @Kosinkadink @christian-byrne
-/.github/ @yoland68 @robinjhuang @webfiltered @pythongosssss @ltdrdata @Kosinkadink @christian-byrne
-/requirements.txt @yoland68 @robinjhuang @webfiltered @pythongosssss @ltdrdata @Kosinkadink @christian-byrne
-/pyproject.toml @yoland68 @robinjhuang @webfiltered @pythongosssss @ltdrdata @Kosinkadink @christian-byrne
+*.md @yoland68 @robinjhuang @webfiltered @pythongosssss @ltdrdata @Kosinkadink @christian-byrne @guill
+/tests/ @yoland68 @robinjhuang @webfiltered @pythongosssss @ltdrdata @Kosinkadink @christian-byrne @guill
+/tests-unit/ @yoland68 @robinjhuang @webfiltered @pythongosssss @ltdrdata @Kosinkadink @christian-byrne @guill
+/notebooks/ @yoland68 @robinjhuang @webfiltered @pythongosssss @ltdrdata @Kosinkadink @christian-byrne @guill
+/script_examples/ @yoland68 @robinjhuang @webfiltered @pythongosssss @ltdrdata @Kosinkadink @christian-byrne @guill
+/.github/ @yoland68 @robinjhuang @webfiltered @pythongosssss @ltdrdata @Kosinkadink @christian-byrne @guill
+/requirements.txt @yoland68 @robinjhuang @webfiltered @pythongosssss @ltdrdata @Kosinkadink @christian-byrne @guill
+/pyproject.toml @yoland68 @robinjhuang @webfiltered @pythongosssss @ltdrdata @Kosinkadink @christian-byrne @guill
# Python web server
-/api_server/ @yoland68 @robinjhuang @webfiltered @pythongosssss @ltdrdata @christian-byrne
-/app/ @yoland68 @robinjhuang @webfiltered @pythongosssss @ltdrdata @christian-byrne
-/utils/ @yoland68 @robinjhuang @webfiltered @pythongosssss @ltdrdata @christian-byrne
+/api_server/ @yoland68 @robinjhuang @webfiltered @pythongosssss @ltdrdata @christian-byrne @guill
+/app/ @yoland68 @robinjhuang @webfiltered @pythongosssss @ltdrdata @christian-byrne @guill
+/utils/ @yoland68 @robinjhuang @webfiltered @pythongosssss @ltdrdata @christian-byrne @guill
# Node developers
-/comfy_extras/ @yoland68 @robinjhuang @pythongosssss @ltdrdata @Kosinkadink @webfiltered @christian-byrne
-/comfy/comfy_types/ @yoland68 @robinjhuang @pythongosssss @ltdrdata @Kosinkadink @webfiltered @christian-byrne
+/comfy_extras/ @yoland68 @robinjhuang @pythongosssss @ltdrdata @Kosinkadink @webfiltered @christian-byrne @guill
+/comfy/comfy_types/ @yoland68 @robinjhuang @pythongosssss @ltdrdata @Kosinkadink @webfiltered @christian-byrne @guill
+/comfy_api_nodes/ @yoland68 @robinjhuang @pythongosssss @ltdrdata @Kosinkadink @webfiltered @christian-byrne @guill
diff --git a/README.md b/README.md
index befc4c006..8024870c2 100644
--- a/README.md
+++ b/README.md
@@ -39,7 +39,7 @@ ComfyUI lets you design and execute advanced stable diffusion pipelines using a
## Get Started
#### [Desktop Application](https://www.comfy.org/download)
-- The easiest way to get started.
+- The easiest way to get started.
- Available on Windows & macOS.
#### [Windows Portable Package](#installing)
@@ -65,17 +65,17 @@ See what ComfyUI can do with the [example workflows](https://comfyanonymous.gith
- [Flux](https://comfyanonymous.github.io/ComfyUI_examples/flux/)
- [Lumina Image 2.0](https://comfyanonymous.github.io/ComfyUI_examples/lumina2/)
- [HiDream](https://comfyanonymous.github.io/ComfyUI_examples/hidream/)
- - [Cosmos Predict2](https://comfyanonymous.github.io/ComfyUI_examples/cosmos_predict2/)
+ - [Qwen Image](https://comfyanonymous.github.io/ComfyUI_examples/qwen_image/)
- Image Editing Models
- [Omnigen 2](https://comfyanonymous.github.io/ComfyUI_examples/omnigen/)
- [Flux Kontext](https://comfyanonymous.github.io/ComfyUI_examples/flux/#flux-kontext-image-editing-model)
- [HiDream E1.1](https://comfyanonymous.github.io/ComfyUI_examples/hidream/#hidream-e11)
+ - [Qwen Image Edit](https://comfyanonymous.github.io/ComfyUI_examples/qwen_image/#edit-model)
- Video Models
- [Stable Video Diffusion](https://comfyanonymous.github.io/ComfyUI_examples/video/)
- [Mochi](https://comfyanonymous.github.io/ComfyUI_examples/mochi/)
- [LTX-Video](https://comfyanonymous.github.io/ComfyUI_examples/ltxv/)
- [Hunyuan Video](https://comfyanonymous.github.io/ComfyUI_examples/hunyuan_video/)
- - [Nvidia Cosmos](https://comfyanonymous.github.io/ComfyUI_examples/cosmos/) and [Cosmos Predict2](https://comfyanonymous.github.io/ComfyUI_examples/cosmos_predict2/)
- [Wan 2.1](https://comfyanonymous.github.io/ComfyUI_examples/wan/)
- [Wan 2.2](https://comfyanonymous.github.io/ComfyUI_examples/wan22/)
- Audio Models
@@ -111,7 +111,7 @@ Workflow examples can be found on the [Examples page](https://comfyanonymous.git
## Release Process
-ComfyUI follows a weekly release cycle every Friday, with three interconnected repositories:
+ComfyUI follows a weekly release cycle targeting Friday but this regularly changes because of model releases or large changes to the codebase. There are three interconnected repositories:
1. **[ComfyUI Core](https://github.com/comfyanonymous/ComfyUI)**
- Releases a new stable version (e.g., v0.7.0)
@@ -190,7 +190,7 @@ comfy install
## Manual Install (Windows, Linux)
-python 3.13 is supported but using 3.12 is recommended because some custom nodes and their dependencies might not support it yet.
+Python 3.13 is very well supported. If you have trouble with some custom node dependencies you can try 3.12
Git clone this repo.
@@ -202,7 +202,7 @@ Put your VAE in: models/vae
### AMD GPUs (Linux only)
AMD users can install rocm and pytorch with pip if you don't have it already installed, this is the command to install the stable version:
-```pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/rocm6.3```
+```pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/rocm6.4```
This is the command to install the nightly with ROCm 6.4 which might have some performance improvements:
@@ -210,33 +210,25 @@ This is the command to install the nightly with ROCm 6.4 which might have some p
### Intel GPUs (Windows and Linux)
-(Option 1) Intel Arc GPU users can install native PyTorch with torch.xpu support using pip (currently available in PyTorch nightly builds). More information can be found [here](https://pytorch.org/docs/main/notes/get_start_xpu.html)
-
-1. To install PyTorch nightly, use the following command:
+(Option 1) Intel Arc GPU users can install native PyTorch with torch.xpu support using pip. More information can be found [here](https://pytorch.org/docs/main/notes/get_start_xpu.html)
+
+1. To install PyTorch xpu, use the following command:
+
+```pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/xpu```
+
+This is the command to install the Pytorch xpu nightly which might have some performance improvements:
```pip install --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/xpu```
-2. Launch ComfyUI by running `python main.py`
-
-
(Option 2) Alternatively, Intel GPUs supported by Intel Extension for PyTorch (IPEX) can leverage IPEX for improved performance.
-1. For Intel® Arc™ A-Series Graphics utilizing IPEX, create a conda environment and use the commands below:
-
-```
-conda install libuv
-pip install torch==2.3.1.post0+cxx11.abi torchvision==0.18.1.post0+cxx11.abi torchaudio==2.3.1.post0+cxx11.abi intel-extension-for-pytorch==2.3.110.post0+xpu --extra-index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/us/ --extra-index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/cn/
-```
-
-For other supported Intel GPUs with IPEX, visit [Installation](https://intel.github.io/intel-extension-for-pytorch/index.html#installation?platform=gpu) for more information.
-
-Additional discussion and help can be found [here](https://github.com/comfyanonymous/ComfyUI/discussions/476).
+1. visit [Installation](https://intel.github.io/intel-extension-for-pytorch/index.html#installation?platform=gpu) for more information.
### NVIDIA
Nvidia users should install stable pytorch using this command:
-```pip install torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cu128```
+```pip install torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cu129```
This is the command to install pytorch nightly instead which might have performance improvements.
@@ -351,7 +343,7 @@ Generate a self-signed certificate (not appropriate for shared/production use) a
Use `--tls-keyfile key.pem --tls-certfile cert.pem` to enable TLS/SSL, the app will now be accessible with `https://...` instead of `http://...`.
-> Note: Windows users can use [alexisrolland/docker-openssl](https://github.com/alexisrolland/docker-openssl) or one of the [3rd party binary distributions](https://wiki.openssl.org/index.php/Binaries) to run the command example above.
+> Note: Windows users can use [alexisrolland/docker-openssl](https://github.com/alexisrolland/docker-openssl) or one of the [3rd party binary distributions](https://wiki.openssl.org/index.php/Binaries) to run the command example above.
If you use a container, note that the volume mount `-v` can be a relative path so `... -v ".\:/openssl-certs" ...` would create the key & cert files in the current directory of your command prompt or powershell terminal.
## Support and dev channel
diff --git a/app/model_manager.py b/app/model_manager.py
index 74d942fb8..ab36bca74 100644
--- a/app/model_manager.py
+++ b/app/model_manager.py
@@ -130,10 +130,21 @@ class ModelFileManager:
for file_name in filenames:
try:
- relative_path = os.path.relpath(os.path.join(dirpath, file_name), directory)
- result.append(relative_path)
- except:
- logging.warning(f"Warning: Unable to access {file_name}. Skipping this file.")
+ full_path = os.path.join(dirpath, file_name)
+ relative_path = os.path.relpath(full_path, directory)
+
+ # Get file metadata
+ file_info = {
+ "name": relative_path,
+ "pathIndex": pathIndex,
+ "modified": os.path.getmtime(full_path), # Add modification time
+ "created": os.path.getctime(full_path), # Add creation time
+ "size": os.path.getsize(full_path) # Add file size
+ }
+ result.append(file_info)
+
+ except Exception as e:
+ logging.warning(f"Warning: Unable to access {file_name}. Error: {e}. Skipping this file.")
continue
for d in subdirs:
@@ -144,7 +155,7 @@ class ModelFileManager:
logging.warning(f"Warning: Unable to access {path}. Skipping this path.")
continue
- return [{"name": f, "pathIndex": pathIndex} for f in result], dirs, time.perf_counter()
+ return result, dirs, time.perf_counter()
def get_model_previews(self, filepath: str) -> list[str | BytesIO]:
dirname = os.path.dirname(filepath)
diff --git a/app/user_manager.py b/app/user_manager.py
index d31da5b9b..a2d376c0c 100644
--- a/app/user_manager.py
+++ b/app/user_manager.py
@@ -20,13 +20,15 @@ class FileInfo(TypedDict):
path: str
size: int
modified: int
+ created: int
def get_file_info(path: str, relative_to: str) -> FileInfo:
return {
"path": os.path.relpath(path, relative_to).replace(os.sep, '/'),
"size": os.path.getsize(path),
- "modified": os.path.getmtime(path)
+ "modified": os.path.getmtime(path),
+ "created": os.path.getctime(path)
}
@@ -361,10 +363,17 @@ class UserManager():
if not overwrite and os.path.exists(path):
return web.Response(status=409, text="File already exists")
- body = await request.read()
+ try:
+ body = await request.read()
- with open(path, "wb") as f:
- f.write(body)
+ with open(path, "wb") as f:
+ f.write(body)
+ except OSError as e:
+ logging.warning(f"Error saving file '{path}': {e}")
+ return web.Response(
+ status=400,
+ reason="Invalid filename. Please avoid special characters like :\\/*?\"<>|"
+ )
user_path = self.get_request_user_filepath(request, None)
if full_info:
diff --git a/comfy/audio_encoders/audio_encoders.py b/comfy/audio_encoders/audio_encoders.py
new file mode 100644
index 000000000..538c21bd5
--- /dev/null
+++ b/comfy/audio_encoders/audio_encoders.py
@@ -0,0 +1,42 @@
+from .wav2vec2 import Wav2Vec2Model
+import comfy.model_management
+import comfy.ops
+import comfy.utils
+import logging
+import torchaudio
+
+
+class AudioEncoderModel():
+ def __init__(self, config):
+ self.load_device = comfy.model_management.text_encoder_device()
+ offload_device = comfy.model_management.text_encoder_offload_device()
+ self.dtype = comfy.model_management.text_encoder_dtype(self.load_device)
+ self.model = Wav2Vec2Model(dtype=self.dtype, device=offload_device, operations=comfy.ops.manual_cast)
+ self.model.eval()
+ self.patcher = comfy.model_patcher.ModelPatcher(self.model, load_device=self.load_device, offload_device=offload_device)
+ self.model_sample_rate = 16000
+
+ def load_sd(self, sd):
+ return self.model.load_state_dict(sd, strict=False)
+
+ def get_sd(self):
+ return self.model.state_dict()
+
+ def encode_audio(self, audio, sample_rate):
+ comfy.model_management.load_model_gpu(self.patcher)
+ audio = torchaudio.functional.resample(audio, sample_rate, self.model_sample_rate)
+ out, all_layers = self.model(audio.to(self.load_device))
+ outputs = {}
+ outputs["encoded_audio"] = out
+ outputs["encoded_audio_all_layers"] = all_layers
+ return outputs
+
+
+def load_audio_encoder_from_sd(sd, prefix=""):
+ audio_encoder = AudioEncoderModel(None)
+ sd = comfy.utils.state_dict_prefix_replace(sd, {"wav2vec2.": ""})
+ m, u = audio_encoder.load_sd(sd)
+ if len(m) > 0:
+ logging.warning("missing audio encoder: {}".format(m))
+
+ return audio_encoder
diff --git a/comfy/audio_encoders/wav2vec2.py b/comfy/audio_encoders/wav2vec2.py
new file mode 100644
index 000000000..de906622a
--- /dev/null
+++ b/comfy/audio_encoders/wav2vec2.py
@@ -0,0 +1,207 @@
+import torch
+import torch.nn as nn
+from comfy.ldm.modules.attention import optimized_attention_masked
+
+
+class LayerNormConv(nn.Module):
+ def __init__(self, in_channels, out_channels, kernel_size, stride, bias=False, dtype=None, device=None, operations=None):
+ super().__init__()
+ self.conv = operations.Conv1d(in_channels, out_channels, kernel_size=kernel_size, stride=stride, bias=bias, device=device, dtype=dtype)
+ self.layer_norm = operations.LayerNorm(out_channels, elementwise_affine=True, device=device, dtype=dtype)
+
+ def forward(self, x):
+ x = self.conv(x)
+ return torch.nn.functional.gelu(self.layer_norm(x.transpose(-2, -1)).transpose(-2, -1))
+
+
+class ConvFeatureEncoder(nn.Module):
+ def __init__(self, conv_dim, dtype=None, device=None, operations=None):
+ super().__init__()
+ self.conv_layers = nn.ModuleList([
+ LayerNormConv(1, conv_dim, kernel_size=10, stride=5, bias=True, device=device, dtype=dtype, operations=operations),
+ LayerNormConv(conv_dim, conv_dim, kernel_size=3, stride=2, bias=True, device=device, dtype=dtype, operations=operations),
+ LayerNormConv(conv_dim, conv_dim, kernel_size=3, stride=2, bias=True, device=device, dtype=dtype, operations=operations),
+ LayerNormConv(conv_dim, conv_dim, kernel_size=3, stride=2, bias=True, device=device, dtype=dtype, operations=operations),
+ LayerNormConv(conv_dim, conv_dim, kernel_size=3, stride=2, bias=True, device=device, dtype=dtype, operations=operations),
+ LayerNormConv(conv_dim, conv_dim, kernel_size=2, stride=2, bias=True, device=device, dtype=dtype, operations=operations),
+ LayerNormConv(conv_dim, conv_dim, kernel_size=2, stride=2, bias=True, device=device, dtype=dtype, operations=operations),
+ ])
+
+ def forward(self, x):
+ x = x.unsqueeze(1)
+
+ for conv in self.conv_layers:
+ x = conv(x)
+
+ return x.transpose(1, 2)
+
+
+class FeatureProjection(nn.Module):
+ def __init__(self, conv_dim, embed_dim, dtype=None, device=None, operations=None):
+ super().__init__()
+ self.layer_norm = operations.LayerNorm(conv_dim, eps=1e-05, device=device, dtype=dtype)
+ self.projection = operations.Linear(conv_dim, embed_dim, device=device, dtype=dtype)
+
+ def forward(self, x):
+ x = self.layer_norm(x)
+ x = self.projection(x)
+ return x
+
+
+class PositionalConvEmbedding(nn.Module):
+ def __init__(self, embed_dim=768, kernel_size=128, groups=16):
+ super().__init__()
+ self.conv = nn.Conv1d(
+ embed_dim,
+ embed_dim,
+ kernel_size=kernel_size,
+ padding=kernel_size // 2,
+ groups=groups,
+ )
+ self.conv = torch.nn.utils.parametrizations.weight_norm(self.conv, name="weight", dim=2)
+ self.activation = nn.GELU()
+
+ def forward(self, x):
+ x = x.transpose(1, 2)
+ x = self.conv(x)[:, :, :-1]
+ x = self.activation(x)
+ x = x.transpose(1, 2)
+ return x
+
+
+class TransformerEncoder(nn.Module):
+ def __init__(
+ self,
+ embed_dim=768,
+ num_heads=12,
+ num_layers=12,
+ mlp_ratio=4.0,
+ dtype=None, device=None, operations=None
+ ):
+ super().__init__()
+
+ self.pos_conv_embed = PositionalConvEmbedding(embed_dim=embed_dim)
+ self.layers = nn.ModuleList([
+ TransformerEncoderLayer(
+ embed_dim=embed_dim,
+ num_heads=num_heads,
+ mlp_ratio=mlp_ratio,
+ device=device, dtype=dtype, operations=operations
+ )
+ for _ in range(num_layers)
+ ])
+
+ self.layer_norm = operations.LayerNorm(embed_dim, eps=1e-05, device=device, dtype=dtype)
+
+ def forward(self, x, mask=None):
+ x = x + self.pos_conv_embed(x)
+ all_x = ()
+ for layer in self.layers:
+ all_x += (x,)
+ x = layer(x, mask)
+ x = self.layer_norm(x)
+ all_x += (x,)
+ return x, all_x
+
+
+class Attention(nn.Module):
+ def __init__(self, embed_dim, num_heads, bias=True, dtype=None, device=None, operations=None):
+ super().__init__()
+ self.embed_dim = embed_dim
+ self.num_heads = num_heads
+ self.head_dim = embed_dim // num_heads
+
+ self.k_proj = operations.Linear(embed_dim, embed_dim, bias=bias, device=device, dtype=dtype)
+ self.v_proj = operations.Linear(embed_dim, embed_dim, bias=bias, device=device, dtype=dtype)
+ self.q_proj = operations.Linear(embed_dim, embed_dim, bias=bias, device=device, dtype=dtype)
+ self.out_proj = operations.Linear(embed_dim, embed_dim, bias=bias, device=device, dtype=dtype)
+
+ def forward(self, x, mask=None):
+ assert (mask is None) # TODO?
+ q = self.q_proj(x)
+ k = self.k_proj(x)
+ v = self.v_proj(x)
+
+ out = optimized_attention_masked(q, k, v, self.num_heads)
+ return self.out_proj(out)
+
+
+class FeedForward(nn.Module):
+ def __init__(self, embed_dim, mlp_ratio, dtype=None, device=None, operations=None):
+ super().__init__()
+ self.intermediate_dense = operations.Linear(embed_dim, int(embed_dim * mlp_ratio), device=device, dtype=dtype)
+ self.output_dense = operations.Linear(int(embed_dim * mlp_ratio), embed_dim, device=device, dtype=dtype)
+
+ def forward(self, x):
+ x = self.intermediate_dense(x)
+ x = torch.nn.functional.gelu(x)
+ x = self.output_dense(x)
+ return x
+
+
+class TransformerEncoderLayer(nn.Module):
+ def __init__(
+ self,
+ embed_dim=768,
+ num_heads=12,
+ mlp_ratio=4.0,
+ dtype=None, device=None, operations=None
+ ):
+ super().__init__()
+
+ self.attention = Attention(embed_dim, num_heads, device=device, dtype=dtype, operations=operations)
+
+ self.layer_norm = operations.LayerNorm(embed_dim, device=device, dtype=dtype)
+ self.feed_forward = FeedForward(embed_dim, mlp_ratio, device=device, dtype=dtype, operations=operations)
+ self.final_layer_norm = operations.LayerNorm(embed_dim, device=device, dtype=dtype)
+
+ def forward(self, x, mask=None):
+ residual = x
+ x = self.layer_norm(x)
+ x = self.attention(x, mask=mask)
+ x = residual + x
+
+ x = x + self.feed_forward(self.final_layer_norm(x))
+ return x
+
+
+class Wav2Vec2Model(nn.Module):
+ """Complete Wav2Vec 2.0 model."""
+
+ def __init__(
+ self,
+ embed_dim=1024,
+ final_dim=256,
+ num_heads=16,
+ num_layers=24,
+ dtype=None, device=None, operations=None
+ ):
+ super().__init__()
+
+ conv_dim = 512
+ self.feature_extractor = ConvFeatureEncoder(conv_dim, device=device, dtype=dtype, operations=operations)
+ self.feature_projection = FeatureProjection(conv_dim, embed_dim, device=device, dtype=dtype, operations=operations)
+
+ self.masked_spec_embed = nn.Parameter(torch.empty(embed_dim, device=device, dtype=dtype))
+
+ self.encoder = TransformerEncoder(
+ embed_dim=embed_dim,
+ num_heads=num_heads,
+ num_layers=num_layers,
+ device=device, dtype=dtype, operations=operations
+ )
+
+ def forward(self, x, mask_time_indices=None, return_dict=False):
+
+ x = torch.mean(x, dim=1)
+
+ x = (x - x.mean()) / torch.sqrt(x.var() + 1e-7)
+
+ features = self.feature_extractor(x)
+ features = self.feature_projection(features)
+
+ batch_size, seq_len, _ = features.shape
+
+ x, all_x = self.encoder(features)
+
+ return x, all_x
diff --git a/comfy/cli_args.py b/comfy/cli_args.py
index 0d760d524..de3e85c08 100644
--- a/comfy/cli_args.py
+++ b/comfy/cli_args.py
@@ -132,6 +132,8 @@ parser.add_argument("--reserve-vram", type=float, default=None, help="Set the am
parser.add_argument("--async-offload", action="store_true", help="Use async weight offloading.")
+parser.add_argument("--force-non-blocking", action="store_true", help="Force ComfyUI to use non-blocking operations for all applicable tensors. This may improve performance on some non-Nvidia systems but can cause issues with some workflows.")
+
parser.add_argument("--default-hashing-function", type=str, choices=['md5', 'sha1', 'sha256', 'sha512'], default='sha256', help="Allows you to choose the hash function to use for duplicate filename / contents comparison. Default is sha256.")
parser.add_argument("--disable-smart-memory", action="store_true", help="Force ComfyUI to agressively offload to regular ram instead of keeping models in vram when it can.")
diff --git a/comfy/clip_model.py b/comfy/clip_model.py
index c8294d483..7e47d8a55 100644
--- a/comfy/clip_model.py
+++ b/comfy/clip_model.py
@@ -97,7 +97,7 @@ class CLIPTextModel_(torch.nn.Module):
self.encoder = CLIPEncoder(num_layers, embed_dim, heads, intermediate_size, intermediate_activation, dtype, device, operations)
self.final_layer_norm = operations.LayerNorm(embed_dim, dtype=dtype, device=device)
- def forward(self, input_tokens=None, attention_mask=None, embeds=None, num_tokens=None, intermediate_output=None, final_layer_norm_intermediate=True, dtype=torch.float32):
+ def forward(self, input_tokens=None, attention_mask=None, embeds=None, num_tokens=None, intermediate_output=None, final_layer_norm_intermediate=True, dtype=torch.float32, embeds_info=[]):
if embeds is not None:
x = embeds + comfy.ops.cast_to(self.embeddings.position_embedding.weight, dtype=dtype, device=embeds.device)
else:
diff --git a/comfy/conds.py b/comfy/conds.py
index 2af2a43a3..5af3e93ea 100644
--- a/comfy/conds.py
+++ b/comfy/conds.py
@@ -1,6 +1,7 @@
import torch
import math
import comfy.utils
+import logging
class CONDRegular:
@@ -10,12 +11,15 @@ class CONDRegular:
def _copy_with(self, cond):
return self.__class__(cond)
- def process_cond(self, batch_size, device, **kwargs):
- return self._copy_with(comfy.utils.repeat_to_batch_size(self.cond, batch_size).to(device))
+ def process_cond(self, batch_size, **kwargs):
+ return self._copy_with(comfy.utils.repeat_to_batch_size(self.cond, batch_size))
def can_concat(self, other):
if self.cond.shape != other.cond.shape:
return False
+ if self.cond.device != other.cond.device:
+ logging.warning("WARNING: conds not on same device, skipping concat.")
+ return False
return True
def concat(self, others):
@@ -29,14 +33,14 @@ class CONDRegular:
class CONDNoiseShape(CONDRegular):
- def process_cond(self, batch_size, device, area, **kwargs):
+ def process_cond(self, batch_size, area, **kwargs):
data = self.cond
if area is not None:
dims = len(area) // 2
for i in range(dims):
data = data.narrow(i + 2, area[i + dims], area[i])
- return self._copy_with(comfy.utils.repeat_to_batch_size(data, batch_size).to(device))
+ return self._copy_with(comfy.utils.repeat_to_batch_size(data, batch_size))
class CONDCrossAttn(CONDRegular):
@@ -51,6 +55,9 @@ class CONDCrossAttn(CONDRegular):
diff = mult_min // min(s1[1], s2[1])
if diff > 4: #arbitrary limit on the padding because it's probably going to impact performance negatively if it's too much
return False
+ if self.cond.device != other.cond.device:
+ logging.warning("WARNING: conds not on same device: skipping concat.")
+ return False
return True
def concat(self, others):
@@ -73,7 +80,7 @@ class CONDConstant(CONDRegular):
def __init__(self, cond):
self.cond = cond
- def process_cond(self, batch_size, device, **kwargs):
+ def process_cond(self, batch_size, **kwargs):
return self._copy_with(self.cond)
def can_concat(self, other):
@@ -92,10 +99,10 @@ class CONDList(CONDRegular):
def __init__(self, cond):
self.cond = cond
- def process_cond(self, batch_size, device, **kwargs):
+ def process_cond(self, batch_size, **kwargs):
out = []
for c in self.cond:
- out.append(comfy.utils.repeat_to_batch_size(c, batch_size).to(device))
+ out.append(comfy.utils.repeat_to_batch_size(c, batch_size))
return self._copy_with(out)
diff --git a/comfy/context_windows.py b/comfy/context_windows.py
new file mode 100644
index 000000000..041f380f9
--- /dev/null
+++ b/comfy/context_windows.py
@@ -0,0 +1,540 @@
+from __future__ import annotations
+from typing import TYPE_CHECKING, Callable
+import torch
+import numpy as np
+import collections
+from dataclasses import dataclass
+from abc import ABC, abstractmethod
+import logging
+import comfy.model_management
+import comfy.patcher_extension
+if TYPE_CHECKING:
+ from comfy.model_base import BaseModel
+ from comfy.model_patcher import ModelPatcher
+ from comfy.controlnet import ControlBase
+
+
+class ContextWindowABC(ABC):
+ def __init__(self):
+ ...
+
+ @abstractmethod
+ def get_tensor(self, full: torch.Tensor) -> torch.Tensor:
+ """
+ Get torch.Tensor applicable to current window.
+ """
+ raise NotImplementedError("Not implemented.")
+
+ @abstractmethod
+ def add_window(self, full: torch.Tensor, to_add: torch.Tensor) -> torch.Tensor:
+ """
+ Apply torch.Tensor of window to the full tensor, in place. Returns reference to updated full tensor, not a copy.
+ """
+ raise NotImplementedError("Not implemented.")
+
+class ContextHandlerABC(ABC):
+ def __init__(self):
+ ...
+
+ @abstractmethod
+ def should_use_context(self, model: BaseModel, conds: list[list[dict]], x_in: torch.Tensor, timestep: torch.Tensor, model_options: dict[str]) -> bool:
+ raise NotImplementedError("Not implemented.")
+
+ @abstractmethod
+ def get_resized_cond(self, cond_in: list[dict], x_in: torch.Tensor, window: ContextWindowABC, device=None) -> list:
+ raise NotImplementedError("Not implemented.")
+
+ @abstractmethod
+ def execute(self, calc_cond_batch: Callable, model: BaseModel, conds: list[list[dict]], x_in: torch.Tensor, timestep: torch.Tensor, model_options: dict[str]):
+ raise NotImplementedError("Not implemented.")
+
+
+
+class IndexListContextWindow(ContextWindowABC):
+ def __init__(self, index_list: list[int], dim: int=0):
+ self.index_list = index_list
+ self.context_length = len(index_list)
+ self.dim = dim
+
+ def get_tensor(self, full: torch.Tensor, device=None, dim=None) -> torch.Tensor:
+ if dim is None:
+ dim = self.dim
+ if dim == 0 and full.shape[dim] == 1:
+ return full
+ idx = [slice(None)] * dim + [self.index_list]
+ return full[idx].to(device)
+
+ def add_window(self, full: torch.Tensor, to_add: torch.Tensor, dim=None) -> torch.Tensor:
+ if dim is None:
+ dim = self.dim
+ idx = [slice(None)] * dim + [self.index_list]
+ full[idx] += to_add
+ return full
+
+
+class IndexListCallbacks:
+ EVALUATE_CONTEXT_WINDOWS = "evaluate_context_windows"
+ COMBINE_CONTEXT_WINDOW_RESULTS = "combine_context_window_results"
+ EXECUTE_START = "execute_start"
+ EXECUTE_CLEANUP = "execute_cleanup"
+
+ def init_callbacks(self):
+ return {}
+
+
+@dataclass
+class ContextSchedule:
+ name: str
+ func: Callable
+
+@dataclass
+class ContextFuseMethod:
+ name: str
+ func: Callable
+
+ContextResults = collections.namedtuple("ContextResults", ['window_idx', 'sub_conds_out', 'sub_conds', 'window'])
+class IndexListContextHandler(ContextHandlerABC):
+ def __init__(self, context_schedule: ContextSchedule, fuse_method: ContextFuseMethod, context_length: int=1, context_overlap: int=0, context_stride: int=1, closed_loop=False, dim=0):
+ self.context_schedule = context_schedule
+ self.fuse_method = fuse_method
+ self.context_length = context_length
+ self.context_overlap = context_overlap
+ self.context_stride = context_stride
+ self.closed_loop = closed_loop
+ self.dim = dim
+ self._step = 0
+
+ self.callbacks = {}
+
+ def should_use_context(self, model: BaseModel, conds: list[list[dict]], x_in: torch.Tensor, timestep: torch.Tensor, model_options: dict[str]) -> bool:
+ # for now, assume first dim is batch - should have stored on BaseModel in actual implementation
+ if x_in.size(self.dim) > self.context_length:
+ logging.info(f"Using context windows {self.context_length} for {x_in.size(self.dim)} frames.")
+ return True
+ return False
+
+ def prepare_control_objects(self, control: ControlBase, device=None) -> ControlBase:
+ if control.previous_controlnet is not None:
+ self.prepare_control_objects(control.previous_controlnet, device)
+ return control
+
+ def get_resized_cond(self, cond_in: list[dict], x_in: torch.Tensor, window: IndexListContextWindow, device=None) -> list:
+ if cond_in is None:
+ return None
+ # reuse or resize cond items to match context requirements
+ resized_cond = []
+ # cond object is a list containing a dict - outer list is irrelevant, so just loop through it
+ for actual_cond in cond_in:
+ resized_actual_cond = actual_cond.copy()
+ # now we are in the inner dict - "pooled_output" is a tensor, "control" is a ControlBase object, "model_conds" is dictionary
+ for key in actual_cond:
+ try:
+ cond_item = actual_cond[key]
+ if isinstance(cond_item, torch.Tensor):
+ # check that tensor is the expected length - x.size(0)
+ if self.dim < cond_item.ndim and cond_item.size(self.dim) == x_in.size(self.dim):
+ # if so, it's subsetting time - tell controls the expected indeces so they can handle them
+ actual_cond_item = window.get_tensor(cond_item)
+ resized_actual_cond[key] = actual_cond_item.to(device)
+ else:
+ resized_actual_cond[key] = cond_item.to(device)
+ # look for control
+ elif key == "control":
+ resized_actual_cond[key] = self.prepare_control_objects(cond_item, device)
+ elif isinstance(cond_item, dict):
+ new_cond_item = cond_item.copy()
+ # when in dictionary, look for tensors and CONDCrossAttn [comfy/conds.py] (has cond attr that is a tensor)
+ for cond_key, cond_value in new_cond_item.items():
+ if isinstance(cond_value, torch.Tensor):
+ if cond_value.ndim < self.dim and cond_value.size(0) == x_in.size(self.dim):
+ new_cond_item[cond_key] = window.get_tensor(cond_value, device)
+ # if has cond that is a Tensor, check if needs to be subset
+ elif hasattr(cond_value, "cond") and isinstance(cond_value.cond, torch.Tensor):
+ if cond_value.cond.ndim < self.dim and cond_value.cond.size(0) == x_in.size(self.dim):
+ new_cond_item[cond_key] = cond_value._copy_with(window.get_tensor(cond_value.cond, device))
+ elif cond_key == "num_video_frames": # for SVD
+ new_cond_item[cond_key] = cond_value._copy_with(cond_value.cond)
+ new_cond_item[cond_key].cond = window.context_length
+ resized_actual_cond[key] = new_cond_item
+ else:
+ resized_actual_cond[key] = cond_item
+ finally:
+ del cond_item # just in case to prevent VRAM issues
+ resized_cond.append(resized_actual_cond)
+ return resized_cond
+
+ def set_step(self, timestep: torch.Tensor, model_options: dict[str]):
+ mask = torch.isclose(model_options["transformer_options"]["sample_sigmas"], timestep, rtol=0.0001)
+ matches = torch.nonzero(mask)
+ if torch.numel(matches) == 0:
+ raise Exception("No sample_sigmas matched current timestep; something went wrong.")
+ self._step = int(matches[0].item())
+
+ def get_context_windows(self, model: BaseModel, x_in: torch.Tensor, model_options: dict[str]) -> list[IndexListContextWindow]:
+ full_length = x_in.size(self.dim) # TODO: choose dim based on model
+ context_windows = self.context_schedule.func(full_length, self, model_options)
+ context_windows = [IndexListContextWindow(window, dim=self.dim) for window in context_windows]
+ return context_windows
+
+ def execute(self, calc_cond_batch: Callable, model: BaseModel, conds: list[list[dict]], x_in: torch.Tensor, timestep: torch.Tensor, model_options: dict[str]):
+ self.set_step(timestep, model_options)
+ context_windows = self.get_context_windows(model, x_in, model_options)
+ enumerated_context_windows = list(enumerate(context_windows))
+
+ conds_final = [torch.zeros_like(x_in) for _ in conds]
+ if self.fuse_method.name == ContextFuseMethods.RELATIVE:
+ counts_final = [torch.ones(get_shape_for_dim(x_in, self.dim), device=x_in.device) for _ in conds]
+ else:
+ counts_final = [torch.zeros(get_shape_for_dim(x_in, self.dim), device=x_in.device) for _ in conds]
+ biases_final = [([0.0] * x_in.shape[self.dim]) for _ in conds]
+
+ for callback in comfy.patcher_extension.get_all_callbacks(IndexListCallbacks.EXECUTE_START, self.callbacks):
+ callback(self, model, x_in, conds, timestep, model_options)
+
+ for enum_window in enumerated_context_windows:
+ results = self.evaluate_context_windows(calc_cond_batch, model, x_in, conds, timestep, [enum_window], model_options)
+ for result in results:
+ self.combine_context_window_results(x_in, result.sub_conds_out, result.sub_conds, result.window, result.window_idx, len(enumerated_context_windows), timestep,
+ conds_final, counts_final, biases_final)
+ try:
+ # finalize conds
+ if self.fuse_method.name == ContextFuseMethods.RELATIVE:
+ # relative is already normalized, so return as is
+ del counts_final
+ return conds_final
+ else:
+ # normalize conds via division by context usage counts
+ for i in range(len(conds_final)):
+ conds_final[i] /= counts_final[i]
+ del counts_final
+ return conds_final
+ finally:
+ for callback in comfy.patcher_extension.get_all_callbacks(IndexListCallbacks.EXECUTE_CLEANUP, self.callbacks):
+ callback(self, model, x_in, conds, timestep, model_options)
+
+ def evaluate_context_windows(self, calc_cond_batch: Callable, model: BaseModel, x_in: torch.Tensor, conds, timestep: torch.Tensor, enumerated_context_windows: list[tuple[int, IndexListContextWindow]],
+ model_options, device=None, first_device=None):
+ results: list[ContextResults] = []
+ for window_idx, window in enumerated_context_windows:
+ # allow processing to end between context window executions for faster Cancel
+ comfy.model_management.throw_exception_if_processing_interrupted()
+
+ for callback in comfy.patcher_extension.get_all_callbacks(IndexListCallbacks.EVALUATE_CONTEXT_WINDOWS, self.callbacks):
+ callback(self, model, x_in, conds, timestep, model_options, window_idx, window, model_options, device, first_device)
+
+ # update exposed params
+ model_options["transformer_options"]["context_window"] = window
+ # get subsections of x, timestep, conds
+ sub_x = window.get_tensor(x_in, device)
+ sub_timestep = window.get_tensor(timestep, device, dim=0)
+ sub_conds = [self.get_resized_cond(cond, x_in, window, device) for cond in conds]
+
+ sub_conds_out = calc_cond_batch(model, sub_conds, sub_x, sub_timestep, model_options)
+ if device is not None:
+ for i in range(len(sub_conds_out)):
+ sub_conds_out[i] = sub_conds_out[i].to(x_in.device)
+ results.append(ContextResults(window_idx, sub_conds_out, sub_conds, window))
+ return results
+
+
+ def combine_context_window_results(self, x_in: torch.Tensor, sub_conds_out, sub_conds, window: IndexListContextWindow, window_idx: int, total_windows: int, timestep: torch.Tensor,
+ conds_final: list[torch.Tensor], counts_final: list[torch.Tensor], biases_final: list[torch.Tensor]):
+ if self.fuse_method.name == ContextFuseMethods.RELATIVE:
+ for pos, idx in enumerate(window.index_list):
+ # bias is the influence of a specific index in relation to the whole context window
+ bias = 1 - abs(idx - (window.index_list[0] + window.index_list[-1]) / 2) / ((window.index_list[-1] - window.index_list[0] + 1e-2) / 2)
+ bias = max(1e-2, bias)
+ # take weighted average relative to total bias of current idx
+ for i in range(len(sub_conds_out)):
+ bias_total = biases_final[i][idx]
+ prev_weight = (bias_total / (bias_total + bias))
+ new_weight = (bias / (bias_total + bias))
+ # account for dims of tensors
+ idx_window = [slice(None)] * self.dim + [idx]
+ pos_window = [slice(None)] * self.dim + [pos]
+ # apply new values
+ conds_final[i][idx_window] = conds_final[i][idx_window] * prev_weight + sub_conds_out[i][pos_window] * new_weight
+ biases_final[i][idx] = bias_total + bias
+ else:
+ # add conds and counts based on weights of fuse method
+ weights = get_context_weights(window.context_length, x_in.shape[self.dim], window.index_list, self, sigma=timestep)
+ weights_tensor = match_weights_to_dim(weights, x_in, self.dim, device=x_in.device)
+ for i in range(len(sub_conds_out)):
+ window.add_window(conds_final[i], sub_conds_out[i] * weights_tensor)
+ window.add_window(counts_final[i], weights_tensor)
+
+ for callback in comfy.patcher_extension.get_all_callbacks(IndexListCallbacks.COMBINE_CONTEXT_WINDOW_RESULTS, self.callbacks):
+ callback(self, x_in, sub_conds_out, sub_conds, window, window_idx, total_windows, timestep, conds_final, counts_final, biases_final)
+
+
+def _prepare_sampling_wrapper(executor, model, noise_shape: torch.Tensor, *args, **kwargs):
+ # limit noise_shape length to context_length for more accurate vram use estimation
+ model_options = kwargs.get("model_options", None)
+ if model_options is None:
+ raise Exception("model_options not found in prepare_sampling_wrapper; this should never happen, something went wrong.")
+ handler: IndexListContextHandler = model_options.get("context_handler", None)
+ if handler is not None:
+ noise_shape = list(noise_shape)
+ noise_shape[handler.dim] = min(noise_shape[handler.dim], handler.context_length)
+ return executor(model, noise_shape, *args, **kwargs)
+
+
+def create_prepare_sampling_wrapper(model: ModelPatcher):
+ model.add_wrapper_with_key(
+ comfy.patcher_extension.WrappersMP.PREPARE_SAMPLING,
+ "ContextWindows_prepare_sampling",
+ _prepare_sampling_wrapper
+ )
+
+
+def match_weights_to_dim(weights: list[float], x_in: torch.Tensor, dim: int, device=None) -> torch.Tensor:
+ total_dims = len(x_in.shape)
+ weights_tensor = torch.Tensor(weights).to(device=device)
+ for _ in range(dim):
+ weights_tensor = weights_tensor.unsqueeze(0)
+ for _ in range(total_dims - dim - 1):
+ weights_tensor = weights_tensor.unsqueeze(-1)
+ return weights_tensor
+
+def get_shape_for_dim(x_in: torch.Tensor, dim: int) -> list[int]:
+ total_dims = len(x_in.shape)
+ shape = []
+ for _ in range(dim):
+ shape.append(1)
+ shape.append(x_in.shape[dim])
+ for _ in range(total_dims - dim - 1):
+ shape.append(1)
+ return shape
+
+class ContextSchedules:
+ UNIFORM_LOOPED = "looped_uniform"
+ UNIFORM_STANDARD = "standard_uniform"
+ STATIC_STANDARD = "standard_static"
+ BATCHED = "batched"
+
+
+# from https://github.com/neggles/animatediff-cli/blob/main/src/animatediff/pipelines/context.py
+def create_windows_uniform_looped(num_frames: int, handler: IndexListContextHandler, model_options: dict[str]):
+ windows = []
+ if num_frames < handler.context_length:
+ windows.append(list(range(num_frames)))
+ return windows
+
+ context_stride = min(handler.context_stride, int(np.ceil(np.log2(num_frames / handler.context_length))) + 1)
+ # obtain uniform windows as normal, looping and all
+ for context_step in 1 << np.arange(context_stride):
+ pad = int(round(num_frames * ordered_halving(handler._step)))
+ for j in range(
+ int(ordered_halving(handler._step) * context_step) + pad,
+ num_frames + pad + (0 if handler.closed_loop else -handler.context_overlap),
+ (handler.context_length * context_step - handler.context_overlap),
+ ):
+ windows.append([e % num_frames for e in range(j, j + handler.context_length * context_step, context_step)])
+
+ return windows
+
+def create_windows_uniform_standard(num_frames: int, handler: IndexListContextHandler, model_options: dict[str]):
+ # unlike looped, uniform_straight does NOT allow windows that loop back to the beginning;
+ # instead, they get shifted to the corresponding end of the frames.
+ # in the case that a window (shifted or not) is identical to the previous one, it gets skipped.
+ windows = []
+ if num_frames <= handler.context_length:
+ windows.append(list(range(num_frames)))
+ return windows
+
+ context_stride = min(handler.context_stride, int(np.ceil(np.log2(num_frames / handler.context_length))) + 1)
+ # first, obtain uniform windows as normal, looping and all
+ for context_step in 1 << np.arange(context_stride):
+ pad = int(round(num_frames * ordered_halving(handler._step)))
+ for j in range(
+ int(ordered_halving(handler._step) * context_step) + pad,
+ num_frames + pad + (-handler.context_overlap),
+ (handler.context_length * context_step - handler.context_overlap),
+ ):
+ windows.append([e % num_frames for e in range(j, j + handler.context_length * context_step, context_step)])
+
+ # now that windows are created, shift any windows that loop, and delete duplicate windows
+ delete_idxs = []
+ win_i = 0
+ while win_i < len(windows):
+ # if window is rolls over itself, need to shift it
+ is_roll, roll_idx = does_window_roll_over(windows[win_i], num_frames)
+ if is_roll:
+ roll_val = windows[win_i][roll_idx] # roll_val might not be 0 for windows of higher strides
+ shift_window_to_end(windows[win_i], num_frames=num_frames)
+ # check if next window (cyclical) is missing roll_val
+ if roll_val not in windows[(win_i+1) % len(windows)]:
+ # need to insert new window here - just insert window starting at roll_val
+ windows.insert(win_i+1, list(range(roll_val, roll_val + handler.context_length)))
+ # delete window if it's not unique
+ for pre_i in range(0, win_i):
+ if windows[win_i] == windows[pre_i]:
+ delete_idxs.append(win_i)
+ break
+ win_i += 1
+
+ # reverse delete_idxs so that they will be deleted in an order that doesn't break idx correlation
+ delete_idxs.reverse()
+ for i in delete_idxs:
+ windows.pop(i)
+
+ return windows
+
+
+def create_windows_static_standard(num_frames: int, handler: IndexListContextHandler, model_options: dict[str]):
+ windows = []
+ if num_frames <= handler.context_length:
+ windows.append(list(range(num_frames)))
+ return windows
+ # always return the same set of windows
+ delta = handler.context_length - handler.context_overlap
+ for start_idx in range(0, num_frames, delta):
+ # if past the end of frames, move start_idx back to allow same context_length
+ ending = start_idx + handler.context_length
+ if ending >= num_frames:
+ final_delta = ending - num_frames
+ final_start_idx = start_idx - final_delta
+ windows.append(list(range(final_start_idx, final_start_idx + handler.context_length)))
+ break
+ windows.append(list(range(start_idx, start_idx + handler.context_length)))
+ return windows
+
+
+def create_windows_batched(num_frames: int, handler: IndexListContextHandler, model_options: dict[str]):
+ windows = []
+ if num_frames <= handler.context_length:
+ windows.append(list(range(num_frames)))
+ return windows
+ # always return the same set of windows;
+ # no overlap, just cut up based on context_length;
+ # last window size will be different if num_frames % opts.context_length != 0
+ for start_idx in range(0, num_frames, handler.context_length):
+ windows.append(list(range(start_idx, min(start_idx + handler.context_length, num_frames))))
+ return windows
+
+
+def create_windows_default(num_frames: int, handler: IndexListContextHandler):
+ return [list(range(num_frames))]
+
+
+CONTEXT_MAPPING = {
+ ContextSchedules.UNIFORM_LOOPED: create_windows_uniform_looped,
+ ContextSchedules.UNIFORM_STANDARD: create_windows_uniform_standard,
+ ContextSchedules.STATIC_STANDARD: create_windows_static_standard,
+ ContextSchedules.BATCHED: create_windows_batched,
+}
+
+
+def get_matching_context_schedule(context_schedule: str) -> ContextSchedule:
+ func = CONTEXT_MAPPING.get(context_schedule, None)
+ if func is None:
+ raise ValueError(f"Unknown context_schedule '{context_schedule}'.")
+ return ContextSchedule(context_schedule, func)
+
+
+def get_context_weights(length: int, full_length: int, idxs: list[int], handler: IndexListContextHandler, sigma: torch.Tensor=None):
+ return handler.fuse_method.func(length, sigma=sigma, handler=handler, full_length=full_length, idxs=idxs)
+
+
+def create_weights_flat(length: int, **kwargs) -> list[float]:
+ # weight is the same for all
+ return [1.0] * length
+
+def create_weights_pyramid(length: int, **kwargs) -> list[float]:
+ # weight is based on the distance away from the edge of the context window;
+ # based on weighted average concept in FreeNoise paper
+ if length % 2 == 0:
+ max_weight = length // 2
+ weight_sequence = list(range(1, max_weight + 1, 1)) + list(range(max_weight, 0, -1))
+ else:
+ max_weight = (length + 1) // 2
+ weight_sequence = list(range(1, max_weight, 1)) + [max_weight] + list(range(max_weight - 1, 0, -1))
+ return weight_sequence
+
+def create_weights_overlap_linear(length: int, full_length: int, idxs: list[int], handler: IndexListContextHandler, **kwargs):
+ # based on code in Kijai's WanVideoWrapper: https://github.com/kijai/ComfyUI-WanVideoWrapper/blob/dbb2523b37e4ccdf45127e5ae33e31362f755c8e/nodes.py#L1302
+ # only expected overlap is given different weights
+ weights_torch = torch.ones((length))
+ # blend left-side on all except first window
+ if min(idxs) > 0:
+ ramp_up = torch.linspace(1e-37, 1, handler.context_overlap)
+ weights_torch[:handler.context_overlap] = ramp_up
+ # blend right-side on all except last window
+ if max(idxs) < full_length-1:
+ ramp_down = torch.linspace(1, 1e-37, handler.context_overlap)
+ weights_torch[-handler.context_overlap:] = ramp_down
+ return weights_torch
+
+class ContextFuseMethods:
+ FLAT = "flat"
+ PYRAMID = "pyramid"
+ RELATIVE = "relative"
+ OVERLAP_LINEAR = "overlap-linear"
+
+ LIST = [PYRAMID, FLAT, OVERLAP_LINEAR]
+ LIST_STATIC = [PYRAMID, RELATIVE, FLAT, OVERLAP_LINEAR]
+
+
+FUSE_MAPPING = {
+ ContextFuseMethods.FLAT: create_weights_flat,
+ ContextFuseMethods.PYRAMID: create_weights_pyramid,
+ ContextFuseMethods.RELATIVE: create_weights_pyramid,
+ ContextFuseMethods.OVERLAP_LINEAR: create_weights_overlap_linear,
+}
+
+def get_matching_fuse_method(fuse_method: str) -> ContextFuseMethod:
+ func = FUSE_MAPPING.get(fuse_method, None)
+ if func is None:
+ raise ValueError(f"Unknown fuse_method '{fuse_method}'.")
+ return ContextFuseMethod(fuse_method, func)
+
+# Returns fraction that has denominator that is a power of 2
+def ordered_halving(val):
+ # get binary value, padded with 0s for 64 bits
+ bin_str = f"{val:064b}"
+ # flip binary value, padding included
+ bin_flip = bin_str[::-1]
+ # convert binary to int
+ as_int = int(bin_flip, 2)
+ # divide by 1 << 64, equivalent to 2**64, or 18446744073709551616,
+ # or b10000000000000000000000000000000000000000000000000000000000000000 (1 with 64 zero's)
+ return as_int / (1 << 64)
+
+
+def get_missing_indexes(windows: list[list[int]], num_frames: int) -> list[int]:
+ all_indexes = list(range(num_frames))
+ for w in windows:
+ for val in w:
+ try:
+ all_indexes.remove(val)
+ except ValueError:
+ pass
+ return all_indexes
+
+
+def does_window_roll_over(window: list[int], num_frames: int) -> tuple[bool, int]:
+ prev_val = -1
+ for i, val in enumerate(window):
+ val = val % num_frames
+ if val < prev_val:
+ return True, i
+ prev_val = val
+ return False, -1
+
+
+def shift_window_to_start(window: list[int], num_frames: int):
+ start_val = window[0]
+ for i in range(len(window)):
+ # 1) subtract each element by start_val to move vals relative to the start of all frames
+ # 2) add num_frames and take modulus to get adjusted vals
+ window[i] = ((window[i] - start_val) + num_frames) % num_frames
+
+
+def shift_window_to_end(window: list[int], num_frames: int):
+ # 1) shift window to start
+ shift_window_to_start(window, num_frames)
+ end_val = window[-1]
+ end_delta = num_frames - end_val - 1
+ for i in range(len(window)):
+ # 2) add end_delta to each val to slide windows to end
+ window[i] = window[i] + end_delta
diff --git a/comfy/controlnet.py b/comfy/controlnet.py
index 9a47b86f2..e3dfedf55 100644
--- a/comfy/controlnet.py
+++ b/comfy/controlnet.py
@@ -28,6 +28,7 @@ import comfy.model_detection
import comfy.model_patcher
import comfy.ops
import comfy.latent_formats
+import comfy.model_base
import comfy.cldm.cldm
import comfy.t2i_adapter.adapter
@@ -35,6 +36,7 @@ import comfy.ldm.cascade.controlnet
import comfy.cldm.mmdit
import comfy.ldm.hydit.controlnet
import comfy.ldm.flux.controlnet
+import comfy.ldm.qwen_image.controlnet
import comfy.cldm.dit_embedder
from typing import TYPE_CHECKING
if TYPE_CHECKING:
@@ -43,7 +45,6 @@ if TYPE_CHECKING:
def broadcast_image_to(tensor, target_batch_size, batched_number):
current_batch_size = tensor.shape[0]
- #print(current_batch_size, target_batch_size)
if current_batch_size == 1:
return tensor
@@ -236,11 +237,11 @@ class ControlNet(ControlBase):
self.cond_hint = None
compression_ratio = self.compression_ratio
if self.vae is not None:
- compression_ratio *= self.vae.downscale_ratio
+ compression_ratio *= self.vae.spacial_compression_encode()
else:
if self.latent_format is not None:
raise ValueError("This Controlnet needs a VAE but none was provided, please use a ControlNetApply node with a VAE input and connect it.")
- self.cond_hint = comfy.utils.common_upscale(self.cond_hint_original, x_noisy.shape[3] * compression_ratio, x_noisy.shape[2] * compression_ratio, self.upscale_algorithm, "center")
+ self.cond_hint = comfy.utils.common_upscale(self.cond_hint_original, x_noisy.shape[-1] * compression_ratio, x_noisy.shape[-2] * compression_ratio, self.upscale_algorithm, "center")
self.cond_hint = self.preprocess_image(self.cond_hint)
if self.vae is not None:
loaded_models = comfy.model_management.loaded_models(only_currently_used=True)
@@ -265,12 +266,12 @@ class ControlNet(ControlBase):
for c in self.extra_conds:
temp = cond.get(c, None)
if temp is not None:
- extra[c] = temp.to(dtype)
+ extra[c] = comfy.model_base.convert_tensor(temp, dtype, x_noisy.device)
timestep = self.model_sampling_current.timestep(t)
x_noisy = self.model_sampling_current.calculate_input(t, x_noisy)
- control = self.control_model(x=x_noisy.to(dtype), hint=self.cond_hint, timesteps=timestep.to(dtype), context=context.to(dtype), **extra)
+ control = self.control_model(x=x_noisy.to(dtype), hint=self.cond_hint, timesteps=timestep.to(dtype), context=comfy.model_management.cast_to_device(context, x_noisy.device, dtype), **extra)
return self.control_merge(control, control_prev, output_dtype=None)
def copy(self):
@@ -582,6 +583,15 @@ def load_controlnet_flux_instantx(sd, model_options={}):
control = ControlNet(control_model, compression_ratio=1, latent_format=latent_format, concat_mask=concat_mask, load_device=load_device, manual_cast_dtype=manual_cast_dtype, extra_conds=extra_conds)
return control
+def load_controlnet_qwen_instantx(sd, model_options={}):
+ model_config, operations, load_device, unet_dtype, manual_cast_dtype, offload_device = controlnet_config(sd, model_options=model_options)
+ control_model = comfy.ldm.qwen_image.controlnet.QwenImageControlNetModel(operations=operations, device=offload_device, dtype=unet_dtype, **model_config.unet_config)
+ control_model = controlnet_load_state_dict(control_model, sd)
+ latent_format = comfy.latent_formats.Wan21()
+ extra_conds = []
+ control = ControlNet(control_model, compression_ratio=1, latent_format=latent_format, load_device=load_device, manual_cast_dtype=manual_cast_dtype, extra_conds=extra_conds)
+ return control
+
def convert_mistoline(sd):
return comfy.utils.state_dict_prefix_replace(sd, {"single_controlnet_blocks.": "controlnet_single_blocks."})
@@ -655,8 +665,11 @@ def load_controlnet_state_dict(state_dict, model=None, model_options={}):
return load_controlnet_sd35(controlnet_data, model_options=model_options) #Stability sd3.5 format
else:
return load_controlnet_mmdit(controlnet_data, model_options=model_options) #SD3 diffusers controlnet
+ elif "transformer_blocks.0.img_mlp.net.0.proj.weight" in controlnet_data:
+ return load_controlnet_qwen_instantx(controlnet_data, model_options=model_options)
elif "controlnet_x_embedder.weight" in controlnet_data:
return load_controlnet_flux_instantx(controlnet_data, model_options=model_options)
+
elif "controlnet_blocks.0.linear.weight" in controlnet_data: #mistoline flux
return load_controlnet_flux_xlabs_mistoline(convert_mistoline(controlnet_data), mistoline=True, model_options=model_options)
diff --git a/comfy/k_diffusion/sampling.py b/comfy/k_diffusion/sampling.py
index a2bc492fd..2d7e09838 100644
--- a/comfy/k_diffusion/sampling.py
+++ b/comfy/k_diffusion/sampling.py
@@ -171,6 +171,16 @@ def offset_first_sigma_for_snr(sigmas, model_sampling, percent_offset=1e-4):
return sigmas
+def ei_h_phi_1(h: torch.Tensor) -> torch.Tensor:
+ """Compute the result of h*phi_1(h) in exponential integrator methods."""
+ return torch.expm1(h)
+
+
+def ei_h_phi_2(h: torch.Tensor) -> torch.Tensor:
+ """Compute the result of h*phi_2(h) in exponential integrator methods."""
+ return (torch.expm1(h) - h) / h
+
+
@torch.no_grad()
def sample_euler(model, x, sigmas, extra_args=None, callback=None, disable=None, s_churn=0., s_tmin=0., s_tmax=float('inf'), s_noise=1.):
"""Implements Algorithm 2 (Euler steps) from Karras et al. (2022)."""
@@ -853,6 +863,11 @@ def sample_dpmpp_2m_sde(model, x, sigmas, extra_args=None, callback=None, disabl
return x
+@torch.no_grad()
+def sample_dpmpp_2m_sde_heun(model, x, sigmas, extra_args=None, callback=None, disable=None, eta=1., s_noise=1., noise_sampler=None, solver_type='heun'):
+ return sample_dpmpp_2m_sde(model, x, sigmas, extra_args=extra_args, callback=callback, disable=disable, eta=eta, s_noise=s_noise, noise_sampler=noise_sampler, solver_type=solver_type)
+
+
@torch.no_grad()
def sample_dpmpp_3m_sde(model, x, sigmas, extra_args=None, callback=None, disable=None, eta=1., s_noise=1., noise_sampler=None):
"""DPM-Solver++(3M) SDE."""
@@ -925,6 +940,16 @@ def sample_dpmpp_3m_sde_gpu(model, x, sigmas, extra_args=None, callback=None, di
return sample_dpmpp_3m_sde(model, x, sigmas, extra_args=extra_args, callback=callback, disable=disable, eta=eta, s_noise=s_noise, noise_sampler=noise_sampler)
+@torch.no_grad()
+def sample_dpmpp_2m_sde_heun_gpu(model, x, sigmas, extra_args=None, callback=None, disable=None, eta=1., s_noise=1., noise_sampler=None, solver_type='heun'):
+ if len(sigmas) <= 1:
+ return x
+ extra_args = {} if extra_args is None else extra_args
+ sigma_min, sigma_max = sigmas[sigmas > 0].min(), sigmas.max()
+ noise_sampler = BrownianTreeNoiseSampler(x, sigma_min, sigma_max, seed=extra_args.get("seed", None), cpu=False) if noise_sampler is None else noise_sampler
+ return sample_dpmpp_2m_sde_heun(model, x, sigmas, extra_args=extra_args, callback=callback, disable=disable, eta=eta, s_noise=s_noise, noise_sampler=noise_sampler, solver_type=solver_type)
+
+
@torch.no_grad()
def sample_dpmpp_2m_sde_gpu(model, x, sigmas, extra_args=None, callback=None, disable=None, eta=1., s_noise=1., noise_sampler=None, solver_type='midpoint'):
if len(sigmas) <= 1:
@@ -1535,13 +1560,12 @@ def sample_er_sde(model, x, sigmas, extra_args=None, callback=None, disable=None
@torch.no_grad()
def sample_seeds_2(model, x, sigmas, extra_args=None, callback=None, disable=None, eta=1., s_noise=1., noise_sampler=None, r=0.5):
"""SEEDS-2 - Stochastic Explicit Exponential Derivative-free Solvers (VP Data Prediction) stage 2.
- arXiv: https://arxiv.org/abs/2305.14267
+ arXiv: https://arxiv.org/abs/2305.14267 (NeurIPS 2023)
"""
extra_args = {} if extra_args is None else extra_args
seed = extra_args.get("seed", None)
noise_sampler = default_noise_sampler(x, seed=seed) if noise_sampler is None else noise_sampler
s_in = x.new_ones([x.shape[0]])
-
inject_noise = eta > 0 and s_noise > 0
model_sampling = model.inner_model.model_patcher.get_model_object('model_sampling')
@@ -1549,55 +1573,53 @@ def sample_seeds_2(model, x, sigmas, extra_args=None, callback=None, disable=Non
lambda_fn = partial(sigma_to_half_log_snr, model_sampling=model_sampling)
sigmas = offset_first_sigma_for_snr(sigmas, model_sampling)
+ fac = 1 / (2 * r)
+
for i in trange(len(sigmas) - 1, disable=disable):
denoised = model(x, sigmas[i] * s_in, **extra_args)
if callback is not None:
callback({'x': x, 'i': i, 'sigma': sigmas[i], 'sigma_hat': sigmas[i], 'denoised': denoised})
+
if sigmas[i + 1] == 0:
x = denoised
- else:
- lambda_s, lambda_t = lambda_fn(sigmas[i]), lambda_fn(sigmas[i + 1])
- h = lambda_t - lambda_s
- h_eta = h * (eta + 1)
- lambda_s_1 = lambda_s + r * h
- fac = 1 / (2 * r)
- sigma_s_1 = sigma_fn(lambda_s_1)
+ continue
- # alpha_t = sigma_t * exp(log(alpha_t / sigma_t)) = sigma_t * exp(lambda_t)
- alpha_s_1 = sigma_s_1 * lambda_s_1.exp()
- alpha_t = sigmas[i + 1] * lambda_t.exp()
+ lambda_s, lambda_t = lambda_fn(sigmas[i]), lambda_fn(sigmas[i + 1])
+ h = lambda_t - lambda_s
+ h_eta = h * (eta + 1)
+ lambda_s_1 = torch.lerp(lambda_s, lambda_t, r)
+ sigma_s_1 = sigma_fn(lambda_s_1)
- coeff_1, coeff_2 = (-r * h_eta).expm1(), (-h_eta).expm1()
- if inject_noise:
- # 0 < r < 1
- noise_coeff_1 = (-2 * r * h * eta).expm1().neg().sqrt()
- noise_coeff_2 = (-r * h * eta).exp() * (-2 * (1 - r) * h * eta).expm1().neg().sqrt()
- noise_1, noise_2 = noise_sampler(sigmas[i], sigma_s_1), noise_sampler(sigma_s_1, sigmas[i + 1])
+ alpha_s_1 = sigma_s_1 * lambda_s_1.exp()
+ alpha_t = sigmas[i + 1] * lambda_t.exp()
- # Step 1
- x_2 = sigma_s_1 / sigmas[i] * (-r * h * eta).exp() * x - alpha_s_1 * coeff_1 * denoised
- if inject_noise:
- x_2 = x_2 + sigma_s_1 * (noise_coeff_1 * noise_1) * s_noise
- denoised_2 = model(x_2, sigma_s_1 * s_in, **extra_args)
+ # Step 1
+ x_2 = sigma_s_1 / sigmas[i] * (-r * h * eta).exp() * x - alpha_s_1 * ei_h_phi_1(-r * h_eta) * denoised
+ if inject_noise:
+ sde_noise = (-2 * r * h * eta).expm1().neg().sqrt() * noise_sampler(sigmas[i], sigma_s_1)
+ x_2 = x_2 + sde_noise * sigma_s_1 * s_noise
+ denoised_2 = model(x_2, sigma_s_1 * s_in, **extra_args)
- # Step 2
- denoised_d = (1 - fac) * denoised + fac * denoised_2
- x = sigmas[i + 1] / sigmas[i] * (-h * eta).exp() * x - alpha_t * coeff_2 * denoised_d
- if inject_noise:
- x = x + sigmas[i + 1] * (noise_coeff_2 * noise_1 + noise_coeff_1 * noise_2) * s_noise
+ # Step 2
+ denoised_d = torch.lerp(denoised, denoised_2, fac)
+ x = sigmas[i + 1] / sigmas[i] * (-h * eta).exp() * x - alpha_t * ei_h_phi_1(-h_eta) * denoised_d
+ if inject_noise:
+ segment_factor = (r - 1) * h * eta
+ sde_noise = sde_noise * segment_factor.exp()
+ sde_noise = sde_noise + segment_factor.mul(2).expm1().neg().sqrt() * noise_sampler(sigma_s_1, sigmas[i + 1])
+ x = x + sde_noise * sigmas[i + 1] * s_noise
return x
@torch.no_grad()
def sample_seeds_3(model, x, sigmas, extra_args=None, callback=None, disable=None, eta=1., s_noise=1., noise_sampler=None, r_1=1./3, r_2=2./3):
"""SEEDS-3 - Stochastic Explicit Exponential Derivative-free Solvers (VP Data Prediction) stage 3.
- arXiv: https://arxiv.org/abs/2305.14267
+ arXiv: https://arxiv.org/abs/2305.14267 (NeurIPS 2023)
"""
extra_args = {} if extra_args is None else extra_args
seed = extra_args.get("seed", None)
noise_sampler = default_noise_sampler(x, seed=seed) if noise_sampler is None else noise_sampler
s_in = x.new_ones([x.shape[0]])
-
inject_noise = eta > 0 and s_noise > 0
model_sampling = model.inner_model.model_patcher.get_model_object('model_sampling')
@@ -1609,45 +1631,49 @@ def sample_seeds_3(model, x, sigmas, extra_args=None, callback=None, disable=Non
denoised = model(x, sigmas[i] * s_in, **extra_args)
if callback is not None:
callback({'x': x, 'i': i, 'sigma': sigmas[i], 'sigma_hat': sigmas[i], 'denoised': denoised})
+
if sigmas[i + 1] == 0:
x = denoised
- else:
- lambda_s, lambda_t = lambda_fn(sigmas[i]), lambda_fn(sigmas[i + 1])
- h = lambda_t - lambda_s
- h_eta = h * (eta + 1)
- lambda_s_1 = lambda_s + r_1 * h
- lambda_s_2 = lambda_s + r_2 * h
- sigma_s_1, sigma_s_2 = sigma_fn(lambda_s_1), sigma_fn(lambda_s_2)
+ continue
- # alpha_t = sigma_t * exp(log(alpha_t / sigma_t)) = sigma_t * exp(lambda_t)
- alpha_s_1 = sigma_s_1 * lambda_s_1.exp()
- alpha_s_2 = sigma_s_2 * lambda_s_2.exp()
- alpha_t = sigmas[i + 1] * lambda_t.exp()
+ lambda_s, lambda_t = lambda_fn(sigmas[i]), lambda_fn(sigmas[i + 1])
+ h = lambda_t - lambda_s
+ h_eta = h * (eta + 1)
+ lambda_s_1 = torch.lerp(lambda_s, lambda_t, r_1)
+ lambda_s_2 = torch.lerp(lambda_s, lambda_t, r_2)
+ sigma_s_1, sigma_s_2 = sigma_fn(lambda_s_1), sigma_fn(lambda_s_2)
- coeff_1, coeff_2, coeff_3 = (-r_1 * h_eta).expm1(), (-r_2 * h_eta).expm1(), (-h_eta).expm1()
- if inject_noise:
- # 0 < r_1 < r_2 < 1
- noise_coeff_1 = (-2 * r_1 * h * eta).expm1().neg().sqrt()
- noise_coeff_2 = (-r_1 * h * eta).exp() * (-2 * (r_2 - r_1) * h * eta).expm1().neg().sqrt()
- noise_coeff_3 = (-r_2 * h * eta).exp() * (-2 * (1 - r_2) * h * eta).expm1().neg().sqrt()
- noise_1, noise_2, noise_3 = noise_sampler(sigmas[i], sigma_s_1), noise_sampler(sigma_s_1, sigma_s_2), noise_sampler(sigma_s_2, sigmas[i + 1])
+ alpha_s_1 = sigma_s_1 * lambda_s_1.exp()
+ alpha_s_2 = sigma_s_2 * lambda_s_2.exp()
+ alpha_t = sigmas[i + 1] * lambda_t.exp()
- # Step 1
- x_2 = sigma_s_1 / sigmas[i] * (-r_1 * h * eta).exp() * x - alpha_s_1 * coeff_1 * denoised
- if inject_noise:
- x_2 = x_2 + sigma_s_1 * (noise_coeff_1 * noise_1) * s_noise
- denoised_2 = model(x_2, sigma_s_1 * s_in, **extra_args)
+ # Step 1
+ x_2 = sigma_s_1 / sigmas[i] * (-r_1 * h * eta).exp() * x - alpha_s_1 * ei_h_phi_1(-r_1 * h_eta) * denoised
+ if inject_noise:
+ sde_noise = (-2 * r_1 * h * eta).expm1().neg().sqrt() * noise_sampler(sigmas[i], sigma_s_1)
+ x_2 = x_2 + sde_noise * sigma_s_1 * s_noise
+ denoised_2 = model(x_2, sigma_s_1 * s_in, **extra_args)
- # Step 2
- x_3 = sigma_s_2 / sigmas[i] * (-r_2 * h * eta).exp() * x - alpha_s_2 * coeff_2 * denoised + (r_2 / r_1) * alpha_s_2 * (coeff_2 / (r_2 * h_eta) + 1) * (denoised_2 - denoised)
- if inject_noise:
- x_3 = x_3 + sigma_s_2 * (noise_coeff_2 * noise_1 + noise_coeff_1 * noise_2) * s_noise
- denoised_3 = model(x_3, sigma_s_2 * s_in, **extra_args)
+ # Step 2
+ a3_2 = r_2 / r_1 * ei_h_phi_2(-r_2 * h_eta)
+ a3_1 = ei_h_phi_1(-r_2 * h_eta) - a3_2
+ x_3 = sigma_s_2 / sigmas[i] * (-r_2 * h * eta).exp() * x - alpha_s_2 * (a3_1 * denoised + a3_2 * denoised_2)
+ if inject_noise:
+ segment_factor = (r_1 - r_2) * h * eta
+ sde_noise = sde_noise * segment_factor.exp()
+ sde_noise = sde_noise + segment_factor.mul(2).expm1().neg().sqrt() * noise_sampler(sigma_s_1, sigma_s_2)
+ x_3 = x_3 + sde_noise * sigma_s_2 * s_noise
+ denoised_3 = model(x_3, sigma_s_2 * s_in, **extra_args)
- # Step 3
- x = sigmas[i + 1] / sigmas[i] * (-h * eta).exp() * x - alpha_t * coeff_3 * denoised + (1. / r_2) * alpha_t * (coeff_3 / h_eta + 1) * (denoised_3 - denoised)
- if inject_noise:
- x = x + sigmas[i + 1] * (noise_coeff_3 * noise_1 + noise_coeff_2 * noise_2 + noise_coeff_1 * noise_3) * s_noise
+ # Step 3
+ b3 = ei_h_phi_2(-h_eta) / r_2
+ b1 = ei_h_phi_1(-h_eta) - b3
+ x = sigmas[i + 1] / sigmas[i] * (-h * eta).exp() * x - alpha_t * (b1 * denoised + b3 * denoised_3)
+ if inject_noise:
+ segment_factor = (r_2 - 1) * h * eta
+ sde_noise = sde_noise * segment_factor.exp()
+ sde_noise = sde_noise + segment_factor.mul(2).expm1().neg().sqrt() * noise_sampler(sigma_s_2, sigmas[i + 1])
+ x = x + sde_noise * sigmas[i + 1] * s_noise
return x
diff --git a/comfy/ldm/ace/model.py b/comfy/ldm/ace/model.py
index 12c524701..41d85eeb5 100644
--- a/comfy/ldm/ace/model.py
+++ b/comfy/ldm/ace/model.py
@@ -19,6 +19,7 @@ import torch
from torch import nn
import comfy.model_management
+import comfy.patcher_extension
from comfy.ldm.lightricks.model import TimestepEmbedding, Timesteps
from .attention import LinearTransformerBlock, t2i_modulate
@@ -343,7 +344,28 @@ class ACEStepTransformer2DModel(nn.Module):
output = self.final_layer(hidden_states, embedded_timestep, output_length)
return output
- def forward(
+ def forward(self,
+ x,
+ timestep,
+ attention_mask=None,
+ context: Optional[torch.Tensor] = None,
+ text_attention_mask: Optional[torch.LongTensor] = None,
+ speaker_embeds: Optional[torch.FloatTensor] = None,
+ lyric_token_idx: Optional[torch.LongTensor] = None,
+ lyric_mask: Optional[torch.LongTensor] = None,
+ block_controlnet_hidden_states: Optional[Union[List[torch.Tensor], torch.Tensor]] = None,
+ controlnet_scale: Union[float, torch.Tensor] = 1.0,
+ lyrics_strength=1.0,
+ **kwargs
+ ):
+ return comfy.patcher_extension.WrapperExecutor.new_class_executor(
+ self._forward,
+ self,
+ comfy.patcher_extension.get_all_wrappers(comfy.patcher_extension.WrappersMP.DIFFUSION_MODEL, kwargs.get("transformer_options", {}))
+ ).execute(x, timestep, attention_mask, context, text_attention_mask, speaker_embeds, lyric_token_idx, lyric_mask, block_controlnet_hidden_states,
+ controlnet_scale, lyrics_strength, **kwargs)
+
+ def _forward(
self,
x,
timestep,
diff --git a/comfy/ldm/aura/mmdit.py b/comfy/ldm/aura/mmdit.py
index 1258ae11f..d7f32b5e8 100644
--- a/comfy/ldm/aura/mmdit.py
+++ b/comfy/ldm/aura/mmdit.py
@@ -9,6 +9,7 @@ import torch.nn.functional as F
from comfy.ldm.modules.attention import optimized_attention
import comfy.ops
+import comfy.patcher_extension
import comfy.ldm.common_dit
def modulate(x, shift, scale):
@@ -436,6 +437,13 @@ class MMDiT(nn.Module):
return x + pos_encoding.reshape(1, -1, self.positional_encoding.shape[-1])
def forward(self, x, timestep, context, transformer_options={}, **kwargs):
+ return comfy.patcher_extension.WrapperExecutor.new_class_executor(
+ self._forward,
+ self,
+ comfy.patcher_extension.get_all_wrappers(comfy.patcher_extension.WrappersMP.DIFFUSION_MODEL, transformer_options)
+ ).execute(x, timestep, context, transformer_options, **kwargs)
+
+ def _forward(self, x, timestep, context, transformer_options={}, **kwargs):
patches_replace = transformer_options.get("patches_replace", {})
# patchify x, add PE
b, c, h, w = x.shape
diff --git a/comfy/ldm/chroma/model.py b/comfy/ldm/chroma/model.py
index 06021d4f2..5cff44dc8 100644
--- a/comfy/ldm/chroma/model.py
+++ b/comfy/ldm/chroma/model.py
@@ -5,6 +5,7 @@ from dataclasses import dataclass
import torch
from torch import Tensor, nn
from einops import rearrange, repeat
+import comfy.patcher_extension
import comfy.ldm.common_dit
from comfy.ldm.flux.layers import (
@@ -253,6 +254,13 @@ class Chroma(nn.Module):
return img
def forward(self, x, timestep, context, guidance, control=None, transformer_options={}, **kwargs):
+ return comfy.patcher_extension.WrapperExecutor.new_class_executor(
+ self._forward,
+ self,
+ comfy.patcher_extension.get_all_wrappers(comfy.patcher_extension.WrappersMP.DIFFUSION_MODEL, transformer_options)
+ ).execute(x, timestep, context, guidance, control, transformer_options, **kwargs)
+
+ def _forward(self, x, timestep, context, guidance, control=None, transformer_options={}, **kwargs):
bs, c, h, w = x.shape
x = comfy.ldm.common_dit.pad_to_patch_size(x, (self.patch_size, self.patch_size))
diff --git a/comfy/ldm/cosmos/cosmos_tokenizer/utils.py b/comfy/ldm/cosmos/cosmos_tokenizer/utils.py
index 3af8d0d05..ca993006f 100644
--- a/comfy/ldm/cosmos/cosmos_tokenizer/utils.py
+++ b/comfy/ldm/cosmos/cosmos_tokenizer/utils.py
@@ -58,7 +58,8 @@ def is_odd(n: int) -> bool:
def nonlinearity(x):
- return x * torch.sigmoid(x)
+ # x * sigmoid(x)
+ return torch.nn.functional.silu(x)
def Normalize(in_channels, num_groups=32):
diff --git a/comfy/ldm/cosmos/model.py b/comfy/ldm/cosmos/model.py
index 4836e0b69..53698b758 100644
--- a/comfy/ldm/cosmos/model.py
+++ b/comfy/ldm/cosmos/model.py
@@ -27,6 +27,8 @@ from torchvision import transforms
from enum import Enum
import logging
+import comfy.patcher_extension
+
from .blocks import (
FinalLayer,
GeneralDITTransformerBlock,
@@ -435,6 +437,42 @@ class GeneralDIT(nn.Module):
latent_condition_sigma: Optional[torch.Tensor] = None,
condition_video_augment_sigma: Optional[torch.Tensor] = None,
**kwargs,
+ ):
+ return comfy.patcher_extension.WrapperExecutor.new_class_executor(
+ self._forward,
+ self,
+ comfy.patcher_extension.get_all_wrappers(comfy.patcher_extension.WrappersMP.DIFFUSION_MODEL, kwargs.get("transformer_options", {}))
+ ).execute(x,
+ timesteps,
+ context,
+ attention_mask,
+ fps,
+ image_size,
+ padding_mask,
+ scalar_feature,
+ data_type,
+ latent_condition,
+ latent_condition_sigma,
+ condition_video_augment_sigma,
+ **kwargs)
+
+ def _forward(
+ self,
+ x: torch.Tensor,
+ timesteps: torch.Tensor,
+ context: torch.Tensor,
+ attention_mask: Optional[torch.Tensor] = None,
+ # crossattn_emb: torch.Tensor,
+ # crossattn_mask: Optional[torch.Tensor] = None,
+ fps: Optional[torch.Tensor] = None,
+ image_size: Optional[torch.Tensor] = None,
+ padding_mask: Optional[torch.Tensor] = None,
+ scalar_feature: Optional[torch.Tensor] = None,
+ data_type: Optional[DataType] = DataType.VIDEO,
+ latent_condition: Optional[torch.Tensor] = None,
+ latent_condition_sigma: Optional[torch.Tensor] = None,
+ condition_video_augment_sigma: Optional[torch.Tensor] = None,
+ **kwargs,
):
"""
Args:
diff --git a/comfy/ldm/cosmos/predict2.py b/comfy/ldm/cosmos/predict2.py
index 316117f77..fcc83ba76 100644
--- a/comfy/ldm/cosmos/predict2.py
+++ b/comfy/ldm/cosmos/predict2.py
@@ -11,6 +11,7 @@ import math
from .position_embedding import VideoRopePosition3DEmb, LearnablePosEmbAxis
from torchvision import transforms
+import comfy.patcher_extension
from comfy.ldm.modules.attention import optimized_attention
def apply_rotary_pos_emb(
@@ -805,7 +806,21 @@ class MiniTrainDIT(nn.Module):
)
return x_B_C_Tt_Hp_Wp
- def forward(
+ def forward(self,
+ x: torch.Tensor,
+ timesteps: torch.Tensor,
+ context: torch.Tensor,
+ fps: Optional[torch.Tensor] = None,
+ padding_mask: Optional[torch.Tensor] = None,
+ **kwargs,
+ ):
+ return comfy.patcher_extension.WrapperExecutor.new_class_executor(
+ self._forward,
+ self,
+ comfy.patcher_extension.get_all_wrappers(comfy.patcher_extension.WrappersMP.DIFFUSION_MODEL, kwargs.get("transformer_options", {}))
+ ).execute(x, timesteps, context, fps, padding_mask, **kwargs)
+
+ def _forward(
self,
x: torch.Tensor,
timesteps: torch.Tensor,
diff --git a/comfy/ldm/flux/model.py b/comfy/ldm/flux/model.py
index 8f4d99f54..1344c3a57 100644
--- a/comfy/ldm/flux/model.py
+++ b/comfy/ldm/flux/model.py
@@ -6,6 +6,7 @@ import torch
from torch import Tensor, nn
from einops import rearrange, repeat
import comfy.ldm.common_dit
+import comfy.patcher_extension
from .layers import (
DoubleStreamBlock,
@@ -157,7 +158,7 @@ class Flux(nn.Module):
if i < len(control_i):
add = control_i[i]
if add is not None:
- img += add
+ img[:, :add.shape[1]] += add
if img.dtype == torch.float16:
img = torch.nan_to_num(img, nan=0.0, posinf=65504, neginf=-65504)
@@ -188,7 +189,7 @@ class Flux(nn.Module):
if i < len(control_o):
add = control_o[i]
if add is not None:
- img[:, txt.shape[1] :, ...] += add
+ img[:, txt.shape[1] : txt.shape[1] + add.shape[1], ...] += add
img = img[:, txt.shape[1] :, ...]
@@ -214,6 +215,13 @@ class Flux(nn.Module):
return img, repeat(img_ids, "h w c -> b (h w) c", b=bs)
def forward(self, x, timestep, context, y=None, guidance=None, ref_latents=None, control=None, transformer_options={}, **kwargs):
+ return comfy.patcher_extension.WrapperExecutor.new_class_executor(
+ self._forward,
+ self,
+ comfy.patcher_extension.get_all_wrappers(comfy.patcher_extension.WrappersMP.DIFFUSION_MODEL, transformer_options)
+ ).execute(x, timestep, context, y, guidance, ref_latents, control, transformer_options, **kwargs)
+
+ def _forward(self, x, timestep, context, y=None, guidance=None, ref_latents=None, control=None, transformer_options={}, **kwargs):
bs, c, h_orig, w_orig = x.shape
patch_size = self.patch_size
@@ -224,19 +232,27 @@ class Flux(nn.Module):
if ref_latents is not None:
h = 0
w = 0
+ index = 0
+ index_ref_method = kwargs.get("ref_latents_method", "offset") == "index"
for ref in ref_latents:
- h_offset = 0
- w_offset = 0
- if ref.shape[-2] + h > ref.shape[-1] + w:
- w_offset = w
+ if index_ref_method:
+ index += 1
+ h_offset = 0
+ w_offset = 0
else:
- h_offset = h
+ index = 1
+ h_offset = 0
+ w_offset = 0
+ if ref.shape[-2] + h > ref.shape[-1] + w:
+ w_offset = w
+ else:
+ h_offset = h
+ h = max(h, ref.shape[-2] + h_offset)
+ w = max(w, ref.shape[-1] + w_offset)
- kontext, kontext_ids = self.process_img(ref, index=1, h_offset=h_offset, w_offset=w_offset)
+ kontext, kontext_ids = self.process_img(ref, index=index, h_offset=h_offset, w_offset=w_offset)
img = torch.cat([img, kontext], dim=1)
img_ids = torch.cat([img_ids, kontext_ids], dim=1)
- h = max(h, ref.shape[-2] + h_offset)
- w = max(w, ref.shape[-1] + w_offset)
txt_ids = torch.zeros((bs, context.shape[1], 3), device=x.device, dtype=x.dtype)
out = self.forward_orig(img, img_ids, context, txt_ids, timestep, y, guidance, control, transformer_options, attn_mask=kwargs.get("attention_mask", None))
diff --git a/comfy/ldm/hidream/model.py b/comfy/ldm/hidream/model.py
index 0305747bf..ae49cf945 100644
--- a/comfy/ldm/hidream/model.py
+++ b/comfy/ldm/hidream/model.py
@@ -13,6 +13,7 @@ from comfy.ldm.flux.layers import LastLayer
from comfy.ldm.modules.attention import optimized_attention
import comfy.model_management
+import comfy.patcher_extension
import comfy.ldm.common_dit
@@ -692,7 +693,23 @@ class HiDreamImageTransformer2DModel(nn.Module):
raise NotImplementedError
return x, x_masks, img_sizes
- def forward(
+ def forward(self,
+ x: torch.Tensor,
+ t: torch.Tensor,
+ y: Optional[torch.Tensor] = None,
+ context: Optional[torch.Tensor] = None,
+ encoder_hidden_states_llama3=None,
+ image_cond=None,
+ control = None,
+ transformer_options = {},
+ ):
+ return comfy.patcher_extension.WrapperExecutor.new_class_executor(
+ self._forward,
+ self,
+ comfy.patcher_extension.get_all_wrappers(comfy.patcher_extension.WrappersMP.DIFFUSION_MODEL, transformer_options)
+ ).execute(x, t, y, context, encoder_hidden_states_llama3, image_cond, control, transformer_options)
+
+ def _forward(
self,
x: torch.Tensor,
t: torch.Tensor,
diff --git a/comfy/ldm/hunyuan3d/model.py b/comfy/ldm/hunyuan3d/model.py
index 4e18358f0..0fa5e78c1 100644
--- a/comfy/ldm/hunyuan3d/model.py
+++ b/comfy/ldm/hunyuan3d/model.py
@@ -7,6 +7,7 @@ from comfy.ldm.flux.layers import (
SingleStreamBlock,
timestep_embedding,
)
+import comfy.patcher_extension
class Hunyuan3Dv2(nn.Module):
@@ -67,6 +68,13 @@ class Hunyuan3Dv2(nn.Module):
self.final_layer = LastLayer(hidden_size, 1, in_channels, dtype=dtype, device=device, operations=operations)
def forward(self, x, timestep, context, guidance=None, transformer_options={}, **kwargs):
+ return comfy.patcher_extension.WrapperExecutor.new_class_executor(
+ self._forward,
+ self,
+ comfy.patcher_extension.get_all_wrappers(comfy.patcher_extension.WrappersMP.DIFFUSION_MODEL, transformer_options)
+ ).execute(x, timestep, context, guidance, transformer_options, **kwargs)
+
+ def _forward(self, x, timestep, context, guidance=None, transformer_options={}, **kwargs):
x = x.movedim(-1, -2)
timestep = 1.0 - timestep
txt = context
diff --git a/comfy/ldm/hunyuan3d/vae.py b/comfy/ldm/hunyuan3d/vae.py
index 0e55ec2b2..760944827 100644
--- a/comfy/ldm/hunyuan3d/vae.py
+++ b/comfy/ldm/hunyuan3d/vae.py
@@ -552,6 +552,11 @@ class FourierEmbedder(nn.Module):
else:
return x
+class CrossAttentionProcessor:
+ def __call__(self, attn, q, k, v):
+ out = comfy.ops.scaled_dot_product_attention(q, k, v)
+ return out
+
class DropPath(nn.Module):
"""Drop paths (Stochastic Depth) per sample (when applied in main path of residual blocks).
"""
diff --git a/comfy/ldm/hunyuan_video/model.py b/comfy/ldm/hunyuan_video/model.py
index fbd8d4196..da1011596 100644
--- a/comfy/ldm/hunyuan_video/model.py
+++ b/comfy/ldm/hunyuan_video/model.py
@@ -1,6 +1,7 @@
#Based on Flux code because of weird hunyuan video code license.
import torch
+import comfy.patcher_extension
import comfy.ldm.flux.layers
import comfy.ldm.modules.diffusionmodules.mmdit
from comfy.ldm.modules.attention import optimized_attention
@@ -348,6 +349,13 @@ class HunyuanVideo(nn.Module):
return repeat(img_ids, "t h w c -> b (t h w) c", b=bs)
def forward(self, x, timestep, context, y, guidance=None, attention_mask=None, guiding_frame_index=None, ref_latent=None, control=None, transformer_options={}, **kwargs):
+ return comfy.patcher_extension.WrapperExecutor.new_class_executor(
+ self._forward,
+ self,
+ comfy.patcher_extension.get_all_wrappers(comfy.patcher_extension.WrappersMP.DIFFUSION_MODEL, transformer_options)
+ ).execute(x, timestep, context, y, guidance, attention_mask, guiding_frame_index, ref_latent, control, transformer_options, **kwargs)
+
+ def _forward(self, x, timestep, context, y, guidance=None, attention_mask=None, guiding_frame_index=None, ref_latent=None, control=None, transformer_options={}, **kwargs):
bs, c, t, h, w = x.shape
img_ids = self.img_ids(x)
txt_ids = torch.zeros((bs, context.shape[1], 3), device=x.device, dtype=x.dtype)
diff --git a/comfy/ldm/lightricks/model.py b/comfy/ldm/lightricks/model.py
index ad9a7daea..aa2ea62b1 100644
--- a/comfy/ldm/lightricks/model.py
+++ b/comfy/ldm/lightricks/model.py
@@ -1,5 +1,6 @@
import torch
from torch import nn
+import comfy.patcher_extension
import comfy.ldm.modules.attention
import comfy.ldm.common_dit
from einops import rearrange
@@ -420,6 +421,13 @@ class LTXVModel(torch.nn.Module):
self.patchifier = SymmetricPatchifier(1)
def forward(self, x, timestep, context, attention_mask, frame_rate=25, transformer_options={}, keyframe_idxs=None, **kwargs):
+ return comfy.patcher_extension.WrapperExecutor.new_class_executor(
+ self._forward,
+ self,
+ comfy.patcher_extension.get_all_wrappers(comfy.patcher_extension.WrappersMP.DIFFUSION_MODEL, transformer_options)
+ ).execute(x, timestep, context, attention_mask, frame_rate, transformer_options, keyframe_idxs, **kwargs)
+
+ def _forward(self, x, timestep, context, attention_mask, frame_rate=25, transformer_options={}, keyframe_idxs=None, **kwargs):
patches_replace = transformer_options.get("patches_replace", {})
orig_shape = list(x.shape)
diff --git a/comfy/ldm/lumina/model.py b/comfy/ldm/lumina/model.py
index f8dc4d7db..e08ed817d 100644
--- a/comfy/ldm/lumina/model.py
+++ b/comfy/ldm/lumina/model.py
@@ -11,6 +11,7 @@ import comfy.ldm.common_dit
from comfy.ldm.modules.diffusionmodules.mmdit import TimestepEmbedder
from comfy.ldm.modules.attention import optimized_attention_masked
from comfy.ldm.flux.layers import EmbedND
+import comfy.patcher_extension
def modulate(x, scale):
@@ -590,8 +591,15 @@ class NextDiT(nn.Module):
return padded_full_embed, mask, img_sizes, l_effective_cap_len, freqs_cis
- # def forward(self, x, t, cap_feats, cap_mask):
def forward(self, x, timesteps, context, num_tokens, attention_mask=None, **kwargs):
+ return comfy.patcher_extension.WrapperExecutor.new_class_executor(
+ self._forward,
+ self,
+ comfy.patcher_extension.get_all_wrappers(comfy.patcher_extension.WrappersMP.DIFFUSION_MODEL, kwargs.get("transformer_options", {}))
+ ).execute(x, timesteps, context, num_tokens, attention_mask, **kwargs)
+
+ # def forward(self, x, t, cap_feats, cap_mask):
+ def _forward(self, x, timesteps, context, num_tokens, attention_mask=None, **kwargs):
t = 1.0 - timesteps
cap_feats = context
cap_mask = attention_mask
diff --git a/comfy/ldm/modules/attention.py b/comfy/ldm/modules/attention.py
index 35d2270ee..043df28df 100644
--- a/comfy/ldm/modules/attention.py
+++ b/comfy/ldm/modules/attention.py
@@ -448,7 +448,7 @@ def attention_pytorch(q, k, v, heads, mask=None, attn_precision=None, skip_resha
mask = mask.unsqueeze(1)
if SDP_BATCH_LIMIT >= b:
- out = torch.nn.functional.scaled_dot_product_attention(q, k, v, attn_mask=mask, dropout_p=0.0, is_causal=False)
+ out = comfy.ops.scaled_dot_product_attention(q, k, v, attn_mask=mask, dropout_p=0.0, is_causal=False)
if not skip_output_reshape:
out = (
out.transpose(1, 2).reshape(b, -1, heads * dim_head)
@@ -461,7 +461,7 @@ def attention_pytorch(q, k, v, heads, mask=None, attn_precision=None, skip_resha
if mask.shape[0] > 1:
m = mask[i : i + SDP_BATCH_LIMIT]
- out[i : i + SDP_BATCH_LIMIT] = torch.nn.functional.scaled_dot_product_attention(
+ out[i : i + SDP_BATCH_LIMIT] = comfy.ops.scaled_dot_product_attention(
q[i : i + SDP_BATCH_LIMIT],
k[i : i + SDP_BATCH_LIMIT],
v[i : i + SDP_BATCH_LIMIT],
diff --git a/comfy/ldm/modules/diffusionmodules/mmdit.py b/comfy/ldm/modules/diffusionmodules/mmdit.py
index eaf3e73a4..4d6beba2d 100644
--- a/comfy/ldm/modules/diffusionmodules/mmdit.py
+++ b/comfy/ldm/modules/diffusionmodules/mmdit.py
@@ -109,7 +109,7 @@ class PatchEmbed(nn.Module):
def modulate(x, shift, scale):
if shift is None:
shift = torch.zeros_like(scale)
- return x * (1 + scale.unsqueeze(1)) + shift.unsqueeze(1)
+ return torch.addcmul(shift.unsqueeze(1), x, 1+ scale.unsqueeze(1))
#################################################################################
@@ -564,10 +564,7 @@ class DismantledBlock(nn.Module):
assert not self.pre_only
attn1 = self.attn.post_attention(attn)
attn2 = self.attn2.post_attention(attn2)
- out1 = gate_msa.unsqueeze(1) * attn1
- out2 = gate_msa2.unsqueeze(1) * attn2
- x = x + out1
- x = x + out2
+ x = gate_cat(x, gate_msa, gate_msa2, attn1, attn2)
x = x + gate_mlp.unsqueeze(1) * self.mlp(
modulate(self.norm2(x), shift_mlp, scale_mlp)
)
@@ -594,6 +591,11 @@ class DismantledBlock(nn.Module):
)
return self.post_attention(attn, *intermediates)
+def gate_cat(x, gate_msa, gate_msa2, attn1, attn2):
+ out1 = gate_msa.unsqueeze(1) * attn1
+ out2 = gate_msa2.unsqueeze(1) * attn2
+ x = torch.stack([x, out1, out2], dim=0).sum(dim=0)
+ return x
def block_mixing(*args, use_checkpoint=True, **kwargs):
if use_checkpoint:
diff --git a/comfy/ldm/modules/diffusionmodules/model.py b/comfy/ldm/modules/diffusionmodules/model.py
index 8162742cf..1fd12b35a 100644
--- a/comfy/ldm/modules/diffusionmodules/model.py
+++ b/comfy/ldm/modules/diffusionmodules/model.py
@@ -36,7 +36,7 @@ def get_timestep_embedding(timesteps, embedding_dim):
def nonlinearity(x):
# swish
- return x*torch.sigmoid(x)
+ return torch.nn.functional.silu(x)
def Normalize(in_channels, num_groups=32):
@@ -285,7 +285,7 @@ def pytorch_attention(q, k, v):
)
try:
- out = torch.nn.functional.scaled_dot_product_attention(q, k, v, attn_mask=None, dropout_p=0.0, is_causal=False)
+ out = comfy.ops.scaled_dot_product_attention(q, k, v, attn_mask=None, dropout_p=0.0, is_causal=False)
out = out.transpose(2, 3).reshape(orig_shape)
except model_management.OOM_EXCEPTION:
logging.warning("scaled_dot_product_attention OOMed: switched to slice attention")
diff --git a/comfy/ldm/qwen_image/controlnet.py b/comfy/ldm/qwen_image/controlnet.py
new file mode 100644
index 000000000..92ac3cf0a
--- /dev/null
+++ b/comfy/ldm/qwen_image/controlnet.py
@@ -0,0 +1,77 @@
+import torch
+import math
+
+from .model import QwenImageTransformer2DModel
+
+
+class QwenImageControlNetModel(QwenImageTransformer2DModel):
+ def __init__(
+ self,
+ extra_condition_channels=0,
+ dtype=None,
+ device=None,
+ operations=None,
+ **kwargs
+ ):
+ super().__init__(final_layer=False, dtype=dtype, device=device, operations=operations, **kwargs)
+ self.main_model_double = 60
+
+ # controlnet_blocks
+ self.controlnet_blocks = torch.nn.ModuleList([])
+ for _ in range(len(self.transformer_blocks)):
+ self.controlnet_blocks.append(operations.Linear(self.inner_dim, self.inner_dim, device=device, dtype=dtype))
+ self.controlnet_x_embedder = operations.Linear(self.in_channels + extra_condition_channels, self.inner_dim, device=device, dtype=dtype)
+
+ def forward(
+ self,
+ x,
+ timesteps,
+ context,
+ attention_mask=None,
+ guidance: torch.Tensor = None,
+ ref_latents=None,
+ hint=None,
+ transformer_options={},
+ **kwargs
+ ):
+ timestep = timesteps
+ encoder_hidden_states = context
+ encoder_hidden_states_mask = attention_mask
+
+ hidden_states, img_ids, orig_shape = self.process_img(x)
+ hint, _, _ = self.process_img(hint)
+
+ txt_start = round(max(((x.shape[-1] + (self.patch_size // 2)) // self.patch_size) // 2, ((x.shape[-2] + (self.patch_size // 2)) // self.patch_size) // 2))
+ txt_ids = torch.arange(txt_start, txt_start + context.shape[1], device=x.device).reshape(1, -1, 1).repeat(x.shape[0], 1, 3)
+ ids = torch.cat((txt_ids, img_ids), dim=1)
+ image_rotary_emb = self.pe_embedder(ids).squeeze(1).unsqueeze(2).to(x.dtype)
+ del ids, txt_ids, img_ids
+
+ hidden_states = self.img_in(hidden_states) + self.controlnet_x_embedder(hint)
+ encoder_hidden_states = self.txt_norm(encoder_hidden_states)
+ encoder_hidden_states = self.txt_in(encoder_hidden_states)
+
+ if guidance is not None:
+ guidance = guidance * 1000
+
+ temb = (
+ self.time_text_embed(timestep, hidden_states)
+ if guidance is None
+ else self.time_text_embed(timestep, guidance, hidden_states)
+ )
+
+ repeat = math.ceil(self.main_model_double / len(self.controlnet_blocks))
+
+ controlnet_block_samples = ()
+ for i, block in enumerate(self.transformer_blocks):
+ encoder_hidden_states, hidden_states = block(
+ hidden_states=hidden_states,
+ encoder_hidden_states=encoder_hidden_states,
+ encoder_hidden_states_mask=encoder_hidden_states_mask,
+ temb=temb,
+ image_rotary_emb=image_rotary_emb,
+ )
+
+ controlnet_block_samples = controlnet_block_samples + (self.controlnet_blocks[i](hidden_states),) * repeat
+
+ return {"input": controlnet_block_samples[:self.main_model_double]}
diff --git a/comfy/ldm/qwen_image/model.py b/comfy/ldm/qwen_image/model.py
new file mode 100644
index 000000000..04071f31c
--- /dev/null
+++ b/comfy/ldm/qwen_image/model.py
@@ -0,0 +1,469 @@
+# https://github.com/QwenLM/Qwen-Image (Apache 2.0)
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from typing import Optional, Tuple
+from einops import repeat
+
+from comfy.ldm.lightricks.model import TimestepEmbedding, Timesteps
+from comfy.ldm.modules.attention import optimized_attention_masked
+from comfy.ldm.flux.layers import EmbedND
+import comfy.ldm.common_dit
+import comfy.patcher_extension
+
+class GELU(nn.Module):
+ def __init__(self, dim_in: int, dim_out: int, approximate: str = "none", bias: bool = True, dtype=None, device=None, operations=None):
+ super().__init__()
+ self.proj = operations.Linear(dim_in, dim_out, bias=bias, dtype=dtype, device=device)
+ self.approximate = approximate
+
+ def forward(self, hidden_states):
+ hidden_states = self.proj(hidden_states)
+ hidden_states = F.gelu(hidden_states, approximate=self.approximate)
+ return hidden_states
+
+
+class FeedForward(nn.Module):
+ def __init__(
+ self,
+ dim: int,
+ dim_out: Optional[int] = None,
+ mult: int = 4,
+ dropout: float = 0.0,
+ inner_dim=None,
+ bias: bool = True,
+ dtype=None, device=None, operations=None
+ ):
+ super().__init__()
+ if inner_dim is None:
+ inner_dim = int(dim * mult)
+ dim_out = dim_out if dim_out is not None else dim
+
+ self.net = nn.ModuleList([])
+ self.net.append(GELU(dim, inner_dim, approximate="tanh", bias=bias, dtype=dtype, device=device, operations=operations))
+ self.net.append(nn.Dropout(dropout))
+ self.net.append(operations.Linear(inner_dim, dim_out, bias=bias, dtype=dtype, device=device))
+
+ def forward(self, hidden_states: torch.Tensor, *args, **kwargs) -> torch.Tensor:
+ for module in self.net:
+ hidden_states = module(hidden_states)
+ return hidden_states
+
+
+def apply_rotary_emb(x, freqs_cis):
+ if x.shape[1] == 0:
+ return x
+
+ t_ = x.reshape(*x.shape[:-1], -1, 1, 2)
+ t_out = freqs_cis[..., 0] * t_[..., 0] + freqs_cis[..., 1] * t_[..., 1]
+ return t_out.reshape(*x.shape)
+
+
+class QwenTimestepProjEmbeddings(nn.Module):
+ def __init__(self, embedding_dim, pooled_projection_dim, dtype=None, device=None, operations=None):
+ super().__init__()
+ self.time_proj = Timesteps(num_channels=256, flip_sin_to_cos=True, downscale_freq_shift=0, scale=1000)
+ self.timestep_embedder = TimestepEmbedding(
+ in_channels=256,
+ time_embed_dim=embedding_dim,
+ dtype=dtype,
+ device=device,
+ operations=operations
+ )
+
+ def forward(self, timestep, hidden_states):
+ timesteps_proj = self.time_proj(timestep)
+ timesteps_emb = self.timestep_embedder(timesteps_proj.to(dtype=hidden_states.dtype))
+ return timesteps_emb
+
+
+class Attention(nn.Module):
+ def __init__(
+ self,
+ query_dim: int,
+ dim_head: int = 64,
+ heads: int = 8,
+ dropout: float = 0.0,
+ bias: bool = False,
+ eps: float = 1e-5,
+ out_bias: bool = True,
+ out_dim: int = None,
+ out_context_dim: int = None,
+ dtype=None,
+ device=None,
+ operations=None
+ ):
+ super().__init__()
+ self.inner_dim = out_dim if out_dim is not None else dim_head * heads
+ self.inner_kv_dim = self.inner_dim
+ self.heads = heads
+ self.dim_head = dim_head
+ self.out_dim = out_dim if out_dim is not None else query_dim
+ self.out_context_dim = out_context_dim if out_context_dim is not None else query_dim
+ self.dropout = dropout
+
+ # Q/K normalization
+ self.norm_q = operations.RMSNorm(dim_head, eps=eps, elementwise_affine=True, dtype=dtype, device=device)
+ self.norm_k = operations.RMSNorm(dim_head, eps=eps, elementwise_affine=True, dtype=dtype, device=device)
+ self.norm_added_q = operations.RMSNorm(dim_head, eps=eps, dtype=dtype, device=device)
+ self.norm_added_k = operations.RMSNorm(dim_head, eps=eps, dtype=dtype, device=device)
+
+ # Image stream projections
+ self.to_q = operations.Linear(query_dim, self.inner_dim, bias=bias, dtype=dtype, device=device)
+ self.to_k = operations.Linear(query_dim, self.inner_kv_dim, bias=bias, dtype=dtype, device=device)
+ self.to_v = operations.Linear(query_dim, self.inner_kv_dim, bias=bias, dtype=dtype, device=device)
+
+ # Text stream projections
+ self.add_q_proj = operations.Linear(query_dim, self.inner_dim, bias=bias, dtype=dtype, device=device)
+ self.add_k_proj = operations.Linear(query_dim, self.inner_kv_dim, bias=bias, dtype=dtype, device=device)
+ self.add_v_proj = operations.Linear(query_dim, self.inner_kv_dim, bias=bias, dtype=dtype, device=device)
+
+ # Output projections
+ self.to_out = nn.ModuleList([
+ operations.Linear(self.inner_dim, self.out_dim, bias=out_bias, dtype=dtype, device=device),
+ nn.Dropout(dropout)
+ ])
+ self.to_add_out = operations.Linear(self.inner_dim, self.out_context_dim, bias=out_bias, dtype=dtype, device=device)
+
+ def forward(
+ self,
+ hidden_states: torch.FloatTensor, # Image stream
+ encoder_hidden_states: torch.FloatTensor = None, # Text stream
+ encoder_hidden_states_mask: torch.FloatTensor = None,
+ attention_mask: Optional[torch.FloatTensor] = None,
+ image_rotary_emb: Optional[torch.Tensor] = None,
+ ) -> Tuple[torch.Tensor, torch.Tensor]:
+ seq_txt = encoder_hidden_states.shape[1]
+
+ img_query = self.to_q(hidden_states).unflatten(-1, (self.heads, -1))
+ img_key = self.to_k(hidden_states).unflatten(-1, (self.heads, -1))
+ img_value = self.to_v(hidden_states).unflatten(-1, (self.heads, -1))
+
+ txt_query = self.add_q_proj(encoder_hidden_states).unflatten(-1, (self.heads, -1))
+ txt_key = self.add_k_proj(encoder_hidden_states).unflatten(-1, (self.heads, -1))
+ txt_value = self.add_v_proj(encoder_hidden_states).unflatten(-1, (self.heads, -1))
+
+ img_query = self.norm_q(img_query)
+ img_key = self.norm_k(img_key)
+ txt_query = self.norm_added_q(txt_query)
+ txt_key = self.norm_added_k(txt_key)
+
+ joint_query = torch.cat([txt_query, img_query], dim=1)
+ joint_key = torch.cat([txt_key, img_key], dim=1)
+ joint_value = torch.cat([txt_value, img_value], dim=1)
+
+ joint_query = apply_rotary_emb(joint_query, image_rotary_emb)
+ joint_key = apply_rotary_emb(joint_key, image_rotary_emb)
+
+ joint_query = joint_query.flatten(start_dim=2)
+ joint_key = joint_key.flatten(start_dim=2)
+ joint_value = joint_value.flatten(start_dim=2)
+
+ joint_hidden_states = optimized_attention_masked(joint_query, joint_key, joint_value, self.heads, attention_mask)
+
+ txt_attn_output = joint_hidden_states[:, :seq_txt, :]
+ img_attn_output = joint_hidden_states[:, seq_txt:, :]
+
+ img_attn_output = self.to_out[0](img_attn_output)
+ img_attn_output = self.to_out[1](img_attn_output)
+ txt_attn_output = self.to_add_out(txt_attn_output)
+
+ return img_attn_output, txt_attn_output
+
+
+class QwenImageTransformerBlock(nn.Module):
+ def __init__(
+ self,
+ dim: int,
+ num_attention_heads: int,
+ attention_head_dim: int,
+ eps: float = 1e-6,
+ dtype=None,
+ device=None,
+ operations=None
+ ):
+ super().__init__()
+ self.dim = dim
+ self.num_attention_heads = num_attention_heads
+ self.attention_head_dim = attention_head_dim
+
+ self.img_mod = nn.Sequential(
+ nn.SiLU(),
+ operations.Linear(dim, 6 * dim, bias=True, dtype=dtype, device=device),
+ )
+ self.img_norm1 = operations.LayerNorm(dim, elementwise_affine=False, eps=eps, dtype=dtype, device=device)
+ self.img_norm2 = operations.LayerNorm(dim, elementwise_affine=False, eps=eps, dtype=dtype, device=device)
+ self.img_mlp = FeedForward(dim=dim, dim_out=dim, dtype=dtype, device=device, operations=operations)
+
+ self.txt_mod = nn.Sequential(
+ nn.SiLU(),
+ operations.Linear(dim, 6 * dim, bias=True, dtype=dtype, device=device),
+ )
+ self.txt_norm1 = operations.LayerNorm(dim, elementwise_affine=False, eps=eps, dtype=dtype, device=device)
+ self.txt_norm2 = operations.LayerNorm(dim, elementwise_affine=False, eps=eps, dtype=dtype, device=device)
+ self.txt_mlp = FeedForward(dim=dim, dim_out=dim, dtype=dtype, device=device, operations=operations)
+
+ self.attn = Attention(
+ query_dim=dim,
+ dim_head=attention_head_dim,
+ heads=num_attention_heads,
+ out_dim=dim,
+ bias=True,
+ eps=eps,
+ dtype=dtype,
+ device=device,
+ operations=operations,
+ )
+
+ def _modulate(self, x: torch.Tensor, mod_params: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:
+ shift, scale, gate = torch.chunk(mod_params, 3, dim=-1)
+ return torch.addcmul(shift.unsqueeze(1), x, 1 + scale.unsqueeze(1)), gate.unsqueeze(1)
+
+ def forward(
+ self,
+ hidden_states: torch.Tensor,
+ encoder_hidden_states: torch.Tensor,
+ encoder_hidden_states_mask: torch.Tensor,
+ temb: torch.Tensor,
+ image_rotary_emb: Optional[Tuple[torch.Tensor, torch.Tensor]] = None,
+ ) -> Tuple[torch.Tensor, torch.Tensor]:
+ img_mod_params = self.img_mod(temb)
+ txt_mod_params = self.txt_mod(temb)
+ img_mod1, img_mod2 = img_mod_params.chunk(2, dim=-1)
+ txt_mod1, txt_mod2 = txt_mod_params.chunk(2, dim=-1)
+
+ img_normed = self.img_norm1(hidden_states)
+ img_modulated, img_gate1 = self._modulate(img_normed, img_mod1)
+ txt_normed = self.txt_norm1(encoder_hidden_states)
+ txt_modulated, txt_gate1 = self._modulate(txt_normed, txt_mod1)
+
+ img_attn_output, txt_attn_output = self.attn(
+ hidden_states=img_modulated,
+ encoder_hidden_states=txt_modulated,
+ encoder_hidden_states_mask=encoder_hidden_states_mask,
+ image_rotary_emb=image_rotary_emb,
+ )
+
+ hidden_states = hidden_states + img_gate1 * img_attn_output
+ encoder_hidden_states = encoder_hidden_states + txt_gate1 * txt_attn_output
+
+ img_normed2 = self.img_norm2(hidden_states)
+ img_modulated2, img_gate2 = self._modulate(img_normed2, img_mod2)
+ hidden_states = torch.addcmul(hidden_states, img_gate2, self.img_mlp(img_modulated2))
+
+ txt_normed2 = self.txt_norm2(encoder_hidden_states)
+ txt_modulated2, txt_gate2 = self._modulate(txt_normed2, txt_mod2)
+ encoder_hidden_states = torch.addcmul(encoder_hidden_states, txt_gate2, self.txt_mlp(txt_modulated2))
+
+ return encoder_hidden_states, hidden_states
+
+
+class LastLayer(nn.Module):
+ def __init__(
+ self,
+ embedding_dim: int,
+ conditioning_embedding_dim: int,
+ elementwise_affine=False,
+ eps=1e-6,
+ bias=True,
+ dtype=None, device=None, operations=None
+ ):
+ super().__init__()
+ self.silu = nn.SiLU()
+ self.linear = operations.Linear(conditioning_embedding_dim, embedding_dim * 2, bias=bias, dtype=dtype, device=device)
+ self.norm = operations.LayerNorm(embedding_dim, eps, elementwise_affine=False, bias=bias, dtype=dtype, device=device)
+
+ def forward(self, x: torch.Tensor, conditioning_embedding: torch.Tensor) -> torch.Tensor:
+ emb = self.linear(self.silu(conditioning_embedding))
+ scale, shift = torch.chunk(emb, 2, dim=1)
+ x = torch.addcmul(shift[:, None, :], self.norm(x), (1 + scale)[:, None, :])
+ return x
+
+
+class QwenImageTransformer2DModel(nn.Module):
+ def __init__(
+ self,
+ patch_size: int = 2,
+ in_channels: int = 64,
+ out_channels: Optional[int] = 16,
+ num_layers: int = 60,
+ attention_head_dim: int = 128,
+ num_attention_heads: int = 24,
+ joint_attention_dim: int = 3584,
+ pooled_projection_dim: int = 768,
+ guidance_embeds: bool = False,
+ axes_dims_rope: Tuple[int, int, int] = (16, 56, 56),
+ image_model=None,
+ final_layer=True,
+ dtype=None,
+ device=None,
+ operations=None,
+ ):
+ super().__init__()
+ self.dtype = dtype
+ self.patch_size = patch_size
+ self.in_channels = in_channels
+ self.out_channels = out_channels or in_channels
+ self.inner_dim = num_attention_heads * attention_head_dim
+
+ self.pe_embedder = EmbedND(dim=attention_head_dim, theta=10000, axes_dim=list(axes_dims_rope))
+
+ self.time_text_embed = QwenTimestepProjEmbeddings(
+ embedding_dim=self.inner_dim,
+ pooled_projection_dim=pooled_projection_dim,
+ dtype=dtype,
+ device=device,
+ operations=operations
+ )
+
+ self.txt_norm = operations.RMSNorm(joint_attention_dim, eps=1e-6, dtype=dtype, device=device)
+ self.img_in = operations.Linear(in_channels, self.inner_dim, dtype=dtype, device=device)
+ self.txt_in = operations.Linear(joint_attention_dim, self.inner_dim, dtype=dtype, device=device)
+
+ self.transformer_blocks = nn.ModuleList([
+ QwenImageTransformerBlock(
+ dim=self.inner_dim,
+ num_attention_heads=num_attention_heads,
+ attention_head_dim=attention_head_dim,
+ dtype=dtype,
+ device=device,
+ operations=operations
+ )
+ for _ in range(num_layers)
+ ])
+
+ if final_layer:
+ self.norm_out = LastLayer(self.inner_dim, self.inner_dim, dtype=dtype, device=device, operations=operations)
+ self.proj_out = operations.Linear(self.inner_dim, patch_size * patch_size * self.out_channels, bias=True, dtype=dtype, device=device)
+
+ def process_img(self, x, index=0, h_offset=0, w_offset=0):
+ bs, c, t, h, w = x.shape
+ patch_size = self.patch_size
+ hidden_states = comfy.ldm.common_dit.pad_to_patch_size(x, (1, self.patch_size, self.patch_size))
+ orig_shape = hidden_states.shape
+ hidden_states = hidden_states.view(orig_shape[0], orig_shape[1], orig_shape[-2] // 2, 2, orig_shape[-1] // 2, 2)
+ hidden_states = hidden_states.permute(0, 2, 4, 1, 3, 5)
+ hidden_states = hidden_states.reshape(orig_shape[0], (orig_shape[-2] // 2) * (orig_shape[-1] // 2), orig_shape[1] * 4)
+ h_len = ((h + (patch_size // 2)) // patch_size)
+ w_len = ((w + (patch_size // 2)) // patch_size)
+
+ h_offset = ((h_offset + (patch_size // 2)) // patch_size)
+ w_offset = ((w_offset + (patch_size // 2)) // patch_size)
+
+ img_ids = torch.zeros((h_len, w_len, 3), device=x.device)
+ img_ids[:, :, 0] = img_ids[:, :, 1] + index
+ img_ids[:, :, 1] = img_ids[:, :, 1] + torch.linspace(h_offset, h_len - 1 + h_offset, steps=h_len, device=x.device, dtype=x.dtype).unsqueeze(1) - (h_len // 2)
+ img_ids[:, :, 2] = img_ids[:, :, 2] + torch.linspace(w_offset, w_len - 1 + w_offset, steps=w_len, device=x.device, dtype=x.dtype).unsqueeze(0) - (w_len // 2)
+ return hidden_states, repeat(img_ids, "h w c -> b (h w) c", b=bs), orig_shape
+
+ def forward(self, x, timestep, context, attention_mask=None, guidance=None, ref_latents=None, transformer_options={}, **kwargs):
+ return comfy.patcher_extension.WrapperExecutor.new_class_executor(
+ self._forward,
+ self,
+ comfy.patcher_extension.get_all_wrappers(comfy.patcher_extension.WrappersMP.DIFFUSION_MODEL, transformer_options)
+ ).execute(x, timestep, context, attention_mask, guidance, ref_latents, transformer_options, **kwargs)
+
+ def _forward(
+ self,
+ x,
+ timesteps,
+ context,
+ attention_mask=None,
+ guidance: torch.Tensor = None,
+ ref_latents=None,
+ transformer_options={},
+ control=None,
+ **kwargs
+ ):
+ timestep = timesteps
+ encoder_hidden_states = context
+ encoder_hidden_states_mask = attention_mask
+
+ hidden_states, img_ids, orig_shape = self.process_img(x)
+ num_embeds = hidden_states.shape[1]
+
+ if ref_latents is not None:
+ h = 0
+ w = 0
+ index = 0
+ index_ref_method = kwargs.get("ref_latents_method", "index") == "index"
+ for ref in ref_latents:
+ if index_ref_method:
+ index += 1
+ h_offset = 0
+ w_offset = 0
+ else:
+ index = 1
+ h_offset = 0
+ w_offset = 0
+ if ref.shape[-2] + h > ref.shape[-1] + w:
+ w_offset = w
+ else:
+ h_offset = h
+ h = max(h, ref.shape[-2] + h_offset)
+ w = max(w, ref.shape[-1] + w_offset)
+
+ kontext, kontext_ids, _ = self.process_img(ref, index=index, h_offset=h_offset, w_offset=w_offset)
+ hidden_states = torch.cat([hidden_states, kontext], dim=1)
+ img_ids = torch.cat([img_ids, kontext_ids], dim=1)
+
+ txt_start = round(max(((x.shape[-1] + (self.patch_size // 2)) // self.patch_size) // 2, ((x.shape[-2] + (self.patch_size // 2)) // self.patch_size) // 2))
+ txt_ids = torch.arange(txt_start, txt_start + context.shape[1], device=x.device).reshape(1, -1, 1).repeat(x.shape[0], 1, 3)
+ ids = torch.cat((txt_ids, img_ids), dim=1)
+ image_rotary_emb = self.pe_embedder(ids).squeeze(1).unsqueeze(2).to(x.dtype)
+ del ids, txt_ids, img_ids
+
+ hidden_states = self.img_in(hidden_states)
+ encoder_hidden_states = self.txt_norm(encoder_hidden_states)
+ encoder_hidden_states = self.txt_in(encoder_hidden_states)
+
+ if guidance is not None:
+ guidance = guidance * 1000
+
+ temb = (
+ self.time_text_embed(timestep, hidden_states)
+ if guidance is None
+ else self.time_text_embed(timestep, guidance, hidden_states)
+ )
+
+ patches_replace = transformer_options.get("patches_replace", {})
+ patches = transformer_options.get("patches", {})
+ blocks_replace = patches_replace.get("dit", {})
+
+ for i, block in enumerate(self.transformer_blocks):
+ if ("double_block", i) in blocks_replace:
+ def block_wrap(args):
+ out = {}
+ out["txt"], out["img"] = block(hidden_states=args["img"], encoder_hidden_states=args["txt"], encoder_hidden_states_mask=encoder_hidden_states_mask, temb=args["vec"], image_rotary_emb=args["pe"])
+ return out
+ out = blocks_replace[("double_block", i)]({"img": hidden_states, "txt": encoder_hidden_states, "vec": temb, "pe": image_rotary_emb}, {"original_block": block_wrap})
+ hidden_states = out["img"]
+ encoder_hidden_states = out["txt"]
+ else:
+ encoder_hidden_states, hidden_states = block(
+ hidden_states=hidden_states,
+ encoder_hidden_states=encoder_hidden_states,
+ encoder_hidden_states_mask=encoder_hidden_states_mask,
+ temb=temb,
+ image_rotary_emb=image_rotary_emb,
+ )
+
+ if "double_block" in patches:
+ for p in patches["double_block"]:
+ out = p({"img": hidden_states, "txt": encoder_hidden_states, "x": x, "block_index": i})
+ hidden_states = out["img"]
+ encoder_hidden_states = out["txt"]
+
+ if control is not None: # Controlnet
+ control_i = control.get("input")
+ if i < len(control_i):
+ add = control_i[i]
+ if add is not None:
+ hidden_states[:, :add.shape[1]] += add
+
+ hidden_states = self.norm_out(hidden_states, temb)
+ hidden_states = self.proj_out(hidden_states)
+
+ hidden_states = hidden_states[:, :num_embeds].view(orig_shape[0], orig_shape[-2] // 2, orig_shape[-1] // 2, orig_shape[1], 2, 2)
+ hidden_states = hidden_states.permute(0, 3, 1, 4, 2, 5)
+ return hidden_states.reshape(orig_shape)[:, :, :, :x.shape[-2], :x.shape[-1]]
diff --git a/comfy/ldm/wan/model.py b/comfy/ldm/wan/model.py
index b9e47e9f7..47857dc2b 100644
--- a/comfy/ldm/wan/model.py
+++ b/comfy/ldm/wan/model.py
@@ -4,13 +4,14 @@ import math
import torch
import torch.nn as nn
-from einops import repeat
+from einops import rearrange
from comfy.ldm.modules.attention import optimized_attention
from comfy.ldm.flux.layers import EmbedND
from comfy.ldm.flux.math import apply_rope
import comfy.ldm.common_dit
import comfy.model_management
+import comfy.patcher_extension
def sinusoidal_embedding_1d(dim, position):
@@ -146,6 +147,18 @@ WAN_CROSSATTENTION_CLASSES = {
}
+def repeat_e(e, x):
+ repeats = 1
+ if e.size(1) > 1:
+ repeats = x.size(1) // e.size(1)
+ if repeats == 1:
+ return e
+ if repeats * e.size(1) == x.size(1):
+ return torch.repeat_interleave(e, repeats, dim=1)
+ else:
+ return torch.repeat_interleave(e, repeats + 1, dim=1)[:, :x.size(1)]
+
+
class WanAttentionBlock(nn.Module):
def __init__(self,
@@ -201,6 +214,7 @@ class WanAttentionBlock(nn.Module):
freqs(Tensor): Rope freqs, shape [1024, C / num_heads / 2]
"""
# assert e.dtype == torch.float32
+
if e.ndim < 4:
e = (comfy.model_management.cast_to(self.modulation, dtype=x.dtype, device=x.device) + e).chunk(6, dim=1)
else:
@@ -209,15 +223,15 @@ class WanAttentionBlock(nn.Module):
# self-attention
y = self.self_attn(
- self.norm1(x) * (1 + e[1]) + e[0],
+ torch.addcmul(repeat_e(e[0], x), self.norm1(x), 1 + repeat_e(e[1], x)),
freqs)
- x = x + y * e[2]
+ x = torch.addcmul(x, y, repeat_e(e[2], x))
# cross-attention & ffn
x = x + self.cross_attn(self.norm3(x), context, context_img_len=context_img_len)
- y = self.ffn(self.norm2(x) * (1 + e[4]) + e[3])
- x = x + y * e[5]
+ y = self.ffn(torch.addcmul(repeat_e(e[3], x), self.norm2(x), 1 + repeat_e(e[4], x)))
+ x = torch.addcmul(x, y, repeat_e(e[5], x))
return x
@@ -331,7 +345,8 @@ class Head(nn.Module):
e = (comfy.model_management.cast_to(self.modulation, dtype=x.dtype, device=x.device) + e.unsqueeze(1)).chunk(2, dim=1)
else:
e = (comfy.model_management.cast_to(self.modulation, dtype=x.dtype, device=x.device).unsqueeze(0) + e.unsqueeze(2)).unbind(2)
- x = (self.head(self.norm(x) * (1 + e[1]) + e[0]))
+
+ x = (self.head(torch.addcmul(repeat_e(e[0], x), self.norm(x), 1 + repeat_e(e[1], x))))
return x
@@ -380,6 +395,7 @@ class WanModel(torch.nn.Module):
cross_attn_norm=True,
eps=1e-6,
flf_pos_embed_token_number=None,
+ in_dim_ref_conv=None,
image_model=None,
device=None,
dtype=None,
@@ -473,6 +489,11 @@ class WanModel(torch.nn.Module):
else:
self.img_emb = None
+ if in_dim_ref_conv is not None:
+ self.ref_conv = operations.Conv2d(in_dim_ref_conv, dim, kernel_size=patch_size[1:], stride=patch_size[1:], device=operation_settings.get("device"), dtype=operation_settings.get("dtype"))
+ else:
+ self.ref_conv = None
+
def forward_orig(
self,
x,
@@ -515,6 +536,13 @@ class WanModel(torch.nn.Module):
e = e.reshape(t.shape[0], -1, e.shape[-1])
e0 = self.time_projection(e).unflatten(2, (6, self.dim))
+ full_ref = None
+ if self.ref_conv is not None:
+ full_ref = kwargs.get("reference_latent", None)
+ if full_ref is not None:
+ full_ref = self.ref_conv(full_ref).flatten(2).transpose(1, 2)
+ x = torch.concat((full_ref, x), dim=1)
+
# context
context = self.text_embedding(context)
@@ -541,31 +569,56 @@ class WanModel(torch.nn.Module):
# head
x = self.head(x, e)
+ if full_ref is not None:
+ x = x[:, full_ref.shape[1]:]
+
# unpatchify
x = self.unpatchify(x, grid_sizes)
return x
- def forward(self, x, timestep, context, clip_fea=None, time_dim_concat=None, transformer_options={}, **kwargs):
- bs, c, t, h, w = x.shape
- x = comfy.ldm.common_dit.pad_to_patch_size(x, self.patch_size)
-
+ def rope_encode(self, t, h, w, t_start=0, steps_t=None, steps_h=None, steps_w=None, device=None, dtype=None):
patch_size = self.patch_size
t_len = ((t + (patch_size[0] // 2)) // patch_size[0])
h_len = ((h + (patch_size[1] // 2)) // patch_size[1])
w_len = ((w + (patch_size[2] // 2)) // patch_size[2])
+ if steps_t is None:
+ steps_t = t_len
+ if steps_h is None:
+ steps_h = h_len
+ if steps_w is None:
+ steps_w = w_len
+
+ img_ids = torch.zeros((steps_t, steps_h, steps_w, 3), device=device, dtype=dtype)
+ img_ids[:, :, :, 0] = img_ids[:, :, :, 0] + torch.linspace(t_start, t_start + (t_len - 1), steps=steps_t, device=device, dtype=dtype).reshape(-1, 1, 1)
+ img_ids[:, :, :, 1] = img_ids[:, :, :, 1] + torch.linspace(0, h_len - 1, steps=steps_h, device=device, dtype=dtype).reshape(1, -1, 1)
+ img_ids[:, :, :, 2] = img_ids[:, :, :, 2] + torch.linspace(0, w_len - 1, steps=steps_w, device=device, dtype=dtype).reshape(1, 1, -1)
+ img_ids = img_ids.reshape(1, -1, img_ids.shape[-1])
+
+ freqs = self.rope_embedder(img_ids).movedim(1, 2)
+ return freqs
+
+ def forward(self, x, timestep, context, clip_fea=None, time_dim_concat=None, transformer_options={}, **kwargs):
+ return comfy.patcher_extension.WrapperExecutor.new_class_executor(
+ self._forward,
+ self,
+ comfy.patcher_extension.get_all_wrappers(comfy.patcher_extension.WrappersMP.DIFFUSION_MODEL, transformer_options)
+ ).execute(x, timestep, context, clip_fea, time_dim_concat, transformer_options, **kwargs)
+
+ def _forward(self, x, timestep, context, clip_fea=None, time_dim_concat=None, transformer_options={}, **kwargs):
+ bs, c, t, h, w = x.shape
+ x = comfy.ldm.common_dit.pad_to_patch_size(x, self.patch_size)
+
+ t_len = t
if time_dim_concat is not None:
time_dim_concat = comfy.ldm.common_dit.pad_to_patch_size(time_dim_concat, self.patch_size)
x = torch.cat([x, time_dim_concat], dim=2)
- t_len = ((x.shape[2] + (patch_size[0] // 2)) // patch_size[0])
+ t_len = x.shape[2]
- img_ids = torch.zeros((t_len, h_len, w_len, 3), device=x.device, dtype=x.dtype)
- img_ids[:, :, :, 0] = img_ids[:, :, :, 0] + torch.linspace(0, t_len - 1, steps=t_len, device=x.device, dtype=x.dtype).reshape(-1, 1, 1)
- img_ids[:, :, :, 1] = img_ids[:, :, :, 1] + torch.linspace(0, h_len - 1, steps=h_len, device=x.device, dtype=x.dtype).reshape(1, -1, 1)
- img_ids[:, :, :, 2] = img_ids[:, :, :, 2] + torch.linspace(0, w_len - 1, steps=w_len, device=x.device, dtype=x.dtype).reshape(1, 1, -1)
- img_ids = repeat(img_ids, "t h w c -> b (t h w) c", b=bs)
+ if self.ref_conv is not None and "reference_latent" in kwargs:
+ t_len += 1
- freqs = self.rope_embedder(img_ids).movedim(1, 2)
+ freqs = self.rope_encode(t_len, h, w, device=x.device, dtype=x.dtype)
return self.forward_orig(x, timestep, context, clip_fea=clip_fea, freqs=freqs, transformer_options=transformer_options, **kwargs)[:, :, :t, :h, :w]
def unpatchify(self, x, grid_sizes):
@@ -738,7 +791,12 @@ class CameraWanModel(WanModel):
operations=None,
):
- super().__init__(model_type='i2v', patch_size=patch_size, text_len=text_len, in_dim=in_dim, dim=dim, ffn_dim=ffn_dim, freq_dim=freq_dim, text_dim=text_dim, out_dim=out_dim, num_heads=num_heads, num_layers=num_layers, window_size=window_size, qk_norm=qk_norm, cross_attn_norm=cross_attn_norm, eps=eps, flf_pos_embed_token_number=flf_pos_embed_token_number, image_model=image_model, device=device, dtype=dtype, operations=operations)
+ if model_type == 'camera':
+ model_type = 'i2v'
+ else:
+ model_type = 't2v'
+
+ super().__init__(model_type=model_type, patch_size=patch_size, text_len=text_len, in_dim=in_dim, dim=dim, ffn_dim=ffn_dim, freq_dim=freq_dim, text_dim=text_dim, out_dim=out_dim, num_heads=num_heads, num_layers=num_layers, window_size=window_size, qk_norm=qk_norm, cross_attn_norm=cross_attn_norm, eps=eps, flf_pos_embed_token_number=flf_pos_embed_token_number, image_model=image_model, device=device, dtype=dtype, operations=operations)
operation_settings = {"operations": operations, "device": device, "dtype": dtype}
self.control_adapter = WanCamAdapter(in_dim_control_adapter, dim, kernel_size=patch_size[1:], stride=patch_size[1:], operation_settings=operation_settings)
@@ -758,8 +816,7 @@ class CameraWanModel(WanModel):
# embeddings
x = self.patch_embedding(x.float()).to(x.dtype)
if self.control_adapter is not None and camera_conditions is not None:
- x_camera = self.control_adapter(camera_conditions).to(x.dtype)
- x = x + x_camera
+ x = x + self.control_adapter(camera_conditions).to(x.dtype)
grid_sizes = x.shape[2:]
x = x.flatten(2).transpose(1, 2)
@@ -797,3 +854,468 @@ class CameraWanModel(WanModel):
# unpatchify
x = self.unpatchify(x, grid_sizes)
return x
+
+
+class CausalConv1d(nn.Module):
+
+ def __init__(self,
+ chan_in,
+ chan_out,
+ kernel_size=3,
+ stride=1,
+ dilation=1,
+ pad_mode='replicate',
+ operations=None,
+ **kwargs):
+ super().__init__()
+
+ self.pad_mode = pad_mode
+ padding = (kernel_size - 1, 0) # T
+ self.time_causal_padding = padding
+
+ self.conv = operations.Conv1d(
+ chan_in,
+ chan_out,
+ kernel_size,
+ stride=stride,
+ dilation=dilation,
+ **kwargs)
+
+ def forward(self, x):
+ x = torch.nn.functional.pad(x, self.time_causal_padding, mode=self.pad_mode)
+ return self.conv(x)
+
+
+class MotionEncoder_tc(nn.Module):
+
+ def __init__(self,
+ in_dim: int,
+ hidden_dim: int,
+ num_heads=int,
+ need_global=True,
+ dtype=None,
+ device=None,
+ operations=None,):
+ factory_kwargs = {"dtype": dtype, "device": device}
+ super().__init__()
+
+ self.num_heads = num_heads
+ self.need_global = need_global
+ self.conv1_local = CausalConv1d(in_dim, hidden_dim // 4 * num_heads, 3, stride=1, operations=operations, **factory_kwargs)
+ if need_global:
+ self.conv1_global = CausalConv1d(
+ in_dim, hidden_dim // 4, 3, stride=1, operations=operations, **factory_kwargs)
+ self.norm1 = operations.LayerNorm(
+ hidden_dim // 4,
+ elementwise_affine=False,
+ eps=1e-6,
+ **factory_kwargs)
+ self.act = nn.SiLU()
+ self.conv2 = CausalConv1d(hidden_dim // 4, hidden_dim // 2, 3, stride=2, operations=operations, **factory_kwargs)
+ self.conv3 = CausalConv1d(hidden_dim // 2, hidden_dim, 3, stride=2, operations=operations, **factory_kwargs)
+
+ if need_global:
+ self.final_linear = operations.Linear(hidden_dim, hidden_dim, **factory_kwargs)
+
+ self.norm1 = operations.LayerNorm(
+ hidden_dim // 4,
+ elementwise_affine=False,
+ eps=1e-6,
+ **factory_kwargs)
+
+ self.norm2 = operations.LayerNorm(
+ hidden_dim // 2,
+ elementwise_affine=False,
+ eps=1e-6,
+ **factory_kwargs)
+
+ self.norm3 = operations.LayerNorm(
+ hidden_dim, elementwise_affine=False, eps=1e-6, **factory_kwargs)
+
+ self.padding_tokens = nn.Parameter(torch.empty(1, 1, 1, hidden_dim, **factory_kwargs))
+
+ def forward(self, x):
+ x = rearrange(x, 'b t c -> b c t')
+ x_ori = x.clone()
+ b, c, t = x.shape
+ x = self.conv1_local(x)
+ x = rearrange(x, 'b (n c) t -> (b n) t c', n=self.num_heads)
+ x = self.norm1(x)
+ x = self.act(x)
+ x = rearrange(x, 'b t c -> b c t')
+ x = self.conv2(x)
+ x = rearrange(x, 'b c t -> b t c')
+ x = self.norm2(x)
+ x = self.act(x)
+ x = rearrange(x, 'b t c -> b c t')
+ x = self.conv3(x)
+ x = rearrange(x, 'b c t -> b t c')
+ x = self.norm3(x)
+ x = self.act(x)
+ x = rearrange(x, '(b n) t c -> b t n c', b=b)
+ padding = comfy.model_management.cast_to(self.padding_tokens, dtype=x.dtype, device=x.device).repeat(b, x.shape[1], 1, 1)
+ x = torch.cat([x, padding], dim=-2)
+ x_local = x.clone()
+
+ if not self.need_global:
+ return x_local
+
+ x = self.conv1_global(x_ori)
+ x = rearrange(x, 'b c t -> b t c')
+ x = self.norm1(x)
+ x = self.act(x)
+ x = rearrange(x, 'b t c -> b c t')
+ x = self.conv2(x)
+ x = rearrange(x, 'b c t -> b t c')
+ x = self.norm2(x)
+ x = self.act(x)
+ x = rearrange(x, 'b t c -> b c t')
+ x = self.conv3(x)
+ x = rearrange(x, 'b c t -> b t c')
+ x = self.norm3(x)
+ x = self.act(x)
+ x = self.final_linear(x)
+ x = rearrange(x, '(b n) t c -> b t n c', b=b)
+
+ return x, x_local
+
+
+class CausalAudioEncoder(nn.Module):
+
+ def __init__(self,
+ dim=5120,
+ num_layers=25,
+ out_dim=2048,
+ video_rate=8,
+ num_token=4,
+ need_global=False,
+ dtype=None,
+ device=None,
+ operations=None):
+ super().__init__()
+ self.encoder = MotionEncoder_tc(
+ in_dim=dim,
+ hidden_dim=out_dim,
+ num_heads=num_token,
+ need_global=need_global, dtype=dtype, device=device, operations=operations)
+ weight = torch.empty((1, num_layers, 1, 1), dtype=dtype, device=device)
+
+ self.weights = torch.nn.Parameter(weight)
+ self.act = torch.nn.SiLU()
+
+ def forward(self, features):
+ # features B * num_layers * dim * video_length
+ weights = self.act(comfy.model_management.cast_to(self.weights, dtype=features.dtype, device=features.device))
+ weights_sum = weights.sum(dim=1, keepdims=True)
+ weighted_feat = ((features * weights) / weights_sum).sum(
+ dim=1) # b dim f
+ weighted_feat = weighted_feat.permute(0, 2, 1) # b f dim
+ res = self.encoder(weighted_feat) # b f n dim
+ return res # b f n dim
+
+
+class AdaLayerNorm(nn.Module):
+ def __init__(self, embedding_dim, output_dim=None, norm_elementwise_affine=False, norm_eps=1e-5, dtype=None, device=None, operations=None):
+ super().__init__()
+
+ output_dim = output_dim or embedding_dim * 2
+
+ self.silu = nn.SiLU()
+ self.linear = operations.Linear(embedding_dim, output_dim, dtype=dtype, device=device)
+ self.norm = operations.LayerNorm(output_dim // 2, norm_eps, norm_elementwise_affine, dtype=dtype, device=device)
+
+ def forward(self, x, temb):
+ temb = self.linear(self.silu(temb))
+ shift, scale = temb.chunk(2, dim=1)
+ shift = shift[:, None, :]
+ scale = scale[:, None, :]
+ x = self.norm(x) * (1 + scale) + shift
+ return x
+
+
+class AudioInjector_WAN(nn.Module):
+
+ def __init__(self,
+ dim=2048,
+ num_heads=32,
+ inject_layer=[0, 27],
+ root_net=None,
+ enable_adain=False,
+ adain_dim=2048,
+ adain_mode=None,
+ dtype=None,
+ device=None,
+ operations=None):
+ super().__init__()
+ self.enable_adain = enable_adain
+ self.adain_mode = adain_mode
+ self.injected_block_id = {}
+ audio_injector_id = 0
+ for inject_id in inject_layer:
+ self.injected_block_id[inject_id] = audio_injector_id
+ audio_injector_id += 1
+
+ self.injector = nn.ModuleList([
+ WanT2VCrossAttention(
+ dim=dim,
+ num_heads=num_heads,
+ qk_norm=True, operation_settings={"operations": operations, "device": device, "dtype": dtype}
+ ) for _ in range(audio_injector_id)
+ ])
+ self.injector_pre_norm_feat = nn.ModuleList([
+ operations.LayerNorm(
+ dim,
+ elementwise_affine=False,
+ eps=1e-6, dtype=dtype, device=device
+ ) for _ in range(audio_injector_id)
+ ])
+ self.injector_pre_norm_vec = nn.ModuleList([
+ operations.LayerNorm(
+ dim,
+ elementwise_affine=False,
+ eps=1e-6, dtype=dtype, device=device
+ ) for _ in range(audio_injector_id)
+ ])
+ if enable_adain:
+ self.injector_adain_layers = nn.ModuleList([
+ AdaLayerNorm(
+ output_dim=dim * 2, embedding_dim=adain_dim, dtype=dtype, device=device, operations=operations)
+ for _ in range(audio_injector_id)
+ ])
+ if adain_mode != "attn_norm":
+ self.injector_adain_output_layers = nn.ModuleList(
+ [operations.Linear(dim, dim, dtype=dtype, device=device) for _ in range(audio_injector_id)])
+
+ def forward(self, x, block_id, audio_emb, audio_emb_global, seq_len):
+ audio_attn_id = self.injected_block_id.get(block_id, None)
+ if audio_attn_id is None:
+ return x
+
+ num_frames = audio_emb.shape[1]
+ input_hidden_states = rearrange(x[:, :seq_len], "b (t n) c -> (b t) n c", t=num_frames)
+ if self.enable_adain and self.adain_mode == "attn_norm":
+ audio_emb_global = rearrange(audio_emb_global, "b t n c -> (b t) n c")
+ adain_hidden_states = self.injector_adain_layers[audio_attn_id](input_hidden_states, temb=audio_emb_global[:, 0])
+ attn_hidden_states = adain_hidden_states
+ else:
+ attn_hidden_states = self.injector_pre_norm_feat[audio_attn_id](input_hidden_states)
+ audio_emb = rearrange(audio_emb, "b t n c -> (b t) n c", t=num_frames)
+ attn_audio_emb = audio_emb
+ residual_out = self.injector[audio_attn_id](x=attn_hidden_states, context=attn_audio_emb)
+ residual_out = rearrange(
+ residual_out, "(b t) n c -> b (t n) c", t=num_frames)
+ x[:, :seq_len] = x[:, :seq_len] + residual_out
+ return x
+
+
+class FramePackMotioner(nn.Module):
+ def __init__(
+ self,
+ inner_dim=1024,
+ num_heads=16, # Used to indicate the number of heads in the backbone network; unrelated to this module's design
+ zip_frame_buckets=[
+ 1, 2, 16
+ ], # Three numbers representing the number of frames sampled for patch operations from the nearest to the farthest frames
+ drop_mode="drop", # If not "drop", it will use "padd", meaning padding instead of deletion
+ dtype=None,
+ device=None,
+ operations=None):
+ super().__init__()
+ self.proj = operations.Conv3d(16, inner_dim, kernel_size=(1, 2, 2), stride=(1, 2, 2), dtype=dtype, device=device)
+ self.proj_2x = operations.Conv3d(16, inner_dim, kernel_size=(2, 4, 4), stride=(2, 4, 4), dtype=dtype, device=device)
+ self.proj_4x = operations.Conv3d(16, inner_dim, kernel_size=(4, 8, 8), stride=(4, 8, 8), dtype=dtype, device=device)
+ self.zip_frame_buckets = zip_frame_buckets
+
+ self.inner_dim = inner_dim
+ self.num_heads = num_heads
+
+ self.drop_mode = drop_mode
+
+ def forward(self, motion_latents, rope_embedder, add_last_motion=2):
+ lat_height, lat_width = motion_latents.shape[3], motion_latents.shape[4]
+ padd_lat = torch.zeros(motion_latents.shape[0], 16, sum(self.zip_frame_buckets), lat_height, lat_width).to(device=motion_latents.device, dtype=motion_latents.dtype)
+ overlap_frame = min(padd_lat.shape[2], motion_latents.shape[2])
+ if overlap_frame > 0:
+ padd_lat[:, :, -overlap_frame:] = motion_latents[:, :, -overlap_frame:]
+
+ if add_last_motion < 2 and self.drop_mode != "drop":
+ zero_end_frame = sum(self.zip_frame_buckets[:len(self.zip_frame_buckets) - add_last_motion - 1])
+ padd_lat[:, :, -zero_end_frame:] = 0
+
+ clean_latents_4x, clean_latents_2x, clean_latents_post = padd_lat[:, :, -sum(self.zip_frame_buckets):, :, :].split(self.zip_frame_buckets[::-1], dim=2) # 16, 2 ,1
+
+ # patchfy
+ clean_latents_post = self.proj(clean_latents_post).flatten(2).transpose(1, 2)
+ clean_latents_2x = self.proj_2x(clean_latents_2x)
+ l_2x_shape = clean_latents_2x.shape
+ clean_latents_2x = clean_latents_2x.flatten(2).transpose(1, 2)
+ clean_latents_4x = self.proj_4x(clean_latents_4x)
+ l_4x_shape = clean_latents_4x.shape
+ clean_latents_4x = clean_latents_4x.flatten(2).transpose(1, 2)
+
+ if add_last_motion < 2 and self.drop_mode == "drop":
+ clean_latents_post = clean_latents_post[:, :
+ 0] if add_last_motion < 2 else clean_latents_post
+ clean_latents_2x = clean_latents_2x[:, :
+ 0] if add_last_motion < 1 else clean_latents_2x
+
+ motion_lat = torch.cat([clean_latents_post, clean_latents_2x, clean_latents_4x], dim=1)
+
+ rope_post = rope_embedder.rope_encode(1, lat_height, lat_width, t_start=-1, device=motion_latents.device, dtype=motion_latents.dtype)
+ rope_2x = rope_embedder.rope_encode(1, lat_height, lat_width, t_start=-3, steps_h=l_2x_shape[-2], steps_w=l_2x_shape[-1], device=motion_latents.device, dtype=motion_latents.dtype)
+ rope_4x = rope_embedder.rope_encode(4, lat_height, lat_width, t_start=-19, steps_h=l_4x_shape[-2], steps_w=l_4x_shape[-1], device=motion_latents.device, dtype=motion_latents.dtype)
+
+ rope = torch.cat([rope_post, rope_2x, rope_4x], dim=1)
+ return motion_lat, rope
+
+
+class WanModel_S2V(WanModel):
+ def __init__(self,
+ model_type='s2v',
+ patch_size=(1, 2, 2),
+ text_len=512,
+ in_dim=16,
+ dim=2048,
+ ffn_dim=8192,
+ freq_dim=256,
+ text_dim=4096,
+ out_dim=16,
+ num_heads=16,
+ num_layers=32,
+ window_size=(-1, -1),
+ qk_norm=True,
+ cross_attn_norm=True,
+ eps=1e-6,
+ audio_dim=1024,
+ num_audio_token=4,
+ enable_adain=True,
+ cond_dim=16,
+ audio_inject_layers=[0, 4, 8, 12, 16, 20, 24, 27, 30, 33, 36, 39],
+ adain_mode="attn_norm",
+ framepack_drop_mode="padd",
+ image_model=None,
+ device=None,
+ dtype=None,
+ operations=None,
+ ):
+
+ super().__init__(model_type='t2v', patch_size=patch_size, text_len=text_len, in_dim=in_dim, dim=dim, ffn_dim=ffn_dim, freq_dim=freq_dim, text_dim=text_dim, out_dim=out_dim, num_heads=num_heads, num_layers=num_layers, window_size=window_size, qk_norm=qk_norm, cross_attn_norm=cross_attn_norm, eps=eps, image_model=image_model, device=device, dtype=dtype, operations=operations)
+
+ self.trainable_cond_mask = operations.Embedding(3, self.dim, device=device, dtype=dtype)
+
+ self.casual_audio_encoder = CausalAudioEncoder(
+ dim=audio_dim,
+ out_dim=self.dim,
+ num_token=num_audio_token,
+ need_global=enable_adain, dtype=dtype, device=device, operations=operations)
+
+ if cond_dim > 0:
+ self.cond_encoder = operations.Conv3d(
+ cond_dim,
+ self.dim,
+ kernel_size=self.patch_size,
+ stride=self.patch_size, device=device, dtype=dtype)
+
+ self.audio_injector = AudioInjector_WAN(
+ dim=self.dim,
+ num_heads=self.num_heads,
+ inject_layer=audio_inject_layers,
+ root_net=self,
+ enable_adain=enable_adain,
+ adain_dim=self.dim,
+ adain_mode=adain_mode,
+ dtype=dtype, device=device, operations=operations
+ )
+
+ self.frame_packer = FramePackMotioner(
+ inner_dim=self.dim,
+ num_heads=self.num_heads,
+ zip_frame_buckets=[1, 2, 16],
+ drop_mode=framepack_drop_mode,
+ dtype=dtype, device=device, operations=operations)
+
+ def forward_orig(
+ self,
+ x,
+ t,
+ context,
+ audio_embed=None,
+ reference_latent=None,
+ control_video=None,
+ reference_motion=None,
+ clip_fea=None,
+ freqs=None,
+ transformer_options={},
+ **kwargs,
+ ):
+ if audio_embed is not None:
+ num_embeds = x.shape[-3] * 4
+ audio_emb_global, audio_emb = self.casual_audio_encoder(audio_embed[:, :, :, :num_embeds])
+ else:
+ audio_emb = None
+
+ # embeddings
+ bs, _, time, height, width = x.shape
+ x = self.patch_embedding(x.float()).to(x.dtype)
+ if control_video is not None:
+ x = x + self.cond_encoder(control_video)
+
+ if t.ndim == 1:
+ t = t.unsqueeze(1).repeat(1, x.shape[2])
+
+ grid_sizes = x.shape[2:]
+ x = x.flatten(2).transpose(1, 2)
+ seq_len = x.size(1)
+
+ cond_mask_weight = comfy.model_management.cast_to(self.trainable_cond_mask.weight, dtype=x.dtype, device=x.device).unsqueeze(1).unsqueeze(1)
+ x = x + cond_mask_weight[0]
+
+ if reference_latent is not None:
+ ref = self.patch_embedding(reference_latent.float()).to(x.dtype)
+ ref = ref.flatten(2).transpose(1, 2)
+ freqs_ref = self.rope_encode(reference_latent.shape[-3], reference_latent.shape[-2], reference_latent.shape[-1], t_start=max(30, time + 9), device=x.device, dtype=x.dtype)
+ ref = ref + cond_mask_weight[1]
+ x = torch.cat([x, ref], dim=1)
+ freqs = torch.cat([freqs, freqs_ref], dim=1)
+ t = torch.cat([t, torch.zeros((t.shape[0], reference_latent.shape[-3]), device=t.device, dtype=t.dtype)], dim=1)
+ del ref, freqs_ref
+
+ if reference_motion is not None:
+ motion_encoded, freqs_motion = self.frame_packer(reference_motion, self)
+ motion_encoded = motion_encoded + cond_mask_weight[2]
+ x = torch.cat([x, motion_encoded], dim=1)
+ freqs = torch.cat([freqs, freqs_motion], dim=1)
+
+ t = torch.repeat_interleave(t, 2, dim=1)
+ t = torch.cat([t, torch.zeros((t.shape[0], 3), device=t.device, dtype=t.dtype)], dim=1)
+ del motion_encoded, freqs_motion
+
+ # time embeddings
+ e = self.time_embedding(
+ sinusoidal_embedding_1d(self.freq_dim, t.flatten()).to(dtype=x[0].dtype))
+ e = e.reshape(t.shape[0], -1, e.shape[-1])
+ e0 = self.time_projection(e).unflatten(2, (6, self.dim))
+
+ # context
+ context = self.text_embedding(context)
+
+ patches_replace = transformer_options.get("patches_replace", {})
+ blocks_replace = patches_replace.get("dit", {})
+ for i, block in enumerate(self.blocks):
+ if ("double_block", i) in blocks_replace:
+ def block_wrap(args):
+ out = {}
+ out["img"] = block(args["img"], context=args["txt"], e=args["vec"], freqs=args["pe"])
+ return out
+ out = blocks_replace[("double_block", i)]({"img": x, "txt": context, "vec": e0, "pe": freqs}, {"original_block": block_wrap})
+ x = out["img"]
+ else:
+ x = block(x, e=e0, freqs=freqs, context=context)
+ if audio_emb is not None:
+ x = self.audio_injector(x, i, audio_emb, audio_emb_global, seq_len)
+ # head
+ x = self.head(x, e)
+
+ # unpatchify
+ x = self.unpatchify(x, grid_sizes)
+ return x
diff --git a/comfy/ldm/wan/vae.py b/comfy/ldm/wan/vae.py
index 6b07840fc..791596938 100644
--- a/comfy/ldm/wan/vae.py
+++ b/comfy/ldm/wan/vae.py
@@ -24,12 +24,17 @@ class CausalConv3d(ops.Conv3d):
self.padding[1], 2 * self.padding[0], 0)
self.padding = (0, 0, 0)
- def forward(self, x, cache_x=None):
+ def forward(self, x, cache_x=None, cache_list=None, cache_idx=None):
+ if cache_list is not None:
+ cache_x = cache_list[cache_idx]
+ cache_list[cache_idx] = None
+
padding = list(self._padding)
if cache_x is not None and self._padding[4] > 0:
cache_x = cache_x.to(x.device)
x = torch.cat([cache_x, x], dim=2)
padding[4] -= cache_x.shape[2]
+ del cache_x
x = F.pad(x, padding)
return super().forward(x)
@@ -166,7 +171,7 @@ class ResidualBlock(nn.Module):
if in_dim != out_dim else nn.Identity()
def forward(self, x, feat_cache=None, feat_idx=[0]):
- h = self.shortcut(x)
+ old_x = x
for layer in self.residual:
if isinstance(layer, CausalConv3d) and feat_cache is not None:
idx = feat_idx[0]
@@ -178,12 +183,12 @@ class ResidualBlock(nn.Module):
cache_x.device), cache_x
],
dim=2)
- x = layer(x, feat_cache[idx])
+ x = layer(x, cache_list=feat_cache, cache_idx=idx)
feat_cache[idx] = cache_x
feat_idx[0] += 1
else:
x = layer(x)
- return x + h
+ return x + self.shortcut(old_x)
class AttentionBlock(nn.Module):
diff --git a/comfy/ldm/wan/vae2_2.py b/comfy/ldm/wan/vae2_2.py
index b9c2d1a26..1f6d584a2 100644
--- a/comfy/ldm/wan/vae2_2.py
+++ b/comfy/ldm/wan/vae2_2.py
@@ -151,7 +151,7 @@ class ResidualBlock(nn.Module):
],
dim=2,
)
- x = layer(x, feat_cache[idx])
+ x = layer(x, cache_list=feat_cache, cache_idx=idx)
feat_cache[idx] = cache_x
feat_idx[0] += 1
else:
diff --git a/comfy/lora.py b/comfy/lora.py
index 387d5c52a..00358884b 100644
--- a/comfy/lora.py
+++ b/comfy/lora.py
@@ -293,6 +293,16 @@ def model_lora_keys_unet(model, key_map={}):
key_lora = k[len("diffusion_model."):-len(".weight")]
key_map["{}".format(key_lora)] = k
+ if isinstance(model, comfy.model_base.QwenImage):
+ for k in sdk:
+ if k.startswith("diffusion_model.") and k.endswith(".weight"): #QwenImage lora format
+ key_lora = k[len("diffusion_model."):-len(".weight")]
+ # Direct mapping for transformer_blocks format (QwenImage LoRA format)
+ key_map["{}".format(key_lora)] = k
+ # Support transformer prefix format
+ key_map["transformer.{}".format(key_lora)] = k
+ key_map["lycoris_{}".format(key_lora.replace(".", "_"))] = k #SimpleTuner lycoris format
+
return key_map
diff --git a/comfy/model_base.py b/comfy/model_base.py
index 8e2cd72a9..39a3344bc 100644
--- a/comfy/model_base.py
+++ b/comfy/model_base.py
@@ -44,6 +44,7 @@ import comfy.ldm.hidream.model
import comfy.ldm.chroma.model
import comfy.ldm.ace.model
import comfy.ldm.omnigen.omnigen2
+import comfy.ldm.qwen_image.model
import comfy.model_management
import comfy.patcher_extension
@@ -108,10 +109,12 @@ def model_sampling(model_config, model_type):
return ModelSampling(model_config)
-def convert_tensor(extra, dtype):
+def convert_tensor(extra, dtype, device):
if hasattr(extra, "dtype"):
if extra.dtype != torch.int and extra.dtype != torch.long:
- extra = extra.to(dtype)
+ extra = comfy.model_management.cast_to_device(extra, device, dtype)
+ else:
+ extra = comfy.model_management.cast_to_device(extra, device, None)
return extra
@@ -149,6 +152,7 @@ class BaseModel(torch.nn.Module):
logging.debug("adm {}".format(self.adm_channels))
self.memory_usage_factor = model_config.memory_usage_factor
self.memory_usage_factor_conds = ()
+ self.memory_usage_shape_process = {}
def apply_model(self, x, t, c_concat=None, c_crossattn=None, control=None, transformer_options={}, **kwargs):
return comfy.patcher_extension.WrapperExecutor.new_class_executor(
@@ -162,7 +166,7 @@ class BaseModel(torch.nn.Module):
xc = self.model_sampling.calculate_input(sigma, x)
if c_concat is not None:
- xc = torch.cat([xc] + [c_concat], dim=1)
+ xc = torch.cat([xc] + [comfy.model_management.cast_to_device(c_concat, xc.device, xc.dtype)], dim=1)
context = c_crossattn
dtype = self.get_dtype()
@@ -171,20 +175,21 @@ class BaseModel(torch.nn.Module):
dtype = self.manual_cast_dtype
xc = xc.to(dtype)
+ device = xc.device
t = self.model_sampling.timestep(t).float()
if context is not None:
- context = context.to(dtype)
+ context = comfy.model_management.cast_to_device(context, device, dtype)
extra_conds = {}
for o in kwargs:
extra = kwargs[o]
if hasattr(extra, "dtype"):
- extra = convert_tensor(extra, dtype)
+ extra = convert_tensor(extra, dtype, device)
elif isinstance(extra, list):
ex = []
for ext in extra:
- ex.append(convert_tensor(ext, dtype))
+ ex.append(convert_tensor(ext, dtype, device))
extra = ex
extra_conds[o] = extra
@@ -348,8 +353,15 @@ class BaseModel(torch.nn.Module):
input_shapes = [input_shape]
for c in self.memory_usage_factor_conds:
shape = cond_shapes.get(c, None)
- if shape is not None and len(shape) > 0:
- input_shapes += shape
+ if shape is not None:
+ if c in self.memory_usage_shape_process:
+ out = []
+ for s in shape:
+ out.append(self.memory_usage_shape_process[c](s))
+ shape = out
+
+ if len(shape) > 0:
+ input_shapes += shape
if comfy.model_management.xformers_enabled() or comfy.model_management.pytorch_attention_flash_attention():
dtype = self.get_dtype()
@@ -400,7 +412,7 @@ class SD21UNCLIP(BaseModel):
unclip_conditioning = kwargs.get("unclip_conditioning", None)
device = kwargs["device"]
if unclip_conditioning is None:
- return torch.zeros((1, self.adm_channels))
+ return torch.zeros((1, self.adm_channels), device=device)
else:
return unclip_adm(unclip_conditioning, device, self.noise_augmentor, kwargs.get("unclip_noise_augment_merge", 0.05), kwargs.get("seed", 0) - 10)
@@ -614,9 +626,11 @@ class IP2P:
if image is None:
image = torch.zeros_like(noise)
+ else:
+ image = image.to(device=device)
if image.shape[1:] != noise.shape[1:]:
- image = utils.common_upscale(image.to(device), noise.shape[-1], noise.shape[-2], "bilinear", "center")
+ image = utils.common_upscale(image, noise.shape[-1], noise.shape[-2], "bilinear", "center")
image = utils.resize_to_batch_size(image, noise.shape[0])
return self.process_ip2p_image_in(image)
@@ -695,7 +709,7 @@ class StableCascade_B(BaseModel):
#size of prior doesn't really matter if zeros because it gets resized but I still want it to get batched
prior = kwargs.get("stable_cascade_prior", torch.zeros((1, 16, (noise.shape[2] * 4) // 42, (noise.shape[3] * 4) // 42), dtype=noise.dtype, layout=noise.layout, device=noise.device))
- out["effnet"] = comfy.conds.CONDRegular(prior)
+ out["effnet"] = comfy.conds.CONDRegular(prior.to(device=noise.device))
out["sca"] = comfy.conds.CONDRegular(torch.zeros((1,)))
return out
@@ -886,6 +900,10 @@ class Flux(BaseModel):
for lat in ref_latents:
latents.append(self.process_latent_in(lat))
out['ref_latents'] = comfy.conds.CONDList(latents)
+
+ ref_latents_method = kwargs.get("reference_latents_method", None)
+ if ref_latents_method is not None:
+ out['ref_latents_method'] = comfy.conds.CONDConstant(ref_latents_method)
return out
def extra_conds_shapes(self, **kwargs):
@@ -1094,9 +1112,10 @@ class WAN21(BaseModel):
shape_image[1] = extra_channels
image = torch.zeros(shape_image, dtype=noise.dtype, layout=noise.layout, device=noise.device)
else:
+ latent_dim = self.latent_format.latent_channels
image = utils.common_upscale(image.to(device), noise.shape[-1], noise.shape[-2], "bilinear", "center")
- for i in range(0, image.shape[1], 16):
- image[:, i: i + 16] = self.process_latent_in(image[:, i: i + 16])
+ for i in range(0, image.shape[1], latent_dim):
+ image[:, i: i + latent_dim] = self.process_latent_in(image[:, i: i + latent_dim])
image = utils.resize_to_batch_size(image, noise.shape[0])
if extra_channels != image.shape[1] + 4:
@@ -1120,7 +1139,11 @@ class WAN21(BaseModel):
mask = mask.repeat(1, 4, 1, 1, 1)
mask = utils.resize_to_batch_size(mask, noise.shape[0])
- return torch.cat((mask, image), dim=1)
+ concat_mask_index = kwargs.get("concat_mask_index", 0)
+ if concat_mask_index != 0:
+ return torch.cat((image[:, :concat_mask_index], mask, image[:, concat_mask_index:]), dim=1)
+ else:
+ return torch.cat((mask, image), dim=1)
def extra_conds(self, **kwargs):
out = super().extra_conds(**kwargs)
@@ -1136,6 +1159,10 @@ class WAN21(BaseModel):
if time_dim_concat is not None:
out['time_dim_concat'] = comfy.conds.CONDRegular(self.process_latent_in(time_dim_concat))
+ reference_latents = kwargs.get("reference_latents", None)
+ if reference_latents is not None:
+ out['reference_latent'] = comfy.conds.CONDRegular(self.process_latent_in(reference_latents[-1])[:, :, 0])
+
return out
@@ -1160,10 +1187,10 @@ class WAN21_Vace(WAN21):
vace_frames_out = []
for j in range(len(vace_frames)):
- vf = vace_frames[j].clone()
+ vf = vace_frames[j].to(device=noise.device, dtype=noise.dtype, copy=True)
for i in range(0, vf.shape[1], 16):
vf[:, i:i + 16] = self.process_latent_in(vf[:, i:i + 16])
- vf = torch.cat([vf, mask[j]], dim=1)
+ vf = torch.cat([vf, mask[j].to(device=noise.device, dtype=noise.dtype)], dim=1)
vace_frames_out.append(vf)
vace_frames = torch.stack(vace_frames_out, dim=1)
@@ -1185,18 +1212,50 @@ class WAN21_Camera(WAN21):
out['camera_conditions'] = comfy.conds.CONDRegular(camera_conditions)
return out
-class WAN22(BaseModel):
+class WAN22_S2V(WAN21):
+ def __init__(self, model_config, model_type=ModelType.FLOW, device=None):
+ super(WAN21, self).__init__(model_config, model_type, device=device, unet_model=comfy.ldm.wan.model.WanModel_S2V)
+ self.memory_usage_factor_conds = ("reference_latent", "reference_motion")
+ self.memory_usage_shape_process = {"reference_motion": lambda shape: [shape[0], shape[1], 1.5, shape[-2], shape[-1]]}
+
+ def extra_conds(self, **kwargs):
+ out = super().extra_conds(**kwargs)
+ audio_embed = kwargs.get("audio_embed", None)
+ if audio_embed is not None:
+ out['audio_embed'] = comfy.conds.CONDRegular(audio_embed)
+
+ reference_latents = kwargs.get("reference_latents", None)
+ if reference_latents is not None:
+ out['reference_latent'] = comfy.conds.CONDRegular(self.process_latent_in(reference_latents[-1]))
+
+ reference_motion = kwargs.get("reference_motion", None)
+ if reference_motion is not None:
+ out['reference_motion'] = comfy.conds.CONDRegular(self.process_latent_in(reference_motion))
+
+ control_video = kwargs.get("control_video", None)
+ if control_video is not None:
+ out['control_video'] = comfy.conds.CONDRegular(self.process_latent_in(control_video))
+ return out
+
+ def extra_conds_shapes(self, **kwargs):
+ out = {}
+ ref_latents = kwargs.get("reference_latents", None)
+ if ref_latents is not None:
+ out['reference_latent'] = list([1, 16, sum(map(lambda a: math.prod(a.size()), ref_latents)) // 16])
+
+ reference_motion = kwargs.get("reference_motion", None)
+ if reference_motion is not None:
+ out['reference_motion'] = reference_motion.shape
+ return out
+
+class WAN22(WAN21):
def __init__(self, model_config, model_type=ModelType.FLOW, image_to_video=False, device=None):
- super().__init__(model_config, model_type, device=device, unet_model=comfy.ldm.wan.model.WanModel)
+ super(WAN21, self).__init__(model_config, model_type, device=device, unet_model=comfy.ldm.wan.model.WanModel)
self.image_to_video = image_to_video
def extra_conds(self, **kwargs):
out = super().extra_conds(**kwargs)
- cross_attn = kwargs.get("cross_attn", None)
- if cross_attn is not None:
- out['c_crossattn'] = comfy.conds.CONDRegular(cross_attn)
-
- denoise_mask = kwargs.get("concat_mask", kwargs.get("denoise_mask", None))
+ denoise_mask = kwargs.get("denoise_mask", None)
if denoise_mask is not None:
out["denoise_mask"] = comfy.conds.CONDRegular(denoise_mask)
return out
@@ -1204,7 +1263,7 @@ class WAN22(BaseModel):
def process_timestep(self, timestep, x, denoise_mask=None, **kwargs):
if denoise_mask is None:
return timestep
- temp_ts = (torch.mean(denoise_mask[:, :, :, ::2, ::2], dim=1, keepdim=True) * timestep.view([timestep.shape[0]] + [1] * (denoise_mask.ndim - 1))).reshape(timestep.shape[0], -1)
+ temp_ts = (torch.mean(denoise_mask[:, :, :, :, :], dim=(1, 3, 4), keepdim=True) * timestep.view([timestep.shape[0]] + [1] * (denoise_mask.ndim - 1))).reshape(timestep.shape[0], -1)
return temp_ts
def scale_latent_inpaint(self, sigma, noise, latent_image, **kwargs):
@@ -1320,3 +1379,32 @@ class Omnigen2(BaseModel):
if ref_latents is not None:
out['ref_latents'] = list([1, 16, sum(map(lambda a: math.prod(a.size()), ref_latents)) // 16])
return out
+
+class QwenImage(BaseModel):
+ def __init__(self, model_config, model_type=ModelType.FLUX, device=None):
+ super().__init__(model_config, model_type, device=device, unet_model=comfy.ldm.qwen_image.model.QwenImageTransformer2DModel)
+ self.memory_usage_factor_conds = ("ref_latents",)
+
+ def extra_conds(self, **kwargs):
+ out = super().extra_conds(**kwargs)
+ cross_attn = kwargs.get("cross_attn", None)
+ if cross_attn is not None:
+ out['c_crossattn'] = comfy.conds.CONDRegular(cross_attn)
+ ref_latents = kwargs.get("reference_latents", None)
+ if ref_latents is not None:
+ latents = []
+ for lat in ref_latents:
+ latents.append(self.process_latent_in(lat))
+ out['ref_latents'] = comfy.conds.CONDList(latents)
+
+ ref_latents_method = kwargs.get("reference_latents_method", None)
+ if ref_latents_method is not None:
+ out['ref_latents_method'] = comfy.conds.CONDConstant(ref_latents_method)
+ return out
+
+ def extra_conds_shapes(self, **kwargs):
+ out = {}
+ ref_latents = kwargs.get("reference_latents", None)
+ if ref_latents is not None:
+ out['ref_latents'] = list([1, 16, sum(map(lambda a: math.prod(a.size()), ref_latents)) // 16])
+ return out
diff --git a/comfy/model_detection.py b/comfy/model_detection.py
index aca10b1af..75552ede9 100644
--- a/comfy/model_detection.py
+++ b/comfy/model_detection.py
@@ -364,7 +364,12 @@ def detect_unet_config(state_dict, key_prefix, metadata=None):
dit_config["vace_in_dim"] = state_dict['{}vace_patch_embedding.weight'.format(key_prefix)].shape[1]
dit_config["vace_layers"] = count_blocks(state_dict_keys, '{}vace_blocks.'.format(key_prefix) + '{}.')
elif '{}control_adapter.conv.weight'.format(key_prefix) in state_dict_keys:
- dit_config["model_type"] = "camera"
+ if '{}img_emb.proj.0.bias'.format(key_prefix) in state_dict_keys:
+ dit_config["model_type"] = "camera"
+ else:
+ dit_config["model_type"] = "camera_2.2"
+ elif '{}casual_audio_encoder.encoder.final_linear.weight'.format(key_prefix) in state_dict_keys:
+ dit_config["model_type"] = "s2v"
else:
if '{}img_emb.proj.0.bias'.format(key_prefix) in state_dict_keys:
dit_config["model_type"] = "i2v"
@@ -373,6 +378,11 @@ def detect_unet_config(state_dict, key_prefix, metadata=None):
flf_weight = state_dict.get('{}img_emb.emb_pos'.format(key_prefix))
if flf_weight is not None:
dit_config["flf_pos_embed_token_number"] = flf_weight.shape[1]
+
+ ref_conv_weight = state_dict.get('{}ref_conv.weight'.format(key_prefix))
+ if ref_conv_weight is not None:
+ dit_config["in_dim_ref_conv"] = ref_conv_weight.shape[1]
+
return dit_config
if '{}latent_in.weight'.format(key_prefix) in state_dict_keys: # Hunyuan 3D
@@ -495,6 +505,13 @@ def detect_unet_config(state_dict, key_prefix, metadata=None):
dit_config["timestep_scale"] = 1000.0
return dit_config
+ if '{}txt_norm.weight'.format(key_prefix) in state_dict_keys: # Qwen Image
+ dit_config = {}
+ dit_config["image_model"] = "qwen_image"
+ dit_config["in_channels"] = state_dict['{}img_in.weight'.format(key_prefix)].shape[1]
+ dit_config["num_layers"] = count_blocks(state_dict_keys, '{}transformer_blocks.'.format(key_prefix) + '{}.')
+ return dit_config
+
if '{}input_blocks.0.0.weight'.format(key_prefix) not in state_dict_keys:
return None
@@ -881,7 +898,7 @@ def convert_diffusers_mmdit(state_dict, output_prefix=""):
depth_single_blocks = count_blocks(state_dict, 'single_transformer_blocks.{}.')
hidden_size = state_dict["x_embedder.bias"].shape[0]
sd_map = comfy.utils.flux_to_diffusers({"depth": depth, "depth_single_blocks": depth_single_blocks, "hidden_size": hidden_size}, output_prefix=output_prefix)
- elif 'transformer_blocks.0.attn.add_q_proj.weight' in state_dict: #SD3
+ elif 'transformer_blocks.0.attn.add_q_proj.weight' in state_dict and 'pos_embed.proj.weight' in state_dict: #SD3
num_blocks = count_blocks(state_dict, 'transformer_blocks.{}.')
depth = state_dict["pos_embed.proj.weight"].shape[0] // 64
sd_map = comfy.utils.mmdit_to_diffusers({"depth": depth, "num_blocks": num_blocks}, output_prefix=output_prefix)
diff --git a/comfy/model_management.py b/comfy/model_management.py
index 9e6149d60..d08aee1fe 100644
--- a/comfy/model_management.py
+++ b/comfy/model_management.py
@@ -78,7 +78,6 @@ try:
torch_version = torch.version.__version__
temp = torch_version.split(".")
torch_version_numeric = (int(temp[0]), int(temp[1]))
- xpu_available = (torch_version_numeric[0] < 2 or (torch_version_numeric[0] == 2 and torch_version_numeric[1] <= 4)) and torch.xpu.is_available()
except:
pass
@@ -102,10 +101,14 @@ if args.directml is not None:
try:
import intel_extension_for_pytorch as ipex # noqa: F401
- _ = torch.xpu.device_count()
- xpu_available = xpu_available or torch.xpu.is_available()
except:
- xpu_available = xpu_available or (hasattr(torch, "xpu") and torch.xpu.is_available())
+ pass
+
+try:
+ _ = torch.xpu.device_count()
+ xpu_available = torch.xpu.is_available()
+except:
+ xpu_available = False
try:
if torch.backends.mps.is_available():
@@ -321,9 +324,9 @@ try:
if torch_version_numeric >= (2, 7): # works on 2.6 but doesn't actually seem to improve much
if any((a in arch) for a in ["gfx90a", "gfx942", "gfx1100", "gfx1101", "gfx1151"]): # TODO: more arches, TODO: gfx950
ENABLE_PYTORCH_ATTENTION = True
- if torch_version_numeric >= (2, 8):
- if any((a in arch) for a in ["gfx1201"]):
- ENABLE_PYTORCH_ATTENTION = True
+# if torch_version_numeric >= (2, 8):
+# if any((a in arch) for a in ["gfx1201"]):
+# ENABLE_PYTORCH_ATTENTION = True
if torch_version_numeric >= (2, 7) and rocm_version >= (6, 4):
if any((a in arch) for a in ["gfx1201", "gfx942", "gfx950"]): # TODO: more arches
SUPPORT_FP8_OPS = True
@@ -340,7 +343,7 @@ if ENABLE_PYTORCH_ATTENTION:
PRIORITIZE_FP16 = False # TODO: remove and replace with something that shows exactly which dtype is faster than the other
try:
- if is_nvidia() and PerformanceFeature.Fp16Accumulation in args.fast:
+ if (is_nvidia() or is_amd()) and PerformanceFeature.Fp16Accumulation in args.fast:
torch.backends.cuda.matmul.allow_fp16_accumulation = True
PRIORITIZE_FP16 = True # TODO: limit to cards where it actually boosts performance
logging.info("Enabled fp16 accumulation.")
@@ -590,7 +593,13 @@ def load_models_gpu(models, memory_required=0, force_patch_weights=False, minimu
else:
minimum_memory_required = max(inference_memory, minimum_memory_required + extra_reserved_memory())
- models = set(models)
+ models_temp = set()
+ for m in models:
+ models_temp.add(m)
+ for mm in m.model_patches_models():
+ models_temp.add(mm)
+
+ models = models_temp
models_to_load = []
@@ -946,10 +955,12 @@ def pick_weight_dtype(dtype, fallback_dtype, device=None):
return dtype
def device_supports_non_blocking(device):
+ if args.force_non_blocking:
+ return True
if is_device_mps(device):
return False #pytorch bug? mps doesn't support non blocking
- if is_intel_xpu():
- return True
+ if is_intel_xpu(): #xpu does support non blocking but it is slower on iGPUs for some reason so disable by default until situation changes
+ return False
if args.deterministic: #TODO: figure out why deterministic breaks non blocking from gpu to cpu (previews)
return False
if directml_enabled:
@@ -1282,10 +1293,10 @@ def should_use_bf16(device=None, model_params=0, prioritize_performance=True, ma
return False
if is_intel_xpu():
- if torch_version_numeric < (2, 6):
+ if torch_version_numeric < (2, 3):
return True
else:
- return torch.xpu.get_device_capability(device)['has_bfloat16_conversions']
+ return torch.xpu.is_bf16_supported()
if is_ascend_npu():
return True
diff --git a/comfy/model_patcher.py b/comfy/model_patcher.py
index 52e76b5f3..a944cb421 100644
--- a/comfy/model_patcher.py
+++ b/comfy/model_patcher.py
@@ -430,6 +430,9 @@ class ModelPatcher:
def set_model_forward_timestep_embed_patch(self, patch):
self.set_model_patch(patch, "forward_timestep_embed_patch")
+ def set_model_double_block_patch(self, patch):
+ self.set_model_patch(patch, "double_block")
+
def add_object_patch(self, name, obj):
self.object_patches[name] = obj
@@ -486,6 +489,30 @@ class ModelPatcher:
if hasattr(wrap_func, "to"):
self.model_options["model_function_wrapper"] = wrap_func.to(device)
+ def model_patches_models(self):
+ to = self.model_options["transformer_options"]
+ models = []
+ if "patches" in to:
+ patches = to["patches"]
+ for name in patches:
+ patch_list = patches[name]
+ for i in range(len(patch_list)):
+ if hasattr(patch_list[i], "models"):
+ models += patch_list[i].models()
+ if "patches_replace" in to:
+ patches = to["patches_replace"]
+ for name in patches:
+ patch_list = patches[name]
+ for k in patch_list:
+ if hasattr(patch_list[k], "models"):
+ models += patch_list[k].models()
+ if "model_function_wrapper" in self.model_options:
+ wrap_func = self.model_options["model_function_wrapper"]
+ if hasattr(wrap_func, "models"):
+ models += wrap_func.models()
+
+ return models
+
def model_dtype(self):
if hasattr(self.model, "get_dtype"):
return self.model.get_dtype()
diff --git a/comfy/ops.py b/comfy/ops.py
index 2cc9bbc27..18e7db705 100644
--- a/comfy/ops.py
+++ b/comfy/ops.py
@@ -24,6 +24,32 @@ import comfy.float
import comfy.rmsnorm
import contextlib
+
+def scaled_dot_product_attention(q, k, v, *args, **kwargs):
+ return torch.nn.functional.scaled_dot_product_attention(q, k, v, *args, **kwargs)
+
+
+try:
+ if torch.cuda.is_available():
+ from torch.nn.attention import SDPBackend, sdpa_kernel
+ import inspect
+ if "set_priority" in inspect.signature(sdpa_kernel).parameters:
+ SDPA_BACKEND_PRIORITY = [
+ SDPBackend.FLASH_ATTENTION,
+ SDPBackend.EFFICIENT_ATTENTION,
+ SDPBackend.MATH,
+ ]
+
+ SDPA_BACKEND_PRIORITY.insert(0, SDPBackend.CUDNN_ATTENTION)
+
+ def scaled_dot_product_attention(q, k, v, *args, **kwargs):
+ with sdpa_kernel(SDPA_BACKEND_PRIORITY, set_priority=True):
+ return torch.nn.functional.scaled_dot_product_attention(q, k, v, *args, **kwargs)
+ else:
+ logging.warning("Torch version too old to set sdpa backend priority.")
+except (ModuleNotFoundError, TypeError):
+ logging.warning("Could not set sdpa backend priority.")
+
cast_to = comfy.model_management.cast_to #TODO: remove once no more references
def cast_to_input(weight, input, non_blocking=False, copy=True):
diff --git a/comfy/patcher_extension.py b/comfy/patcher_extension.py
index 965958f4c..46cc7b2a8 100644
--- a/comfy/patcher_extension.py
+++ b/comfy/patcher_extension.py
@@ -50,6 +50,7 @@ class WrappersMP:
OUTER_SAMPLE = "outer_sample"
PREPARE_SAMPLING = "prepare_sampling"
SAMPLER_SAMPLE = "sampler_sample"
+ PREDICT_NOISE = "predict_noise"
CALC_COND_BATCH = "calc_cond_batch"
APPLY_MODEL = "apply_model"
DIFFUSION_MODEL = "diffusion_model"
diff --git a/comfy/rmsnorm.py b/comfy/rmsnorm.py
index 66ae8321d..555542a46 100644
--- a/comfy/rmsnorm.py
+++ b/comfy/rmsnorm.py
@@ -1,6 +1,7 @@
import torch
import comfy.model_management
import numbers
+import logging
RMSNorm = None
@@ -9,6 +10,7 @@ try:
RMSNorm = torch.nn.RMSNorm
except:
rms_norm_torch = None
+ logging.warning("Please update pytorch to use native RMSNorm")
def rms_norm(x, weight=None, eps=1e-6):
diff --git a/comfy/sampler_helpers.py b/comfy/sampler_helpers.py
index 8dbc41455..e46971afb 100644
--- a/comfy/sampler_helpers.py
+++ b/comfy/sampler_helpers.py
@@ -149,7 +149,7 @@ def cleanup_models(conds, models):
cleanup_additional_models(set(control_cleanup))
-def prepare_model_patcher(model: 'ModelPatcher', conds, model_options: dict):
+def prepare_model_patcher(model: ModelPatcher, conds, model_options: dict):
'''
Registers hooks from conds.
'''
@@ -158,8 +158,8 @@ def prepare_model_patcher(model: 'ModelPatcher', conds, model_options: dict):
for k in conds:
get_hooks_from_cond(conds[k], hooks)
# add wrappers and callbacks from ModelPatcher to transformer_options
- model_options["transformer_options"]["wrappers"] = comfy.patcher_extension.copy_nested_dicts(model.wrappers)
- model_options["transformer_options"]["callbacks"] = comfy.patcher_extension.copy_nested_dicts(model.callbacks)
+ comfy.patcher_extension.merge_nested_dicts(model_options["transformer_options"].setdefault("wrappers", {}), model.wrappers, copy_dict1=False)
+ comfy.patcher_extension.merge_nested_dicts(model_options["transformer_options"].setdefault("callbacks", {}), model.callbacks, copy_dict1=False)
# begin registering hooks
registered = comfy.hooks.HookGroup()
target_dict = comfy.hooks.create_target_dict(comfy.hooks.EnumWeightTarget.Model)
diff --git a/comfy/samplers.py b/comfy/samplers.py
old mode 100644
new mode 100755
index e93d2a315..b3202cec6
--- a/comfy/samplers.py
+++ b/comfy/samplers.py
@@ -16,6 +16,8 @@ import comfy.sampler_helpers
import comfy.model_patcher
import comfy.patcher_extension
import comfy.hooks
+import comfy.context_windows
+import comfy.utils
import scipy.stats
import numpy
@@ -60,7 +62,7 @@ def get_area_and_mult(conds, x_in, timestep_in):
if "mask_strength" in conds:
mask_strength = conds["mask_strength"]
mask = conds['mask']
- assert (mask.shape[1:] == x_in.shape[2:])
+ # assert (mask.shape[1:] == x_in.shape[2:])
mask = mask[:input_x.shape[0]]
if area is not None:
@@ -68,7 +70,7 @@ def get_area_and_mult(conds, x_in, timestep_in):
mask = mask.narrow(i + 1, area[len(dims) + i], area[i])
mask = mask * mask_strength
- mask = mask.unsqueeze(1).repeat(input_x.shape[0] // mask.shape[0], input_x.shape[1], 1, 1)
+ mask = mask.unsqueeze(1).repeat((input_x.shape[0] // mask.shape[0], input_x.shape[1]) + (1, ) * (mask.ndim - 1))
else:
mask = torch.ones_like(input_x)
mult = mask * strength
@@ -89,7 +91,7 @@ def get_area_and_mult(conds, x_in, timestep_in):
conditioning = {}
model_conds = conds["model_conds"]
for c in model_conds:
- conditioning[c] = model_conds[c].process_cond(batch_size=x_in.shape[0], device=x_in.device, area=area)
+ conditioning[c] = model_conds[c].process_cond(batch_size=x_in.shape[0], area=area)
hooks = conds.get('hooks', None)
control = conds.get('control', None)
@@ -198,14 +200,20 @@ def finalize_default_conds(model: 'BaseModel', hooked_to_run: dict[comfy.hooks.H
hooked_to_run.setdefault(p.hooks, list())
hooked_to_run[p.hooks] += [(p, i)]
-def calc_cond_batch(model: 'BaseModel', conds: list[list[dict]], x_in: torch.Tensor, timestep, model_options):
+def calc_cond_batch(model: BaseModel, conds: list[list[dict]], x_in: torch.Tensor, timestep, model_options: dict[str]):
+ handler: comfy.context_windows.ContextHandlerABC = model_options.get("context_handler", None)
+ if handler is None or not handler.should_use_context(model, conds, x_in, timestep, model_options):
+ return _calc_cond_batch_outer(model, conds, x_in, timestep, model_options)
+ return handler.execute(_calc_cond_batch_outer, model, conds, x_in, timestep, model_options)
+
+def _calc_cond_batch_outer(model: BaseModel, conds: list[list[dict]], x_in: torch.Tensor, timestep, model_options):
executor = comfy.patcher_extension.WrapperExecutor.new_executor(
_calc_cond_batch,
comfy.patcher_extension.get_all_wrappers(comfy.patcher_extension.WrappersMP.CALC_COND_BATCH, model_options, is_model_options=True)
)
return executor.execute(model, conds, x_in, timestep, model_options)
-def _calc_cond_batch(model: 'BaseModel', conds: list[list[dict]], x_in: torch.Tensor, timestep, model_options):
+def _calc_cond_batch(model: BaseModel, conds: list[list[dict]], x_in: torch.Tensor, timestep, model_options):
out_conds = []
out_counts = []
# separate conds by matching hooks
@@ -546,7 +554,10 @@ def resolve_areas_and_cond_masks_multidim(conditions, dims, device):
if len(mask.shape) == len(dims):
mask = mask.unsqueeze(0)
if mask.shape[1:] != dims:
- mask = torch.nn.functional.interpolate(mask.unsqueeze(1), size=dims, mode='bilinear', align_corners=False).squeeze(1)
+ if mask.ndim < 4:
+ mask = comfy.utils.common_upscale(mask.unsqueeze(1), dims[-1], dims[-2], 'bilinear', 'none').squeeze(1)
+ else:
+ mask = comfy.utils.common_upscale(mask, dims[-1], dims[-2], 'bilinear', 'none')
if modified.get("set_area_to_bounds", False): #TODO: handle dim != 2
bounds = torch.max(torch.abs(mask),dim=0).values.unsqueeze(0)
@@ -718,7 +729,7 @@ class Sampler:
KSAMPLER_NAMES = ["euler", "euler_cfg_pp", "euler_ancestral", "euler_ancestral_cfg_pp", "heun", "heunpp2","dpm_2", "dpm_2_ancestral",
"lms", "dpm_fast", "dpm_adaptive", "dpmpp_2s_ancestral", "dpmpp_2s_ancestral_cfg_pp", "dpmpp_sde", "dpmpp_sde_gpu",
- "dpmpp_2m", "dpmpp_2m_cfg_pp", "dpmpp_2m_sde", "dpmpp_2m_sde_gpu", "dpmpp_3m_sde", "dpmpp_3m_sde_gpu", "ddpm", "lcm",
+ "dpmpp_2m", "dpmpp_2m_cfg_pp", "dpmpp_2m_sde", "dpmpp_2m_sde_gpu", "dpmpp_2m_sde_heun", "dpmpp_2m_sde_heun_gpu", "dpmpp_3m_sde", "dpmpp_3m_sde_gpu", "ddpm", "lcm",
"ipndm", "ipndm_v", "deis", "res_multistep", "res_multistep_cfg_pp", "res_multistep_ancestral", "res_multistep_ancestral_cfg_pp",
"gradient_estimation", "gradient_estimation_cfg_pp", "er_sde", "seeds_2", "seeds_3", "sa_solver", "sa_solver_pece"]
@@ -946,7 +957,14 @@ class CFGGuider:
self.original_conds[k] = comfy.sampler_helpers.convert_cond(conds[k])
def __call__(self, *args, **kwargs):
- return self.predict_noise(*args, **kwargs)
+ return self.outer_predict_noise(*args, **kwargs)
+
+ def outer_predict_noise(self, x, timestep, model_options={}, seed=None):
+ return comfy.patcher_extension.WrapperExecutor.new_class_executor(
+ self.predict_noise,
+ self,
+ comfy.patcher_extension.get_all_wrappers(comfy.patcher_extension.WrappersMP.PREDICT_NOISE, self.model_options, is_model_options=True)
+ ).execute(x, timestep, model_options, seed)
def predict_noise(self, x, timestep, model_options={}, seed=None):
return sampling_function(self.inner_model, x, timestep, self.conds.get("negative", None), self.conds.get("positive", None), self.cfg, model_options=model_options, seed=seed)
diff --git a/comfy/sd.py b/comfy/sd.py
index 44d1a9e06..014f797ca 100644
--- a/comfy/sd.py
+++ b/comfy/sd.py
@@ -47,6 +47,7 @@ import comfy.text_encoders.wan
import comfy.text_encoders.hidream
import comfy.text_encoders.ace
import comfy.text_encoders.omnigen2
+import comfy.text_encoders.qwen_image
import comfy.model_patcher
import comfy.lora
@@ -783,6 +784,7 @@ class CLIPType(Enum):
CHROMA = 15
ACE = 16
OMNIGEN2 = 17
+ QWEN_IMAGE = 18
def load_clip(ckpt_paths, embedding_directory=None, clip_type=CLIPType.STABLE_DIFFUSION, model_options={}):
@@ -803,6 +805,7 @@ class TEModel(Enum):
T5_XXL_OLD = 8
GEMMA_2_2B = 9
QWEN25_3B = 10
+ QWEN25_7B = 11
def detect_te_model(sd):
if "text_model.encoder.layers.30.mlp.fc1.weight" in sd:
@@ -824,7 +827,11 @@ def detect_te_model(sd):
if 'model.layers.0.post_feedforward_layernorm.weight' in sd:
return TEModel.GEMMA_2_2B
if 'model.layers.0.self_attn.k_proj.bias' in sd:
- return TEModel.QWEN25_3B
+ weight = sd['model.layers.0.self_attn.k_proj.bias']
+ if weight.shape[0] == 256:
+ return TEModel.QWEN25_3B
+ if weight.shape[0] == 512:
+ return TEModel.QWEN25_7B
if "model.layers.0.post_attention_layernorm.weight" in sd:
return TEModel.LLAMA3_8
return None
@@ -929,6 +936,9 @@ def load_text_encoder_state_dicts(state_dicts=[], embedding_directory=None, clip
elif te_model == TEModel.QWEN25_3B:
clip_target.clip = comfy.text_encoders.omnigen2.te(**llama_detect(clip_data))
clip_target.tokenizer = comfy.text_encoders.omnigen2.Omnigen2Tokenizer
+ elif te_model == TEModel.QWEN25_7B:
+ clip_target.clip = comfy.text_encoders.qwen_image.te(**llama_detect(clip_data))
+ clip_target.tokenizer = comfy.text_encoders.qwen_image.QwenImageTokenizer
else:
# clip_l
if clip_type == CLIPType.SD3:
diff --git a/comfy/sd1_clip.py b/comfy/sd1_clip.py
index ade340fd1..f8a7c2a1b 100644
--- a/comfy/sd1_clip.py
+++ b/comfy/sd1_clip.py
@@ -204,17 +204,19 @@ class SDClipModel(torch.nn.Module, ClipTokenWeightEncoder):
tokens_embed = self.transformer.get_input_embeddings()(tokens_embed, out_dtype=torch.float32)
index = 0
pad_extra = 0
+ embeds_info = []
for o in other_embeds:
emb = o[1]
if torch.is_tensor(emb):
emb = {"type": "embedding", "data": emb}
+ extra = None
emb_type = emb.get("type", None)
if emb_type == "embedding":
emb = emb.get("data", None)
else:
if hasattr(self.transformer, "preprocess_embed"):
- emb = self.transformer.preprocess_embed(emb, device=device)
+ emb, extra = self.transformer.preprocess_embed(emb, device=device)
else:
emb = None
@@ -229,6 +231,7 @@ class SDClipModel(torch.nn.Module, ClipTokenWeightEncoder):
tokens_embed = torch.cat([tokens_embed[:, :ind], emb, tokens_embed[:, ind:]], dim=1)
attention_mask = attention_mask[:ind] + [1] * emb_shape + attention_mask[ind:]
index += emb_shape - 1
+ embeds_info.append({"type": emb_type, "index": ind, "size": emb_shape, "extra": extra})
else:
index += -1
pad_extra += emb_shape
@@ -243,11 +246,11 @@ class SDClipModel(torch.nn.Module, ClipTokenWeightEncoder):
attention_masks.append(attention_mask)
num_tokens.append(sum(attention_mask))
- return torch.cat(embeds_out), torch.tensor(attention_masks, device=device, dtype=torch.long), num_tokens
+ return torch.cat(embeds_out), torch.tensor(attention_masks, device=device, dtype=torch.long), num_tokens, embeds_info
def forward(self, tokens):
device = self.transformer.get_input_embeddings().weight.device
- embeds, attention_mask, num_tokens = self.process_tokens(tokens, device)
+ embeds, attention_mask, num_tokens, embeds_info = self.process_tokens(tokens, device)
attention_mask_model = None
if self.enable_attention_masks:
@@ -258,7 +261,7 @@ class SDClipModel(torch.nn.Module, ClipTokenWeightEncoder):
else:
intermediate_output = self.layer_idx
- outputs = self.transformer(None, attention_mask_model, embeds=embeds, num_tokens=num_tokens, intermediate_output=intermediate_output, final_layer_norm_intermediate=self.layer_norm_hidden_state, dtype=torch.float32)
+ outputs = self.transformer(None, attention_mask_model, embeds=embeds, num_tokens=num_tokens, intermediate_output=intermediate_output, final_layer_norm_intermediate=self.layer_norm_hidden_state, dtype=torch.float32, embeds_info=embeds_info)
if self.layer == "last":
z = outputs[0].float()
@@ -531,7 +534,10 @@ class SDTokenizer:
min_padding = tokenizer_options.get("{}_min_padding".format(self.embedding_key), self.min_padding)
text = escape_important(text)
- parsed_weights = token_weights(text, 1.0)
+ if kwargs.get("disable_weights", False):
+ parsed_weights = [(text, 1.0)]
+ else:
+ parsed_weights = token_weights(text, 1.0)
# tokenize words
tokens = []
diff --git a/comfy/supported_models.py b/comfy/supported_models.py
index 7e7c75d7a..75dad277d 100644
--- a/comfy/supported_models.py
+++ b/comfy/supported_models.py
@@ -19,6 +19,7 @@ import comfy.text_encoders.lumina2
import comfy.text_encoders.wan
import comfy.text_encoders.ace
import comfy.text_encoders.omnigen2
+import comfy.text_encoders.qwen_image
from . import supported_models_base
from . import latent_formats
@@ -699,7 +700,7 @@ class Flux(supported_models_base.BASE):
unet_extra_config = {}
latent_format = latent_formats.Flux
- memory_usage_factor = 2.8
+ memory_usage_factor = 3.1 # TODO: debug why flux mem usage is so weird on windows.
supported_inference_dtypes = [torch.bfloat16, torch.float16, torch.float32]
@@ -1045,6 +1046,18 @@ class WAN21_Camera(WAN21_T2V):
def get_model(self, state_dict, prefix="", device=None):
out = model_base.WAN21_Camera(self, image_to_video=False, device=device)
return out
+
+class WAN22_Camera(WAN21_T2V):
+ unet_config = {
+ "image_model": "wan2.1",
+ "model_type": "camera_2.2",
+ "in_dim": 36,
+ }
+
+ def get_model(self, state_dict, prefix="", device=None):
+ out = model_base.WAN21_Camera(self, image_to_video=False, device=device)
+ return out
+
class WAN21_Vace(WAN21_T2V):
unet_config = {
"image_model": "wan2.1",
@@ -1059,6 +1072,19 @@ class WAN21_Vace(WAN21_T2V):
out = model_base.WAN21_Vace(self, image_to_video=False, device=device)
return out
+class WAN22_S2V(WAN21_T2V):
+ unet_config = {
+ "image_model": "wan2.1",
+ "model_type": "s2v",
+ }
+
+ def __init__(self, unet_config):
+ super().__init__(unet_config)
+
+ def get_model(self, state_dict, prefix="", device=None):
+ out = model_base.WAN22_S2V(self, device=device)
+ return out
+
class WAN22_T2V(WAN21_T2V):
unet_config = {
"image_model": "wan2.1",
@@ -1240,7 +1266,36 @@ class Omnigen2(supported_models_base.BASE):
hunyuan_detect = comfy.text_encoders.hunyuan_video.llama_detect(state_dict, "{}qwen25_3b.transformer.".format(pref))
return supported_models_base.ClipTarget(comfy.text_encoders.omnigen2.Omnigen2Tokenizer, comfy.text_encoders.omnigen2.te(**hunyuan_detect))
+class QwenImage(supported_models_base.BASE):
+ unet_config = {
+ "image_model": "qwen_image",
+ }
-models = [LotusD, Stable_Zero123, SD15_instructpix2pix, SD15, SD20, SD21UnclipL, SD21UnclipH, SDXL_instructpix2pix, SDXLRefiner, SDXL, SSD1B, KOALA_700M, KOALA_1B, Segmind_Vega, SD_X4Upscaler, Stable_Cascade_C, Stable_Cascade_B, SV3D_u, SV3D_p, SD3, StableAudio, AuraFlow, PixArtAlpha, PixArtSigma, HunyuanDiT, HunyuanDiT1, FluxInpaint, Flux, FluxSchnell, GenmoMochi, LTXV, HunyuanVideoSkyreelsI2V, HunyuanVideoI2V, HunyuanVideo, CosmosT2V, CosmosI2V, CosmosT2IPredict2, CosmosI2VPredict2, Lumina2, WAN22_T2V, WAN21_T2V, WAN21_I2V, WAN21_FunControl2V, WAN21_Vace, WAN21_Camera, Hunyuan3Dv2mini, Hunyuan3Dv2, Hunyuan3Dv2_1, HiDream, Chroma, ACEStep, Omnigen2]
+ sampling_settings = {
+ "multiplier": 1.0,
+ "shift": 1.15,
+ }
+
+ memory_usage_factor = 1.8 #TODO
+
+ unet_extra_config = {}
+ latent_format = latent_formats.Wan21
+
+ supported_inference_dtypes = [torch.bfloat16, torch.float32]
+
+ vae_key_prefix = ["vae."]
+ text_encoder_key_prefix = ["text_encoders."]
+
+ def get_model(self, state_dict, prefix="", device=None):
+ out = model_base.QwenImage(self, device=device)
+ return out
+
+ def clip_target(self, state_dict={}):
+ pref = self.text_encoder_key_prefix[0]
+ hunyuan_detect = comfy.text_encoders.hunyuan_video.llama_detect(state_dict, "{}qwen25_7b.transformer.".format(pref))
+ return supported_models_base.ClipTarget(comfy.text_encoders.qwen_image.QwenImageTokenizer, comfy.text_encoders.qwen_image.te(**hunyuan_detect))
+
+
+models = [LotusD, Stable_Zero123, SD15_instructpix2pix, SD15, SD20, SD21UnclipL, SD21UnclipH, SDXL_instructpix2pix, SDXLRefiner, SDXL, SSD1B, KOALA_700M, KOALA_1B, Segmind_Vega, SD_X4Upscaler, Stable_Cascade_C, Stable_Cascade_B, SV3D_u, SV3D_p, SD3, StableAudio, AuraFlow, PixArtAlpha, PixArtSigma, HunyuanDiT, HunyuanDiT1, FluxInpaint, Flux, FluxSchnell, GenmoMochi, LTXV, HunyuanVideoSkyreelsI2V, HunyuanVideoI2V, HunyuanVideo, CosmosT2V, CosmosI2V, CosmosT2IPredict2, CosmosI2VPredict2, Lumina2, WAN22_T2V, WAN21_T2V, WAN21_I2V, WAN21_FunControl2V, WAN21_Vace, WAN21_Camera, WAN22_Camera, WAN22_S2V, Hunyuan3Dv2mini, Hunyuan3Dv2, Hunyuan3Dv2_1, HiDream, Chroma, ACEStep, Omnigen2, QwenImage]
models += [SVD_img2vid]
diff --git a/comfy/text_encoders/bert.py b/comfy/text_encoders/bert.py
index 551b03162..ed4638a9a 100644
--- a/comfy/text_encoders/bert.py
+++ b/comfy/text_encoders/bert.py
@@ -116,7 +116,7 @@ class BertModel_(torch.nn.Module):
self.embeddings = BertEmbeddings(config_dict["vocab_size"], config_dict["max_position_embeddings"], config_dict["type_vocab_size"], config_dict["pad_token_id"], embed_dim, layer_norm_eps, dtype, device, operations)
self.encoder = BertEncoder(config_dict["num_hidden_layers"], embed_dim, config_dict["intermediate_size"], config_dict["num_attention_heads"], layer_norm_eps, dtype, device, operations)
- def forward(self, input_tokens, attention_mask=None, embeds=None, num_tokens=None, intermediate_output=None, final_layer_norm_intermediate=True, dtype=None):
+ def forward(self, input_tokens, attention_mask=None, embeds=None, num_tokens=None, intermediate_output=None, final_layer_norm_intermediate=True, dtype=None, embeds_info=[]):
x = self.embeddings(input_tokens, embeds=embeds, dtype=dtype)
mask = None
if attention_mask is not None:
diff --git a/comfy/text_encoders/llama.py b/comfy/text_encoders/llama.py
index 7fbd0f604..4c976058f 100644
--- a/comfy/text_encoders/llama.py
+++ b/comfy/text_encoders/llama.py
@@ -2,12 +2,14 @@ import torch
import torch.nn as nn
from dataclasses import dataclass
from typing import Optional, Any
+import math
from comfy.ldm.modules.attention import optimized_attention_for_device
import comfy.model_management
import comfy.ldm.common_dit
import comfy.model_management
+from . import qwen_vl
@dataclass
class Llama2Config:
@@ -25,6 +27,7 @@ class Llama2Config:
rms_norm_add = False
mlp_activation = "silu"
qkv_bias = False
+ rope_dims = None
@dataclass
class Qwen25_3BConfig:
@@ -42,6 +45,25 @@ class Qwen25_3BConfig:
rms_norm_add = False
mlp_activation = "silu"
qkv_bias = True
+ rope_dims = None
+
+@dataclass
+class Qwen25_7BVLI_Config:
+ vocab_size: int = 152064
+ hidden_size: int = 3584
+ intermediate_size: int = 18944
+ num_hidden_layers: int = 28
+ num_attention_heads: int = 28
+ num_key_value_heads: int = 4
+ max_position_embeddings: int = 128000
+ rms_norm_eps: float = 1e-6
+ rope_theta: float = 1000000.0
+ transformer_type: str = "llama"
+ head_dim = 128
+ rms_norm_add = False
+ mlp_activation = "silu"
+ qkv_bias = True
+ rope_dims = [16, 24, 24]
@dataclass
class Gemma2_2B_Config:
@@ -59,6 +81,7 @@ class Gemma2_2B_Config:
rms_norm_add = True
mlp_activation = "gelu_pytorch_tanh"
qkv_bias = False
+ rope_dims = None
class RMSNorm(nn.Module):
def __init__(self, dim: int, eps: float = 1e-5, add=False, device=None, dtype=None):
@@ -83,24 +106,30 @@ def rotate_half(x):
return torch.cat((-x2, x1), dim=-1)
-def precompute_freqs_cis(head_dim, seq_len, theta, device=None):
+def precompute_freqs_cis(head_dim, position_ids, theta, rope_dims=None, device=None):
theta_numerator = torch.arange(0, head_dim, 2, device=device).float()
inv_freq = 1.0 / (theta ** (theta_numerator / head_dim))
- position_ids = torch.arange(0, seq_len, device=device).unsqueeze(0)
-
inv_freq_expanded = inv_freq[None, :, None].float().expand(position_ids.shape[0], -1, 1)
position_ids_expanded = position_ids[:, None, :].float()
freqs = (inv_freq_expanded.float() @ position_ids_expanded.float()).transpose(1, 2)
emb = torch.cat((freqs, freqs), dim=-1)
cos = emb.cos()
sin = emb.sin()
+ if rope_dims is not None and position_ids.shape[0] > 1:
+ mrope_section = rope_dims * 2
+ cos = torch.cat([m[i % 3] for i, m in enumerate(cos.split(mrope_section, dim=-1))], dim=-1).unsqueeze(0)
+ sin = torch.cat([m[i % 3] for i, m in enumerate(sin.split(mrope_section, dim=-1))], dim=-1).unsqueeze(0)
+ else:
+ cos = cos.unsqueeze(1)
+ sin = sin.unsqueeze(1)
+
return (cos, sin)
def apply_rope(xq, xk, freqs_cis):
- cos = freqs_cis[0].unsqueeze(1)
- sin = freqs_cis[1].unsqueeze(1)
+ cos = freqs_cis[0]
+ sin = freqs_cis[1]
q_embed = (xq * cos) + (rotate_half(xq) * sin)
k_embed = (xk * cos) + (rotate_half(xk) * sin)
return q_embed, k_embed
@@ -260,7 +289,7 @@ class Llama2_(nn.Module):
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps, add=config.rms_norm_add, device=device, dtype=dtype)
# self.lm_head = ops.Linear(config.hidden_size, config.vocab_size, bias=False, device=device, dtype=dtype)
- def forward(self, x, attention_mask=None, embeds=None, num_tokens=None, intermediate_output=None, final_layer_norm_intermediate=True, dtype=None):
+ def forward(self, x, attention_mask=None, embeds=None, num_tokens=None, intermediate_output=None, final_layer_norm_intermediate=True, dtype=None, position_ids=None, embeds_info=[]):
if embeds is not None:
x = embeds
else:
@@ -269,9 +298,13 @@ class Llama2_(nn.Module):
if self.normalize_in:
x *= self.config.hidden_size ** 0.5
+ if position_ids is None:
+ position_ids = torch.arange(0, x.shape[1], device=x.device).unsqueeze(0)
+
freqs_cis = precompute_freqs_cis(self.config.head_dim,
- x.shape[1],
+ position_ids,
self.config.rope_theta,
+ self.config.rope_dims,
device=x.device)
mask = None
@@ -348,6 +381,45 @@ class Qwen25_3B(BaseLlama, torch.nn.Module):
self.model = Llama2_(config, device=device, dtype=dtype, ops=operations)
self.dtype = dtype
+class Qwen25_7BVLI(BaseLlama, torch.nn.Module):
+ def __init__(self, config_dict, dtype, device, operations):
+ super().__init__()
+ config = Qwen25_7BVLI_Config(**config_dict)
+ self.num_layers = config.num_hidden_layers
+
+ self.model = Llama2_(config, device=device, dtype=dtype, ops=operations)
+ self.visual = qwen_vl.Qwen2VLVisionTransformer(hidden_size=1280, output_hidden_size=config.hidden_size, device=device, dtype=dtype, ops=operations)
+ self.dtype = dtype
+
+ def preprocess_embed(self, embed, device):
+ if embed["type"] == "image":
+ image, grid = qwen_vl.process_qwen2vl_images(embed["data"])
+ return self.visual(image.to(device, dtype=torch.float32), grid), grid
+ return None, None
+
+ def forward(self, x, attention_mask=None, embeds=None, num_tokens=None, intermediate_output=None, final_layer_norm_intermediate=True, dtype=None, embeds_info=[]):
+ grid = None
+ for e in embeds_info:
+ if e.get("type") == "image":
+ grid = e.get("extra", None)
+ position_ids = torch.zeros((3, embeds.shape[1]), device=embeds.device)
+ start = e.get("index")
+ position_ids[:, :start] = torch.arange(0, start, device=embeds.device)
+ end = e.get("size") + start
+ len_max = int(grid.max()) // 2
+ start_next = len_max + start
+ position_ids[:, end:] = torch.arange(start_next, start_next + (embeds.shape[1] - end), device=embeds.device)
+ position_ids[0, start:end] = start
+ max_d = int(grid[0][1]) // 2
+ position_ids[1, start:end] = torch.arange(start, start + max_d, device=embeds.device).unsqueeze(1).repeat(1, math.ceil((end - start) / max_d)).flatten(0)[:end - start]
+ max_d = int(grid[0][2]) // 2
+ position_ids[2, start:end] = torch.arange(start, start + max_d, device=embeds.device).unsqueeze(0).repeat(math.ceil((end - start) / max_d), 1).flatten(0)[:end - start]
+
+ if grid is None:
+ position_ids = None
+
+ return super().forward(x, attention_mask=attention_mask, embeds=embeds, num_tokens=num_tokens, intermediate_output=intermediate_output, final_layer_norm_intermediate=final_layer_norm_intermediate, dtype=dtype, position_ids=position_ids)
+
class Gemma2_2B(BaseLlama, torch.nn.Module):
def __init__(self, config_dict, dtype, device, operations):
super().__init__()
diff --git a/comfy/text_encoders/qwen_image.py b/comfy/text_encoders/qwen_image.py
new file mode 100644
index 000000000..6646b1003
--- /dev/null
+++ b/comfy/text_encoders/qwen_image.py
@@ -0,0 +1,85 @@
+from transformers import Qwen2Tokenizer
+from comfy import sd1_clip
+import comfy.text_encoders.llama
+import os
+import torch
+import numbers
+
+class Qwen25_7BVLITokenizer(sd1_clip.SDTokenizer):
+ def __init__(self, embedding_directory=None, tokenizer_data={}):
+ tokenizer_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "qwen25_tokenizer")
+ super().__init__(tokenizer_path, pad_with_end=False, embedding_size=3584, embedding_key='qwen25_7b', tokenizer_class=Qwen2Tokenizer, has_start_token=False, has_end_token=False, pad_to_max_length=False, max_length=99999999, min_length=1, pad_token=151643, tokenizer_data=tokenizer_data)
+
+
+class QwenImageTokenizer(sd1_clip.SD1Tokenizer):
+ def __init__(self, embedding_directory=None, tokenizer_data={}):
+ super().__init__(embedding_directory=embedding_directory, tokenizer_data=tokenizer_data, name="qwen25_7b", tokenizer=Qwen25_7BVLITokenizer)
+ self.llama_template = "<|im_start|>system\nDescribe the image by detailing the color, shape, size, texture, quantity, text, spatial relationships of the objects and background:<|im_end|>\n<|im_start|>user\n{}<|im_end|>\n<|im_start|>assistant\n"
+ self.llama_template_images = "<|im_start|>system\nDescribe the key features of the input image (color, shape, size, texture, objects, background), then explain how the user's text instruction should alter or modify the image. Generate a new image that meets the user's requirements while maintaining consistency with the original input where appropriate.<|im_end|>\n<|im_start|>user\n<|vision_start|><|image_pad|><|vision_end|>{}<|im_end|>\n<|im_start|>assistant\n"
+
+ def tokenize_with_weights(self, text, return_word_ids=False, llama_template=None, images=[], **kwargs):
+ if llama_template is None:
+ if len(images) > 0:
+ llama_text = self.llama_template_images.format(text)
+ else:
+ llama_text = self.llama_template.format(text)
+ else:
+ llama_text = llama_template.format(text)
+ tokens = super().tokenize_with_weights(llama_text, return_word_ids=return_word_ids, disable_weights=True, **kwargs)
+ key_name = next(iter(tokens))
+ embed_count = 0
+ qwen_tokens = tokens[key_name]
+ for r in qwen_tokens:
+ for i in range(len(r)):
+ if r[i][0] == 151655:
+ if len(images) > embed_count:
+ r[i] = ({"type": "image", "data": images[embed_count], "original_type": "image"},) + r[i][1:]
+ embed_count += 1
+ return tokens
+
+
+class Qwen25_7BVLIModel(sd1_clip.SDClipModel):
+ def __init__(self, device="cpu", layer="last", layer_idx=None, dtype=None, attention_mask=True, model_options={}):
+ super().__init__(device=device, layer=layer, layer_idx=layer_idx, textmodel_json_config={}, dtype=dtype, special_tokens={"pad": 151643}, layer_norm_hidden_state=False, model_class=comfy.text_encoders.llama.Qwen25_7BVLI, enable_attention_masks=attention_mask, return_attention_masks=attention_mask, model_options=model_options)
+
+
+class QwenImageTEModel(sd1_clip.SD1ClipModel):
+ def __init__(self, device="cpu", dtype=None, model_options={}):
+ super().__init__(device=device, dtype=dtype, name="qwen25_7b", clip_model=Qwen25_7BVLIModel, model_options=model_options)
+
+ def encode_token_weights(self, token_weight_pairs):
+ out, pooled, extra = super().encode_token_weights(token_weight_pairs)
+ tok_pairs = token_weight_pairs["qwen25_7b"][0]
+ count_im_start = 0
+ for i, v in enumerate(tok_pairs):
+ elem = v[0]
+ if not torch.is_tensor(elem):
+ if isinstance(elem, numbers.Integral):
+ if elem == 151644 and count_im_start < 2:
+ template_end = i
+ count_im_start += 1
+
+ if out.shape[1] > (template_end + 3):
+ if tok_pairs[template_end + 1][0] == 872:
+ if tok_pairs[template_end + 2][0] == 198:
+ template_end += 3
+
+ out = out[:, template_end:]
+
+ extra["attention_mask"] = extra["attention_mask"][:, template_end:]
+ if extra["attention_mask"].sum() == torch.numel(extra["attention_mask"]):
+ extra.pop("attention_mask") # attention mask is useless if no masked elements
+
+ return out, pooled, extra
+
+
+def te(dtype_llama=None, llama_scaled_fp8=None):
+ class QwenImageTEModel_(QwenImageTEModel):
+ def __init__(self, device="cpu", dtype=None, model_options={}):
+ if llama_scaled_fp8 is not None and "scaled_fp8" not in model_options:
+ model_options = model_options.copy()
+ model_options["scaled_fp8"] = llama_scaled_fp8
+ if dtype_llama is not None:
+ dtype = dtype_llama
+ super().__init__(device=device, dtype=dtype, model_options=model_options)
+ return QwenImageTEModel_
diff --git a/comfy/text_encoders/qwen_vl.py b/comfy/text_encoders/qwen_vl.py
new file mode 100644
index 000000000..3b18ce730
--- /dev/null
+++ b/comfy/text_encoders/qwen_vl.py
@@ -0,0 +1,428 @@
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from typing import Optional, Tuple
+import math
+from comfy.ldm.modules.attention import optimized_attention_for_device
+
+
+def process_qwen2vl_images(
+ images: torch.Tensor,
+ min_pixels: int = 3136,
+ max_pixels: int = 12845056,
+ patch_size: int = 14,
+ temporal_patch_size: int = 2,
+ merge_size: int = 2,
+ image_mean: list = None,
+ image_std: list = None,
+):
+ if image_mean is None:
+ image_mean = [0.48145466, 0.4578275, 0.40821073]
+ if image_std is None:
+ image_std = [0.26862954, 0.26130258, 0.27577711]
+
+ batch_size, height, width, channels = images.shape
+ device = images.device
+ # dtype = images.dtype
+
+ images = images.permute(0, 3, 1, 2)
+
+ grid_thw_list = []
+ img = images[0]
+
+ factor = patch_size * merge_size
+
+ h_bar = round(height / factor) * factor
+ w_bar = round(width / factor) * factor
+
+ if h_bar * w_bar > max_pixels:
+ beta = math.sqrt((height * width) / max_pixels)
+ h_bar = max(factor, math.floor(height / beta / factor) * factor)
+ w_bar = max(factor, math.floor(width / beta / factor) * factor)
+ elif h_bar * w_bar < min_pixels:
+ beta = math.sqrt(min_pixels / (height * width))
+ h_bar = math.ceil(height * beta / factor) * factor
+ w_bar = math.ceil(width * beta / factor) * factor
+
+ img_resized = F.interpolate(
+ img.unsqueeze(0),
+ size=(h_bar, w_bar),
+ mode='bilinear',
+ align_corners=False
+ ).squeeze(0)
+
+ normalized = img_resized.clone()
+ for c in range(3):
+ normalized[c] = (img_resized[c] - image_mean[c]) / image_std[c]
+
+ grid_h = h_bar // patch_size
+ grid_w = w_bar // patch_size
+ grid_thw = torch.tensor([1, grid_h, grid_w], device=device, dtype=torch.long)
+
+ pixel_values = normalized
+ grid_thw_list.append(grid_thw)
+ image_grid_thw = torch.stack(grid_thw_list)
+
+ grid_t = 1
+ channel = pixel_values.shape[0]
+ pixel_values = pixel_values.unsqueeze(0).repeat(2, 1, 1, 1)
+
+ patches = pixel_values.reshape(
+ grid_t,
+ temporal_patch_size,
+ channel,
+ grid_h // merge_size,
+ merge_size,
+ patch_size,
+ grid_w // merge_size,
+ merge_size,
+ patch_size,
+ )
+
+ patches = patches.permute(0, 3, 6, 4, 7, 2, 1, 5, 8)
+ flatten_patches = patches.reshape(
+ grid_t * grid_h * grid_w,
+ channel * temporal_patch_size * patch_size * patch_size
+ )
+
+ return flatten_patches, image_grid_thw
+
+
+class VisionPatchEmbed(nn.Module):
+ def __init__(
+ self,
+ patch_size: int = 14,
+ temporal_patch_size: int = 2,
+ in_channels: int = 3,
+ embed_dim: int = 3584,
+ device=None,
+ dtype=None,
+ ops=None,
+ ):
+ super().__init__()
+ self.patch_size = patch_size
+ self.temporal_patch_size = temporal_patch_size
+ self.in_channels = in_channels
+ self.embed_dim = embed_dim
+
+ kernel_size = [temporal_patch_size, patch_size, patch_size]
+ self.proj = ops.Conv3d(
+ in_channels,
+ embed_dim,
+ kernel_size=kernel_size,
+ stride=kernel_size,
+ bias=False,
+ device=device,
+ dtype=dtype
+ )
+
+ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
+ hidden_states = hidden_states.view(
+ -1, self.in_channels, self.temporal_patch_size, self.patch_size, self.patch_size
+ )
+ hidden_states = self.proj(hidden_states)
+ return hidden_states.view(-1, self.embed_dim)
+
+
+def rotate_half(x):
+ x1 = x[..., : x.shape[-1] // 2]
+ x2 = x[..., x.shape[-1] // 2 :]
+ return torch.cat((-x2, x1), dim=-1)
+
+
+def apply_rotary_pos_emb_vision(q, k, cos, sin):
+ cos, sin = cos.unsqueeze(-2).float(), sin.unsqueeze(-2).float()
+ q_embed = (q * cos) + (rotate_half(q) * sin)
+ k_embed = (k * cos) + (rotate_half(k) * sin)
+ return q_embed, k_embed
+
+
+class VisionRotaryEmbedding(nn.Module):
+ def __init__(self, dim: int, theta: float = 10000.0):
+ super().__init__()
+ self.dim = dim
+ self.theta = theta
+
+ def forward(self, seqlen: int, device) -> torch.Tensor:
+ inv_freq = 1.0 / (self.theta ** (torch.arange(0, self.dim, 2, dtype=torch.float, device=device) / self.dim))
+ seq = torch.arange(seqlen, device=inv_freq.device, dtype=inv_freq.dtype)
+ freqs = torch.outer(seq, inv_freq)
+ return freqs
+
+
+class PatchMerger(nn.Module):
+ def __init__(self, dim: int, context_dim: int, spatial_merge_size: int = 2, device=None, dtype=None, ops=None):
+ super().__init__()
+ self.hidden_size = context_dim * (spatial_merge_size ** 2)
+ self.ln_q = ops.RMSNorm(context_dim, eps=1e-6, device=device, dtype=dtype)
+ self.mlp = nn.Sequential(
+ ops.Linear(self.hidden_size, self.hidden_size, device=device, dtype=dtype),
+ nn.GELU(),
+ ops.Linear(self.hidden_size, dim, device=device, dtype=dtype),
+ )
+
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
+ x = self.ln_q(x).reshape(-1, self.hidden_size)
+ x = self.mlp(x)
+ return x
+
+
+class VisionAttention(nn.Module):
+ def __init__(self, hidden_size: int, num_heads: int, device=None, dtype=None, ops=None):
+ super().__init__()
+ self.hidden_size = hidden_size
+ self.num_heads = num_heads
+ self.head_dim = hidden_size // num_heads
+ self.scaling = self.head_dim ** -0.5
+
+ self.qkv = ops.Linear(hidden_size, hidden_size * 3, bias=True, device=device, dtype=dtype)
+ self.proj = ops.Linear(hidden_size, hidden_size, bias=True, device=device, dtype=dtype)
+
+ def forward(
+ self,
+ hidden_states: torch.Tensor,
+ position_embeddings: Optional[Tuple[torch.Tensor, torch.Tensor]] = None,
+ cu_seqlens=None,
+ optimized_attention=None,
+ ) -> torch.Tensor:
+ if hidden_states.dim() == 2:
+ seq_length, _ = hidden_states.shape
+ batch_size = 1
+ hidden_states = hidden_states.unsqueeze(0)
+ else:
+ batch_size, seq_length, _ = hidden_states.shape
+
+ qkv = self.qkv(hidden_states)
+ qkv = qkv.reshape(batch_size, seq_length, 3, self.num_heads, self.head_dim)
+ query_states, key_states, value_states = qkv.reshape(seq_length, 3, self.num_heads, -1).permute(1, 0, 2, 3).unbind(0)
+
+ if position_embeddings is not None:
+ cos, sin = position_embeddings
+ query_states, key_states = apply_rotary_pos_emb_vision(query_states, key_states, cos, sin)
+
+ query_states = query_states.transpose(0, 1).unsqueeze(0)
+ key_states = key_states.transpose(0, 1).unsqueeze(0)
+ value_states = value_states.transpose(0, 1).unsqueeze(0)
+
+ lengths = cu_seqlens[1:] - cu_seqlens[:-1]
+ splits = [
+ torch.split(tensor, lengths.tolist(), dim=2) for tensor in (query_states, key_states, value_states)
+ ]
+
+ attn_outputs = [
+ optimized_attention(q, k, v, self.num_heads, skip_reshape=True)
+ for q, k, v in zip(*splits)
+ ]
+ attn_output = torch.cat(attn_outputs, dim=1)
+ attn_output = attn_output.reshape(seq_length, -1)
+ attn_output = self.proj(attn_output)
+
+ return attn_output
+
+
+class VisionMLP(nn.Module):
+ def __init__(self, hidden_size: int, intermediate_size: int, device=None, dtype=None, ops=None):
+ super().__init__()
+ self.gate_proj = ops.Linear(hidden_size, intermediate_size, bias=True, device=device, dtype=dtype)
+ self.up_proj = ops.Linear(hidden_size, intermediate_size, bias=True, device=device, dtype=dtype)
+ self.down_proj = ops.Linear(intermediate_size, hidden_size, bias=True, device=device, dtype=dtype)
+ self.act_fn = nn.SiLU()
+
+ def forward(self, hidden_state):
+ return self.down_proj(self.act_fn(self.gate_proj(hidden_state)) * self.up_proj(hidden_state))
+
+
+class VisionBlock(nn.Module):
+ def __init__(self, hidden_size: int, intermediate_size: int, num_heads: int, device=None, dtype=None, ops=None):
+ super().__init__()
+ self.norm1 = ops.RMSNorm(hidden_size, eps=1e-6, device=device, dtype=dtype)
+ self.norm2 = ops.RMSNorm(hidden_size, eps=1e-6, device=device, dtype=dtype)
+ self.attn = VisionAttention(hidden_size, num_heads, device=device, dtype=dtype, ops=ops)
+ self.mlp = VisionMLP(hidden_size, intermediate_size, device=device, dtype=dtype, ops=ops)
+
+ def forward(
+ self,
+ hidden_states: torch.Tensor,
+ position_embeddings: Optional[Tuple[torch.Tensor, torch.Tensor]] = None,
+ cu_seqlens=None,
+ optimized_attention=None,
+ ) -> torch.Tensor:
+ residual = hidden_states
+ hidden_states = self.norm1(hidden_states)
+ hidden_states = self.attn(hidden_states, position_embeddings, cu_seqlens, optimized_attention)
+ hidden_states = residual + hidden_states
+
+ residual = hidden_states
+ hidden_states = self.norm2(hidden_states)
+ hidden_states = self.mlp(hidden_states)
+ hidden_states = residual + hidden_states
+
+ return hidden_states
+
+
+class Qwen2VLVisionTransformer(nn.Module):
+ def __init__(
+ self,
+ hidden_size: int = 3584,
+ output_hidden_size: int = 3584,
+ intermediate_size: int = 3420,
+ num_heads: int = 16,
+ num_layers: int = 32,
+ patch_size: int = 14,
+ temporal_patch_size: int = 2,
+ spatial_merge_size: int = 2,
+ window_size: int = 112,
+ device=None,
+ dtype=None,
+ ops=None
+ ):
+ super().__init__()
+ self.hidden_size = hidden_size
+ self.patch_size = patch_size
+ self.spatial_merge_size = spatial_merge_size
+ self.window_size = window_size
+ self.fullatt_block_indexes = [7, 15, 23, 31]
+
+ self.patch_embed = VisionPatchEmbed(
+ patch_size=patch_size,
+ temporal_patch_size=temporal_patch_size,
+ in_channels=3,
+ embed_dim=hidden_size,
+ device=device,
+ dtype=dtype,
+ ops=ops,
+ )
+
+ head_dim = hidden_size // num_heads
+ self.rotary_pos_emb = VisionRotaryEmbedding(head_dim // 2)
+
+ self.blocks = nn.ModuleList([
+ VisionBlock(hidden_size, intermediate_size, num_heads, device, dtype, ops)
+ for _ in range(num_layers)
+ ])
+
+ self.merger = PatchMerger(
+ dim=output_hidden_size,
+ context_dim=hidden_size,
+ spatial_merge_size=spatial_merge_size,
+ device=device,
+ dtype=dtype,
+ ops=ops,
+ )
+
+ def get_window_index(self, grid_thw):
+ window_index = []
+ cu_window_seqlens = [0]
+ window_index_id = 0
+ vit_merger_window_size = self.window_size // self.spatial_merge_size // self.patch_size
+
+ for grid_t, grid_h, grid_w in grid_thw:
+ llm_grid_h = grid_h // self.spatial_merge_size
+ llm_grid_w = grid_w // self.spatial_merge_size
+
+ index = torch.arange(grid_t * llm_grid_h * llm_grid_w).reshape(grid_t, llm_grid_h, llm_grid_w)
+
+ pad_h = vit_merger_window_size - llm_grid_h % vit_merger_window_size
+ pad_w = vit_merger_window_size - llm_grid_w % vit_merger_window_size
+ num_windows_h = (llm_grid_h + pad_h) // vit_merger_window_size
+ num_windows_w = (llm_grid_w + pad_w) // vit_merger_window_size
+
+ index_padded = F.pad(index, (0, pad_w, 0, pad_h), "constant", -100)
+ index_padded = index_padded.reshape(
+ grid_t,
+ num_windows_h,
+ vit_merger_window_size,
+ num_windows_w,
+ vit_merger_window_size,
+ )
+ index_padded = index_padded.permute(0, 1, 3, 2, 4).reshape(
+ grid_t,
+ num_windows_h * num_windows_w,
+ vit_merger_window_size,
+ vit_merger_window_size,
+ )
+
+ seqlens = (index_padded != -100).sum([2, 3]).reshape(-1)
+ index_padded = index_padded.reshape(-1)
+ index_new = index_padded[index_padded != -100]
+ window_index.append(index_new + window_index_id)
+
+ cu_seqlens_tmp = seqlens.cumsum(0) * self.spatial_merge_size * self.spatial_merge_size + cu_window_seqlens[-1]
+ cu_window_seqlens.extend(cu_seqlens_tmp.tolist())
+ window_index_id += (grid_t * llm_grid_h * llm_grid_w).item()
+
+ window_index = torch.cat(window_index, dim=0)
+ return window_index, cu_window_seqlens
+
+ def get_position_embeddings(self, grid_thw, device):
+ pos_ids = []
+
+ for t, h, w in grid_thw:
+ hpos_ids = torch.arange(h, device=device).unsqueeze(1).expand(-1, w)
+ hpos_ids = hpos_ids.reshape(
+ h // self.spatial_merge_size,
+ self.spatial_merge_size,
+ w // self.spatial_merge_size,
+ self.spatial_merge_size,
+ )
+ hpos_ids = hpos_ids.permute(0, 2, 1, 3).flatten()
+
+ wpos_ids = torch.arange(w, device=device).unsqueeze(0).expand(h, -1)
+ wpos_ids = wpos_ids.reshape(
+ h // self.spatial_merge_size,
+ self.spatial_merge_size,
+ w // self.spatial_merge_size,
+ self.spatial_merge_size,
+ )
+ wpos_ids = wpos_ids.permute(0, 2, 1, 3).flatten()
+
+ pos_ids.append(torch.stack([hpos_ids, wpos_ids], dim=-1).repeat(t, 1))
+
+ pos_ids = torch.cat(pos_ids, dim=0)
+ max_grid_size = grid_thw[:, 1:].max()
+ rotary_pos_emb_full = self.rotary_pos_emb(max_grid_size, device)
+ return rotary_pos_emb_full[pos_ids].flatten(1)
+
+ def forward(
+ self,
+ pixel_values: torch.Tensor,
+ image_grid_thw: Optional[torch.Tensor] = None,
+ ) -> torch.Tensor:
+ optimized_attention = optimized_attention_for_device(pixel_values.device, mask=False, small_input=True)
+
+ hidden_states = self.patch_embed(pixel_values)
+
+ window_index, cu_window_seqlens = self.get_window_index(image_grid_thw)
+ cu_window_seqlens = torch.tensor(cu_window_seqlens, device=hidden_states.device)
+ cu_window_seqlens = torch.unique_consecutive(cu_window_seqlens)
+
+ position_embeddings = self.get_position_embeddings(image_grid_thw, hidden_states.device)
+
+ seq_len, _ = hidden_states.size()
+ spatial_merge_unit = self.spatial_merge_size * self.spatial_merge_size
+
+ hidden_states = hidden_states.reshape(seq_len // spatial_merge_unit, spatial_merge_unit, -1)
+ hidden_states = hidden_states[window_index, :, :]
+ hidden_states = hidden_states.reshape(seq_len, -1)
+
+ position_embeddings = position_embeddings.reshape(seq_len // spatial_merge_unit, spatial_merge_unit, -1)
+ position_embeddings = position_embeddings[window_index, :, :]
+ position_embeddings = position_embeddings.reshape(seq_len, -1)
+ position_embeddings = torch.cat((position_embeddings, position_embeddings), dim=-1)
+ position_embeddings = (position_embeddings.cos(), position_embeddings.sin())
+
+ cu_seqlens = torch.repeat_interleave(image_grid_thw[:, 1] * image_grid_thw[:, 2], image_grid_thw[:, 0]).cumsum(
+ dim=0,
+ dtype=torch.int32,
+ )
+ cu_seqlens = F.pad(cu_seqlens, (1, 0), value=0)
+
+ for i, block in enumerate(self.blocks):
+ if i in self.fullatt_block_indexes:
+ cu_seqlens_now = cu_seqlens
+ else:
+ cu_seqlens_now = cu_window_seqlens
+ hidden_states = block(hidden_states, position_embeddings, cu_seqlens_now, optimized_attention=optimized_attention)
+
+ hidden_states = self.merger(hidden_states)
+ return hidden_states
diff --git a/comfy/text_encoders/t5.py b/comfy/text_encoders/t5.py
index 36bf35309..e8588992a 100644
--- a/comfy/text_encoders/t5.py
+++ b/comfy/text_encoders/t5.py
@@ -199,7 +199,7 @@ class T5Stack(torch.nn.Module):
self.final_layer_norm = T5LayerNorm(model_dim, dtype=dtype, device=device, operations=operations)
# self.dropout = nn.Dropout(config.dropout_rate)
- def forward(self, x, attention_mask=None, intermediate_output=None, final_layer_norm_intermediate=True, dtype=None):
+ def forward(self, x, attention_mask=None, intermediate_output=None, final_layer_norm_intermediate=True, dtype=None, embeds_info=[]):
mask = None
if attention_mask is not None:
mask = 1.0 - attention_mask.to(x.dtype).reshape((attention_mask.shape[0], 1, -1, attention_mask.shape[-1])).expand(attention_mask.shape[0], 1, attention_mask.shape[-1], attention_mask.shape[-1])
diff --git a/comfy/weight_adapter/lokr.py b/comfy/weight_adapter/lokr.py
index 49b0be55f..563c835f5 100644
--- a/comfy/weight_adapter/lokr.py
+++ b/comfy/weight_adapter/lokr.py
@@ -97,6 +97,9 @@ class LoKrAdapter(WeightAdapterBase):
(mat1, mat2, alpha, None, None, None, None, None, None)
)
+ def to_train(self):
+ return LokrDiff(self.weights)
+
@classmethod
def load(
cls,
diff --git a/comfy/weight_adapter/lora.py b/comfy/weight_adapter/lora.py
index 729dbd9e6..47aa17d13 100644
--- a/comfy/weight_adapter/lora.py
+++ b/comfy/weight_adapter/lora.py
@@ -96,6 +96,7 @@ class LoRAAdapter(WeightAdapterBase):
diffusers3_lora = "{}.lora.up.weight".format(x)
mochi_lora = "{}.lora_B".format(x)
transformers_lora = "{}.lora_linear_layer.up.weight".format(x)
+ qwen_default_lora = "{}.lora_B.default.weight".format(x)
A_name = None
if regular_lora in lora.keys():
@@ -122,6 +123,10 @@ class LoRAAdapter(WeightAdapterBase):
A_name = transformers_lora
B_name = "{}.lora_linear_layer.down.weight".format(x)
mid_name = None
+ elif qwen_default_lora in lora.keys():
+ A_name = qwen_default_lora
+ B_name = "{}.lora_A.default.weight".format(x)
+ mid_name = None
if A_name is not None:
mid = None
diff --git a/comfy_api/generate_api_stubs.py b/comfy_api/generate_api_stubs.py
new file mode 100644
index 000000000..604a7eced
--- /dev/null
+++ b/comfy_api/generate_api_stubs.py
@@ -0,0 +1,86 @@
+#!/usr/bin/env python3
+"""
+Script to generate .pyi stub files for the synchronous API wrappers.
+This allows generating stubs without running the full ComfyUI application.
+"""
+
+import os
+import sys
+import logging
+import importlib
+
+# Add ComfyUI to path so we can import modules
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from comfy_api.internal.async_to_sync import AsyncToSyncConverter
+from comfy_api.version_list import supported_versions
+
+
+def generate_stubs_for_module(module_name: str) -> None:
+ """Generate stub files for a specific module that exports ComfyAPI and ComfyAPISync."""
+ try:
+ # Import the module
+ module = importlib.import_module(module_name)
+
+ # Check if module has ComfyAPISync (the sync wrapper)
+ if hasattr(module, "ComfyAPISync"):
+ # Module already has a sync class
+ api_class = getattr(module, "ComfyAPI", None)
+ sync_class = getattr(module, "ComfyAPISync")
+
+ if api_class:
+ # Generate the stub file
+ AsyncToSyncConverter.generate_stub_file(api_class, sync_class)
+ logging.info(f"Generated stub file for {module_name}")
+ else:
+ logging.warning(
+ f"Module {module_name} has ComfyAPISync but no ComfyAPI"
+ )
+
+ elif hasattr(module, "ComfyAPI"):
+ # Module only has async API, need to create sync wrapper first
+ from comfy_api.internal.async_to_sync import create_sync_class
+
+ api_class = getattr(module, "ComfyAPI")
+ sync_class = create_sync_class(api_class)
+
+ # Generate the stub file
+ AsyncToSyncConverter.generate_stub_file(api_class, sync_class)
+ logging.info(f"Generated stub file for {module_name}")
+ else:
+ logging.warning(
+ f"Module {module_name} does not export ComfyAPI or ComfyAPISync"
+ )
+
+ except Exception as e:
+ logging.error(f"Failed to generate stub for {module_name}: {e}")
+ import traceback
+
+ traceback.print_exc()
+
+
+def main():
+ """Main function to generate all API stub files."""
+ logging.basicConfig(level=logging.INFO)
+
+ logging.info("Starting stub generation...")
+
+ # Dynamically get module names from supported_versions
+ api_modules = []
+ for api_class in supported_versions:
+ # Extract module name from the class
+ module_name = api_class.__module__
+ if module_name not in api_modules:
+ api_modules.append(module_name)
+
+ logging.info(f"Found {len(api_modules)} API modules: {api_modules}")
+
+ # Generate stubs for each module
+ for module_name in api_modules:
+ generate_stubs_for_module(module_name)
+
+ logging.info("Stub generation complete!")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/comfy_api/input/__init__.py b/comfy_api/input/__init__.py
index 66667946f..68ff78270 100644
--- a/comfy_api/input/__init__.py
+++ b/comfy_api/input/__init__.py
@@ -1,8 +1,16 @@
-from .basic_types import ImageInput, AudioInput
-from .video_types import VideoInput
+# This file only exists for backwards compatibility.
+from comfy_api.latest._input import (
+ ImageInput,
+ AudioInput,
+ MaskInput,
+ LatentInput,
+ VideoInput,
+)
__all__ = [
"ImageInput",
"AudioInput",
+ "MaskInput",
+ "LatentInput",
"VideoInput",
]
diff --git a/comfy_api/input/basic_types.py b/comfy_api/input/basic_types.py
index 033fb7e27..5eadce86a 100644
--- a/comfy_api/input/basic_types.py
+++ b/comfy_api/input/basic_types.py
@@ -1,20 +1,14 @@
-import torch
-from typing import TypedDict
-
-ImageInput = torch.Tensor
-"""
-An image in format [B, H, W, C] where B is the batch size, C is the number of channels,
-"""
-
-class AudioInput(TypedDict):
- """
- TypedDict representing audio input.
- """
-
- waveform: torch.Tensor
- """
- Tensor in the format [B, C, T] where B is the batch size, C is the number of channels,
- """
-
- sample_rate: int
+# This file only exists for backwards compatibility.
+from comfy_api.latest._input.basic_types import (
+ ImageInput,
+ AudioInput,
+ MaskInput,
+ LatentInput,
+)
+__all__ = [
+ "ImageInput",
+ "AudioInput",
+ "MaskInput",
+ "LatentInput",
+]
diff --git a/comfy_api/input/video_types.py b/comfy_api/input/video_types.py
index 5d95dc507..9ace78cbc 100644
--- a/comfy_api/input/video_types.py
+++ b/comfy_api/input/video_types.py
@@ -1,85 +1,6 @@
-from __future__ import annotations
-from abc import ABC, abstractmethod
-from typing import Optional, Union
-import io
-import av
-from comfy_api.util import VideoContainer, VideoCodec, VideoComponents
+# This file only exists for backwards compatibility.
+from comfy_api.latest._input.video_types import VideoInput
-class VideoInput(ABC):
- """
- Abstract base class for video input types.
- """
-
- @abstractmethod
- def get_components(self) -> VideoComponents:
- """
- Abstract method to get the video components (images, audio, and frame rate).
-
- Returns:
- VideoComponents containing images, audio, and frame rate
- """
- pass
-
- @abstractmethod
- def save_to(
- self,
- path: str,
- format: VideoContainer = VideoContainer.AUTO,
- codec: VideoCodec = VideoCodec.AUTO,
- metadata: Optional[dict] = None
- ):
- """
- Abstract method to save the video input to a file.
- """
- pass
-
- def get_stream_source(self) -> Union[str, io.BytesIO]:
- """
- Get a streamable source for the video. This allows processing without
- loading the entire video into memory.
-
- Returns:
- Either a file path (str) or a BytesIO object that can be opened with av.
-
- Default implementation creates a BytesIO buffer, but subclasses should
- override this for better performance when possible.
- """
- buffer = io.BytesIO()
- self.save_to(buffer)
- buffer.seek(0)
- return buffer
-
- # Provide a default implementation, but subclasses can provide optimized versions
- # if possible.
- def get_dimensions(self) -> tuple[int, int]:
- """
- Returns the dimensions of the video input.
-
- Returns:
- Tuple of (width, height)
- """
- components = self.get_components()
- return components.images.shape[2], components.images.shape[1]
-
- def get_duration(self) -> float:
- """
- Returns the duration of the video in seconds.
-
- Returns:
- Duration in seconds
- """
- components = self.get_components()
- frame_count = components.images.shape[0]
- return float(frame_count / components.frame_rate)
-
- def get_container_format(self) -> str:
- """
- Returns the container format of the video (e.g., 'mp4', 'mov', 'avi').
-
- Returns:
- Container format as string
- """
- # Default implementation - subclasses should override for better performance
- source = self.get_stream_source()
- with av.open(source, mode="r") as container:
- return container.format.name
+__all__ = [
+ "VideoInput",
+]
diff --git a/comfy_api/input_impl/__init__.py b/comfy_api/input_impl/__init__.py
index 02901b8b9..b78ff0c08 100644
--- a/comfy_api/input_impl/__init__.py
+++ b/comfy_api/input_impl/__init__.py
@@ -1,7 +1,7 @@
-from .video_types import VideoFromFile, VideoFromComponents
+# This file only exists for backwards compatibility.
+from comfy_api.latest._input_impl import VideoFromFile, VideoFromComponents
__all__ = [
- # Implementations
"VideoFromFile",
"VideoFromComponents",
]
diff --git a/comfy_api/input_impl/video_types.py b/comfy_api/input_impl/video_types.py
index 91e7c1bfa..bd2e56ad5 100644
--- a/comfy_api/input_impl/video_types.py
+++ b/comfy_api/input_impl/video_types.py
@@ -1,324 +1,2 @@
-from __future__ import annotations
-from av.container import InputContainer
-from av.subtitles.stream import SubtitleStream
-from fractions import Fraction
-from typing import Optional
-from comfy_api.input import AudioInput
-import av
-import io
-import json
-import numpy as np
-import torch
-from comfy_api.input import VideoInput
-from comfy_api.util import VideoContainer, VideoCodec, VideoComponents
-
-
-def container_to_output_format(container_format: str | None) -> str | None:
- """
- A container's `format` may be a comma-separated list of formats.
- E.g., iso container's `format` may be `mov,mp4,m4a,3gp,3g2,mj2`.
- However, writing to a file/stream with `av.open` requires a single format,
- or `None` to auto-detect.
- """
- if not container_format:
- return None # Auto-detect
-
- if "," not in container_format:
- return container_format
-
- formats = container_format.split(",")
- return formats[0]
-
-
-def get_open_write_kwargs(
- dest: str | io.BytesIO, container_format: str, to_format: str | None
-) -> dict:
- """Get kwargs for writing a `VideoFromFile` to a file/stream with `av.open`"""
- open_kwargs = {
- "mode": "w",
- # If isobmff, preserve custom metadata tags (workflow, prompt, extra_pnginfo)
- "options": {"movflags": "use_metadata_tags"},
- }
-
- is_write_to_buffer = isinstance(dest, io.BytesIO)
- if is_write_to_buffer:
- # Set output format explicitly, since it cannot be inferred from file extension
- if to_format == VideoContainer.AUTO:
- to_format = container_format.lower()
- elif isinstance(to_format, str):
- to_format = to_format.lower()
- open_kwargs["format"] = container_to_output_format(to_format)
-
- return open_kwargs
-
-
-class VideoFromFile(VideoInput):
- """
- Class representing video input from a file.
- """
-
- def __init__(self, file: str | io.BytesIO):
- """
- Initialize the VideoFromFile object based off of either a path on disk or a BytesIO object
- containing the file contents.
- """
- self.__file = file
-
- def get_stream_source(self) -> str | io.BytesIO:
- """
- Return the underlying file source for efficient streaming.
- This avoids unnecessary memory copies when the source is already a file path.
- """
- if isinstance(self.__file, io.BytesIO):
- self.__file.seek(0)
- return self.__file
-
- def get_dimensions(self) -> tuple[int, int]:
- """
- Returns the dimensions of the video input.
-
- Returns:
- Tuple of (width, height)
- """
- if isinstance(self.__file, io.BytesIO):
- self.__file.seek(0) # Reset the BytesIO object to the beginning
- with av.open(self.__file, mode='r') as container:
- for stream in container.streams:
- if stream.type == 'video':
- assert isinstance(stream, av.VideoStream)
- return stream.width, stream.height
- raise ValueError(f"No video stream found in file '{self.__file}'")
-
- def get_duration(self) -> float:
- """
- Returns the duration of the video in seconds.
-
- Returns:
- Duration in seconds
- """
- if isinstance(self.__file, io.BytesIO):
- self.__file.seek(0)
- with av.open(self.__file, mode="r") as container:
- if container.duration is not None:
- return float(container.duration / av.time_base)
-
- # Fallback: calculate from frame count and frame rate
- video_stream = next(
- (s for s in container.streams if s.type == "video"), None
- )
- if video_stream and video_stream.frames and video_stream.average_rate:
- return float(video_stream.frames / video_stream.average_rate)
-
- # Last resort: decode frames to count them
- if video_stream and video_stream.average_rate:
- frame_count = 0
- container.seek(0)
- for packet in container.demux(video_stream):
- for _ in packet.decode():
- frame_count += 1
- if frame_count > 0:
- return float(frame_count / video_stream.average_rate)
-
- raise ValueError(f"Could not determine duration for file '{self.__file}'")
-
- def get_container_format(self) -> str:
- """
- Returns the container format of the video (e.g., 'mp4', 'mov', 'avi').
-
- Returns:
- Container format as string
- """
- if isinstance(self.__file, io.BytesIO):
- self.__file.seek(0)
- with av.open(self.__file, mode='r') as container:
- return container.format.name
-
- def get_components_internal(self, container: InputContainer) -> VideoComponents:
- # Get video frames
- frames = []
- for frame in container.decode(video=0):
- img = frame.to_ndarray(format='rgb24') # shape: (H, W, 3)
- img = torch.from_numpy(img) / 255.0 # shape: (H, W, 3)
- frames.append(img)
-
- images = torch.stack(frames) if len(frames) > 0 else torch.zeros(0, 3, 0, 0)
-
- # Get frame rate
- video_stream = next(s for s in container.streams if s.type == 'video')
- frame_rate = Fraction(video_stream.average_rate) if video_stream and video_stream.average_rate else Fraction(1)
-
- # Get audio if available
- audio = None
- try:
- container.seek(0) # Reset the container to the beginning
- for stream in container.streams:
- if stream.type != 'audio':
- continue
- assert isinstance(stream, av.AudioStream)
- audio_frames = []
- for packet in container.demux(stream):
- for frame in packet.decode():
- assert isinstance(frame, av.AudioFrame)
- audio_frames.append(frame.to_ndarray()) # shape: (channels, samples)
- if len(audio_frames) > 0:
- audio_data = np.concatenate(audio_frames, axis=1) # shape: (channels, total_samples)
- audio_tensor = torch.from_numpy(audio_data).unsqueeze(0) # shape: (1, channels, total_samples)
- audio = AudioInput({
- "waveform": audio_tensor,
- "sample_rate": int(stream.sample_rate) if stream.sample_rate else 1,
- })
- except StopIteration:
- pass # No audio stream
-
- metadata = container.metadata
- return VideoComponents(images=images, audio=audio, frame_rate=frame_rate, metadata=metadata)
-
- def get_components(self) -> VideoComponents:
- if isinstance(self.__file, io.BytesIO):
- self.__file.seek(0) # Reset the BytesIO object to the beginning
- with av.open(self.__file, mode='r') as container:
- return self.get_components_internal(container)
- raise ValueError(f"No video stream found in file '{self.__file}'")
-
- def save_to(
- self,
- path: str | io.BytesIO,
- format: VideoContainer = VideoContainer.AUTO,
- codec: VideoCodec = VideoCodec.AUTO,
- metadata: Optional[dict] = None
- ):
- if isinstance(self.__file, io.BytesIO):
- self.__file.seek(0) # Reset the BytesIO object to the beginning
- with av.open(self.__file, mode='r') as container:
- container_format = container.format.name
- video_encoding = container.streams.video[0].codec.name if len(container.streams.video) > 0 else None
- reuse_streams = True
- if format != VideoContainer.AUTO and format not in container_format.split(","):
- reuse_streams = False
- if codec != VideoCodec.AUTO and codec != video_encoding and video_encoding is not None:
- reuse_streams = False
-
- if not reuse_streams:
- components = self.get_components_internal(container)
- video = VideoFromComponents(components)
- return video.save_to(
- path,
- format=format,
- codec=codec,
- metadata=metadata
- )
-
- streams = container.streams
-
- open_kwargs = get_open_write_kwargs(path, container_format, format)
- with av.open(path, **open_kwargs) as output_container:
- # Copy over the original metadata
- for key, value in container.metadata.items():
- if metadata is None or key not in metadata:
- output_container.metadata[key] = value
-
- # Add our new metadata
- if metadata is not None:
- for key, value in metadata.items():
- if isinstance(value, str):
- output_container.metadata[key] = value
- else:
- output_container.metadata[key] = json.dumps(value)
-
- # Add streams to the new container
- stream_map = {}
- for stream in streams:
- if isinstance(stream, (av.VideoStream, av.AudioStream, SubtitleStream)):
- out_stream = output_container.add_stream_from_template(template=stream, opaque=True)
- stream_map[stream] = out_stream
-
- # Write packets to the new container
- for packet in container.demux():
- if packet.stream in stream_map and packet.dts is not None:
- packet.stream = stream_map[packet.stream]
- output_container.mux(packet)
-
-class VideoFromComponents(VideoInput):
- """
- Class representing video input from tensors.
- """
-
- def __init__(self, components: VideoComponents):
- self.__components = components
-
- def get_components(self) -> VideoComponents:
- return VideoComponents(
- images=self.__components.images,
- audio=self.__components.audio,
- frame_rate=self.__components.frame_rate
- )
-
- def save_to(
- self,
- path: str,
- format: VideoContainer = VideoContainer.AUTO,
- codec: VideoCodec = VideoCodec.AUTO,
- metadata: Optional[dict] = None
- ):
- if format != VideoContainer.AUTO and format != VideoContainer.MP4:
- raise ValueError("Only MP4 format is supported for now")
- if codec != VideoCodec.AUTO and codec != VideoCodec.H264:
- raise ValueError("Only H264 codec is supported for now")
- with av.open(path, mode='w', options={'movflags': 'use_metadata_tags'}) as output:
- # Add metadata before writing any streams
- if metadata is not None:
- for key, value in metadata.items():
- output.metadata[key] = json.dumps(value)
-
- frame_rate = Fraction(round(self.__components.frame_rate * 1000), 1000)
- # Create a video stream
- video_stream = output.add_stream('h264', rate=frame_rate)
- video_stream.width = self.__components.images.shape[2]
- video_stream.height = self.__components.images.shape[1]
- video_stream.pix_fmt = 'yuv420p'
-
- # Create an audio stream
- audio_sample_rate = 1
- audio_stream: Optional[av.AudioStream] = None
- if self.__components.audio:
- audio_sample_rate = int(self.__components.audio['sample_rate'])
- audio_stream = output.add_stream('aac', rate=audio_sample_rate)
- audio_stream.sample_rate = audio_sample_rate
- audio_stream.format = 'fltp'
-
- # Encode video
- for i, frame in enumerate(self.__components.images):
- img = (frame * 255).clamp(0, 255).byte().cpu().numpy() # shape: (H, W, 3)
- frame = av.VideoFrame.from_ndarray(img, format='rgb24')
- frame = frame.reformat(format='yuv420p') # Convert to YUV420P as required by h264
- packet = video_stream.encode(frame)
- output.mux(packet)
-
- # Flush video
- packet = video_stream.encode(None)
- output.mux(packet)
-
- if audio_stream and self.__components.audio:
- # Encode audio
- samples_per_frame = int(audio_sample_rate / frame_rate)
- num_frames = self.__components.audio['waveform'].shape[2] // samples_per_frame
- for i in range(num_frames):
- start = i * samples_per_frame
- end = start + samples_per_frame
- # TODO(Feature) - Add support for stereo audio
- chunk = (
- self.__components.audio["waveform"][0, 0, start:end]
- .unsqueeze(0)
- .contiguous()
- .numpy()
- )
- audio_frame = av.AudioFrame.from_ndarray(chunk, format='fltp', layout='mono')
- audio_frame.sample_rate = audio_sample_rate
- audio_frame.pts = i * samples_per_frame
- for packet in audio_stream.encode(audio_frame):
- output.mux(packet)
-
- # Flush audio
- for packet in audio_stream.encode(None):
- output.mux(packet)
-
+# This file only exists for backwards compatibility.
+from comfy_api.latest._input_impl.video_types import * # noqa: F403
diff --git a/comfy_api/internal/__init__.py b/comfy_api/internal/__init__.py
new file mode 100644
index 000000000..4ca02e320
--- /dev/null
+++ b/comfy_api/internal/__init__.py
@@ -0,0 +1,150 @@
+# Internal infrastructure for ComfyAPI
+from .api_registry import (
+ ComfyAPIBase as ComfyAPIBase,
+ ComfyAPIWithVersion as ComfyAPIWithVersion,
+ register_versions as register_versions,
+ get_all_versions as get_all_versions,
+)
+
+import asyncio
+from dataclasses import asdict
+from typing import Callable, Optional
+
+
+def first_real_override(cls: type, name: str, *, base: type=None) -> Optional[Callable]:
+ """Return the *callable* override of `name` visible on `cls`, or None if every
+ implementation up to (and including) `base` is the placeholder defined on `base`.
+
+ If base is not provided, it will assume cls has a GET_BASE_CLASS
+ """
+ if base is None:
+ if not hasattr(cls, "GET_BASE_CLASS"):
+ raise ValueError("base is required if cls does not have a GET_BASE_CLASS; is this a valid ComfyNode subclass?")
+ base = cls.GET_BASE_CLASS()
+ base_attr = getattr(base, name, None)
+ if base_attr is None:
+ return None
+ base_func = base_attr.__func__
+ for c in cls.mro(): # NodeB, NodeA, ComfyNode, object …
+ if c is base: # reached the placeholder – we're done
+ break
+ if name in c.__dict__: # first class that *defines* the attr
+ func = getattr(c, name).__func__
+ if func is not base_func: # real override
+ return getattr(cls, name) # bound to *cls*
+ return None
+
+
+class _ComfyNodeInternal:
+ """Class that all V3-based APIs inherit from for ComfyNode.
+
+ This is intended to only be referenced within execution.py, as it has to handle all V3 APIs going forward."""
+ @classmethod
+ def GET_NODE_INFO_V1(cls):
+ ...
+
+
+class _NodeOutputInternal:
+ """Class that all V3-based APIs inherit from for NodeOutput.
+
+ This is intended to only be referenced within execution.py, as it has to handle all V3 APIs going forward."""
+ ...
+
+
+def as_pruned_dict(dataclass_obj):
+ '''Return dict of dataclass object with pruned None values.'''
+ return prune_dict(asdict(dataclass_obj))
+
+def prune_dict(d: dict):
+ return {k: v for k,v in d.items() if v is not None}
+
+
+def is_class(obj):
+ '''
+ Returns True if is a class type.
+ Returns False if is a class instance.
+ '''
+ return isinstance(obj, type)
+
+
+def copy_class(cls: type) -> type:
+ '''
+ Copy a class and its attributes.
+ '''
+ if cls is None:
+ return None
+ cls_dict = {
+ k: v for k, v in cls.__dict__.items()
+ if k not in ('__dict__', '__weakref__', '__module__', '__doc__')
+ }
+ # new class
+ new_cls = type(
+ cls.__name__,
+ (cls,),
+ cls_dict
+ )
+ # metadata preservation
+ new_cls.__module__ = cls.__module__
+ new_cls.__doc__ = cls.__doc__
+ return new_cls
+
+
+class classproperty(object):
+ def __init__(self, f):
+ self.f = f
+ def __get__(self, obj, owner):
+ return self.f(owner)
+
+
+# NOTE: this was ai generated and validated by hand
+def shallow_clone_class(cls, new_name=None):
+ '''
+ Shallow clone a class while preserving super() functionality.
+ '''
+ new_name = new_name or f"{cls.__name__}Clone"
+ # Include the original class in the bases to maintain proper inheritance
+ new_bases = (cls,) + cls.__bases__
+ return type(new_name, new_bases, dict(cls.__dict__))
+
+# NOTE: this was ai generated and validated by hand
+def lock_class(cls):
+ '''
+ Lock a class so that its top-levelattributes cannot be modified.
+ '''
+ # Locked instance __setattr__
+ def locked_instance_setattr(self, name, value):
+ raise AttributeError(
+ f"Cannot set attribute '{name}' on immutable instance of {type(self).__name__}"
+ )
+ # Locked metaclass
+ class LockedMeta(type(cls)):
+ def __setattr__(cls_, name, value):
+ raise AttributeError(
+ f"Cannot modify class attribute '{name}' on locked class '{cls_.__name__}'"
+ )
+ # Rebuild class with locked behavior
+ locked_dict = dict(cls.__dict__)
+ locked_dict['__setattr__'] = locked_instance_setattr
+
+ return LockedMeta(cls.__name__, cls.__bases__, locked_dict)
+
+
+def make_locked_method_func(type_obj, func, class_clone):
+ """
+ Returns a function that, when called with **inputs, will execute:
+ getattr(type_obj, func).__func__(lock_class(class_clone), **inputs)
+
+ Supports both synchronous and asynchronous methods.
+ """
+ locked_class = lock_class(class_clone)
+ method = getattr(type_obj, func).__func__
+
+ # Check if the original method is async
+ if asyncio.iscoroutinefunction(method):
+ async def wrapped_async_func(**inputs):
+ return await method(locked_class, **inputs)
+ return wrapped_async_func
+ else:
+ def wrapped_func(**inputs):
+ return method(locked_class, **inputs)
+ return wrapped_func
diff --git a/comfy_api/internal/api_registry.py b/comfy_api/internal/api_registry.py
new file mode 100644
index 000000000..7e3375cf6
--- /dev/null
+++ b/comfy_api/internal/api_registry.py
@@ -0,0 +1,39 @@
+from typing import Type, List, NamedTuple
+from comfy_api.internal.singleton import ProxiedSingleton
+from packaging import version as packaging_version
+
+
+class ComfyAPIBase(ProxiedSingleton):
+ def __init__(self):
+ pass
+
+
+class ComfyAPIWithVersion(NamedTuple):
+ version: str
+ api_class: Type[ComfyAPIBase]
+
+
+def parse_version(version_str: str) -> packaging_version.Version:
+ """
+ Parses a version string into a packaging_version.Version object.
+ Raises ValueError if the version string is invalid.
+ """
+ if version_str == "latest":
+ return packaging_version.parse("9999999.9999999.9999999")
+ return packaging_version.parse(version_str)
+
+
+registered_versions: List[ComfyAPIWithVersion] = []
+
+
+def register_versions(versions: List[ComfyAPIWithVersion]):
+ versions.sort(key=lambda x: parse_version(x.version))
+ global registered_versions
+ registered_versions = versions
+
+
+def get_all_versions() -> List[ComfyAPIWithVersion]:
+ """
+ Returns a list of all registered ComfyAPI versions.
+ """
+ return registered_versions
diff --git a/comfy_api/internal/async_to_sync.py b/comfy_api/internal/async_to_sync.py
new file mode 100644
index 000000000..f5f805a62
--- /dev/null
+++ b/comfy_api/internal/async_to_sync.py
@@ -0,0 +1,987 @@
+import asyncio
+import concurrent.futures
+import contextvars
+import functools
+import inspect
+import logging
+import os
+import textwrap
+import threading
+from enum import Enum
+from typing import Optional, Type, get_origin, get_args
+
+
+class TypeTracker:
+ """Tracks types discovered during stub generation for automatic import generation."""
+
+ def __init__(self):
+ self.discovered_types = {} # type_name -> (module, qualname)
+ self.builtin_types = {
+ "Any",
+ "Dict",
+ "List",
+ "Optional",
+ "Tuple",
+ "Union",
+ "Set",
+ "Sequence",
+ "cast",
+ "NamedTuple",
+ "str",
+ "int",
+ "float",
+ "bool",
+ "None",
+ "bytes",
+ "object",
+ "type",
+ "dict",
+ "list",
+ "tuple",
+ "set",
+ }
+ self.already_imported = (
+ set()
+ ) # Track types already imported to avoid duplicates
+
+ def track_type(self, annotation):
+ """Track a type annotation and record its module/import info."""
+ if annotation is None or annotation is type(None):
+ return
+
+ # Skip builtins and typing module types we already import
+ type_name = getattr(annotation, "__name__", None)
+ if type_name and (
+ type_name in self.builtin_types or type_name in self.already_imported
+ ):
+ return
+
+ # Get module and qualname
+ module = getattr(annotation, "__module__", None)
+ qualname = getattr(annotation, "__qualname__", type_name or "")
+
+ # Skip types from typing module (they're already imported)
+ if module == "typing":
+ return
+
+ # Skip UnionType and GenericAlias from types module as they're handled specially
+ if module == "types" and type_name in ("UnionType", "GenericAlias"):
+ return
+
+ if module and module not in ["builtins", "__main__"]:
+ # Store the type info
+ if type_name:
+ self.discovered_types[type_name] = (module, qualname)
+
+ def get_imports(self, main_module_name: str) -> list[str]:
+ """Generate import statements for all discovered types."""
+ imports = []
+ imports_by_module = {}
+
+ for type_name, (module, qualname) in sorted(self.discovered_types.items()):
+ # Skip types from the main module (they're already imported)
+ if main_module_name and module == main_module_name:
+ continue
+
+ if module not in imports_by_module:
+ imports_by_module[module] = []
+ if type_name not in imports_by_module[module]: # Avoid duplicates
+ imports_by_module[module].append(type_name)
+
+ # Generate import statements
+ for module, types in sorted(imports_by_module.items()):
+ if len(types) == 1:
+ imports.append(f"from {module} import {types[0]}")
+ else:
+ imports.append(f"from {module} import {', '.join(sorted(set(types)))}")
+
+ return imports
+
+
+class AsyncToSyncConverter:
+ """
+ Provides utilities to convert async classes to sync classes with proper type hints.
+ """
+
+ _thread_pool: Optional[concurrent.futures.ThreadPoolExecutor] = None
+ _thread_pool_lock = threading.Lock()
+ _thread_pool_initialized = False
+
+ @classmethod
+ def get_thread_pool(cls, max_workers=None) -> concurrent.futures.ThreadPoolExecutor:
+ """Get or create the shared thread pool with proper thread-safe initialization."""
+ # Fast path - check if already initialized without acquiring lock
+ if cls._thread_pool_initialized:
+ assert cls._thread_pool is not None, "Thread pool should be initialized"
+ return cls._thread_pool
+
+ # Slow path - acquire lock and create pool if needed
+ with cls._thread_pool_lock:
+ if not cls._thread_pool_initialized:
+ cls._thread_pool = concurrent.futures.ThreadPoolExecutor(
+ max_workers=max_workers, thread_name_prefix="async_to_sync_"
+ )
+ cls._thread_pool_initialized = True
+
+ # This should never be None at this point, but add assertion for type checker
+ assert cls._thread_pool is not None
+ return cls._thread_pool
+
+ @classmethod
+ def run_async_in_thread(cls, coro_func, *args, **kwargs):
+ """
+ Run an async function in a separate thread from the thread pool.
+ Blocks until the async function completes.
+ Properly propagates contextvars between threads and manages event loops.
+ """
+ # Capture current context - this includes all context variables
+ context = contextvars.copy_context()
+
+ # Store the result and any exception that occurs
+ result_container: dict = {"result": None, "exception": None}
+
+ # Function that runs in the thread pool
+ def run_in_thread():
+ # Create new event loop for this thread
+ loop = asyncio.new_event_loop()
+ asyncio.set_event_loop(loop)
+
+ try:
+ # Create the coroutine within the context
+ async def run_with_context():
+ # The coroutine function might access context variables
+ return await coro_func(*args, **kwargs)
+
+ # Run the coroutine with the captured context
+ # This ensures all context variables are available in the async function
+ result = context.run(loop.run_until_complete, run_with_context())
+ result_container["result"] = result
+ except Exception as e:
+ # Store the exception to re-raise in the calling thread
+ result_container["exception"] = e
+ finally:
+ # Ensure event loop is properly closed to prevent warnings
+ try:
+ # Cancel any remaining tasks
+ pending = asyncio.all_tasks(loop)
+ for task in pending:
+ task.cancel()
+
+ # Run the loop briefly to handle cancellations
+ if pending:
+ loop.run_until_complete(
+ asyncio.gather(*pending, return_exceptions=True)
+ )
+ except Exception:
+ pass # Ignore errors during cleanup
+
+ # Close the event loop
+ loop.close()
+
+ # Clear the event loop from the thread
+ asyncio.set_event_loop(None)
+
+ # Submit to thread pool and wait for result
+ thread_pool = cls.get_thread_pool()
+ future = thread_pool.submit(run_in_thread)
+ future.result() # Wait for completion
+
+ # Re-raise any exception that occurred in the thread
+ if result_container["exception"] is not None:
+ raise result_container["exception"]
+
+ return result_container["result"]
+
+ @classmethod
+ def create_sync_class(cls, async_class: Type, thread_pool_size=10) -> Type:
+ """
+ Creates a new class with synchronous versions of all async methods.
+
+ Args:
+ async_class: The async class to convert
+ thread_pool_size: Size of thread pool to use
+
+ Returns:
+ A new class with sync versions of all async methods
+ """
+ sync_class_name = "ComfyAPISyncStub"
+ cls.get_thread_pool(thread_pool_size)
+
+ # Create a proper class with docstrings and proper base classes
+ sync_class_dict = {
+ "__doc__": async_class.__doc__,
+ "__module__": async_class.__module__,
+ "__qualname__": sync_class_name,
+ "__orig_class__": async_class, # Store original class for typing references
+ }
+
+ # Create __init__ method
+ def __init__(self, *args, **kwargs):
+ self._async_instance = async_class(*args, **kwargs)
+
+ # Handle annotated class attributes (like execution: Execution)
+ # Get all annotations from the class hierarchy
+ all_annotations = {}
+ for base_class in reversed(inspect.getmro(async_class)):
+ if hasattr(base_class, "__annotations__"):
+ all_annotations.update(base_class.__annotations__)
+
+ # For each annotated attribute, check if it needs to be created or wrapped
+ for attr_name, attr_type in all_annotations.items():
+ if hasattr(self._async_instance, attr_name):
+ # Attribute exists on the instance
+ attr = getattr(self._async_instance, attr_name)
+ # Check if this attribute needs a sync wrapper
+ if hasattr(attr, "__class__"):
+ from comfy_api.internal.singleton import ProxiedSingleton
+
+ if isinstance(attr, ProxiedSingleton):
+ # Create a sync version of this attribute
+ try:
+ sync_attr_class = cls.create_sync_class(attr.__class__)
+ # Create instance of the sync wrapper with the async instance
+ sync_attr = object.__new__(sync_attr_class) # type: ignore
+ sync_attr._async_instance = attr
+ setattr(self, attr_name, sync_attr)
+ except Exception:
+ # If we can't create a sync version, keep the original
+ setattr(self, attr_name, attr)
+ else:
+ # Not async, just copy the reference
+ setattr(self, attr_name, attr)
+ else:
+ # Attribute doesn't exist, but is annotated - create it
+ # This handles cases like execution: Execution
+ if isinstance(attr_type, type):
+ # Check if the type is defined as an inner class
+ if hasattr(async_class, attr_type.__name__):
+ inner_class = getattr(async_class, attr_type.__name__)
+ from comfy_api.internal.singleton import ProxiedSingleton
+
+ # Create an instance of the inner class
+ try:
+ # For ProxiedSingleton classes, get or create the singleton instance
+ if issubclass(inner_class, ProxiedSingleton):
+ async_instance = inner_class.get_instance()
+ else:
+ async_instance = inner_class()
+
+ # Create sync wrapper
+ sync_attr_class = cls.create_sync_class(inner_class)
+ sync_attr = object.__new__(sync_attr_class) # type: ignore
+ sync_attr._async_instance = async_instance
+ setattr(self, attr_name, sync_attr)
+ # Also set on the async instance for consistency
+ setattr(self._async_instance, attr_name, async_instance)
+ except Exception as e:
+ logging.warning(
+ f"Failed to create instance for {attr_name}: {e}"
+ )
+
+ # Handle other instance attributes that might not be annotated
+ for name, attr in inspect.getmembers(self._async_instance):
+ if name.startswith("_") or hasattr(self, name):
+ continue
+
+ # If attribute is an instance of a class, and that class is defined in the original class
+ # we need to check if it needs a sync wrapper
+ if isinstance(attr, object) and not isinstance(
+ attr, (str, int, float, bool, list, dict, tuple)
+ ):
+ from comfy_api.internal.singleton import ProxiedSingleton
+
+ if isinstance(attr, ProxiedSingleton):
+ # Create a sync version of this nested class
+ try:
+ sync_attr_class = cls.create_sync_class(attr.__class__)
+ # Create instance of the sync wrapper with the async instance
+ sync_attr = object.__new__(sync_attr_class) # type: ignore
+ sync_attr._async_instance = attr
+ setattr(self, name, sync_attr)
+ except Exception:
+ # If we can't create a sync version, keep the original
+ setattr(self, name, attr)
+
+ sync_class_dict["__init__"] = __init__
+
+ # Process methods from the async class
+ for name, method in inspect.getmembers(
+ async_class, predicate=inspect.isfunction
+ ):
+ if name.startswith("_"):
+ continue
+
+ # Extract the actual return type from a coroutine
+ if inspect.iscoroutinefunction(method):
+ # Create sync version of async method with proper signature
+ @functools.wraps(method)
+ def sync_method(self, *args, _method_name=name, **kwargs):
+ async_method = getattr(self._async_instance, _method_name)
+ return AsyncToSyncConverter.run_async_in_thread(
+ async_method, *args, **kwargs
+ )
+
+ # Add to the class dict
+ sync_class_dict[name] = sync_method
+ else:
+ # For regular methods, create a proxy method
+ @functools.wraps(method)
+ def proxy_method(self, *args, _method_name=name, **kwargs):
+ method = getattr(self._async_instance, _method_name)
+ return method(*args, **kwargs)
+
+ # Add to the class dict
+ sync_class_dict[name] = proxy_method
+
+ # Handle property access
+ for name, prop in inspect.getmembers(
+ async_class, lambda x: isinstance(x, property)
+ ):
+
+ def make_property(name, prop_obj):
+ def getter(self):
+ value = getattr(self._async_instance, name)
+ if inspect.iscoroutinefunction(value):
+
+ def sync_fn(*args, **kwargs):
+ return AsyncToSyncConverter.run_async_in_thread(
+ value, *args, **kwargs
+ )
+
+ return sync_fn
+ return value
+
+ def setter(self, value):
+ setattr(self._async_instance, name, value)
+
+ return property(getter, setter if prop_obj.fset else None)
+
+ sync_class_dict[name] = make_property(name, prop)
+
+ # Create the class
+ sync_class = type(sync_class_name, (object,), sync_class_dict)
+
+ return sync_class
+
+ @classmethod
+ def _format_type_annotation(
+ cls, annotation, type_tracker: Optional[TypeTracker] = None
+ ) -> str:
+ """Convert a type annotation to its string representation for stub files."""
+ if (
+ annotation is inspect.Parameter.empty
+ or annotation is inspect.Signature.empty
+ ):
+ return "Any"
+
+ # Handle None type
+ if annotation is type(None):
+ return "None"
+
+ # Track the type if we have a tracker
+ if type_tracker:
+ type_tracker.track_type(annotation)
+
+ # Try using typing.get_origin/get_args for Python 3.8+
+ try:
+ origin = get_origin(annotation)
+ args = get_args(annotation)
+
+ if origin is not None:
+ # Track the origin type
+ if type_tracker:
+ type_tracker.track_type(origin)
+
+ # Get the origin name
+ origin_name = getattr(origin, "__name__", str(origin))
+ if "." in origin_name:
+ origin_name = origin_name.split(".")[-1]
+
+ # Special handling for types.UnionType (Python 3.10+ pipe operator)
+ # Convert to old-style Union for compatibility
+ if str(origin) == "" or origin_name == "UnionType":
+ origin_name = "Union"
+
+ # Format arguments recursively
+ if args:
+ formatted_args = []
+ for arg in args:
+ # Track each type in the union
+ if type_tracker:
+ type_tracker.track_type(arg)
+ formatted_args.append(cls._format_type_annotation(arg, type_tracker))
+ return f"{origin_name}[{', '.join(formatted_args)}]"
+ else:
+ return origin_name
+ except (AttributeError, TypeError):
+ # Fallback for older Python versions or non-generic types
+ pass
+
+ # Handle generic types the old way for compatibility
+ if hasattr(annotation, "__origin__") and hasattr(annotation, "__args__"):
+ origin = annotation.__origin__
+ origin_name = (
+ origin.__name__
+ if hasattr(origin, "__name__")
+ else str(origin).split("'")[1]
+ )
+
+ # Format each type argument
+ args = []
+ for arg in annotation.__args__:
+ args.append(cls._format_type_annotation(arg, type_tracker))
+
+ return f"{origin_name}[{', '.join(args)}]"
+
+ # Handle regular types with __name__
+ if hasattr(annotation, "__name__"):
+ return annotation.__name__
+
+ # Handle special module types (like types from typing module)
+ if hasattr(annotation, "__module__") and hasattr(annotation, "__qualname__"):
+ # For types like typing.Literal, typing.TypedDict, etc.
+ return annotation.__qualname__
+
+ # Last resort: string conversion with cleanup
+ type_str = str(annotation)
+
+ # Clean up common patterns more robustly
+ if type_str.startswith(""):
+ type_str = type_str[8:-2] # Remove ""
+
+ # Remove module prefixes for common modules
+ for prefix in ["typing.", "builtins.", "types."]:
+ if type_str.startswith(prefix):
+ type_str = type_str[len(prefix) :]
+
+ # Handle special cases
+ if type_str in ("_empty", "inspect._empty"):
+ return "None"
+
+ # Fix NoneType (this should rarely be needed now)
+ if type_str == "NoneType":
+ return "None"
+
+ return type_str
+
+ @classmethod
+ def _extract_coroutine_return_type(cls, annotation):
+ """Extract the actual return type from a Coroutine annotation."""
+ if hasattr(annotation, "__args__") and len(annotation.__args__) > 2:
+ # Coroutine[Any, Any, ReturnType] -> extract ReturnType
+ return annotation.__args__[2]
+ return annotation
+
+ @classmethod
+ def _format_parameter_default(cls, default_value) -> str:
+ """Format a parameter's default value for stub files."""
+ if default_value is inspect.Parameter.empty:
+ return ""
+ elif default_value is None:
+ return " = None"
+ elif isinstance(default_value, bool):
+ return f" = {default_value}"
+ elif default_value == {}:
+ return " = {}"
+ elif default_value == []:
+ return " = []"
+ else:
+ return f" = {default_value}"
+
+ @classmethod
+ def _format_method_parameters(
+ cls,
+ sig: inspect.Signature,
+ skip_self: bool = True,
+ type_hints: Optional[dict] = None,
+ type_tracker: Optional[TypeTracker] = None,
+ ) -> str:
+ """Format method parameters for stub files."""
+ params = []
+ if type_hints is None:
+ type_hints = {}
+
+ for i, (param_name, param) in enumerate(sig.parameters.items()):
+ if i == 0 and param_name == "self" and skip_self:
+ params.append("self")
+ else:
+ # Get type annotation from type hints if available, otherwise from signature
+ annotation = type_hints.get(param_name, param.annotation)
+ type_str = cls._format_type_annotation(annotation, type_tracker)
+
+ # Get default value
+ default_str = cls._format_parameter_default(param.default)
+
+ # Combine parameter parts
+ if annotation is inspect.Parameter.empty:
+ params.append(f"{param_name}: Any{default_str}")
+ else:
+ params.append(f"{param_name}: {type_str}{default_str}")
+
+ return ", ".join(params)
+
+ @classmethod
+ def _generate_method_signature(
+ cls,
+ method_name: str,
+ method,
+ is_async: bool = False,
+ type_tracker: Optional[TypeTracker] = None,
+ ) -> str:
+ """Generate a complete method signature for stub files."""
+ sig = inspect.signature(method)
+
+ # Try to get evaluated type hints to resolve string annotations
+ try:
+ from typing import get_type_hints
+ type_hints = get_type_hints(method)
+ except Exception:
+ # Fallback to empty dict if we can't get type hints
+ type_hints = {}
+
+ # For async methods, extract the actual return type
+ return_annotation = type_hints.get('return', sig.return_annotation)
+ if is_async and inspect.iscoroutinefunction(method):
+ return_annotation = cls._extract_coroutine_return_type(return_annotation)
+
+ # Format parameters with type hints
+ params_str = cls._format_method_parameters(sig, type_hints=type_hints, type_tracker=type_tracker)
+
+ # Format return type
+ return_type = cls._format_type_annotation(return_annotation, type_tracker)
+ if return_annotation is inspect.Signature.empty:
+ return_type = "None"
+
+ return f"def {method_name}({params_str}) -> {return_type}: ..."
+
+ @classmethod
+ def _generate_imports(
+ cls, async_class: Type, type_tracker: TypeTracker
+ ) -> list[str]:
+ """Generate import statements for the stub file."""
+ imports = []
+
+ # Add standard typing imports
+ imports.append(
+ "from typing import Any, Dict, List, Optional, Tuple, Union, Set, Sequence, cast, NamedTuple"
+ )
+
+ # Add imports from the original module
+ if async_class.__module__ != "builtins":
+ module = inspect.getmodule(async_class)
+ additional_types = []
+
+ if module:
+ # Check if module has __all__ defined
+ module_all = getattr(module, "__all__", None)
+
+ for name, obj in sorted(inspect.getmembers(module)):
+ if isinstance(obj, type):
+ # Skip if __all__ is defined and this name isn't in it
+ # unless it's already been tracked as used in type annotations
+ if module_all is not None and name not in module_all:
+ # Check if this type was actually used in annotations
+ if name not in type_tracker.discovered_types:
+ continue
+
+ # Check for NamedTuple
+ if issubclass(obj, tuple) and hasattr(obj, "_fields"):
+ additional_types.append(name)
+ # Mark as already imported
+ type_tracker.already_imported.add(name)
+ # Check for Enum
+ elif issubclass(obj, Enum) and name != "Enum":
+ additional_types.append(name)
+ # Mark as already imported
+ type_tracker.already_imported.add(name)
+
+ if additional_types:
+ type_imports = ", ".join([async_class.__name__] + additional_types)
+ imports.append(f"from {async_class.__module__} import {type_imports}")
+ else:
+ imports.append(
+ f"from {async_class.__module__} import {async_class.__name__}"
+ )
+
+ # Add imports for all discovered types
+ # Pass the main module name to avoid duplicate imports
+ imports.extend(
+ type_tracker.get_imports(main_module_name=async_class.__module__)
+ )
+
+ # Add base module import if needed
+ if hasattr(inspect.getmodule(async_class), "__name__"):
+ module_name = inspect.getmodule(async_class).__name__
+ if "." in module_name:
+ base_module = module_name.split(".")[0]
+ # Only add if not already importing from it
+ if not any(imp.startswith(f"from {base_module}") for imp in imports):
+ imports.append(f"import {base_module}")
+
+ return imports
+
+ @classmethod
+ def _get_class_attributes(cls, async_class: Type) -> list[tuple[str, Type]]:
+ """Extract class attributes that are classes themselves."""
+ class_attributes = []
+
+ # Look for class attributes that are classes
+ for name, attr in sorted(inspect.getmembers(async_class)):
+ if isinstance(attr, type) and not name.startswith("_"):
+ class_attributes.append((name, attr))
+ elif (
+ hasattr(async_class, "__annotations__")
+ and name in async_class.__annotations__
+ ):
+ annotation = async_class.__annotations__[name]
+ if isinstance(annotation, type):
+ class_attributes.append((name, annotation))
+
+ return class_attributes
+
+ @classmethod
+ def _generate_inner_class_stub(
+ cls,
+ name: str,
+ attr: Type,
+ indent: str = " ",
+ type_tracker: Optional[TypeTracker] = None,
+ ) -> list[str]:
+ """Generate stub for an inner class."""
+ stub_lines = []
+ stub_lines.append(f"{indent}class {name}Sync:")
+
+ # Add docstring if available
+ if hasattr(attr, "__doc__") and attr.__doc__:
+ stub_lines.extend(
+ cls._format_docstring_for_stub(attr.__doc__, f"{indent} ")
+ )
+
+ # Add __init__ if it exists
+ if hasattr(attr, "__init__"):
+ try:
+ init_method = getattr(attr, "__init__")
+ init_sig = inspect.signature(init_method)
+
+ # Try to get type hints
+ try:
+ from typing import get_type_hints
+ init_hints = get_type_hints(init_method)
+ except Exception:
+ init_hints = {}
+
+ # Format parameters
+ params_str = cls._format_method_parameters(
+ init_sig, type_hints=init_hints, type_tracker=type_tracker
+ )
+ # Add __init__ docstring if available (before the method)
+ if hasattr(init_method, "__doc__") and init_method.__doc__:
+ stub_lines.extend(
+ cls._format_docstring_for_stub(
+ init_method.__doc__, f"{indent} "
+ )
+ )
+ stub_lines.append(
+ f"{indent} def __init__({params_str}) -> None: ..."
+ )
+ except (ValueError, TypeError):
+ stub_lines.append(
+ f"{indent} def __init__(self, *args, **kwargs) -> None: ..."
+ )
+
+ # Add methods to the inner class
+ has_methods = False
+ for method_name, method in sorted(
+ inspect.getmembers(attr, predicate=inspect.isfunction)
+ ):
+ if method_name.startswith("_"):
+ continue
+
+ has_methods = True
+ try:
+ # Add method docstring if available (before the method signature)
+ if method.__doc__:
+ stub_lines.extend(
+ cls._format_docstring_for_stub(method.__doc__, f"{indent} ")
+ )
+
+ method_sig = cls._generate_method_signature(
+ method_name, method, is_async=True, type_tracker=type_tracker
+ )
+ stub_lines.append(f"{indent} {method_sig}")
+ except (ValueError, TypeError):
+ stub_lines.append(
+ f"{indent} def {method_name}(self, *args, **kwargs): ..."
+ )
+
+ if not has_methods:
+ stub_lines.append(f"{indent} pass")
+
+ return stub_lines
+
+ @classmethod
+ def _format_docstring_for_stub(
+ cls, docstring: str, indent: str = " "
+ ) -> list[str]:
+ """Format a docstring for inclusion in a stub file with proper indentation."""
+ if not docstring:
+ return []
+
+ # First, dedent the docstring to remove any existing indentation
+ dedented = textwrap.dedent(docstring).strip()
+
+ # Split into lines
+ lines = dedented.split("\n")
+
+ # Build the properly indented docstring
+ result = []
+ result.append(f'{indent}"""')
+
+ for line in lines:
+ if line.strip(): # Non-empty line
+ result.append(f"{indent}{line}")
+ else: # Empty line
+ result.append("")
+
+ result.append(f'{indent}"""')
+ return result
+
+ @classmethod
+ def _post_process_stub_content(cls, stub_content: list[str]) -> list[str]:
+ """Post-process stub content to fix any remaining issues."""
+ processed = []
+
+ for line in stub_content:
+ # Skip processing imports
+ if line.startswith(("from ", "import ")):
+ processed.append(line)
+ continue
+
+ # Fix method signatures missing return types
+ if (
+ line.strip().startswith("def ")
+ and line.strip().endswith(": ...")
+ and ") -> " not in line
+ ):
+ # Add -> None for methods without return annotation
+ line = line.replace(": ...", " -> None: ...")
+
+ processed.append(line)
+
+ return processed
+
+ @classmethod
+ def generate_stub_file(cls, async_class: Type, sync_class: Type) -> None:
+ """
+ Generate a .pyi stub file for the sync class to help IDEs with type checking.
+ """
+ try:
+ # Only generate stub if we can determine module path
+ if async_class.__module__ == "__main__":
+ return
+
+ module = inspect.getmodule(async_class)
+ if not module:
+ return
+
+ module_path = module.__file__
+ if not module_path:
+ return
+
+ # Create stub file path in a 'generated' subdirectory
+ module_dir = os.path.dirname(module_path)
+ stub_dir = os.path.join(module_dir, "generated")
+
+ # Ensure the generated directory exists
+ os.makedirs(stub_dir, exist_ok=True)
+
+ module_name = os.path.basename(module_path)
+ if module_name.endswith(".py"):
+ module_name = module_name[:-3]
+
+ sync_stub_path = os.path.join(stub_dir, f"{sync_class.__name__}.pyi")
+
+ # Create a type tracker for this stub generation
+ type_tracker = TypeTracker()
+
+ stub_content = []
+
+ # We'll generate imports after processing all methods to capture all types
+ # Leave a placeholder for imports
+ imports_placeholder_index = len(stub_content)
+ stub_content.append("") # Will be replaced with imports later
+
+ # Class definition
+ stub_content.append(f"class {sync_class.__name__}:")
+
+ # Docstring
+ if async_class.__doc__:
+ stub_content.extend(
+ cls._format_docstring_for_stub(async_class.__doc__, " ")
+ )
+
+ # Generate __init__
+ try:
+ init_method = async_class.__init__
+ init_signature = inspect.signature(init_method)
+
+ # Try to get type hints for __init__
+ try:
+ from typing import get_type_hints
+ init_hints = get_type_hints(init_method)
+ except Exception:
+ init_hints = {}
+
+ # Format parameters
+ params_str = cls._format_method_parameters(
+ init_signature, type_hints=init_hints, type_tracker=type_tracker
+ )
+ # Add __init__ docstring if available (before the method)
+ if hasattr(init_method, "__doc__") and init_method.__doc__:
+ stub_content.extend(
+ cls._format_docstring_for_stub(init_method.__doc__, " ")
+ )
+ stub_content.append(f" def __init__({params_str}) -> None: ...")
+ except (ValueError, TypeError):
+ stub_content.append(
+ " def __init__(self, *args, **kwargs) -> None: ..."
+ )
+
+ stub_content.append("") # Add newline after __init__
+
+ # Get class attributes
+ class_attributes = cls._get_class_attributes(async_class)
+
+ # Generate inner classes
+ for name, attr in class_attributes:
+ inner_class_stub = cls._generate_inner_class_stub(
+ name, attr, type_tracker=type_tracker
+ )
+ stub_content.extend(inner_class_stub)
+ stub_content.append("") # Add newline after the inner class
+
+ # Add methods to the main class
+ processed_methods = set() # Keep track of methods we've processed
+ for name, method in sorted(
+ inspect.getmembers(async_class, predicate=inspect.isfunction)
+ ):
+ if name.startswith("_") or name in processed_methods:
+ continue
+
+ processed_methods.add(name)
+
+ try:
+ method_sig = cls._generate_method_signature(
+ name, method, is_async=True, type_tracker=type_tracker
+ )
+
+ # Add docstring if available (before the method signature for proper formatting)
+ if method.__doc__:
+ stub_content.extend(
+ cls._format_docstring_for_stub(method.__doc__, " ")
+ )
+
+ stub_content.append(f" {method_sig}")
+
+ stub_content.append("") # Add newline after each method
+
+ except (ValueError, TypeError):
+ # If we can't get the signature, just add a simple stub
+ stub_content.append(f" def {name}(self, *args, **kwargs): ...")
+ stub_content.append("") # Add newline
+
+ # Add properties
+ for name, prop in sorted(
+ inspect.getmembers(async_class, lambda x: isinstance(x, property))
+ ):
+ stub_content.append(" @property")
+ stub_content.append(f" def {name}(self) -> Any: ...")
+ if prop.fset:
+ stub_content.append(f" @{name}.setter")
+ stub_content.append(
+ f" def {name}(self, value: Any) -> None: ..."
+ )
+ stub_content.append("") # Add newline after each property
+
+ # Add placeholders for the nested class instances
+ # Check the actual attribute names from class annotations and attributes
+ attribute_mappings = {}
+
+ # First check annotations for typed attributes (including from parent classes)
+ # Collect all annotations from the class hierarchy
+ all_annotations = {}
+ for base_class in reversed(inspect.getmro(async_class)):
+ if hasattr(base_class, "__annotations__"):
+ all_annotations.update(base_class.__annotations__)
+
+ for attr_name, attr_type in sorted(all_annotations.items()):
+ for class_name, class_type in class_attributes:
+ # If the class type matches the annotated type
+ if (
+ attr_type == class_type
+ or (hasattr(attr_type, "__name__") and attr_type.__name__ == class_name)
+ or (isinstance(attr_type, str) and attr_type == class_name)
+ ):
+ attribute_mappings[class_name] = attr_name
+
+ # Remove the extra checking - annotations should be sufficient
+
+ # Add the attribute declarations with proper names
+ for class_name, class_type in class_attributes:
+ # Check if there's a mapping from annotation
+ attr_name = attribute_mappings.get(class_name, class_name)
+ # Use the annotation name if it exists, even if the attribute doesn't exist yet
+ # This is because the attribute might be created at runtime
+ stub_content.append(f" {attr_name}: {class_name}Sync")
+
+ stub_content.append("") # Add a final newline
+
+ # Now generate imports with all discovered types
+ imports = cls._generate_imports(async_class, type_tracker)
+
+ # Deduplicate imports while preserving order
+ seen = set()
+ unique_imports = []
+ for imp in imports:
+ if imp not in seen:
+ seen.add(imp)
+ unique_imports.append(imp)
+ else:
+ logging.warning(f"Duplicate import detected: {imp}")
+
+ # Replace the placeholder with actual imports
+ stub_content[imports_placeholder_index : imports_placeholder_index + 1] = (
+ unique_imports
+ )
+
+ # Post-process stub content
+ stub_content = cls._post_process_stub_content(stub_content)
+
+ # Write stub file
+ with open(sync_stub_path, "w") as f:
+ f.write("\n".join(stub_content))
+
+ logging.info(f"Generated stub file: {sync_stub_path}")
+
+ except Exception as e:
+ # If stub generation fails, log the error but don't break the main functionality
+ logging.error(
+ f"Error generating stub file for {sync_class.__name__}: {str(e)}"
+ )
+ import traceback
+
+ logging.error(traceback.format_exc())
+
+
+def create_sync_class(async_class: Type, thread_pool_size=10) -> Type:
+ """
+ Creates a sync version of an async class
+
+ Args:
+ async_class: The async class to convert
+ thread_pool_size: Size of thread pool to use
+
+ Returns:
+ A new class with sync versions of all async methods
+ """
+ return AsyncToSyncConverter.create_sync_class(async_class, thread_pool_size)
diff --git a/comfy_api/internal/singleton.py b/comfy_api/internal/singleton.py
new file mode 100644
index 000000000..75f16f98e
--- /dev/null
+++ b/comfy_api/internal/singleton.py
@@ -0,0 +1,33 @@
+from typing import Type, TypeVar
+
+class SingletonMetaclass(type):
+ T = TypeVar("T", bound="SingletonMetaclass")
+ _instances = {}
+
+ def __call__(cls, *args, **kwargs):
+ if cls not in cls._instances:
+ cls._instances[cls] = super(SingletonMetaclass, cls).__call__(
+ *args, **kwargs
+ )
+ return cls._instances[cls]
+
+ def inject_instance(cls: Type[T], instance: T) -> None:
+ assert cls not in SingletonMetaclass._instances, (
+ "Cannot inject instance after first instantiation"
+ )
+ SingletonMetaclass._instances[cls] = instance
+
+ def get_instance(cls: Type[T], *args, **kwargs) -> T:
+ """
+ Gets the singleton instance of the class, creating it if it doesn't exist.
+ """
+ if cls not in SingletonMetaclass._instances:
+ SingletonMetaclass._instances[cls] = super(
+ SingletonMetaclass, cls
+ ).__call__(*args, **kwargs)
+ return cls._instances[cls]
+
+
+class ProxiedSingleton(object, metaclass=SingletonMetaclass):
+ def __init__(self):
+ super().__init__()
diff --git a/comfy_api/latest/__init__.py b/comfy_api/latest/__init__.py
new file mode 100644
index 000000000..2cee65aa9
--- /dev/null
+++ b/comfy_api/latest/__init__.py
@@ -0,0 +1,124 @@
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from typing import Type, TYPE_CHECKING
+from comfy_api.internal import ComfyAPIBase
+from comfy_api.internal.singleton import ProxiedSingleton
+from comfy_api.internal.async_to_sync import create_sync_class
+from comfy_api.latest._input import ImageInput, AudioInput, MaskInput, LatentInput, VideoInput
+from comfy_api.latest._input_impl import VideoFromFile, VideoFromComponents
+from comfy_api.latest._util import VideoCodec, VideoContainer, VideoComponents
+from comfy_api.latest._io import _IO as io #noqa: F401
+from comfy_api.latest._ui import _UI as ui #noqa: F401
+# from comfy_api.latest._resources import _RESOURCES as resources #noqa: F401
+from comfy_execution.utils import get_executing_context
+from comfy_execution.progress import get_progress_state, PreviewImageTuple
+from PIL import Image
+from comfy.cli_args import args
+import numpy as np
+
+
+class ComfyAPI_latest(ComfyAPIBase):
+ VERSION = "latest"
+ STABLE = False
+
+ class Execution(ProxiedSingleton):
+ async def set_progress(
+ self,
+ value: float,
+ max_value: float,
+ node_id: str | None = None,
+ preview_image: Image.Image | ImageInput | None = None,
+ ignore_size_limit: bool = False,
+ ) -> None:
+ """
+ Update the progress bar displayed in the ComfyUI interface.
+
+ This function allows custom nodes and API calls to report their progress
+ back to the user interface, providing visual feedback during long operations.
+
+ Migration from previous API: comfy.utils.PROGRESS_BAR_HOOK
+ """
+ executing_context = get_executing_context()
+ if node_id is None and executing_context is not None:
+ node_id = executing_context.node_id
+ if node_id is None:
+ raise ValueError("node_id must be provided if not in executing context")
+
+ # Convert preview_image to PreviewImageTuple if needed
+ to_display: PreviewImageTuple | Image.Image | ImageInput | None = preview_image
+ if to_display is not None:
+ # First convert to PIL Image if needed
+ if isinstance(to_display, ImageInput):
+ # Convert ImageInput (torch.Tensor) to PIL Image
+ # Handle tensor shape [B, H, W, C] -> get first image if batch
+ tensor = to_display
+ if len(tensor.shape) == 4:
+ tensor = tensor[0]
+
+ # Convert to numpy array and scale to 0-255
+ image_np = (tensor.cpu().numpy() * 255).astype(np.uint8)
+ to_display = Image.fromarray(image_np)
+
+ if isinstance(to_display, Image.Image):
+ # Detect image format from PIL Image
+ image_format = to_display.format if to_display.format else "JPEG"
+ # Use None for preview_size if ignore_size_limit is True
+ preview_size = None if ignore_size_limit else args.preview_size
+ to_display = (image_format, to_display, preview_size)
+
+ get_progress_state().update_progress(
+ node_id=node_id,
+ value=value,
+ max_value=max_value,
+ image=to_display,
+ )
+
+ execution: Execution
+
+class ComfyExtension(ABC):
+ async def on_load(self) -> None:
+ """
+ Called when an extension is loaded.
+ This should be used to initialize any global resources neeeded by the extension.
+ """
+
+ @abstractmethod
+ async def get_node_list(self) -> list[type[io.ComfyNode]]:
+ """
+ Returns a list of nodes that this extension provides.
+ """
+
+class Input:
+ Image = ImageInput
+ Audio = AudioInput
+ Mask = MaskInput
+ Latent = LatentInput
+ Video = VideoInput
+
+class InputImpl:
+ VideoFromFile = VideoFromFile
+ VideoFromComponents = VideoFromComponents
+
+class Types:
+ VideoCodec = VideoCodec
+ VideoContainer = VideoContainer
+ VideoComponents = VideoComponents
+
+ComfyAPI = ComfyAPI_latest
+
+# Create a synchronous version of the API
+if TYPE_CHECKING:
+ import comfy_api.latest.generated.ComfyAPISyncStub # type: ignore
+
+ ComfyAPISync: Type[comfy_api.latest.generated.ComfyAPISyncStub.ComfyAPISyncStub]
+ComfyAPISync = create_sync_class(ComfyAPI_latest)
+
+__all__ = [
+ "ComfyAPI",
+ "ComfyAPISync",
+ "Input",
+ "InputImpl",
+ "Types",
+ "ComfyExtension",
+]
diff --git a/comfy_api/latest/_input/__init__.py b/comfy_api/latest/_input/__init__.py
new file mode 100644
index 000000000..14f0e72f4
--- /dev/null
+++ b/comfy_api/latest/_input/__init__.py
@@ -0,0 +1,10 @@
+from .basic_types import ImageInput, AudioInput, MaskInput, LatentInput
+from .video_types import VideoInput
+
+__all__ = [
+ "ImageInput",
+ "AudioInput",
+ "VideoInput",
+ "MaskInput",
+ "LatentInput",
+]
diff --git a/comfy_api/latest/_input/basic_types.py b/comfy_api/latest/_input/basic_types.py
new file mode 100644
index 000000000..245c6cbb1
--- /dev/null
+++ b/comfy_api/latest/_input/basic_types.py
@@ -0,0 +1,42 @@
+import torch
+from typing import TypedDict, List, Optional
+
+ImageInput = torch.Tensor
+"""
+An image in format [B, H, W, C] where B is the batch size, C is the number of channels,
+"""
+
+MaskInput = torch.Tensor
+"""
+A mask in format [B, H, W] where B is the batch size
+"""
+
+class AudioInput(TypedDict):
+ """
+ TypedDict representing audio input.
+ """
+
+ waveform: torch.Tensor
+ """
+ Tensor in the format [B, C, T] where B is the batch size, C is the number of channels,
+ """
+
+ sample_rate: int
+
+class LatentInput(TypedDict):
+ """
+ TypedDict representing latent input.
+ """
+
+ samples: torch.Tensor
+ """
+ Tensor in the format [B, C, H, W] where B is the batch size, C is the number of channels,
+ H is the height, and W is the width.
+ """
+
+ noise_mask: Optional[MaskInput]
+ """
+ Optional noise mask tensor in the same format as samples.
+ """
+
+ batch_index: Optional[List[int]]
diff --git a/comfy_api/latest/_input/video_types.py b/comfy_api/latest/_input/video_types.py
new file mode 100644
index 000000000..5d95dc507
--- /dev/null
+++ b/comfy_api/latest/_input/video_types.py
@@ -0,0 +1,85 @@
+from __future__ import annotations
+from abc import ABC, abstractmethod
+from typing import Optional, Union
+import io
+import av
+from comfy_api.util import VideoContainer, VideoCodec, VideoComponents
+
+class VideoInput(ABC):
+ """
+ Abstract base class for video input types.
+ """
+
+ @abstractmethod
+ def get_components(self) -> VideoComponents:
+ """
+ Abstract method to get the video components (images, audio, and frame rate).
+
+ Returns:
+ VideoComponents containing images, audio, and frame rate
+ """
+ pass
+
+ @abstractmethod
+ def save_to(
+ self,
+ path: str,
+ format: VideoContainer = VideoContainer.AUTO,
+ codec: VideoCodec = VideoCodec.AUTO,
+ metadata: Optional[dict] = None
+ ):
+ """
+ Abstract method to save the video input to a file.
+ """
+ pass
+
+ def get_stream_source(self) -> Union[str, io.BytesIO]:
+ """
+ Get a streamable source for the video. This allows processing without
+ loading the entire video into memory.
+
+ Returns:
+ Either a file path (str) or a BytesIO object that can be opened with av.
+
+ Default implementation creates a BytesIO buffer, but subclasses should
+ override this for better performance when possible.
+ """
+ buffer = io.BytesIO()
+ self.save_to(buffer)
+ buffer.seek(0)
+ return buffer
+
+ # Provide a default implementation, but subclasses can provide optimized versions
+ # if possible.
+ def get_dimensions(self) -> tuple[int, int]:
+ """
+ Returns the dimensions of the video input.
+
+ Returns:
+ Tuple of (width, height)
+ """
+ components = self.get_components()
+ return components.images.shape[2], components.images.shape[1]
+
+ def get_duration(self) -> float:
+ """
+ Returns the duration of the video in seconds.
+
+ Returns:
+ Duration in seconds
+ """
+ components = self.get_components()
+ frame_count = components.images.shape[0]
+ return float(frame_count / components.frame_rate)
+
+ def get_container_format(self) -> str:
+ """
+ Returns the container format of the video (e.g., 'mp4', 'mov', 'avi').
+
+ Returns:
+ Container format as string
+ """
+ # Default implementation - subclasses should override for better performance
+ source = self.get_stream_source()
+ with av.open(source, mode="r") as container:
+ return container.format.name
diff --git a/comfy_api/latest/_input_impl/__init__.py b/comfy_api/latest/_input_impl/__init__.py
new file mode 100644
index 000000000..02901b8b9
--- /dev/null
+++ b/comfy_api/latest/_input_impl/__init__.py
@@ -0,0 +1,7 @@
+from .video_types import VideoFromFile, VideoFromComponents
+
+__all__ = [
+ # Implementations
+ "VideoFromFile",
+ "VideoFromComponents",
+]
diff --git a/comfy_api/latest/_input_impl/video_types.py b/comfy_api/latest/_input_impl/video_types.py
new file mode 100644
index 000000000..f646504c8
--- /dev/null
+++ b/comfy_api/latest/_input_impl/video_types.py
@@ -0,0 +1,308 @@
+from __future__ import annotations
+from av.container import InputContainer
+from av.subtitles.stream import SubtitleStream
+from fractions import Fraction
+from typing import Optional
+from comfy_api.latest._input import AudioInput, VideoInput
+import av
+import io
+import json
+import numpy as np
+import math
+import torch
+from comfy_api.latest._util import VideoContainer, VideoCodec, VideoComponents
+
+
+def container_to_output_format(container_format: str | None) -> str | None:
+ """
+ A container's `format` may be a comma-separated list of formats.
+ E.g., iso container's `format` may be `mov,mp4,m4a,3gp,3g2,mj2`.
+ However, writing to a file/stream with `av.open` requires a single format,
+ or `None` to auto-detect.
+ """
+ if not container_format:
+ return None # Auto-detect
+
+ if "," not in container_format:
+ return container_format
+
+ formats = container_format.split(",")
+ return formats[0]
+
+
+def get_open_write_kwargs(
+ dest: str | io.BytesIO, container_format: str, to_format: str | None
+) -> dict:
+ """Get kwargs for writing a `VideoFromFile` to a file/stream with `av.open`"""
+ open_kwargs = {
+ "mode": "w",
+ # If isobmff, preserve custom metadata tags (workflow, prompt, extra_pnginfo)
+ "options": {"movflags": "use_metadata_tags"},
+ }
+
+ is_write_to_buffer = isinstance(dest, io.BytesIO)
+ if is_write_to_buffer:
+ # Set output format explicitly, since it cannot be inferred from file extension
+ if to_format == VideoContainer.AUTO:
+ to_format = container_format.lower()
+ elif isinstance(to_format, str):
+ to_format = to_format.lower()
+ open_kwargs["format"] = container_to_output_format(to_format)
+
+ return open_kwargs
+
+
+class VideoFromFile(VideoInput):
+ """
+ Class representing video input from a file.
+ """
+
+ def __init__(self, file: str | io.BytesIO):
+ """
+ Initialize the VideoFromFile object based off of either a path on disk or a BytesIO object
+ containing the file contents.
+ """
+ self.__file = file
+
+ def get_stream_source(self) -> str | io.BytesIO:
+ """
+ Return the underlying file source for efficient streaming.
+ This avoids unnecessary memory copies when the source is already a file path.
+ """
+ if isinstance(self.__file, io.BytesIO):
+ self.__file.seek(0)
+ return self.__file
+
+ def get_dimensions(self) -> tuple[int, int]:
+ """
+ Returns the dimensions of the video input.
+
+ Returns:
+ Tuple of (width, height)
+ """
+ if isinstance(self.__file, io.BytesIO):
+ self.__file.seek(0) # Reset the BytesIO object to the beginning
+ with av.open(self.__file, mode='r') as container:
+ for stream in container.streams:
+ if stream.type == 'video':
+ assert isinstance(stream, av.VideoStream)
+ return stream.width, stream.height
+ raise ValueError(f"No video stream found in file '{self.__file}'")
+
+ def get_duration(self) -> float:
+ """
+ Returns the duration of the video in seconds.
+
+ Returns:
+ Duration in seconds
+ """
+ if isinstance(self.__file, io.BytesIO):
+ self.__file.seek(0)
+ with av.open(self.__file, mode="r") as container:
+ if container.duration is not None:
+ return float(container.duration / av.time_base)
+
+ # Fallback: calculate from frame count and frame rate
+ video_stream = next(
+ (s for s in container.streams if s.type == "video"), None
+ )
+ if video_stream and video_stream.frames and video_stream.average_rate:
+ return float(video_stream.frames / video_stream.average_rate)
+
+ # Last resort: decode frames to count them
+ if video_stream and video_stream.average_rate:
+ frame_count = 0
+ container.seek(0)
+ for packet in container.demux(video_stream):
+ for _ in packet.decode():
+ frame_count += 1
+ if frame_count > 0:
+ return float(frame_count / video_stream.average_rate)
+
+ raise ValueError(f"Could not determine duration for file '{self.__file}'")
+
+ def get_container_format(self) -> str:
+ """
+ Returns the container format of the video (e.g., 'mp4', 'mov', 'avi').
+
+ Returns:
+ Container format as string
+ """
+ if isinstance(self.__file, io.BytesIO):
+ self.__file.seek(0)
+ with av.open(self.__file, mode='r') as container:
+ return container.format.name
+
+ def get_components_internal(self, container: InputContainer) -> VideoComponents:
+ # Get video frames
+ frames = []
+ for frame in container.decode(video=0):
+ img = frame.to_ndarray(format='rgb24') # shape: (H, W, 3)
+ img = torch.from_numpy(img) / 255.0 # shape: (H, W, 3)
+ frames.append(img)
+
+ images = torch.stack(frames) if len(frames) > 0 else torch.zeros(0, 3, 0, 0)
+
+ # Get frame rate
+ video_stream = next(s for s in container.streams if s.type == 'video')
+ frame_rate = Fraction(video_stream.average_rate) if video_stream and video_stream.average_rate else Fraction(1)
+
+ # Get audio if available
+ audio = None
+ try:
+ container.seek(0) # Reset the container to the beginning
+ for stream in container.streams:
+ if stream.type != 'audio':
+ continue
+ assert isinstance(stream, av.AudioStream)
+ audio_frames = []
+ for packet in container.demux(stream):
+ for frame in packet.decode():
+ assert isinstance(frame, av.AudioFrame)
+ audio_frames.append(frame.to_ndarray()) # shape: (channels, samples)
+ if len(audio_frames) > 0:
+ audio_data = np.concatenate(audio_frames, axis=1) # shape: (channels, total_samples)
+ audio_tensor = torch.from_numpy(audio_data).unsqueeze(0) # shape: (1, channels, total_samples)
+ audio = AudioInput({
+ "waveform": audio_tensor,
+ "sample_rate": int(stream.sample_rate) if stream.sample_rate else 1,
+ })
+ except StopIteration:
+ pass # No audio stream
+
+ metadata = container.metadata
+ return VideoComponents(images=images, audio=audio, frame_rate=frame_rate, metadata=metadata)
+
+ def get_components(self) -> VideoComponents:
+ if isinstance(self.__file, io.BytesIO):
+ self.__file.seek(0) # Reset the BytesIO object to the beginning
+ with av.open(self.__file, mode='r') as container:
+ return self.get_components_internal(container)
+ raise ValueError(f"No video stream found in file '{self.__file}'")
+
+ def save_to(
+ self,
+ path: str | io.BytesIO,
+ format: VideoContainer = VideoContainer.AUTO,
+ codec: VideoCodec = VideoCodec.AUTO,
+ metadata: Optional[dict] = None
+ ):
+ if isinstance(self.__file, io.BytesIO):
+ self.__file.seek(0) # Reset the BytesIO object to the beginning
+ with av.open(self.__file, mode='r') as container:
+ container_format = container.format.name
+ video_encoding = container.streams.video[0].codec.name if len(container.streams.video) > 0 else None
+ reuse_streams = True
+ if format != VideoContainer.AUTO and format not in container_format.split(","):
+ reuse_streams = False
+ if codec != VideoCodec.AUTO and codec != video_encoding and video_encoding is not None:
+ reuse_streams = False
+
+ if not reuse_streams:
+ components = self.get_components_internal(container)
+ video = VideoFromComponents(components)
+ return video.save_to(
+ path,
+ format=format,
+ codec=codec,
+ metadata=metadata
+ )
+
+ streams = container.streams
+
+ open_kwargs = get_open_write_kwargs(path, container_format, format)
+ with av.open(path, **open_kwargs) as output_container:
+ # Copy over the original metadata
+ for key, value in container.metadata.items():
+ if metadata is None or key not in metadata:
+ output_container.metadata[key] = value
+
+ # Add our new metadata
+ if metadata is not None:
+ for key, value in metadata.items():
+ if isinstance(value, str):
+ output_container.metadata[key] = value
+ else:
+ output_container.metadata[key] = json.dumps(value)
+
+ # Add streams to the new container
+ stream_map = {}
+ for stream in streams:
+ if isinstance(stream, (av.VideoStream, av.AudioStream, SubtitleStream)):
+ out_stream = output_container.add_stream_from_template(template=stream, opaque=True)
+ stream_map[stream] = out_stream
+
+ # Write packets to the new container
+ for packet in container.demux():
+ if packet.stream in stream_map and packet.dts is not None:
+ packet.stream = stream_map[packet.stream]
+ output_container.mux(packet)
+
+class VideoFromComponents(VideoInput):
+ """
+ Class representing video input from tensors.
+ """
+
+ def __init__(self, components: VideoComponents):
+ self.__components = components
+
+ def get_components(self) -> VideoComponents:
+ return VideoComponents(
+ images=self.__components.images,
+ audio=self.__components.audio,
+ frame_rate=self.__components.frame_rate
+ )
+
+ def save_to(
+ self,
+ path: str,
+ format: VideoContainer = VideoContainer.AUTO,
+ codec: VideoCodec = VideoCodec.AUTO,
+ metadata: Optional[dict] = None
+ ):
+ if format != VideoContainer.AUTO and format != VideoContainer.MP4:
+ raise ValueError("Only MP4 format is supported for now")
+ if codec != VideoCodec.AUTO and codec != VideoCodec.H264:
+ raise ValueError("Only H264 codec is supported for now")
+ with av.open(path, mode='w', options={'movflags': 'use_metadata_tags'}) as output:
+ # Add metadata before writing any streams
+ if metadata is not None:
+ for key, value in metadata.items():
+ output.metadata[key] = json.dumps(value)
+
+ frame_rate = Fraction(round(self.__components.frame_rate * 1000), 1000)
+ # Create a video stream
+ video_stream = output.add_stream('h264', rate=frame_rate)
+ video_stream.width = self.__components.images.shape[2]
+ video_stream.height = self.__components.images.shape[1]
+ video_stream.pix_fmt = 'yuv420p'
+
+ # Create an audio stream
+ audio_sample_rate = 1
+ audio_stream: Optional[av.AudioStream] = None
+ if self.__components.audio:
+ audio_sample_rate = int(self.__components.audio['sample_rate'])
+ audio_stream = output.add_stream('aac', rate=audio_sample_rate)
+
+ # Encode video
+ for i, frame in enumerate(self.__components.images):
+ img = (frame * 255).clamp(0, 255).byte().cpu().numpy() # shape: (H, W, 3)
+ frame = av.VideoFrame.from_ndarray(img, format='rgb24')
+ frame = frame.reformat(format='yuv420p') # Convert to YUV420P as required by h264
+ packet = video_stream.encode(frame)
+ output.mux(packet)
+
+ # Flush video
+ packet = video_stream.encode(None)
+ output.mux(packet)
+
+ if audio_stream and self.__components.audio:
+ waveform = self.__components.audio['waveform']
+ waveform = waveform[:, :, :math.ceil((audio_sample_rate / frame_rate) * self.__components.images.shape[0])]
+ frame = av.AudioFrame.from_ndarray(waveform.movedim(2, 1).reshape(1, -1).float().numpy(), format='flt', layout='mono' if waveform.shape[1] == 1 else 'stereo')
+ frame.sample_rate = audio_sample_rate
+ frame.pts = 0
+ output.mux(audio_stream.encode(frame))
+
+ # Flush encoder
+ output.mux(audio_stream.encode(None))
diff --git a/comfy_api/latest/_io.py b/comfy_api/latest/_io.py
new file mode 100644
index 000000000..e0ee943a7
--- /dev/null
+++ b/comfy_api/latest/_io.py
@@ -0,0 +1,1631 @@
+from __future__ import annotations
+
+import copy
+import inspect
+from abc import ABC, abstractmethod
+from collections import Counter
+from dataclasses import asdict, dataclass
+from enum import Enum
+from typing import Any, Callable, Literal, TypedDict, TypeVar, TYPE_CHECKING
+from typing_extensions import NotRequired, final
+
+# used for type hinting
+import torch
+
+if TYPE_CHECKING:
+ from spandrel import ImageModelDescriptor
+ from comfy.clip_vision import ClipVisionModel
+ from comfy.clip_vision import Output as ClipVisionOutput_
+ from comfy.controlnet import ControlNet
+ from comfy.hooks import HookGroup, HookKeyframeGroup
+ from comfy.model_patcher import ModelPatcher
+ from comfy.samplers import CFGGuider, Sampler
+ from comfy.sd import CLIP, VAE
+ from comfy.sd import StyleModel as StyleModel_
+ from comfy_api.input import VideoInput
+from comfy_api.internal import (_ComfyNodeInternal, _NodeOutputInternal, classproperty, copy_class, first_real_override, is_class,
+ prune_dict, shallow_clone_class)
+from comfy_api.latest._resources import Resources, ResourcesLocal
+from comfy_execution.graph_utils import ExecutionBlocker
+
+# from comfy_extras.nodes_images import SVG as SVG_ # NOTE: needs to be moved before can be imported due to circular reference
+
+class FolderType(str, Enum):
+ input = "input"
+ output = "output"
+ temp = "temp"
+
+
+class UploadType(str, Enum):
+ image = "image_upload"
+ audio = "audio_upload"
+ video = "video_upload"
+ model = "file_upload"
+
+
+class RemoteOptions:
+ def __init__(self, route: str, refresh_button: bool, control_after_refresh: Literal["first", "last"]="first",
+ timeout: int=None, max_retries: int=None, refresh: int=None):
+ self.route = route
+ """The route to the remote source."""
+ self.refresh_button = refresh_button
+ """Specifies whether to show a refresh button in the UI below the widget."""
+ self.control_after_refresh = control_after_refresh
+ """Specifies the control after the refresh button is clicked. If "first", the first item will be automatically selected, and so on."""
+ self.timeout = timeout
+ """The maximum amount of time to wait for a response from the remote source in milliseconds."""
+ self.max_retries = max_retries
+ """The maximum number of retries before aborting the request."""
+ self.refresh = refresh
+ """The TTL of the remote input's value in milliseconds. Specifies the interval at which the remote input's value is refreshed."""
+
+ def as_dict(self):
+ return prune_dict({
+ "route": self.route,
+ "refresh_button": self.refresh_button,
+ "control_after_refresh": self.control_after_refresh,
+ "timeout": self.timeout,
+ "max_retries": self.max_retries,
+ "refresh": self.refresh,
+ })
+
+
+class NumberDisplay(str, Enum):
+ number = "number"
+ slider = "slider"
+
+
+class _StringIOType(str):
+ def __ne__(self, value: object) -> bool:
+ if self == "*" or value == "*":
+ return False
+ if not isinstance(value, str):
+ return True
+ a = frozenset(self.split(","))
+ b = frozenset(value.split(","))
+ return not (b.issubset(a) or a.issubset(b))
+
+class _ComfyType(ABC):
+ Type = Any
+ io_type: str = None
+
+# NOTE: this is a workaround to make the decorator return the correct type
+T = TypeVar("T", bound=type)
+def comfytype(io_type: str, **kwargs):
+ '''
+ Decorator to mark nested classes as ComfyType; io_type will be bound to the class.
+
+ A ComfyType may have the following attributes:
+ - Type =
+ - class Input(Input): ...
+ - class Output(Output): ...
+ '''
+ def decorator(cls: T) -> T:
+ if isinstance(cls, _ComfyType) or issubclass(cls, _ComfyType):
+ # clone Input and Output classes to avoid modifying the original class
+ new_cls = cls
+ if hasattr(new_cls, "Input"):
+ new_cls.Input = copy_class(new_cls.Input)
+ if hasattr(new_cls, "Output"):
+ new_cls.Output = copy_class(new_cls.Output)
+ else:
+ # copy class attributes except for special ones that shouldn't be in type()
+ cls_dict = {
+ k: v for k, v in cls.__dict__.items()
+ if k not in ('__dict__', '__weakref__', '__module__', '__doc__')
+ }
+ # new class
+ new_cls: ComfyTypeIO = type(
+ cls.__name__,
+ (cls, ComfyTypeIO),
+ cls_dict
+ )
+ # metadata preservation
+ new_cls.__module__ = cls.__module__
+ new_cls.__doc__ = cls.__doc__
+ # assign ComfyType attributes, if needed
+ # NOTE: use __ne__ trick for io_type (see node_typing.IO.__ne__ for details)
+ new_cls.io_type = _StringIOType(io_type)
+ if hasattr(new_cls, "Input") and new_cls.Input is not None:
+ new_cls.Input.Parent = new_cls
+ if hasattr(new_cls, "Output") and new_cls.Output is not None:
+ new_cls.Output.Parent = new_cls
+ return new_cls
+ return decorator
+
+def Custom(io_type: str) -> type[ComfyTypeIO]:
+ '''Create a ComfyType for a custom io_type.'''
+ @comfytype(io_type=io_type)
+ class CustomComfyType(ComfyTypeIO):
+ ...
+ return CustomComfyType
+
+class _IO_V3:
+ '''
+ Base class for V3 Inputs and Outputs.
+ '''
+ Parent: _ComfyType = None
+
+ def __init__(self):
+ pass
+
+ @property
+ def io_type(self):
+ return self.Parent.io_type
+
+ @property
+ def Type(self):
+ return self.Parent.Type
+
+class Input(_IO_V3):
+ '''
+ Base class for a V3 Input.
+ '''
+ def __init__(self, id: str, display_name: str=None, optional=False, tooltip: str=None, lazy: bool=None, extra_dict=None):
+ super().__init__()
+ self.id = id
+ self.display_name = display_name
+ self.optional = optional
+ self.tooltip = tooltip
+ self.lazy = lazy
+ self.extra_dict = extra_dict if extra_dict is not None else {}
+
+ def as_dict(self):
+ return prune_dict({
+ "display_name": self.display_name,
+ "optional": self.optional,
+ "tooltip": self.tooltip,
+ "lazy": self.lazy,
+ }) | prune_dict(self.extra_dict)
+
+ def get_io_type(self):
+ return _StringIOType(self.io_type)
+
+class WidgetInput(Input):
+ '''
+ Base class for a V3 Input with widget.
+ '''
+ def __init__(self, id: str, display_name: str=None, optional=False, tooltip: str=None, lazy: bool=None,
+ default: Any=None,
+ socketless: bool=None, widget_type: str=None, force_input: bool=None, extra_dict=None):
+ super().__init__(id, display_name, optional, tooltip, lazy, extra_dict)
+ self.default = default
+ self.socketless = socketless
+ self.widget_type = widget_type
+ self.force_input = force_input
+
+ def as_dict(self):
+ return super().as_dict() | prune_dict({
+ "default": self.default,
+ "socketless": self.socketless,
+ "widgetType": self.widget_type,
+ "forceInput": self.force_input,
+ })
+
+ def get_io_type(self):
+ return self.widget_type if self.widget_type is not None else super().get_io_type()
+
+
+class Output(_IO_V3):
+ def __init__(self, id: str=None, display_name: str=None, tooltip: str=None,
+ is_output_list=False):
+ self.id = id
+ self.display_name = display_name
+ self.tooltip = tooltip
+ self.is_output_list = is_output_list
+
+ def as_dict(self):
+ return prune_dict({
+ "display_name": self.display_name,
+ "tooltip": self.tooltip,
+ "is_output_list": self.is_output_list,
+ })
+
+ def get_io_type(self):
+ return self.io_type
+
+
+class ComfyTypeI(_ComfyType):
+ '''ComfyType subclass that only has a default Input class - intended for types that only have Inputs.'''
+ class Input(Input):
+ ...
+
+class ComfyTypeIO(ComfyTypeI):
+ '''ComfyType subclass that has default Input and Output classes; useful for types with both Inputs and Outputs.'''
+ class Output(Output):
+ ...
+
+
+@comfytype(io_type="BOOLEAN")
+class Boolean(ComfyTypeIO):
+ Type = bool
+
+ class Input(WidgetInput):
+ '''Boolean input.'''
+ def __init__(self, id: str, display_name: str=None, optional=False, tooltip: str=None, lazy: bool=None,
+ default: bool=None, label_on: str=None, label_off: str=None,
+ socketless: bool=None, force_input: bool=None):
+ super().__init__(id, display_name, optional, tooltip, lazy, default, socketless, None, force_input)
+ self.label_on = label_on
+ self.label_off = label_off
+ self.default: bool
+
+ def as_dict(self):
+ return super().as_dict() | prune_dict({
+ "label_on": self.label_on,
+ "label_off": self.label_off,
+ })
+
+@comfytype(io_type="INT")
+class Int(ComfyTypeIO):
+ Type = int
+
+ class Input(WidgetInput):
+ '''Integer input.'''
+ def __init__(self, id: str, display_name: str=None, optional=False, tooltip: str=None, lazy: bool=None,
+ default: int=None, min: int=None, max: int=None, step: int=None, control_after_generate: bool=None,
+ display_mode: NumberDisplay=None, socketless: bool=None, force_input: bool=None):
+ super().__init__(id, display_name, optional, tooltip, lazy, default, socketless, None, force_input)
+ self.min = min
+ self.max = max
+ self.step = step
+ self.control_after_generate = control_after_generate
+ self.display_mode = display_mode
+ self.default: int
+
+ def as_dict(self):
+ return super().as_dict() | prune_dict({
+ "min": self.min,
+ "max": self.max,
+ "step": self.step,
+ "control_after_generate": self.control_after_generate,
+ "display": self.display_mode.value if self.display_mode else None,
+ })
+
+@comfytype(io_type="FLOAT")
+class Float(ComfyTypeIO):
+ Type = float
+
+ class Input(WidgetInput):
+ '''Float input.'''
+ def __init__(self, id: str, display_name: str=None, optional=False, tooltip: str=None, lazy: bool=None,
+ default: float=None, min: float=None, max: float=None, step: float=None, round: float=None,
+ display_mode: NumberDisplay=None, socketless: bool=None, force_input: bool=None):
+ super().__init__(id, display_name, optional, tooltip, lazy, default, socketless, None, force_input)
+ self.min = min
+ self.max = max
+ self.step = step
+ self.round = round
+ self.display_mode = display_mode
+ self.default: float
+
+ def as_dict(self):
+ return super().as_dict() | prune_dict({
+ "min": self.min,
+ "max": self.max,
+ "step": self.step,
+ "round": self.round,
+ "display": self.display_mode,
+ })
+
+@comfytype(io_type="STRING")
+class String(ComfyTypeIO):
+ Type = str
+
+ class Input(WidgetInput):
+ '''String input.'''
+ def __init__(self, id: str, display_name: str=None, optional=False, tooltip: str=None, lazy: bool=None,
+ multiline=False, placeholder: str=None, default: str=None, dynamic_prompts: bool=None,
+ socketless: bool=None, force_input: bool=None):
+ super().__init__(id, display_name, optional, tooltip, lazy, default, socketless, None, force_input)
+ self.multiline = multiline
+ self.placeholder = placeholder
+ self.dynamic_prompts = dynamic_prompts
+ self.default: str
+
+ def as_dict(self):
+ return super().as_dict() | prune_dict({
+ "multiline": self.multiline,
+ "placeholder": self.placeholder,
+ "dynamicPrompts": self.dynamic_prompts,
+ })
+
+@comfytype(io_type="COMBO")
+class Combo(ComfyTypeI):
+ Type = str
+ class Input(WidgetInput):
+ """Combo input (dropdown)."""
+ Type = str
+ def __init__(self, id: str, options: list[str]=None, display_name: str=None, optional=False, tooltip: str=None, lazy: bool=None,
+ default: str=None, control_after_generate: bool=None,
+ upload: UploadType=None, image_folder: FolderType=None,
+ remote: RemoteOptions=None,
+ socketless: bool=None):
+ super().__init__(id, display_name, optional, tooltip, lazy, default, socketless)
+ self.multiselect = False
+ self.options = options
+ self.control_after_generate = control_after_generate
+ self.upload = upload
+ self.image_folder = image_folder
+ self.remote = remote
+ self.default: str
+
+ def as_dict(self):
+ return super().as_dict() | prune_dict({
+ "multiselect": self.multiselect,
+ "options": self.options,
+ "control_after_generate": self.control_after_generate,
+ **({self.upload.value: True} if self.upload is not None else {}),
+ "image_folder": self.image_folder.value if self.image_folder else None,
+ "remote": self.remote.as_dict() if self.remote else None,
+ })
+
+
+@comfytype(io_type="COMBO")
+class MultiCombo(ComfyTypeI):
+ '''Multiselect Combo input (dropdown for selecting potentially more than one value).'''
+ # TODO: something is wrong with the serialization, frontend does not recognize it as multiselect
+ Type = list[str]
+ class Input(Combo.Input):
+ def __init__(self, id: str, options: list[str], display_name: str=None, optional=False, tooltip: str=None, lazy: bool=None,
+ default: list[str]=None, placeholder: str=None, chip: bool=None, control_after_generate: bool=None,
+ socketless: bool=None):
+ super().__init__(id, options, display_name, optional, tooltip, lazy, default, control_after_generate, socketless=socketless)
+ self.multiselect = True
+ self.placeholder = placeholder
+ self.chip = chip
+ self.default: list[str]
+
+ def as_dict(self):
+ to_return = super().as_dict() | prune_dict({
+ "multi_select": self.multiselect,
+ "placeholder": self.placeholder,
+ "chip": self.chip,
+ })
+ return to_return
+
+@comfytype(io_type="IMAGE")
+class Image(ComfyTypeIO):
+ Type = torch.Tensor
+
+
+@comfytype(io_type="WAN_CAMERA_EMBEDDING")
+class WanCameraEmbedding(ComfyTypeIO):
+ Type = torch.Tensor
+
+
+@comfytype(io_type="WEBCAM")
+class Webcam(ComfyTypeIO):
+ Type = str
+
+ class Input(WidgetInput):
+ """Webcam input."""
+ Type = str
+ def __init__(
+ self, id: str, display_name: str=None, optional=False,
+ tooltip: str=None, lazy: bool=None, default: str=None, socketless: bool=None
+ ):
+ super().__init__(id, display_name, optional, tooltip, lazy, default, socketless)
+
+
+@comfytype(io_type="MASK")
+class Mask(ComfyTypeIO):
+ Type = torch.Tensor
+
+@comfytype(io_type="LATENT")
+class Latent(ComfyTypeIO):
+ '''Latents are stored as a dictionary.'''
+ class LatentDict(TypedDict):
+ samples: torch.Tensor
+ '''Latent tensors.'''
+ noise_mask: NotRequired[torch.Tensor]
+ batch_index: NotRequired[list[int]]
+ type: NotRequired[str]
+ '''Only needed if dealing with these types: audio, hunyuan3dv2'''
+ Type = LatentDict
+
+@comfytype(io_type="CONDITIONING")
+class Conditioning(ComfyTypeIO):
+ class PooledDict(TypedDict):
+ pooled_output: torch.Tensor
+ '''Pooled output from CLIP.'''
+ control: NotRequired[ControlNet]
+ '''ControlNet to apply to conditioning.'''
+ control_apply_to_uncond: NotRequired[bool]
+ '''Whether to apply ControlNet to matching negative conditioning at sample time, if applicable.'''
+ cross_attn_controlnet: NotRequired[torch.Tensor]
+ '''CrossAttn from CLIP to use for controlnet only.'''
+ pooled_output_controlnet: NotRequired[torch.Tensor]
+ '''Pooled output from CLIP to use for controlnet only.'''
+ gligen: NotRequired[tuple[str, Gligen, list[tuple[torch.Tensor, int, ...]]]]
+ '''GLIGEN to apply to conditioning.'''
+ area: NotRequired[tuple[int, ...] | tuple[str, float, ...]]
+ '''Set area of conditioning. First half of values apply to dimensions, the second half apply to coordinates.
+ By default, the dimensions are based on total pixel amount, but the first value can be set to "percentage" to use a percentage of the image size instead.
+
+ (1024, 1024, 0, 0) would apply conditioning to the top-left 1024x1024 pixels.
+
+ ("percentage", 0.5, 0.5, 0, 0) would apply conditioning to the top-left 50% of the image.''' # TODO: verify its actually top-left
+ strength: NotRequired[float]
+ '''Strength of conditioning. Default strength is 1.0.'''
+ mask: NotRequired[torch.Tensor]
+ '''Mask to apply conditioning to.'''
+ mask_strength: NotRequired[float]
+ '''Strength of conditioning mask. Default strength is 1.0.'''
+ set_area_to_bounds: NotRequired[bool]
+ '''Whether conditioning mask should determine bounds of area - if set to false, latents are sampled at full resolution and result is applied in mask.'''
+ concat_latent_image: NotRequired[torch.Tensor]
+ '''Used for inpainting and specific models.'''
+ concat_mask: NotRequired[torch.Tensor]
+ '''Used for inpainting and specific models.'''
+ concat_image: NotRequired[torch.Tensor]
+ '''Used by SD_4XUpscale_Conditioning.'''
+ noise_augmentation: NotRequired[float]
+ '''Used by SD_4XUpscale_Conditioning.'''
+ hooks: NotRequired[HookGroup]
+ '''Applies hooks to conditioning.'''
+ default: NotRequired[bool]
+ '''Whether to this conditioning is 'default'; default conditioning gets applied to any areas of the image that have no masks/areas applied, assuming at least one area/mask is present during sampling.'''
+ start_percent: NotRequired[float]
+ '''Determines relative step to begin applying conditioning, expressed as a float between 0.0 and 1.0.'''
+ end_percent: NotRequired[float]
+ '''Determines relative step to end applying conditioning, expressed as a float between 0.0 and 1.0.'''
+ clip_start_percent: NotRequired[float]
+ '''Internal variable for conditioning scheduling - start of application, expressed as a float between 0.0 and 1.0.'''
+ clip_end_percent: NotRequired[float]
+ '''Internal variable for conditioning scheduling - end of application, expressed as a float between 0.0 and 1.0.'''
+ attention_mask: NotRequired[torch.Tensor]
+ '''Masks text conditioning; used by StyleModel among others.'''
+ attention_mask_img_shape: NotRequired[tuple[int, ...]]
+ '''Masks text conditioning; used by StyleModel among others.'''
+ unclip_conditioning: NotRequired[list[dict]]
+ '''Used by unCLIP.'''
+ conditioning_lyrics: NotRequired[torch.Tensor]
+ '''Used by AceT5Model.'''
+ seconds_start: NotRequired[float]
+ '''Used by StableAudio.'''
+ seconds_total: NotRequired[float]
+ '''Used by StableAudio.'''
+ lyrics_strength: NotRequired[float]
+ '''Used by AceStepAudio.'''
+ width: NotRequired[int]
+ '''Used by certain models (e.g. CLIPTextEncodeSDXL/Refiner, PixArtAlpha).'''
+ height: NotRequired[int]
+ '''Used by certain models (e.g. CLIPTextEncodeSDXL/Refiner, PixArtAlpha).'''
+ aesthetic_score: NotRequired[float]
+ '''Used by CLIPTextEncodeSDXL/Refiner.'''
+ crop_w: NotRequired[int]
+ '''Used by CLIPTextEncodeSDXL.'''
+ crop_h: NotRequired[int]
+ '''Used by CLIPTextEncodeSDXL.'''
+ target_width: NotRequired[int]
+ '''Used by CLIPTextEncodeSDXL.'''
+ target_height: NotRequired[int]
+ '''Used by CLIPTextEncodeSDXL.'''
+ reference_latents: NotRequired[list[torch.Tensor]]
+ '''Used by ReferenceLatent.'''
+ guidance: NotRequired[float]
+ '''Used by Flux-like models with guidance embed.'''
+ guiding_frame_index: NotRequired[int]
+ '''Used by Hunyuan ImageToVideo.'''
+ ref_latent: NotRequired[torch.Tensor]
+ '''Used by Hunyuan ImageToVideo.'''
+ keyframe_idxs: NotRequired[list[int]]
+ '''Used by LTXV.'''
+ frame_rate: NotRequired[float]
+ '''Used by LTXV.'''
+ stable_cascade_prior: NotRequired[torch.Tensor]
+ '''Used by StableCascade.'''
+ elevation: NotRequired[list[float]]
+ '''Used by SV3D.'''
+ azimuth: NotRequired[list[float]]
+ '''Used by SV3D.'''
+ motion_bucket_id: NotRequired[int]
+ '''Used by SVD-like models.'''
+ fps: NotRequired[int]
+ '''Used by SVD-like models.'''
+ augmentation_level: NotRequired[float]
+ '''Used by SVD-like models.'''
+ clip_vision_output: NotRequired[ClipVisionOutput_]
+ '''Used by WAN-like models.'''
+ vace_frames: NotRequired[torch.Tensor]
+ '''Used by WAN VACE.'''
+ vace_mask: NotRequired[torch.Tensor]
+ '''Used by WAN VACE.'''
+ vace_strength: NotRequired[float]
+ '''Used by WAN VACE.'''
+ camera_conditions: NotRequired[Any] # TODO: assign proper type once defined
+ '''Used by WAN Camera.'''
+ time_dim_concat: NotRequired[torch.Tensor]
+ '''Used by WAN Phantom Subject.'''
+
+ CondList = list[tuple[torch.Tensor, PooledDict]]
+ Type = CondList
+
+@comfytype(io_type="SAMPLER")
+class Sampler(ComfyTypeIO):
+ if TYPE_CHECKING:
+ Type = Sampler
+
+@comfytype(io_type="SIGMAS")
+class Sigmas(ComfyTypeIO):
+ Type = torch.Tensor
+
+@comfytype(io_type="NOISE")
+class Noise(ComfyTypeIO):
+ Type = torch.Tensor
+
+@comfytype(io_type="GUIDER")
+class Guider(ComfyTypeIO):
+ if TYPE_CHECKING:
+ Type = CFGGuider
+
+@comfytype(io_type="CLIP")
+class Clip(ComfyTypeIO):
+ if TYPE_CHECKING:
+ Type = CLIP
+
+@comfytype(io_type="CONTROL_NET")
+class ControlNet(ComfyTypeIO):
+ if TYPE_CHECKING:
+ Type = ControlNet
+
+@comfytype(io_type="VAE")
+class Vae(ComfyTypeIO):
+ if TYPE_CHECKING:
+ Type = VAE
+
+@comfytype(io_type="MODEL")
+class Model(ComfyTypeIO):
+ if TYPE_CHECKING:
+ Type = ModelPatcher
+
+@comfytype(io_type="CLIP_VISION")
+class ClipVision(ComfyTypeIO):
+ if TYPE_CHECKING:
+ Type = ClipVisionModel
+
+@comfytype(io_type="CLIP_VISION_OUTPUT")
+class ClipVisionOutput(ComfyTypeIO):
+ if TYPE_CHECKING:
+ Type = ClipVisionOutput_
+
+@comfytype(io_type="STYLE_MODEL")
+class StyleModel(ComfyTypeIO):
+ if TYPE_CHECKING:
+ Type = StyleModel_
+
+@comfytype(io_type="GLIGEN")
+class Gligen(ComfyTypeIO):
+ '''ModelPatcher that wraps around a 'Gligen' model.'''
+ if TYPE_CHECKING:
+ Type = ModelPatcher
+
+@comfytype(io_type="UPSCALE_MODEL")
+class UpscaleModel(ComfyTypeIO):
+ if TYPE_CHECKING:
+ Type = ImageModelDescriptor
+
+@comfytype(io_type="AUDIO")
+class Audio(ComfyTypeIO):
+ class AudioDict(TypedDict):
+ waveform: torch.Tensor
+ sampler_rate: int
+ Type = AudioDict
+
+@comfytype(io_type="VIDEO")
+class Video(ComfyTypeIO):
+ if TYPE_CHECKING:
+ Type = VideoInput
+
+@comfytype(io_type="SVG")
+class SVG(ComfyTypeIO):
+ Type = Any # TODO: SVG class is defined in comfy_extras/nodes_images.py, causing circular reference; should be moved to somewhere else before referenced directly in v3
+
+@comfytype(io_type="LORA_MODEL")
+class LoraModel(ComfyTypeIO):
+ Type = dict[str, torch.Tensor]
+
+@comfytype(io_type="LOSS_MAP")
+class LossMap(ComfyTypeIO):
+ class LossMapDict(TypedDict):
+ loss: list[torch.Tensor]
+ Type = LossMapDict
+
+@comfytype(io_type="VOXEL")
+class Voxel(ComfyTypeIO):
+ Type = Any # TODO: VOXEL class is defined in comfy_extras/nodes_hunyuan3d.py; should be moved to somewhere else before referenced directly in v3
+
+@comfytype(io_type="MESH")
+class Mesh(ComfyTypeIO):
+ Type = Any # TODO: MESH class is defined in comfy_extras/nodes_hunyuan3d.py; should be moved to somewhere else before referenced directly in v3
+
+@comfytype(io_type="HOOKS")
+class Hooks(ComfyTypeIO):
+ if TYPE_CHECKING:
+ Type = HookGroup
+
+@comfytype(io_type="HOOK_KEYFRAMES")
+class HookKeyframes(ComfyTypeIO):
+ if TYPE_CHECKING:
+ Type = HookKeyframeGroup
+
+@comfytype(io_type="TIMESTEPS_RANGE")
+class TimestepsRange(ComfyTypeIO):
+ '''Range defined by start and endpoint, between 0.0 and 1.0.'''
+ Type = tuple[int, int]
+
+@comfytype(io_type="LATENT_OPERATION")
+class LatentOperation(ComfyTypeIO):
+ Type = Callable[[torch.Tensor], torch.Tensor]
+
+@comfytype(io_type="FLOW_CONTROL")
+class FlowControl(ComfyTypeIO):
+ # NOTE: only used in testing_nodes right now
+ Type = tuple[str, Any]
+
+@comfytype(io_type="ACCUMULATION")
+class Accumulation(ComfyTypeIO):
+ # NOTE: only used in testing_nodes right now
+ class AccumulationDict(TypedDict):
+ accum: list[Any]
+ Type = AccumulationDict
+
+
+@comfytype(io_type="LOAD3D_CAMERA")
+class Load3DCamera(ComfyTypeIO):
+ class CameraInfo(TypedDict):
+ position: dict[str, float | int]
+ target: dict[str, float | int]
+ zoom: int
+ cameraType: str
+
+ Type = CameraInfo
+
+
+@comfytype(io_type="LOAD_3D")
+class Load3D(ComfyTypeIO):
+ """3D models are stored as a dictionary."""
+ class Model3DDict(TypedDict):
+ image: str
+ mask: str
+ normal: str
+ camera_info: Load3DCamera.CameraInfo
+ recording: NotRequired[str]
+
+ Type = Model3DDict
+
+
+@comfytype(io_type="LOAD_3D_ANIMATION")
+class Load3DAnimation(Load3D):
+ ...
+
+
+@comfytype(io_type="PHOTOMAKER")
+class Photomaker(ComfyTypeIO):
+ Type = Any
+
+
+@comfytype(io_type="POINT")
+class Point(ComfyTypeIO):
+ Type = Any # NOTE: I couldn't find any references in core code to POINT io_type. Does this exist?
+
+@comfytype(io_type="FACE_ANALYSIS")
+class FaceAnalysis(ComfyTypeIO):
+ Type = Any # NOTE: I couldn't find any references in core code to POINT io_type. Does this exist?
+
+@comfytype(io_type="BBOX")
+class BBOX(ComfyTypeIO):
+ Type = Any # NOTE: I couldn't find any references in core code to POINT io_type. Does this exist?
+
+@comfytype(io_type="SEGS")
+class SEGS(ComfyTypeIO):
+ Type = Any # NOTE: I couldn't find any references in core code to POINT io_type. Does this exist?
+
+@comfytype(io_type="*")
+class AnyType(ComfyTypeIO):
+ Type = Any
+
+@comfytype(io_type="MODEL_PATCH")
+class MODEL_PATCH(ComfyTypeIO):
+ Type = Any
+
+@comfytype(io_type="AUDIO_ENCODER")
+class AudioEncoder(ComfyTypeIO):
+ Type = Any
+
+@comfytype(io_type="AUDIO_ENCODER_OUTPUT")
+class AudioEncoderOutput(ComfyTypeIO):
+ Type = Any
+
+@comfytype(io_type="COMFY_MULTITYPED_V3")
+class MultiType:
+ Type = Any
+ class Input(Input):
+ '''
+ Input that permits more than one input type; if `id` is an instance of `ComfyType.Input`, then that input will be used to create a widget (if applicable) with overridden values.
+ '''
+ def __init__(self, id: str | Input, types: list[type[_ComfyType] | _ComfyType], display_name: str=None, optional=False, tooltip: str=None, lazy: bool=None, extra_dict=None):
+ # if id is an Input, then use that Input with overridden values
+ self.input_override = None
+ if isinstance(id, Input):
+ self.input_override = copy.copy(id)
+ optional = id.optional if id.optional is True else optional
+ tooltip = id.tooltip if id.tooltip is not None else tooltip
+ display_name = id.display_name if id.display_name is not None else display_name
+ lazy = id.lazy if id.lazy is not None else lazy
+ id = id.id
+ # if is a widget input, make sure widget_type is set appropriately
+ if isinstance(self.input_override, WidgetInput):
+ self.input_override.widget_type = self.input_override.get_io_type()
+ super().__init__(id, display_name, optional, tooltip, lazy, extra_dict)
+ self._io_types = types
+
+ @property
+ def io_types(self) -> list[type[Input]]:
+ '''
+ Returns list of Input class types permitted.
+ '''
+ io_types = []
+ for x in self._io_types:
+ if not is_class(x):
+ io_types.append(type(x))
+ else:
+ io_types.append(x)
+ return io_types
+
+ def get_io_type(self):
+ # ensure types are unique and order is preserved
+ str_types = [x.io_type for x in self.io_types]
+ if self.input_override is not None:
+ str_types.insert(0, self.input_override.get_io_type())
+ return ",".join(list(dict.fromkeys(str_types)))
+
+ def as_dict(self):
+ if self.input_override is not None:
+ return self.input_override.as_dict() | super().as_dict()
+ else:
+ return super().as_dict()
+
+class DynamicInput(Input, ABC):
+ '''
+ Abstract class for dynamic input registration.
+ '''
+ @abstractmethod
+ def get_dynamic(self) -> list[Input]:
+ ...
+
+class DynamicOutput(Output, ABC):
+ '''
+ Abstract class for dynamic output registration.
+ '''
+ def __init__(self, id: str=None, display_name: str=None, tooltip: str=None,
+ is_output_list=False):
+ super().__init__(id, display_name, tooltip, is_output_list)
+
+ @abstractmethod
+ def get_dynamic(self) -> list[Output]:
+ ...
+
+
+@comfytype(io_type="COMFY_AUTOGROW_V3")
+class AutogrowDynamic(ComfyTypeI):
+ Type = list[Any]
+ class Input(DynamicInput):
+ def __init__(self, id: str, template_input: Input, min: int=1, max: int=None,
+ display_name: str=None, optional=False, tooltip: str=None, lazy: bool=None, extra_dict=None):
+ super().__init__(id, display_name, optional, tooltip, lazy, extra_dict)
+ self.template_input = template_input
+ if min is not None:
+ assert(min >= 1)
+ if max is not None:
+ assert(max >= 1)
+ self.min = min
+ self.max = max
+
+ def get_dynamic(self) -> list[Input]:
+ curr_count = 1
+ new_inputs = []
+ for i in range(self.min):
+ new_input = copy.copy(self.template_input)
+ new_input.id = f"{new_input.id}{curr_count}_${self.id}_ag$"
+ if new_input.display_name is not None:
+ new_input.display_name = f"{new_input.display_name}{curr_count}"
+ new_input.optional = self.optional or new_input.optional
+ if isinstance(self.template_input, WidgetInput):
+ new_input.force_input = True
+ new_inputs.append(new_input)
+ curr_count += 1
+ # pretend to expand up to max
+ for i in range(curr_count-1, self.max):
+ new_input = copy.copy(self.template_input)
+ new_input.id = f"{new_input.id}{curr_count}_${self.id}_ag$"
+ if new_input.display_name is not None:
+ new_input.display_name = f"{new_input.display_name}{curr_count}"
+ new_input.optional = True
+ if isinstance(self.template_input, WidgetInput):
+ new_input.force_input = True
+ new_inputs.append(new_input)
+ curr_count += 1
+ return new_inputs
+
+@comfytype(io_type="COMFY_COMBODYNAMIC_V3")
+class ComboDynamic(ComfyTypeI):
+ class Input(DynamicInput):
+ def __init__(self, id: str):
+ pass
+
+@comfytype(io_type="COMFY_MATCHTYPE_V3")
+class MatchType(ComfyTypeIO):
+ class Template:
+ def __init__(self, template_id: str, allowed_types: _ComfyType | list[_ComfyType]):
+ self.template_id = template_id
+ self.allowed_types = [allowed_types] if isinstance(allowed_types, _ComfyType) else allowed_types
+
+ def as_dict(self):
+ return {
+ "template_id": self.template_id,
+ "allowed_types": "".join(t.io_type for t in self.allowed_types),
+ }
+
+ class Input(DynamicInput):
+ def __init__(self, id: str, template: MatchType.Template,
+ display_name: str=None, optional=False, tooltip: str=None, lazy: bool=None, extra_dict=None):
+ super().__init__(id, display_name, optional, tooltip, lazy, extra_dict)
+ self.template = template
+
+ def get_dynamic(self) -> list[Input]:
+ return [self]
+
+ def as_dict(self):
+ return super().as_dict() | prune_dict({
+ "template": self.template.as_dict(),
+ })
+
+ class Output(DynamicOutput):
+ def __init__(self, id: str, template: MatchType.Template, display_name: str=None, tooltip: str=None,
+ is_output_list=False):
+ super().__init__(id, display_name, tooltip, is_output_list)
+ self.template = template
+
+ def get_dynamic(self) -> list[Output]:
+ return [self]
+
+ def as_dict(self):
+ return super().as_dict() | prune_dict({
+ "template": self.template.as_dict(),
+ })
+
+
+class HiddenHolder:
+ def __init__(self, unique_id: str, prompt: Any,
+ extra_pnginfo: Any, dynprompt: Any,
+ auth_token_comfy_org: str, api_key_comfy_org: str, **kwargs):
+ self.unique_id = unique_id
+ """UNIQUE_ID is the unique identifier of the node, and matches the id property of the node on the client side. It is commonly used in client-server communications (see messages)."""
+ self.prompt = prompt
+ """PROMPT is the complete prompt sent by the client to the server. See the prompt object for a full description."""
+ self.extra_pnginfo = extra_pnginfo
+ """EXTRA_PNGINFO is a dictionary that will be copied into the metadata of any .png files saved. Custom nodes can store additional information in this dictionary for saving (or as a way to communicate with a downstream node)."""
+ self.dynprompt = dynprompt
+ """DYNPROMPT is an instance of comfy_execution.graph.DynamicPrompt. It differs from PROMPT in that it may mutate during the course of execution in response to Node Expansion."""
+ self.auth_token_comfy_org = auth_token_comfy_org
+ """AUTH_TOKEN_COMFY_ORG is a token acquired from signing into a ComfyOrg account on frontend."""
+ self.api_key_comfy_org = api_key_comfy_org
+ """API_KEY_COMFY_ORG is an API Key generated by ComfyOrg that allows skipping signing into a ComfyOrg account on frontend."""
+
+ def __getattr__(self, key: str):
+ '''If hidden variable not found, return None.'''
+ return None
+
+ @classmethod
+ def from_dict(cls, d: dict | None):
+ if d is None:
+ d = {}
+ return cls(
+ unique_id=d.get(Hidden.unique_id, None),
+ prompt=d.get(Hidden.prompt, None),
+ extra_pnginfo=d.get(Hidden.extra_pnginfo, None),
+ dynprompt=d.get(Hidden.dynprompt, None),
+ auth_token_comfy_org=d.get(Hidden.auth_token_comfy_org, None),
+ api_key_comfy_org=d.get(Hidden.api_key_comfy_org, None),
+ )
+
+class Hidden(str, Enum):
+ '''
+ Enumerator for requesting hidden variables in nodes.
+ '''
+ unique_id = "UNIQUE_ID"
+ """UNIQUE_ID is the unique identifier of the node, and matches the id property of the node on the client side. It is commonly used in client-server communications (see messages)."""
+ prompt = "PROMPT"
+ """PROMPT is the complete prompt sent by the client to the server. See the prompt object for a full description."""
+ extra_pnginfo = "EXTRA_PNGINFO"
+ """EXTRA_PNGINFO is a dictionary that will be copied into the metadata of any .png files saved. Custom nodes can store additional information in this dictionary for saving (or as a way to communicate with a downstream node)."""
+ dynprompt = "DYNPROMPT"
+ """DYNPROMPT is an instance of comfy_execution.graph.DynamicPrompt. It differs from PROMPT in that it may mutate during the course of execution in response to Node Expansion."""
+ auth_token_comfy_org = "AUTH_TOKEN_COMFY_ORG"
+ """AUTH_TOKEN_COMFY_ORG is a token acquired from signing into a ComfyOrg account on frontend."""
+ api_key_comfy_org = "API_KEY_COMFY_ORG"
+ """API_KEY_COMFY_ORG is an API Key generated by ComfyOrg that allows skipping signing into a ComfyOrg account on frontend."""
+
+
+@dataclass
+class NodeInfoV1:
+ input: dict=None
+ input_order: dict[str, list[str]]=None
+ output: list[str]=None
+ output_is_list: list[bool]=None
+ output_name: list[str]=None
+ output_tooltips: list[str]=None
+ name: str=None
+ display_name: str=None
+ description: str=None
+ python_module: Any=None
+ category: str=None
+ output_node: bool=None
+ deprecated: bool=None
+ experimental: bool=None
+ api_node: bool=None
+
+@dataclass
+class NodeInfoV3:
+ input: dict=None
+ output: dict=None
+ hidden: list[str]=None
+ name: str=None
+ display_name: str=None
+ description: str=None
+ category: str=None
+ output_node: bool=None
+ deprecated: bool=None
+ experimental: bool=None
+ api_node: bool=None
+
+
+@dataclass
+class Schema:
+ """Definition of V3 node properties."""
+
+ node_id: str
+ """ID of node - should be globally unique. If this is a custom node, add a prefix or postfix to avoid name clashes."""
+ display_name: str = None
+ """Display name of node."""
+ category: str = "sd"
+ """The category of the node, as per the "Add Node" menu."""
+ inputs: list[Input]=None
+ outputs: list[Output]=None
+ hidden: list[Hidden]=None
+ description: str=""
+ """Node description, shown as a tooltip when hovering over the node."""
+ is_input_list: bool = False
+ """A flag indicating if this node implements the additional code necessary to deal with OUTPUT_IS_LIST nodes.
+
+ All inputs of ``type`` will become ``list[type]``, regardless of how many items are passed in. This also affects ``check_lazy_status``.
+
+ From the docs:
+
+ A node can also override the default input behaviour and receive the whole list in a single call. This is done by setting a class attribute `INPUT_IS_LIST` to ``True``.
+
+ Comfy Docs: https://docs.comfy.org/custom-nodes/backend/lists#list-processing
+ """
+ is_output_node: bool=False
+ """Flags this node as an output node, causing any inputs it requires to be executed.
+
+ If a node is not connected to any output nodes, that node will not be executed. Usage::
+
+ From the docs:
+
+ By default, a node is not considered an output. Set ``OUTPUT_NODE = True`` to specify that it is.
+
+ Comfy Docs: https://docs.comfy.org/custom-nodes/backend/server_overview#output-node
+ """
+ is_deprecated: bool=False
+ """Flags a node as deprecated, indicating to users that they should find alternatives to this node."""
+ is_experimental: bool=False
+ """Flags a node as experimental, informing users that it may change or not work as expected."""
+ is_api_node: bool=False
+ """Flags a node as an API node. See: https://docs.comfy.org/tutorials/api-nodes/overview."""
+ not_idempotent: bool=False
+ """Flags a node as not idempotent; when True, the node will run and not reuse the cached outputs when identical inputs are provided on a different node in the graph."""
+ enable_expand: bool=False
+ """Flags a node as expandable, allowing NodeOutput to include 'expand' property."""
+
+ def validate(self):
+ '''Validate the schema:
+ - verify ids on inputs and outputs are unique - both internally and in relation to each other
+ '''
+ input_ids = [i.id for i in self.inputs] if self.inputs is not None else []
+ output_ids = [o.id for o in self.outputs] if self.outputs is not None else []
+ input_set = set(input_ids)
+ output_set = set(output_ids)
+ issues = []
+ # verify ids are unique per list
+ if len(input_set) != len(input_ids):
+ issues.append(f"Input ids must be unique, but {[item for item, count in Counter(input_ids).items() if count > 1]} are not.")
+ if len(output_set) != len(output_ids):
+ issues.append(f"Output ids must be unique, but {[item for item, count in Counter(output_ids).items() if count > 1]} are not.")
+ # verify ids are unique between lists
+ intersection = input_set & output_set
+ if len(intersection) > 0:
+ issues.append(f"Ids must be unique between inputs and outputs, but {intersection} are not.")
+ if len(issues) > 0:
+ raise ValueError("\n".join(issues))
+
+ def finalize(self):
+ """Add hidden based on selected schema options, and give outputs without ids default ids."""
+ # if is an api_node, will need key-related hidden
+ if self.is_api_node:
+ if self.hidden is None:
+ self.hidden = []
+ if Hidden.auth_token_comfy_org not in self.hidden:
+ self.hidden.append(Hidden.auth_token_comfy_org)
+ if Hidden.api_key_comfy_org not in self.hidden:
+ self.hidden.append(Hidden.api_key_comfy_org)
+ # if is an output_node, will need prompt and extra_pnginfo
+ if self.is_output_node:
+ if self.hidden is None:
+ self.hidden = []
+ if Hidden.prompt not in self.hidden:
+ self.hidden.append(Hidden.prompt)
+ if Hidden.extra_pnginfo not in self.hidden:
+ self.hidden.append(Hidden.extra_pnginfo)
+ # give outputs without ids default ids
+ if self.outputs is not None:
+ for i, output in enumerate(self.outputs):
+ if output.id is None:
+ output.id = f"_{i}_{output.io_type}_"
+
+ def get_v1_info(self, cls) -> NodeInfoV1:
+ # get V1 inputs
+ input = {
+ "required": {}
+ }
+ if self.inputs:
+ for i in self.inputs:
+ if isinstance(i, DynamicInput):
+ dynamic_inputs = i.get_dynamic()
+ for d in dynamic_inputs:
+ add_to_dict_v1(d, input)
+ else:
+ add_to_dict_v1(i, input)
+ if self.hidden:
+ for hidden in self.hidden:
+ input.setdefault("hidden", {})[hidden.name] = (hidden.value,)
+ # create separate lists from output fields
+ output = []
+ output_is_list = []
+ output_name = []
+ output_tooltips = []
+ if self.outputs:
+ for o in self.outputs:
+ output.append(o.io_type)
+ output_is_list.append(o.is_output_list)
+ output_name.append(o.display_name if o.display_name else o.io_type)
+ output_tooltips.append(o.tooltip if o.tooltip else None)
+
+ info = NodeInfoV1(
+ input=input,
+ input_order={key: list(value.keys()) for (key, value) in input.items()},
+ output=output,
+ output_is_list=output_is_list,
+ output_name=output_name,
+ output_tooltips=output_tooltips,
+ name=self.node_id,
+ display_name=self.display_name,
+ category=self.category,
+ description=self.description,
+ output_node=self.is_output_node,
+ deprecated=self.is_deprecated,
+ experimental=self.is_experimental,
+ api_node=self.is_api_node,
+ python_module=getattr(cls, "RELATIVE_PYTHON_MODULE", "nodes")
+ )
+ return info
+
+
+ def get_v3_info(self, cls) -> NodeInfoV3:
+ input_dict = {}
+ output_dict = {}
+ hidden_list = []
+ # TODO: make sure dynamic types will be handled correctly
+ if self.inputs:
+ for input in self.inputs:
+ add_to_dict_v3(input, input_dict)
+ if self.outputs:
+ for output in self.outputs:
+ add_to_dict_v3(output, output_dict)
+ if self.hidden:
+ for hidden in self.hidden:
+ hidden_list.append(hidden.value)
+
+ info = NodeInfoV3(
+ input=input_dict,
+ output=output_dict,
+ hidden=hidden_list,
+ name=self.node_id,
+ display_name=self.display_name,
+ description=self.description,
+ category=self.category,
+ output_node=self.is_output_node,
+ deprecated=self.is_deprecated,
+ experimental=self.is_experimental,
+ api_node=self.is_api_node,
+ python_module=getattr(cls, "RELATIVE_PYTHON_MODULE", "nodes")
+ )
+ return info
+
+
+def add_to_dict_v1(i: Input, input: dict):
+ key = "optional" if i.optional else "required"
+ as_dict = i.as_dict()
+ # for v1, we don't want to include the optional key
+ as_dict.pop("optional", None)
+ input.setdefault(key, {})[i.id] = (i.get_io_type(), as_dict)
+
+def add_to_dict_v3(io: Input | Output, d: dict):
+ d[io.id] = (io.get_io_type(), io.as_dict())
+
+
+
+class _ComfyNodeBaseInternal(_ComfyNodeInternal):
+ """Common base class for storing internal methods and properties; DO NOT USE for defining nodes."""
+
+ RELATIVE_PYTHON_MODULE = None
+ SCHEMA = None
+
+ # filled in during execution
+ resources: Resources = None
+ hidden: HiddenHolder = None
+
+ @classmethod
+ @abstractmethod
+ def define_schema(cls) -> Schema:
+ """Override this function with one that returns a Schema instance."""
+ raise NotImplementedError
+
+ @classmethod
+ @abstractmethod
+ def execute(cls, **kwargs) -> NodeOutput:
+ """Override this function with one that performs node's actions."""
+ raise NotImplementedError
+
+ @classmethod
+ def validate_inputs(cls, **kwargs) -> bool:
+ """Optionally, define this function to validate inputs; equivalent to V1's VALIDATE_INPUTS."""
+ raise NotImplementedError
+
+ @classmethod
+ def fingerprint_inputs(cls, **kwargs) -> Any:
+ """Optionally, define this function to fingerprint inputs; equivalent to V1's IS_CHANGED."""
+ raise NotImplementedError
+
+ @classmethod
+ def check_lazy_status(cls, **kwargs) -> list[str]:
+ """Optionally, define this function to return a list of input names that should be evaluated.
+
+ This basic mixin impl. requires all inputs.
+
+ :kwargs: All node inputs will be included here. If the input is ``None``, it should be assumed that it has not yet been evaluated. \
+ When using ``INPUT_IS_LIST = True``, unevaluated will instead be ``(None,)``.
+
+ Params should match the nodes execution ``FUNCTION`` (self, and all inputs by name).
+ Will be executed repeatedly until it returns an empty list, or all requested items were already evaluated (and sent as params).
+
+ Comfy Docs: https://docs.comfy.org/custom-nodes/backend/lazy_evaluation#defining-check-lazy-status
+ """
+ return [name for name in kwargs if kwargs[name] is None]
+
+ def __init__(self):
+ self.local_resources: ResourcesLocal = None
+ self.__class__.VALIDATE_CLASS()
+
+ @classmethod
+ def GET_BASE_CLASS(cls):
+ return _ComfyNodeBaseInternal
+
+ @final
+ @classmethod
+ def VALIDATE_CLASS(cls):
+ if first_real_override(cls, "define_schema") is None:
+ raise Exception(f"No define_schema function was defined for node class {cls.__name__}.")
+ if first_real_override(cls, "execute") is None:
+ raise Exception(f"No execute function was defined for node class {cls.__name__}.")
+
+ @classproperty
+ def FUNCTION(cls): # noqa
+ if inspect.iscoroutinefunction(cls.execute):
+ return "EXECUTE_NORMALIZED_ASYNC"
+ return "EXECUTE_NORMALIZED"
+
+ @final
+ @classmethod
+ def EXECUTE_NORMALIZED(cls, *args, **kwargs) -> NodeOutput:
+ to_return = cls.execute(*args, **kwargs)
+ if to_return is None:
+ to_return = NodeOutput()
+ elif isinstance(to_return, NodeOutput):
+ pass
+ elif isinstance(to_return, tuple):
+ to_return = NodeOutput(*to_return)
+ elif isinstance(to_return, dict):
+ to_return = NodeOutput.from_dict(to_return)
+ elif isinstance(to_return, ExecutionBlocker):
+ to_return = NodeOutput(block_execution=to_return.message)
+ else:
+ raise Exception(f"Invalid return type from node: {type(to_return)}")
+ if to_return.expand is not None and not cls.SCHEMA.enable_expand:
+ raise Exception(f"Node {cls.__name__} is not expandable, but expand included in NodeOutput; developer should set enable_expand=True on node's Schema to allow this.")
+ return to_return
+
+ @final
+ @classmethod
+ async def EXECUTE_NORMALIZED_ASYNC(cls, *args, **kwargs) -> NodeOutput:
+ to_return = await cls.execute(*args, **kwargs)
+ if to_return is None:
+ to_return = NodeOutput()
+ elif isinstance(to_return, NodeOutput):
+ pass
+ elif isinstance(to_return, tuple):
+ to_return = NodeOutput(*to_return)
+ elif isinstance(to_return, dict):
+ to_return = NodeOutput.from_dict(to_return)
+ elif isinstance(to_return, ExecutionBlocker):
+ to_return = NodeOutput(block_execution=to_return.message)
+ else:
+ raise Exception(f"Invalid return type from node: {type(to_return)}")
+ if to_return.expand is not None and not cls.SCHEMA.enable_expand:
+ raise Exception(f"Node {cls.__name__} is not expandable, but expand included in NodeOutput; developer should set enable_expand=True on node's Schema to allow this.")
+ return to_return
+
+ @final
+ @classmethod
+ def PREPARE_CLASS_CLONE(cls, hidden_inputs: dict) -> type[ComfyNode]:
+ """Creates clone of real node class to prevent monkey-patching."""
+ c_type: type[ComfyNode] = cls if is_class(cls) else type(cls)
+ type_clone: type[ComfyNode] = shallow_clone_class(c_type)
+ # set hidden
+ type_clone.hidden = HiddenHolder.from_dict(hidden_inputs)
+ return type_clone
+
+ @final
+ @classmethod
+ def GET_NODE_INFO_V3(cls) -> dict[str, Any]:
+ schema = cls.GET_SCHEMA()
+ info = schema.get_v3_info(cls)
+ return asdict(info)
+ #############################################
+ # V1 Backwards Compatibility code
+ #--------------------------------------------
+ @final
+ @classmethod
+ def GET_NODE_INFO_V1(cls) -> dict[str, Any]:
+ schema = cls.GET_SCHEMA()
+ info = schema.get_v1_info(cls)
+ return asdict(info)
+
+ _DESCRIPTION = None
+ @final
+ @classproperty
+ def DESCRIPTION(cls): # noqa
+ if cls._DESCRIPTION is None:
+ cls.GET_SCHEMA()
+ return cls._DESCRIPTION
+
+ _CATEGORY = None
+ @final
+ @classproperty
+ def CATEGORY(cls): # noqa
+ if cls._CATEGORY is None:
+ cls.GET_SCHEMA()
+ return cls._CATEGORY
+
+ _EXPERIMENTAL = None
+ @final
+ @classproperty
+ def EXPERIMENTAL(cls): # noqa
+ if cls._EXPERIMENTAL is None:
+ cls.GET_SCHEMA()
+ return cls._EXPERIMENTAL
+
+ _DEPRECATED = None
+ @final
+ @classproperty
+ def DEPRECATED(cls): # noqa
+ if cls._DEPRECATED is None:
+ cls.GET_SCHEMA()
+ return cls._DEPRECATED
+
+ _API_NODE = None
+ @final
+ @classproperty
+ def API_NODE(cls): # noqa
+ if cls._API_NODE is None:
+ cls.GET_SCHEMA()
+ return cls._API_NODE
+
+ _OUTPUT_NODE = None
+ @final
+ @classproperty
+ def OUTPUT_NODE(cls): # noqa
+ if cls._OUTPUT_NODE is None:
+ cls.GET_SCHEMA()
+ return cls._OUTPUT_NODE
+
+ _INPUT_IS_LIST = None
+ @final
+ @classproperty
+ def INPUT_IS_LIST(cls): # noqa
+ if cls._INPUT_IS_LIST is None:
+ cls.GET_SCHEMA()
+ return cls._INPUT_IS_LIST
+ _OUTPUT_IS_LIST = None
+
+ @final
+ @classproperty
+ def OUTPUT_IS_LIST(cls): # noqa
+ if cls._OUTPUT_IS_LIST is None:
+ cls.GET_SCHEMA()
+ return cls._OUTPUT_IS_LIST
+
+ _RETURN_TYPES = None
+ @final
+ @classproperty
+ def RETURN_TYPES(cls): # noqa
+ if cls._RETURN_TYPES is None:
+ cls.GET_SCHEMA()
+ return cls._RETURN_TYPES
+
+ _RETURN_NAMES = None
+ @final
+ @classproperty
+ def RETURN_NAMES(cls): # noqa
+ if cls._RETURN_NAMES is None:
+ cls.GET_SCHEMA()
+ return cls._RETURN_NAMES
+
+ _OUTPUT_TOOLTIPS = None
+ @final
+ @classproperty
+ def OUTPUT_TOOLTIPS(cls): # noqa
+ if cls._OUTPUT_TOOLTIPS is None:
+ cls.GET_SCHEMA()
+ return cls._OUTPUT_TOOLTIPS
+
+ _NOT_IDEMPOTENT = None
+ @final
+ @classproperty
+ def NOT_IDEMPOTENT(cls): # noqa
+ if cls._NOT_IDEMPOTENT is None:
+ cls.GET_SCHEMA()
+ return cls._NOT_IDEMPOTENT
+
+ @final
+ @classmethod
+ def INPUT_TYPES(cls, include_hidden=True, return_schema=False) -> dict[str, dict] | tuple[dict[str, dict], Schema]:
+ schema = cls.FINALIZE_SCHEMA()
+ info = schema.get_v1_info(cls)
+ input = info.input
+ if not include_hidden:
+ input.pop("hidden", None)
+ if return_schema:
+ return input, schema
+ return input
+
+ @final
+ @classmethod
+ def FINALIZE_SCHEMA(cls):
+ """Call define_schema and finalize it."""
+ schema = cls.define_schema()
+ schema.finalize()
+ return schema
+
+ @final
+ @classmethod
+ def GET_SCHEMA(cls) -> Schema:
+ """Validate node class, finalize schema, validate schema, and set expected class properties."""
+ cls.VALIDATE_CLASS()
+ schema = cls.FINALIZE_SCHEMA()
+ schema.validate()
+ if cls._DESCRIPTION is None:
+ cls._DESCRIPTION = schema.description
+ if cls._CATEGORY is None:
+ cls._CATEGORY = schema.category
+ if cls._EXPERIMENTAL is None:
+ cls._EXPERIMENTAL = schema.is_experimental
+ if cls._DEPRECATED is None:
+ cls._DEPRECATED = schema.is_deprecated
+ if cls._API_NODE is None:
+ cls._API_NODE = schema.is_api_node
+ if cls._OUTPUT_NODE is None:
+ cls._OUTPUT_NODE = schema.is_output_node
+ if cls._INPUT_IS_LIST is None:
+ cls._INPUT_IS_LIST = schema.is_input_list
+ if cls._NOT_IDEMPOTENT is None:
+ cls._NOT_IDEMPOTENT = schema.not_idempotent
+
+ if cls._RETURN_TYPES is None:
+ output = []
+ output_name = []
+ output_is_list = []
+ output_tooltips = []
+ if schema.outputs:
+ for o in schema.outputs:
+ output.append(o.io_type)
+ output_name.append(o.display_name if o.display_name else o.io_type)
+ output_is_list.append(o.is_output_list)
+ output_tooltips.append(o.tooltip if o.tooltip else None)
+
+ cls._RETURN_TYPES = output
+ cls._RETURN_NAMES = output_name
+ cls._OUTPUT_IS_LIST = output_is_list
+ cls._OUTPUT_TOOLTIPS = output_tooltips
+ cls.SCHEMA = schema
+ return schema
+ #--------------------------------------------
+ #############################################
+
+
+class ComfyNode(_ComfyNodeBaseInternal):
+ """Common base class for all V3 nodes."""
+
+ @classmethod
+ @abstractmethod
+ def define_schema(cls) -> Schema:
+ """Override this function with one that returns a Schema instance."""
+ raise NotImplementedError
+
+ @classmethod
+ @abstractmethod
+ def execute(cls, **kwargs) -> NodeOutput:
+ """Override this function with one that performs node's actions."""
+ raise NotImplementedError
+
+ @classmethod
+ def validate_inputs(cls, **kwargs) -> bool:
+ """Optionally, define this function to validate inputs; equivalent to V1's VALIDATE_INPUTS."""
+ raise NotImplementedError
+
+ @classmethod
+ def fingerprint_inputs(cls, **kwargs) -> Any:
+ """Optionally, define this function to fingerprint inputs; equivalent to V1's IS_CHANGED."""
+ raise NotImplementedError
+
+ @classmethod
+ def check_lazy_status(cls, **kwargs) -> list[str]:
+ """Optionally, define this function to return a list of input names that should be evaluated.
+
+ This basic mixin impl. requires all inputs.
+
+ :kwargs: All node inputs will be included here. If the input is ``None``, it should be assumed that it has not yet been evaluated. \
+ When using ``INPUT_IS_LIST = True``, unevaluated will instead be ``(None,)``.
+
+ Params should match the nodes execution ``FUNCTION`` (self, and all inputs by name).
+ Will be executed repeatedly until it returns an empty list, or all requested items were already evaluated (and sent as params).
+
+ Comfy Docs: https://docs.comfy.org/custom-nodes/backend/lazy_evaluation#defining-check-lazy-status
+ """
+ return [name for name in kwargs if kwargs[name] is None]
+
+ @final
+ @classmethod
+ def GET_BASE_CLASS(cls):
+ """DO NOT override this class. Will break things in execution.py."""
+ return ComfyNode
+
+
+class NodeOutput(_NodeOutputInternal):
+ '''
+ Standardized output of a node; can pass in any number of args and/or a UIOutput into 'ui' kwarg.
+ '''
+ def __init__(self, *args: Any, ui: _UIOutput | dict=None, expand: dict=None, block_execution: str=None):
+ self.args = args
+ self.ui = ui
+ self.expand = expand
+ self.block_execution = block_execution
+
+ @property
+ def result(self):
+ return self.args if len(self.args) > 0 else None
+
+ @classmethod
+ def from_dict(cls, data: dict[str, Any]) -> "NodeOutput":
+ args = ()
+ ui = None
+ expand = None
+ if "result" in data:
+ result = data["result"]
+ if isinstance(result, ExecutionBlocker):
+ return cls(block_execution=result.message)
+ args = result
+ if "ui" in data:
+ ui = data["ui"]
+ if "expand" in data:
+ expand = data["expand"]
+ return cls(args=args, ui=ui, expand=expand)
+
+ def __getitem__(self, index) -> Any:
+ return self.args[index]
+
+class _UIOutput(ABC):
+ def __init__(self):
+ pass
+
+ @abstractmethod
+ def as_dict(self) -> dict:
+ ...
+
+
+class _IO:
+ FolderType = FolderType
+ UploadType = UploadType
+ RemoteOptions = RemoteOptions
+ NumberDisplay = NumberDisplay
+
+ comfytype = staticmethod(comfytype)
+ Custom = staticmethod(Custom)
+ Input = Input
+ WidgetInput = WidgetInput
+ Output = Output
+ ComfyTypeI = ComfyTypeI
+ ComfyTypeIO = ComfyTypeIO
+ #---------------------------------
+ # Supported Types
+ Boolean = Boolean
+ Int = Int
+ Float = Float
+ String = String
+ Combo = Combo
+ MultiCombo = MultiCombo
+ Image = Image
+ WanCameraEmbedding = WanCameraEmbedding
+ Webcam = Webcam
+ Mask = Mask
+ Latent = Latent
+ Conditioning = Conditioning
+ Sampler = Sampler
+ Sigmas = Sigmas
+ Noise = Noise
+ Guider = Guider
+ Clip = Clip
+ ControlNet = ControlNet
+ Vae = Vae
+ Model = Model
+ ClipVision = ClipVision
+ ClipVisionOutput = ClipVisionOutput
+ AudioEncoderOutput = AudioEncoderOutput
+ StyleModel = StyleModel
+ Gligen = Gligen
+ UpscaleModel = UpscaleModel
+ Audio = Audio
+ Video = Video
+ SVG = SVG
+ LoraModel = LoraModel
+ LossMap = LossMap
+ Voxel = Voxel
+ Mesh = Mesh
+ Hooks = Hooks
+ HookKeyframes = HookKeyframes
+ TimestepsRange = TimestepsRange
+ LatentOperation = LatentOperation
+ FlowControl = FlowControl
+ Accumulation = Accumulation
+ Load3DCamera = Load3DCamera
+ Load3D = Load3D
+ Load3DAnimation = Load3DAnimation
+ Photomaker = Photomaker
+ Point = Point
+ FaceAnalysis = FaceAnalysis
+ BBOX = BBOX
+ SEGS = SEGS
+ AnyType = AnyType
+ MultiType = MultiType
+ #---------------------------------
+ HiddenHolder = HiddenHolder
+ Hidden = Hidden
+ NodeInfoV1 = NodeInfoV1
+ NodeInfoV3 = NodeInfoV3
+ Schema = Schema
+ ComfyNode = ComfyNode
+ NodeOutput = NodeOutput
+ add_to_dict_v1 = staticmethod(add_to_dict_v1)
+ add_to_dict_v3 = staticmethod(add_to_dict_v3)
diff --git a/comfy_api/latest/_resources.py b/comfy_api/latest/_resources.py
new file mode 100644
index 000000000..a6bdda972
--- /dev/null
+++ b/comfy_api/latest/_resources.py
@@ -0,0 +1,72 @@
+from __future__ import annotations
+import comfy.utils
+import folder_paths
+import logging
+from abc import ABC, abstractmethod
+from typing import Any
+import torch
+
+class ResourceKey(ABC):
+ Type = Any
+ def __init__(self):
+ ...
+
+class TorchDictFolderFilename(ResourceKey):
+ '''Key for requesting a torch file via file_name from a folder category.'''
+ Type = dict[str, torch.Tensor]
+ def __init__(self, folder_name: str, file_name: str):
+ self.folder_name = folder_name
+ self.file_name = file_name
+
+ def __hash__(self):
+ return hash((self.folder_name, self.file_name))
+
+ def __eq__(self, other: object) -> bool:
+ if not isinstance(other, TorchDictFolderFilename):
+ return False
+ return self.folder_name == other.folder_name and self.file_name == other.file_name
+
+ def __str__(self):
+ return f"{self.folder_name} -> {self.file_name}"
+
+class Resources(ABC):
+ def __init__(self):
+ ...
+
+ @abstractmethod
+ def get(self, key: ResourceKey, default: Any=...) -> Any:
+ pass
+
+class ResourcesLocal(Resources):
+ def __init__(self):
+ super().__init__()
+ self.local_resources: dict[ResourceKey, Any] = {}
+
+ def get(self, key: ResourceKey, default: Any=...) -> Any:
+ cached = self.local_resources.get(key, None)
+ if cached is not None:
+ logging.info(f"Using cached resource '{key}'")
+ return cached
+ logging.info(f"Loading resource '{key}'")
+ to_return = None
+ if isinstance(key, TorchDictFolderFilename):
+ if default is ...:
+ to_return = comfy.utils.load_torch_file(folder_paths.get_full_path_or_raise(key.folder_name, key.file_name), safe_load=True)
+ else:
+ full_path = folder_paths.get_full_path(key.folder_name, key.file_name)
+ if full_path is not None:
+ to_return = comfy.utils.load_torch_file(full_path, safe_load=True)
+
+ if to_return is not None:
+ self.local_resources[key] = to_return
+ return to_return
+ if default is not ...:
+ return default
+ raise Exception(f"Unsupported resource key type: {type(key)}")
+
+
+class _RESOURCES:
+ ResourceKey = ResourceKey
+ TorchDictFolderFilename = TorchDictFolderFilename
+ Resources = Resources
+ ResourcesLocal = ResourcesLocal
diff --git a/comfy_api/latest/_ui.py b/comfy_api/latest/_ui.py
new file mode 100644
index 000000000..26a55615f
--- /dev/null
+++ b/comfy_api/latest/_ui.py
@@ -0,0 +1,463 @@
+from __future__ import annotations
+
+import json
+import os
+import random
+from io import BytesIO
+from typing import Type
+
+import av
+import numpy as np
+import torch
+try:
+ import torchaudio
+ TORCH_AUDIO_AVAILABLE = True
+except:
+ TORCH_AUDIO_AVAILABLE = False
+from PIL import Image as PILImage
+from PIL.PngImagePlugin import PngInfo
+
+import folder_paths
+
+# used for image preview
+from comfy.cli_args import args
+from comfy_api.latest._io import ComfyNode, FolderType, Image, _UIOutput
+
+
+class SavedResult(dict):
+ def __init__(self, filename: str, subfolder: str, type: FolderType):
+ super().__init__(filename=filename, subfolder=subfolder,type=type.value)
+
+ @property
+ def filename(self) -> str:
+ return self["filename"]
+
+ @property
+ def subfolder(self) -> str:
+ return self["subfolder"]
+
+ @property
+ def type(self) -> FolderType:
+ return FolderType(self["type"])
+
+
+class SavedImages(_UIOutput):
+ """A UI output class to represent one or more saved images, potentially animated."""
+ def __init__(self, results: list[SavedResult], is_animated: bool = False):
+ super().__init__()
+ self.results = results
+ self.is_animated = is_animated
+
+ def as_dict(self) -> dict:
+ data = {"images": self.results}
+ if self.is_animated:
+ data["animated"] = (True,)
+ return data
+
+
+class SavedAudios(_UIOutput):
+ """UI wrapper around one or more audio files on disk (FLAC / MP3 / Opus)."""
+ def __init__(self, results: list[SavedResult]):
+ super().__init__()
+ self.results = results
+
+ def as_dict(self) -> dict:
+ return {"audio": self.results}
+
+
+def _get_directory_by_folder_type(folder_type: FolderType) -> str:
+ if folder_type == FolderType.input:
+ return folder_paths.get_input_directory()
+ if folder_type == FolderType.output:
+ return folder_paths.get_output_directory()
+ return folder_paths.get_temp_directory()
+
+
+class ImageSaveHelper:
+ """A helper class with static methods to handle image saving and metadata."""
+
+ @staticmethod
+ def _convert_tensor_to_pil(image_tensor: torch.Tensor) -> PILImage.Image:
+ """Converts a single torch tensor to a PIL Image."""
+ return PILImage.fromarray(np.clip(255.0 * image_tensor.cpu().numpy(), 0, 255).astype(np.uint8))
+
+ @staticmethod
+ def _create_png_metadata(cls: Type[ComfyNode] | None) -> PngInfo | None:
+ """Creates a PngInfo object with prompt and extra_pnginfo."""
+ if args.disable_metadata or cls is None or not cls.hidden:
+ return None
+ metadata = PngInfo()
+ if cls.hidden.prompt:
+ metadata.add_text("prompt", json.dumps(cls.hidden.prompt))
+ if cls.hidden.extra_pnginfo:
+ for x in cls.hidden.extra_pnginfo:
+ metadata.add_text(x, json.dumps(cls.hidden.extra_pnginfo[x]))
+ return metadata
+
+ @staticmethod
+ def _create_animated_png_metadata(cls: Type[ComfyNode] | None) -> PngInfo | None:
+ """Creates a PngInfo object with prompt and extra_pnginfo for animated PNGs (APNG)."""
+ if args.disable_metadata or cls is None or not cls.hidden:
+ return None
+ metadata = PngInfo()
+ if cls.hidden.prompt:
+ metadata.add(
+ b"comf",
+ "prompt".encode("latin-1", "strict")
+ + b"\0"
+ + json.dumps(cls.hidden.prompt).encode("latin-1", "strict"),
+ after_idat=True,
+ )
+ if cls.hidden.extra_pnginfo:
+ for x in cls.hidden.extra_pnginfo:
+ metadata.add(
+ b"comf",
+ x.encode("latin-1", "strict")
+ + b"\0"
+ + json.dumps(cls.hidden.extra_pnginfo[x]).encode("latin-1", "strict"),
+ after_idat=True,
+ )
+ return metadata
+
+ @staticmethod
+ def _create_webp_metadata(pil_image: PILImage.Image, cls: Type[ComfyNode] | None) -> PILImage.Exif:
+ """Creates EXIF metadata bytes for WebP images."""
+ exif_data = pil_image.getexif()
+ if args.disable_metadata or cls is None or cls.hidden is None:
+ return exif_data
+ if cls.hidden.prompt is not None:
+ exif_data[0x0110] = "prompt:{}".format(json.dumps(cls.hidden.prompt)) # EXIF 0x0110 = Model
+ if cls.hidden.extra_pnginfo is not None:
+ inital_exif_tag = 0x010F # EXIF 0x010f = Make
+ for key, value in cls.hidden.extra_pnginfo.items():
+ exif_data[inital_exif_tag] = "{}:{}".format(key, json.dumps(value))
+ inital_exif_tag -= 1
+ return exif_data
+
+ @staticmethod
+ def save_images(
+ images, filename_prefix: str, folder_type: FolderType, cls: Type[ComfyNode] | None, compress_level = 4,
+ ) -> list[SavedResult]:
+ """Saves a batch of images as individual PNG files."""
+ full_output_folder, filename, counter, subfolder, _ = folder_paths.get_save_image_path(
+ filename_prefix, _get_directory_by_folder_type(folder_type), images[0].shape[1], images[0].shape[0]
+ )
+ results = []
+ metadata = ImageSaveHelper._create_png_metadata(cls)
+ for batch_number, image_tensor in enumerate(images):
+ img = ImageSaveHelper._convert_tensor_to_pil(image_tensor)
+ filename_with_batch_num = filename.replace("%batch_num%", str(batch_number))
+ file = f"{filename_with_batch_num}_{counter:05}_.png"
+ img.save(os.path.join(full_output_folder, file), pnginfo=metadata, compress_level=compress_level)
+ results.append(SavedResult(file, subfolder, folder_type))
+ counter += 1
+ return results
+
+ @staticmethod
+ def get_save_images_ui(images, filename_prefix: str, cls: Type[ComfyNode] | None, compress_level=4) -> SavedImages:
+ """Saves a batch of images and returns a UI object for the node output."""
+ return SavedImages(
+ ImageSaveHelper.save_images(
+ images,
+ filename_prefix=filename_prefix,
+ folder_type=FolderType.output,
+ cls=cls,
+ compress_level=compress_level,
+ )
+ )
+
+ @staticmethod
+ def save_animated_png(
+ images, filename_prefix: str, folder_type: FolderType, cls: Type[ComfyNode] | None, fps: float, compress_level: int
+ ) -> SavedResult:
+ """Saves a batch of images as a single animated PNG."""
+ full_output_folder, filename, counter, subfolder, _ = folder_paths.get_save_image_path(
+ filename_prefix, _get_directory_by_folder_type(folder_type), images[0].shape[1], images[0].shape[0]
+ )
+ pil_images = [ImageSaveHelper._convert_tensor_to_pil(img) for img in images]
+ metadata = ImageSaveHelper._create_animated_png_metadata(cls)
+ file = f"{filename}_{counter:05}_.png"
+ save_path = os.path.join(full_output_folder, file)
+ pil_images[0].save(
+ save_path,
+ pnginfo=metadata,
+ compress_level=compress_level,
+ save_all=True,
+ duration=int(1000.0 / fps),
+ append_images=pil_images[1:],
+ )
+ return SavedResult(file, subfolder, folder_type)
+
+ @staticmethod
+ def get_save_animated_png_ui(
+ images, filename_prefix: str, cls: Type[ComfyNode] | None, fps: float, compress_level: int
+ ) -> SavedImages:
+ """Saves an animated PNG and returns a UI object for the node output."""
+ result = ImageSaveHelper.save_animated_png(
+ images,
+ filename_prefix=filename_prefix,
+ folder_type=FolderType.output,
+ cls=cls,
+ fps=fps,
+ compress_level=compress_level,
+ )
+ return SavedImages([result], is_animated=len(images) > 1)
+
+ @staticmethod
+ def save_animated_webp(
+ images,
+ filename_prefix: str,
+ folder_type: FolderType,
+ cls: Type[ComfyNode] | None,
+ fps: float,
+ lossless: bool,
+ quality: int,
+ method: int,
+ ) -> SavedResult:
+ """Saves a batch of images as a single animated WebP."""
+ full_output_folder, filename, counter, subfolder, _ = folder_paths.get_save_image_path(
+ filename_prefix, _get_directory_by_folder_type(folder_type), images[0].shape[1], images[0].shape[0]
+ )
+ pil_images = [ImageSaveHelper._convert_tensor_to_pil(img) for img in images]
+ pil_exif = ImageSaveHelper._create_webp_metadata(pil_images[0], cls)
+ file = f"{filename}_{counter:05}_.webp"
+ pil_images[0].save(
+ os.path.join(full_output_folder, file),
+ save_all=True,
+ duration=int(1000.0 / fps),
+ append_images=pil_images[1:],
+ exif=pil_exif,
+ lossless=lossless,
+ quality=quality,
+ method=method,
+ )
+ return SavedResult(file, subfolder, folder_type)
+
+ @staticmethod
+ def get_save_animated_webp_ui(
+ images,
+ filename_prefix: str,
+ cls: Type[ComfyNode] | None,
+ fps: float,
+ lossless: bool,
+ quality: int,
+ method: int,
+ ) -> SavedImages:
+ """Saves an animated WebP and returns a UI object for the node output."""
+ result = ImageSaveHelper.save_animated_webp(
+ images,
+ filename_prefix=filename_prefix,
+ folder_type=FolderType.output,
+ cls=cls,
+ fps=fps,
+ lossless=lossless,
+ quality=quality,
+ method=method,
+ )
+ return SavedImages([result], is_animated=len(images) > 1)
+
+
+class AudioSaveHelper:
+ """A helper class with static methods to handle audio saving and metadata."""
+ _OPUS_RATES = [8000, 12000, 16000, 24000, 48000]
+
+ @staticmethod
+ def save_audio(
+ audio: dict,
+ filename_prefix: str,
+ folder_type: FolderType,
+ cls: Type[ComfyNode] | None,
+ format: str = "flac",
+ quality: str = "128k",
+ ) -> list[SavedResult]:
+ full_output_folder, filename, counter, subfolder, _ = folder_paths.get_save_image_path(
+ filename_prefix, _get_directory_by_folder_type(folder_type)
+ )
+
+ metadata = {}
+ if not args.disable_metadata and cls is not None:
+ if cls.hidden.prompt is not None:
+ metadata["prompt"] = json.dumps(cls.hidden.prompt)
+ if cls.hidden.extra_pnginfo is not None:
+ for x in cls.hidden.extra_pnginfo:
+ metadata[x] = json.dumps(cls.hidden.extra_pnginfo[x])
+
+ results = []
+ for batch_number, waveform in enumerate(audio["waveform"].cpu()):
+ filename_with_batch_num = filename.replace("%batch_num%", str(batch_number))
+ file = f"{filename_with_batch_num}_{counter:05}_.{format}"
+ output_path = os.path.join(full_output_folder, file)
+
+ # Use original sample rate initially
+ sample_rate = audio["sample_rate"]
+
+ # Handle Opus sample rate requirements
+ if format == "opus":
+ if sample_rate > 48000:
+ sample_rate = 48000
+ elif sample_rate not in AudioSaveHelper._OPUS_RATES:
+ # Find the next highest supported rate
+ for rate in sorted(AudioSaveHelper._OPUS_RATES):
+ if rate > sample_rate:
+ sample_rate = rate
+ break
+ if sample_rate not in AudioSaveHelper._OPUS_RATES: # Fallback if still not supported
+ sample_rate = 48000
+
+ # Resample if necessary
+ if sample_rate != audio["sample_rate"]:
+ if not TORCH_AUDIO_AVAILABLE:
+ raise Exception("torchaudio is not available; cannot resample audio.")
+ waveform = torchaudio.functional.resample(waveform, audio["sample_rate"], sample_rate)
+
+ # Create output with specified format
+ output_buffer = BytesIO()
+ output_container = av.open(output_buffer, mode="w", format=format)
+
+ # Set metadata on the container
+ for key, value in metadata.items():
+ output_container.metadata[key] = value
+
+ # Set up the output stream with appropriate properties
+ if format == "opus":
+ out_stream = output_container.add_stream("libopus", rate=sample_rate)
+ if quality == "64k":
+ out_stream.bit_rate = 64000
+ elif quality == "96k":
+ out_stream.bit_rate = 96000
+ elif quality == "128k":
+ out_stream.bit_rate = 128000
+ elif quality == "192k":
+ out_stream.bit_rate = 192000
+ elif quality == "320k":
+ out_stream.bit_rate = 320000
+ elif format == "mp3":
+ out_stream = output_container.add_stream("libmp3lame", rate=sample_rate)
+ if quality == "V0":
+ # TODO i would really love to support V3 and V5 but there doesn't seem to be a way to set the qscale level, the property below is a bool
+ out_stream.codec_context.qscale = 1
+ elif quality == "128k":
+ out_stream.bit_rate = 128000
+ elif quality == "320k":
+ out_stream.bit_rate = 320000
+ else: # format == "flac":
+ out_stream = output_container.add_stream("flac", rate=sample_rate)
+
+ frame = av.AudioFrame.from_ndarray(
+ waveform.movedim(0, 1).reshape(1, -1).float().numpy(),
+ format="flt",
+ layout="mono" if waveform.shape[0] == 1 else "stereo",
+ )
+ frame.sample_rate = sample_rate
+ frame.pts = 0
+ output_container.mux(out_stream.encode(frame))
+
+ # Flush encoder
+ output_container.mux(out_stream.encode(None))
+
+ # Close containers
+ output_container.close()
+
+ # Write the output to file
+ output_buffer.seek(0)
+ with open(output_path, "wb") as f:
+ f.write(output_buffer.getbuffer())
+
+ results.append(SavedResult(file, subfolder, folder_type))
+ counter += 1
+
+ return results
+
+ @staticmethod
+ def get_save_audio_ui(
+ audio, filename_prefix: str, cls: Type[ComfyNode] | None, format: str = "flac", quality: str = "128k",
+ ) -> SavedAudios:
+ """Save and instantly wrap for UI."""
+ return SavedAudios(
+ AudioSaveHelper.save_audio(
+ audio,
+ filename_prefix=filename_prefix,
+ folder_type=FolderType.output,
+ cls=cls,
+ format=format,
+ quality=quality,
+ )
+ )
+
+
+class PreviewImage(_UIOutput):
+ def __init__(self, image: Image.Type, animated: bool = False, cls: Type[ComfyNode] = None, **kwargs):
+ self.values = ImageSaveHelper.save_images(
+ image,
+ filename_prefix="ComfyUI_temp_" + ''.join(random.choice("abcdefghijklmnopqrstupvxyz") for _ in range(5)),
+ folder_type=FolderType.temp,
+ cls=cls,
+ compress_level=1,
+ )
+ self.animated = animated
+
+ def as_dict(self):
+ return {
+ "images": self.values,
+ "animated": (self.animated,)
+ }
+
+
+class PreviewMask(PreviewImage):
+ def __init__(self, mask: PreviewMask.Type, animated: bool=False, cls: ComfyNode=None, **kwargs):
+ preview = mask.reshape((-1, 1, mask.shape[-2], mask.shape[-1])).movedim(1, -1).expand(-1, -1, -1, 3)
+ super().__init__(preview, animated, cls, **kwargs)
+
+
+class PreviewAudio(_UIOutput):
+ def __init__(self, audio: dict, cls: Type[ComfyNode] = None, **kwargs):
+ self.values = AudioSaveHelper.save_audio(
+ audio,
+ filename_prefix="ComfyUI_temp_" + "".join(random.choice("abcdefghijklmnopqrstuvwxyz") for _ in range(5)),
+ folder_type=FolderType.temp,
+ cls=cls,
+ format="flac",
+ quality="128k",
+ )
+
+ def as_dict(self) -> dict:
+ return {"audio": self.values}
+
+
+class PreviewVideo(_UIOutput):
+ def __init__(self, values: list[SavedResult | dict], **kwargs):
+ self.values = values
+
+ def as_dict(self):
+ return {"images": self.values, "animated": (True,)}
+
+
+class PreviewUI3D(_UIOutput):
+ def __init__(self, model_file, camera_info, **kwargs):
+ self.model_file = model_file
+ self.camera_info = camera_info
+
+ def as_dict(self):
+ return {"result": [self.model_file, self.camera_info]}
+
+
+class PreviewText(_UIOutput):
+ def __init__(self, value: str, **kwargs):
+ self.value = value
+
+ def as_dict(self):
+ return {"text": (self.value,)}
+
+
+class _UI:
+ SavedResult = SavedResult
+ SavedImages = SavedImages
+ SavedAudios = SavedAudios
+ ImageSaveHelper = ImageSaveHelper
+ AudioSaveHelper = AudioSaveHelper
+ PreviewImage = PreviewImage
+ PreviewMask = PreviewMask
+ PreviewAudio = PreviewAudio
+ PreviewVideo = PreviewVideo
+ PreviewUI3D = PreviewUI3D
+ PreviewText = PreviewText
diff --git a/comfy_api/latest/_util/__init__.py b/comfy_api/latest/_util/__init__.py
new file mode 100644
index 000000000..9019c46db
--- /dev/null
+++ b/comfy_api/latest/_util/__init__.py
@@ -0,0 +1,8 @@
+from .video_types import VideoContainer, VideoCodec, VideoComponents
+
+__all__ = [
+ # Utility Types
+ "VideoContainer",
+ "VideoCodec",
+ "VideoComponents",
+]
diff --git a/comfy_api/latest/_util/video_types.py b/comfy_api/latest/_util/video_types.py
new file mode 100644
index 000000000..c3e3d8e3a
--- /dev/null
+++ b/comfy_api/latest/_util/video_types.py
@@ -0,0 +1,52 @@
+from __future__ import annotations
+from dataclasses import dataclass
+from enum import Enum
+from fractions import Fraction
+from typing import Optional
+from comfy_api.latest._input import ImageInput, AudioInput
+
+class VideoCodec(str, Enum):
+ AUTO = "auto"
+ H264 = "h264"
+
+ @classmethod
+ def as_input(cls) -> list[str]:
+ """
+ Returns a list of codec names that can be used as node input.
+ """
+ return [member.value for member in cls]
+
+class VideoContainer(str, Enum):
+ AUTO = "auto"
+ MP4 = "mp4"
+
+ @classmethod
+ def as_input(cls) -> list[str]:
+ """
+ Returns a list of container names that can be used as node input.
+ """
+ return [member.value for member in cls]
+
+ @classmethod
+ def get_extension(cls, value) -> str:
+ """
+ Returns the file extension for the container.
+ """
+ if isinstance(value, str):
+ value = cls(value)
+ if value == VideoContainer.MP4 or value == VideoContainer.AUTO:
+ return "mp4"
+ return ""
+
+@dataclass
+class VideoComponents:
+ """
+ Dataclass representing the components of a video.
+ """
+
+ images: ImageInput
+ frame_rate: Fraction
+ audio: Optional[AudioInput] = None
+ metadata: Optional[dict] = None
+
+
diff --git a/comfy_api/latest/generated/ComfyAPISyncStub.pyi b/comfy_api/latest/generated/ComfyAPISyncStub.pyi
new file mode 100644
index 000000000..525c074dd
--- /dev/null
+++ b/comfy_api/latest/generated/ComfyAPISyncStub.pyi
@@ -0,0 +1,20 @@
+from typing import Any, Dict, List, Optional, Tuple, Union, Set, Sequence, cast, NamedTuple
+from comfy_api.latest import ComfyAPI_latest
+from PIL.Image import Image
+from torch import Tensor
+class ComfyAPISyncStub:
+ def __init__(self) -> None: ...
+
+ class ExecutionSync:
+ def __init__(self) -> None: ...
+ """
+ Update the progress bar displayed in the ComfyUI interface.
+
+ This function allows custom nodes and API calls to report their progress
+ back to the user interface, providing visual feedback during long operations.
+
+ Migration from previous API: comfy.utils.PROGRESS_BAR_HOOK
+ """
+ def set_progress(self, value: float, max_value: float, node_id: Union[str, None] = None, preview_image: Union[Image, Tensor, None] = None, ignore_size_limit: bool = False) -> None: ...
+
+ execution: ExecutionSync
diff --git a/comfy_api/util.py b/comfy_api/util.py
new file mode 100644
index 000000000..1aa9606d2
--- /dev/null
+++ b/comfy_api/util.py
@@ -0,0 +1,8 @@
+# This file only exists for backwards compatibility.
+from comfy_api.latest._util import VideoCodec, VideoContainer, VideoComponents
+
+__all__ = [
+ "VideoCodec",
+ "VideoContainer",
+ "VideoComponents",
+]
diff --git a/comfy_api/util/__init__.py b/comfy_api/util/__init__.py
index 9019c46db..4c8a89d1e 100644
--- a/comfy_api/util/__init__.py
+++ b/comfy_api/util/__init__.py
@@ -1,7 +1,7 @@
-from .video_types import VideoContainer, VideoCodec, VideoComponents
+# This file only exists for backwards compatibility.
+from comfy_api.latest._util import VideoContainer, VideoCodec, VideoComponents
__all__ = [
- # Utility Types
"VideoContainer",
"VideoCodec",
"VideoComponents",
diff --git a/comfy_api/util/video_types.py b/comfy_api/util/video_types.py
index d09663db9..68c780d64 100644
--- a/comfy_api/util/video_types.py
+++ b/comfy_api/util/video_types.py
@@ -1,51 +1,12 @@
-from __future__ import annotations
-from dataclasses import dataclass
-from enum import Enum
-from fractions import Fraction
-from typing import Optional
-from comfy_api.input import ImageInput, AudioInput
-
-class VideoCodec(str, Enum):
- AUTO = "auto"
- H264 = "h264"
-
- @classmethod
- def as_input(cls) -> list[str]:
- """
- Returns a list of codec names that can be used as node input.
- """
- return [member.value for member in cls]
-
-class VideoContainer(str, Enum):
- AUTO = "auto"
- MP4 = "mp4"
-
- @classmethod
- def as_input(cls) -> list[str]:
- """
- Returns a list of container names that can be used as node input.
- """
- return [member.value for member in cls]
-
- @classmethod
- def get_extension(cls, value) -> str:
- """
- Returns the file extension for the container.
- """
- if isinstance(value, str):
- value = cls(value)
- if value == VideoContainer.MP4 or value == VideoContainer.AUTO:
- return "mp4"
- return ""
-
-@dataclass
-class VideoComponents:
- """
- Dataclass representing the components of a video.
- """
-
- images: ImageInput
- frame_rate: Fraction
- audio: Optional[AudioInput] = None
- metadata: Optional[dict] = None
+# This file only exists for backwards compatibility.
+from comfy_api.latest._util.video_types import (
+ VideoContainer,
+ VideoCodec,
+ VideoComponents,
+)
+__all__ = [
+ "VideoContainer",
+ "VideoCodec",
+ "VideoComponents",
+]
diff --git a/comfy_api/v0_0_1/__init__.py b/comfy_api/v0_0_1/__init__.py
new file mode 100644
index 000000000..93608771d
--- /dev/null
+++ b/comfy_api/v0_0_1/__init__.py
@@ -0,0 +1,42 @@
+from comfy_api.v0_0_2 import (
+ ComfyAPIAdapter_v0_0_2,
+ Input as Input_v0_0_2,
+ InputImpl as InputImpl_v0_0_2,
+ Types as Types_v0_0_2,
+)
+from typing import Type, TYPE_CHECKING
+from comfy_api.internal.async_to_sync import create_sync_class
+
+
+# This version only exists to serve as a template for future version adapters.
+# There is no reason anyone should ever use it.
+class ComfyAPIAdapter_v0_0_1(ComfyAPIAdapter_v0_0_2):
+ VERSION = "0.0.1"
+ STABLE = True
+
+class Input(Input_v0_0_2):
+ pass
+
+class InputImpl(InputImpl_v0_0_2):
+ pass
+
+class Types(Types_v0_0_2):
+ pass
+
+ComfyAPI = ComfyAPIAdapter_v0_0_1
+
+# Create a synchronous version of the API
+if TYPE_CHECKING:
+ from comfy_api.v0_0_1.generated.ComfyAPISyncStub import ComfyAPISyncStub # type: ignore
+
+ ComfyAPISync: Type[ComfyAPISyncStub]
+
+ComfyAPISync = create_sync_class(ComfyAPIAdapter_v0_0_1)
+
+__all__ = [
+ "ComfyAPI",
+ "ComfyAPISync",
+ "Input",
+ "InputImpl",
+ "Types",
+]
diff --git a/comfy_api/v0_0_1/generated/ComfyAPISyncStub.pyi b/comfy_api/v0_0_1/generated/ComfyAPISyncStub.pyi
new file mode 100644
index 000000000..270030324
--- /dev/null
+++ b/comfy_api/v0_0_1/generated/ComfyAPISyncStub.pyi
@@ -0,0 +1,20 @@
+from typing import Any, Dict, List, Optional, Tuple, Union, Set, Sequence, cast, NamedTuple
+from comfy_api.v0_0_1 import ComfyAPIAdapter_v0_0_1
+from PIL.Image import Image
+from torch import Tensor
+class ComfyAPISyncStub:
+ def __init__(self) -> None: ...
+
+ class ExecutionSync:
+ def __init__(self) -> None: ...
+ """
+ Update the progress bar displayed in the ComfyUI interface.
+
+ This function allows custom nodes and API calls to report their progress
+ back to the user interface, providing visual feedback during long operations.
+
+ Migration from previous API: comfy.utils.PROGRESS_BAR_HOOK
+ """
+ def set_progress(self, value: float, max_value: float, node_id: Union[str, None] = None, preview_image: Union[Image, Tensor, None] = None, ignore_size_limit: bool = False) -> None: ...
+
+ execution: ExecutionSync
diff --git a/comfy_api/v0_0_2/__init__.py b/comfy_api/v0_0_2/__init__.py
new file mode 100644
index 000000000..de0f95001
--- /dev/null
+++ b/comfy_api/v0_0_2/__init__.py
@@ -0,0 +1,45 @@
+from comfy_api.latest import (
+ ComfyAPI_latest,
+ Input as Input_latest,
+ InputImpl as InputImpl_latest,
+ Types as Types_latest,
+)
+from typing import Type, TYPE_CHECKING
+from comfy_api.internal.async_to_sync import create_sync_class
+from comfy_api.latest import io, ui, ComfyExtension #noqa: F401
+
+
+class ComfyAPIAdapter_v0_0_2(ComfyAPI_latest):
+ VERSION = "0.0.2"
+ STABLE = False
+
+
+class Input(Input_latest):
+ pass
+
+
+class InputImpl(InputImpl_latest):
+ pass
+
+
+class Types(Types_latest):
+ pass
+
+
+ComfyAPI = ComfyAPIAdapter_v0_0_2
+
+# Create a synchronous version of the API
+if TYPE_CHECKING:
+ from comfy_api.v0_0_2.generated.ComfyAPISyncStub import ComfyAPISyncStub # type: ignore
+
+ ComfyAPISync: Type[ComfyAPISyncStub]
+ComfyAPISync = create_sync_class(ComfyAPIAdapter_v0_0_2)
+
+__all__ = [
+ "ComfyAPI",
+ "ComfyAPISync",
+ "Input",
+ "InputImpl",
+ "Types",
+ "ComfyExtension",
+]
diff --git a/comfy_api/v0_0_2/generated/ComfyAPISyncStub.pyi b/comfy_api/v0_0_2/generated/ComfyAPISyncStub.pyi
new file mode 100644
index 000000000..7fcec685e
--- /dev/null
+++ b/comfy_api/v0_0_2/generated/ComfyAPISyncStub.pyi
@@ -0,0 +1,20 @@
+from typing import Any, Dict, List, Optional, Tuple, Union, Set, Sequence, cast, NamedTuple
+from comfy_api.v0_0_2 import ComfyAPIAdapter_v0_0_2
+from PIL.Image import Image
+from torch import Tensor
+class ComfyAPISyncStub:
+ def __init__(self) -> None: ...
+
+ class ExecutionSync:
+ def __init__(self) -> None: ...
+ """
+ Update the progress bar displayed in the ComfyUI interface.
+
+ This function allows custom nodes and API calls to report their progress
+ back to the user interface, providing visual feedback during long operations.
+
+ Migration from previous API: comfy.utils.PROGRESS_BAR_HOOK
+ """
+ def set_progress(self, value: float, max_value: float, node_id: Union[str, None] = None, preview_image: Union[Image, Tensor, None] = None, ignore_size_limit: bool = False) -> None: ...
+
+ execution: ExecutionSync
diff --git a/comfy_api/version_list.py b/comfy_api/version_list.py
new file mode 100644
index 000000000..7cb1871d5
--- /dev/null
+++ b/comfy_api/version_list.py
@@ -0,0 +1,12 @@
+from comfy_api.latest import ComfyAPI_latest
+from comfy_api.v0_0_2 import ComfyAPIAdapter_v0_0_2
+from comfy_api.v0_0_1 import ComfyAPIAdapter_v0_0_1
+from comfy_api.internal import ComfyAPIBase
+from typing import List, Type
+
+supported_versions: List[Type[ComfyAPIBase]] = [
+ ComfyAPI_latest,
+ ComfyAPIAdapter_v0_0_2,
+ ComfyAPIAdapter_v0_0_1,
+]
+
diff --git a/comfy_api_nodes/apinode_utils.py b/comfy_api_nodes/apinode_utils.py
index 788e2803f..f953f86df 100644
--- a/comfy_api_nodes/apinode_utils.py
+++ b/comfy_api_nodes/apinode_utils.py
@@ -1,4 +1,5 @@
from __future__ import annotations
+import aiohttp
import io
import logging
import mimetypes
@@ -21,7 +22,6 @@ from server import PromptServer
import numpy as np
from PIL import Image
-import requests
import torch
import math
import base64
@@ -30,7 +30,7 @@ from io import BytesIO
import av
-def download_url_to_video_output(video_url: str, timeout: int = None) -> VideoFromFile:
+async def download_url_to_video_output(video_url: str, timeout: int = None) -> VideoFromFile:
"""Downloads a video from a URL and returns a `VIDEO` output.
Args:
@@ -39,7 +39,7 @@ def download_url_to_video_output(video_url: str, timeout: int = None) -> VideoFr
Returns:
A Comfy node `VIDEO` output.
"""
- video_io = download_url_to_bytesio(video_url, timeout)
+ video_io = await download_url_to_bytesio(video_url, timeout)
if video_io is None:
error_msg = f"Failed to download video from {video_url}"
logging.error(error_msg)
@@ -62,7 +62,7 @@ def downscale_image_tensor(image, total_pixels=1536 * 1024) -> torch.Tensor:
return s
-def validate_and_cast_response(
+async def validate_and_cast_response(
response, timeout: int = None, node_id: Union[str, None] = None
) -> torch.Tensor:
"""Validates and casts a response to a torch.Tensor.
@@ -86,35 +86,24 @@ def validate_and_cast_response(
image_tensors: list[torch.Tensor] = []
# Process each image in the data array
- for image_data in data:
- image_url = image_data.url
- b64_data = image_data.b64_json
+ async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=timeout)) as session:
+ for img_data in data:
+ img_bytes: bytes
+ if img_data.b64_json:
+ img_bytes = base64.b64decode(img_data.b64_json)
+ elif img_data.url:
+ if node_id:
+ PromptServer.instance.send_progress_text(f"Result URL: {img_data.url}", node_id)
+ async with session.get(img_data.url) as resp:
+ if resp.status != 200:
+ raise ValueError("Failed to download generated image")
+ img_bytes = await resp.read()
+ else:
+ raise ValueError("Invalid image payload – neither URL nor base64 data present.")
- if not image_url and not b64_data:
- raise ValueError("No image was generated in the response")
-
- if b64_data:
- img_data = base64.b64decode(b64_data)
- img = Image.open(io.BytesIO(img_data))
-
- elif image_url:
- if node_id:
- PromptServer.instance.send_progress_text(
- f"Result URL: {image_url}", node_id
- )
- img_response = requests.get(image_url, timeout=timeout)
- if img_response.status_code != 200:
- raise ValueError("Failed to download the image")
- img = Image.open(io.BytesIO(img_response.content))
-
- img = img.convert("RGBA")
-
- # Convert to numpy array, normalize to float32 between 0 and 1
- img_array = np.array(img).astype(np.float32) / 255.0
- img_tensor = torch.from_numpy(img_array)
-
- # Add to list of tensors
- image_tensors.append(img_tensor)
+ pil_img = Image.open(BytesIO(img_bytes)).convert("RGBA")
+ arr = np.asarray(pil_img).astype(np.float32) / 255.0
+ image_tensors.append(torch.from_numpy(arr))
return torch.stack(image_tensors, dim=0)
@@ -175,7 +164,7 @@ def mimetype_to_extension(mime_type: str) -> str:
return mime_type.split("/")[-1].lower()
-def download_url_to_bytesio(url: str, timeout: int = None) -> BytesIO:
+async def download_url_to_bytesio(url: str, timeout: int = None) -> BytesIO:
"""Downloads content from a URL using requests and returns it as BytesIO.
Args:
@@ -185,9 +174,11 @@ def download_url_to_bytesio(url: str, timeout: int = None) -> BytesIO:
Returns:
BytesIO object containing the downloaded content.
"""
- response = requests.get(url, stream=True, timeout=timeout)
- response.raise_for_status() # Raises HTTPError for bad responses (4XX or 5XX)
- return BytesIO(response.content)
+ timeout_cfg = aiohttp.ClientTimeout(total=timeout) if timeout else None
+ async with aiohttp.ClientSession(timeout=timeout_cfg) as session:
+ async with session.get(url) as resp:
+ resp.raise_for_status() # Raises HTTPError for bad responses (4XX or 5XX)
+ return BytesIO(await resp.read())
def bytesio_to_image_tensor(image_bytesio: BytesIO, mode: str = "RGBA") -> torch.Tensor:
@@ -210,15 +201,15 @@ def bytesio_to_image_tensor(image_bytesio: BytesIO, mode: str = "RGBA") -> torch
return torch.from_numpy(image_array).unsqueeze(0)
-def download_url_to_image_tensor(url: str, timeout: int = None) -> torch.Tensor:
+async def download_url_to_image_tensor(url: str, timeout: int = None) -> torch.Tensor:
"""Downloads an image from a URL and returns a [B, H, W, C] tensor."""
- image_bytesio = download_url_to_bytesio(url, timeout)
+ image_bytesio = await download_url_to_bytesio(url, timeout)
return bytesio_to_image_tensor(image_bytesio)
-def process_image_response(response: requests.Response) -> torch.Tensor:
+def process_image_response(response_content: bytes | str) -> torch.Tensor:
"""Uses content from a Response object and converts it to a torch.Tensor"""
- return bytesio_to_image_tensor(BytesIO(response.content))
+ return bytesio_to_image_tensor(BytesIO(response_content))
def _tensor_to_pil(image: torch.Tensor, total_pixels: int = 2048 * 2048) -> Image.Image:
@@ -336,10 +327,10 @@ def text_filepath_to_data_uri(filepath: str) -> str:
return f"data:{mime_type};base64,{base64_string}"
-def upload_file_to_comfyapi(
+async def upload_file_to_comfyapi(
file_bytes_io: BytesIO,
filename: str,
- upload_mime_type: str,
+ upload_mime_type: Optional[str],
auth_kwargs: Optional[dict[str, str]] = None,
) -> str:
"""
@@ -354,7 +345,10 @@ def upload_file_to_comfyapi(
Returns:
The download URL for the uploaded file.
"""
- request_object = UploadRequest(file_name=filename, content_type=upload_mime_type)
+ if upload_mime_type is None:
+ request_object = UploadRequest(file_name=filename)
+ else:
+ request_object = UploadRequest(file_name=filename, content_type=upload_mime_type)
operation = SynchronousOperation(
endpoint=ApiEndpoint(
path="/customers/storage",
@@ -366,12 +360,8 @@ def upload_file_to_comfyapi(
auth_kwargs=auth_kwargs,
)
- response: UploadResponse = operation.execute()
- upload_response = ApiClient.upload_file(
- response.upload_url, file_bytes_io, content_type=upload_mime_type
- )
- upload_response.raise_for_status()
-
+ response: UploadResponse = await operation.execute()
+ await ApiClient.upload_file(response.upload_url, file_bytes_io, content_type=upload_mime_type)
return response.download_url
@@ -399,7 +389,7 @@ def video_to_base64_string(
return base64.b64encode(video_bytes_io.getvalue()).decode("utf-8")
-def upload_video_to_comfyapi(
+async def upload_video_to_comfyapi(
video: VideoInput,
auth_kwargs: Optional[dict[str, str]] = None,
container: VideoContainer = VideoContainer.MP4,
@@ -439,9 +429,7 @@ def upload_video_to_comfyapi(
video.save_to(video_bytes_io, format=container, codec=codec)
video_bytes_io.seek(0)
- return upload_file_to_comfyapi(
- video_bytes_io, filename, upload_mime_type, auth_kwargs
- )
+ return await upload_file_to_comfyapi(video_bytes_io, filename, upload_mime_type, auth_kwargs)
def audio_tensor_to_contiguous_ndarray(waveform: torch.Tensor) -> np.ndarray:
@@ -501,7 +489,7 @@ def audio_ndarray_to_bytesio(
return audio_bytes_io
-def upload_audio_to_comfyapi(
+async def upload_audio_to_comfyapi(
audio: AudioInput,
auth_kwargs: Optional[dict[str, str]] = None,
container_format: str = "mp4",
@@ -527,7 +515,7 @@ def upload_audio_to_comfyapi(
audio_data_np, sample_rate, container_format, codec_name
)
- return upload_file_to_comfyapi(audio_bytes_io, filename, mime_type, auth_kwargs)
+ return await upload_file_to_comfyapi(audio_bytes_io, filename, mime_type, auth_kwargs)
def audio_to_base64_string(
@@ -544,7 +532,7 @@ def audio_to_base64_string(
return base64.b64encode(audio_bytes).decode("utf-8")
-def upload_images_to_comfyapi(
+async def upload_images_to_comfyapi(
image: torch.Tensor,
max_images=8,
auth_kwargs: Optional[dict[str, str]] = None,
@@ -561,55 +549,15 @@ def upload_images_to_comfyapi(
mime_type: Optional MIME type for the image.
"""
# if batch, try to upload each file if max_images is greater than 0
- idx_image = 0
download_urls: list[str] = []
is_batch = len(image.shape) > 3
- batch_length = 1
- if is_batch:
- batch_length = image.shape[0]
- while True:
- curr_image = image
- if len(image.shape) > 3:
- curr_image = image[idx_image]
- # get BytesIO version of image
- img_binary = tensor_to_bytesio(curr_image, mime_type=mime_type)
- # first, request upload/download urls from comfy API
- if not mime_type:
- request_object = UploadRequest(file_name=img_binary.name)
- else:
- request_object = UploadRequest(
- file_name=img_binary.name, content_type=mime_type
- )
- operation = SynchronousOperation(
- endpoint=ApiEndpoint(
- path="/customers/storage",
- method=HttpMethod.POST,
- request_model=UploadRequest,
- response_model=UploadResponse,
- ),
- request=request_object,
- auth_kwargs=auth_kwargs,
- )
- response = operation.execute()
+ batch_len = image.shape[0] if is_batch else 1
- upload_response = ApiClient.upload_file(
- response.upload_url, img_binary, content_type=mime_type
- )
- # verify success
- try:
- upload_response.raise_for_status()
- except requests.exceptions.HTTPError as e:
- raise ValueError(f"Could not upload one or more images: {e}") from e
- # add download_url to list
- download_urls.append(response.download_url)
-
- idx_image += 1
- # stop uploading additional files if done
- if is_batch and max_images > 0:
- if idx_image >= max_images:
- break
- if idx_image >= batch_length:
- break
+ for idx in range(min(batch_len, max_images)):
+ tensor = image[idx] if is_batch else image
+ img_io = tensor_to_bytesio(tensor, mime_type=mime_type)
+ url = await upload_file_to_comfyapi(img_io, img_io.name, mime_type, auth_kwargs)
+ download_urls.append(url)
return download_urls
diff --git a/comfy_api_nodes/apis/__init__.py b/comfy_api_nodes/apis/__init__.py
index 086028abe..7a09df55b 100644
--- a/comfy_api_nodes/apis/__init__.py
+++ b/comfy_api_nodes/apis/__init__.py
@@ -1,6 +1,6 @@
# generated by datamodel-codegen:
# filename: filtered-openapi.yaml
-# timestamp: 2025-07-06T09:47:31+00:00
+# timestamp: 2025-07-30T08:54:00+00:00
from __future__ import annotations
@@ -37,6 +37,99 @@ class AuditLog(BaseModel):
)
+class BFLAsyncResponse(BaseModel):
+ id: str = Field(..., title='Id')
+ polling_url: str = Field(..., title='Polling Url')
+
+
+class BFLAsyncWebhookResponse(BaseModel):
+ id: str = Field(..., title='Id')
+ status: str = Field(..., title='Status')
+ webhook_url: str = Field(..., title='Webhook Url')
+
+
+class CannyHighThreshold(RootModel[int]):
+ root: int = Field(
+ ...,
+ description='High threshold for Canny edge detection',
+ ge=0,
+ le=500,
+ title='Canny High Threshold',
+ )
+
+
+class CannyLowThreshold(RootModel[int]):
+ root: int = Field(
+ ...,
+ description='Low threshold for Canny edge detection',
+ ge=0,
+ le=500,
+ title='Canny Low Threshold',
+ )
+
+
+class Guidance(RootModel[float]):
+ root: float = Field(
+ ...,
+ description='Guidance strength for the image generation process',
+ ge=1.0,
+ le=100.0,
+ title='Guidance',
+ )
+
+
+class Steps(RootModel[int]):
+ root: int = Field(
+ ...,
+ description='Number of steps for the image generation process',
+ ge=15,
+ le=50,
+ title='Steps',
+ )
+
+
+class WebhookUrl(RootModel[AnyUrl]):
+ root: AnyUrl = Field(
+ ..., description='URL to receive webhook notifications', title='Webhook Url'
+ )
+
+
+class BFLFluxKontextMaxGenerateRequest(BaseModel):
+ guidance: Optional[float] = Field(
+ 3, description='The guidance scale for generation', ge=1.0, le=20.0
+ )
+ input_image: str = Field(..., description='Base64 encoded image to be edited')
+ prompt: str = Field(
+ ..., description='The text prompt describing what to edit on the image'
+ )
+ steps: Optional[int] = Field(
+ 50, description='Number of inference steps', ge=1, le=50
+ )
+
+
+class BFLFluxKontextMaxGenerateResponse(BaseModel):
+ id: str = Field(..., description='Job ID for tracking')
+ polling_url: str = Field(..., description='URL to poll for results')
+
+
+class BFLFluxKontextProGenerateRequest(BaseModel):
+ guidance: Optional[float] = Field(
+ 3, description='The guidance scale for generation', ge=1.0, le=20.0
+ )
+ input_image: str = Field(..., description='Base64 encoded image to be edited')
+ prompt: str = Field(
+ ..., description='The text prompt describing what to edit on the image'
+ )
+ steps: Optional[int] = Field(
+ 50, description='Number of inference steps', ge=1, le=50
+ )
+
+
+class BFLFluxKontextProGenerateResponse(BaseModel):
+ id: str = Field(..., description='Job ID for tracking')
+ polling_url: str = Field(..., description='URL to poll for results')
+
+
class OutputFormat(str, Enum):
jpeg = 'jpeg'
png = 'png'
@@ -68,6 +161,67 @@ class BFLFluxPro11GenerateResponse(BaseModel):
polling_url: str = Field(..., description='URL to poll for results')
+class Bottom(RootModel[int]):
+ root: int = Field(
+ ...,
+ description='Number of pixels to expand at the bottom of the image',
+ ge=0,
+ le=2048,
+ title='Bottom',
+ )
+
+
+class Guidance2(RootModel[float]):
+ root: float = Field(
+ ...,
+ description='Guidance strength for the image generation process',
+ ge=1.5,
+ le=100.0,
+ title='Guidance',
+ )
+
+
+class Left(RootModel[int]):
+ root: int = Field(
+ ...,
+ description='Number of pixels to expand on the left side of the image',
+ ge=0,
+ le=2048,
+ title='Left',
+ )
+
+
+class Right(RootModel[int]):
+ root: int = Field(
+ ...,
+ description='Number of pixels to expand on the right side of the image',
+ ge=0,
+ le=2048,
+ title='Right',
+ )
+
+
+class Steps2(RootModel[int]):
+ root: int = Field(
+ ...,
+ description='Number of steps for the image generation process',
+ examples=[50],
+ ge=15,
+ le=50,
+ title='Steps',
+ )
+
+
+class Top(RootModel[int]):
+ root: int = Field(
+ ...,
+ description='Number of pixels to expand at the top of the image',
+ ge=0,
+ le=2048,
+ title='Top',
+ )
+
+
class BFLFluxProGenerateRequest(BaseModel):
guidance_scale: Optional[float] = Field(
None, description='The guidance scale for generation.', ge=1.0, le=20.0
@@ -96,7 +250,71 @@ class BFLFluxProGenerateResponse(BaseModel):
polling_url: str = Field(..., description='URL to poll for the generation result.')
+class BFLOutputFormat(str, Enum):
+ jpeg = 'jpeg'
+ png = 'png'
+
+
+class BFLValidationError(BaseModel):
+ loc: List[Union[str, int]] = Field(..., title='Location')
+ msg: str = Field(..., title='Message')
+ type: str = Field(..., title='Error Type')
+
+
class Status(str, Enum):
+ success = 'success'
+ not_found = 'not_found'
+ error = 'error'
+
+
+class ClaimMyNodeRequest(BaseModel):
+ GH_TOKEN: str = Field(
+ ..., description='GitHub token to verify if the user owns the repo of the node'
+ )
+
+
+class ComfyNode(BaseModel):
+ category: Optional[str] = Field(
+ None,
+ description='UI category where the node is listed, used for grouping nodes.',
+ )
+ comfy_node_name: Optional[str] = Field(
+ None, description='Unique identifier for the node'
+ )
+ deprecated: Optional[bool] = Field(
+ None,
+ description='Indicates if the node is deprecated. Deprecated nodes are hidden in the UI.',
+ )
+ description: Optional[str] = Field(
+ None, description="Brief description of the node's functionality or purpose."
+ )
+ experimental: Optional[bool] = Field(
+ None,
+ description='Indicates if the node is experimental, subject to changes or removal.',
+ )
+ function: Optional[str] = Field(
+ None, description='Name of the entry-point function to execute the node.'
+ )
+ input_types: Optional[str] = Field(None, description='Defines input parameters')
+ output_is_list: Optional[List[bool]] = Field(
+ None, description='Boolean values indicating if each output is a list.'
+ )
+ return_names: Optional[str] = Field(
+ None, description='Names of the outputs for clarity in workflows.'
+ )
+ return_types: Optional[str] = Field(
+ None, description='Specifies the types of outputs produced by the node.'
+ )
+
+
+class ComfyNodeCloudBuildInfo(BaseModel):
+ build_id: Optional[str] = None
+ location: Optional[str] = None
+ project_id: Optional[str] = None
+ project_number: Optional[str] = None
+
+
+class Status1(str, Enum):
in_progress = 'in_progress'
completed = 'completed'
incomplete = 'incomplete'
@@ -113,7 +331,7 @@ class ComputerToolCall(BaseModel):
description='An identifier used when responding to the tool call with output.\n',
)
id: str = Field(..., description='The unique ID of the computer call.')
- status: Status = Field(
+ status: Status1 = Field(
...,
description='The status of the item. One of `in_progress`, `completed`, or\n`incomplete`. Populated when items are returned via API.\n',
)
@@ -156,6 +374,7 @@ class Customer(BaseModel):
None, description='The date and time the user was created'
)
email: Optional[str] = Field(None, description='The email address for this user')
+ has_fund: Optional[bool] = Field(None, description='Whether the user has funds')
id: str = Field(..., description='The firebase UID of the user')
is_admin: Optional[bool] = Field(None, description='Whether the user is an admin')
metronome_id: Optional[str] = Field(None, description='The Metronome customer ID')
@@ -194,6 +413,16 @@ class Type2(str, Enum):
message = 'message'
+class Error(BaseModel):
+ details: Optional[List[str]] = Field(
+ None,
+ description='Optional detailed information about the error or hints for resolving it.',
+ )
+ message: Optional[str] = Field(
+ None, description='A clear and concise description of the error.'
+ )
+
+
class ErrorResponse(BaseModel):
error: str
message: str
@@ -221,7 +450,7 @@ class Result(BaseModel):
)
-class Status1(str, Enum):
+class Status2(str, Enum):
in_progress = 'in_progress'
searching = 'searching'
completed = 'completed'
@@ -241,7 +470,7 @@ class FileSearchToolCall(BaseModel):
results: Optional[List[Result]] = Field(
None, description='The results of the file search tool call.\n'
)
- status: Status1 = Field(
+ status: Status2 = Field(
...,
description='The status of the file search tool call. One of `in_progress`, \n`searching`, `incomplete` or `failed`,\n',
)
@@ -266,7 +495,7 @@ class FunctionTool(BaseModel):
type: Literal['FunctionTool'] = Field(..., description='The type of tool')
-class Status2(str, Enum):
+class Status3(str, Enum):
in_progress = 'in_progress'
completed = 'completed'
incomplete = 'incomplete'
@@ -288,7 +517,7 @@ class FunctionToolCall(BaseModel):
None, description='The unique ID of the function tool call.\n'
)
name: str = Field(..., description='The name of the function to run.\n')
- status: Optional[Status2] = Field(
+ status: Optional[Status3] = Field(
None,
description='The status of the item. One of `in_progress`, `completed`, or\n`incomplete`. Populated when items are returned via API.\n',
)
@@ -442,6 +671,95 @@ class GeminiVideoMetadata(BaseModel):
startOffset: Optional[GeminiOffset] = None
+class GitCommitSummary(BaseModel):
+ author: Optional[str] = Field(None, description='The author of the commit')
+ branch_name: Optional[str] = Field(
+ None, description='The branch where the commit was made'
+ )
+ commit_hash: Optional[str] = Field(None, description='The hash of the commit')
+ commit_name: Optional[str] = Field(None, description='The name of the commit')
+ status_summary: Optional[Dict[str, str]] = Field(
+ None, description='A map of operating system to status pairs'
+ )
+ timestamp: Optional[datetime] = Field(
+ None, description='The timestamp when the commit was made'
+ )
+
+
+class GithubEnterprise(BaseModel):
+ avatar_url: str = Field(..., description='URL to the enterprise avatar')
+ created_at: datetime = Field(..., description='When the enterprise was created')
+ description: Optional[str] = Field(None, description='The enterprise description')
+ html_url: str = Field(..., description='The HTML URL of the enterprise')
+ id: int = Field(..., description='The enterprise ID')
+ name: str = Field(..., description='The enterprise name')
+ node_id: str = Field(..., description='The enterprise node ID')
+ slug: str = Field(..., description='The enterprise slug')
+ updated_at: datetime = Field(
+ ..., description='When the enterprise was last updated'
+ )
+ website_url: Optional[str] = Field(None, description='The enterprise website URL')
+
+
+class RepositorySelection(str, Enum):
+ selected = 'selected'
+ all = 'all'
+
+
+class GithubOrganization(BaseModel):
+ avatar_url: str = Field(..., description="URL to the organization's avatar")
+ description: Optional[str] = Field(None, description='The organization description')
+ events_url: str = Field(..., description="The API URL of the organization's events")
+ hooks_url: str = Field(..., description="The API URL of the organization's hooks")
+ id: int = Field(..., description='The organization ID')
+ issues_url: str = Field(..., description="The API URL of the organization's issues")
+ login: str = Field(..., description="The organization's login name")
+ members_url: str = Field(
+ ..., description="The API URL of the organization's members"
+ )
+ node_id: str = Field(..., description='The organization node ID')
+ public_members_url: str = Field(
+ ..., description="The API URL of the organization's public members"
+ )
+ repos_url: str = Field(
+ ..., description="The API URL of the organization's repositories"
+ )
+ url: str = Field(..., description='The API URL of the organization')
+
+
+class State(str, Enum):
+ uploaded = 'uploaded'
+ open = 'open'
+
+
+class Action(str, Enum):
+ published = 'published'
+ unpublished = 'unpublished'
+ created = 'created'
+ edited = 'edited'
+ deleted = 'deleted'
+ prereleased = 'prereleased'
+ released = 'released'
+
+
+class Type7(str, Enum):
+ Bot = 'Bot'
+ User = 'User'
+ Organization = 'Organization'
+
+
+class GithubUser(BaseModel):
+ avatar_url: str = Field(..., description="URL to the user's avatar")
+ gravatar_id: Optional[str] = Field(None, description="The user's gravatar ID")
+ html_url: str = Field(..., description='The HTML URL of the user')
+ id: int = Field(..., description="The user's ID")
+ login: str = Field(..., description="The user's login name")
+ node_id: str = Field(..., description="The user's node ID")
+ site_admin: bool = Field(..., description='Whether the user is a site admin')
+ type: Type7 = Field(..., description='The type of user')
+ url: str = Field(..., description='The API URL of the user')
+
+
class IdeogramColorPalette1(BaseModel):
name: str = Field(..., description='Name of the preset color palette')
@@ -689,7 +1007,7 @@ class Includable(str, Enum):
computer_call_output_output_image_url = 'computer_call_output.output.image_url'
-class Type7(str, Enum):
+class Type8(str, Enum):
input_file = 'input_file'
@@ -703,7 +1021,7 @@ class InputFileContent(BaseModel):
filename: Optional[str] = Field(
None, description='The name of the file to be sent to the model.'
)
- type: Type7 = Field(
+ type: Type8 = Field(
..., description='The type of the input item. Always `input_file`.'
)
@@ -714,7 +1032,7 @@ class Detail(str, Enum):
auto = 'auto'
-class Type8(str, Enum):
+class Type9(str, Enum):
input_image = 'input_image'
@@ -730,7 +1048,7 @@ class InputImageContent(BaseModel):
None,
description='The URL of the image to be sent to the model. A fully qualified URL or base64 encoded image in a data URL.',
)
- type: Type8 = Field(
+ type: Type9 = Field(
..., description='The type of the input item. Always `input_image`.'
)
@@ -741,17 +1059,17 @@ class Role3(str, Enum):
developer = 'developer'
-class Type9(str, Enum):
+class Type10(str, Enum):
message = 'message'
-class Type10(str, Enum):
+class Type11(str, Enum):
input_text = 'input_text'
class InputTextContent(BaseModel):
text: str = Field(..., description='The text input to the model.')
- type: Type10 = Field(
+ type: Type11 = Field(
..., description='The type of the input item. Always `input_text`.'
)
@@ -923,7 +1241,7 @@ class ResourcePackType(str, Enum):
constant_period = 'constant_period'
-class Status4(str, Enum):
+class Status5(str, Enum):
toBeOnline = 'toBeOnline'
online = 'online'
expired = 'expired'
@@ -949,7 +1267,7 @@ class ResourcePackSubscribeInfo(BaseModel):
None,
description='Resource package type (decreasing_total=decreasing total, constant_period=constant periodicity)',
)
- status: Optional[Status4] = Field(None, description='Resource Package Status')
+ status: Optional[Status5] = Field(None, description='Resource Package Status')
total_quantity: Optional[float] = Field(None, description='Total quantity')
@@ -997,6 +1315,7 @@ class KlingTaskStatus(str, Enum):
class KlingTextToVideoModelName(str, Enum):
kling_v1 = 'kling-v1'
kling_v1_6 = 'kling-v1-6'
+ kling_v2_1_master = 'kling-v2-1-master'
class KlingVideoGenAspectRatio(str, Enum):
@@ -1029,6 +1348,8 @@ class KlingVideoGenModelName(str, Enum):
kling_v1_5 = 'kling-v1-5'
kling_v1_6 = 'kling-v1-6'
kling_v2_master = 'kling-v2-master'
+ kling_v2_1 = 'kling-v2-1'
+ kling_v2_1_master = 'kling-v2-1-master'
class KlingVideoResult(BaseModel):
@@ -1113,7 +1434,7 @@ class LumaError(BaseModel):
detail: Optional[str] = Field(None, description='The error message')
-class Type11(str, Enum):
+class Type12(str, Enum):
generation = 'generation'
@@ -1153,7 +1474,7 @@ class LumaImageRef(BaseModel):
)
-class Type12(str, Enum):
+class Type13(str, Enum):
image = 'image'
@@ -1223,6 +1544,36 @@ class LumaVideoModelOutputResolution(
root: Union[LumaVideoModelOutputResolution1, str]
+class MachineStats(BaseModel):
+ cpu_capacity: Optional[str] = Field(None, description='Total CPU on the machine.')
+ disk_capacity: Optional[str] = Field(
+ None, description='Total disk capacity on the machine.'
+ )
+ gpu_type: Optional[str] = Field(
+ None, description='The GPU type. eg. NVIDIA Tesla K80'
+ )
+ initial_cpu: Optional[str] = Field(
+ None, description='Initial CPU available before the job starts.'
+ )
+ initial_disk: Optional[str] = Field(
+ None, description='Initial disk available before the job starts.'
+ )
+ initial_ram: Optional[str] = Field(
+ None, description='Initial RAM available before the job starts.'
+ )
+ machine_name: Optional[str] = Field(None, description='Name of the machine.')
+ memory_capacity: Optional[str] = Field(
+ None, description='Total memory on the machine.'
+ )
+ os_version: Optional[str] = Field(
+ None, description='The operating system version. eg. Ubuntu Linux 20.04'
+ )
+ pip_freeze: Optional[str] = Field(None, description='The pip freeze output')
+ vram_time_series: Optional[Dict[str, Any]] = Field(
+ None, description='Time series of VRAM usage.'
+ )
+
+
class MinimaxBaseResponse(BaseModel):
status_code: int = Field(
...,
@@ -1251,7 +1602,7 @@ class MinimaxFileRetrieveResponse(BaseModel):
file: File
-class Status5(str, Enum):
+class Status6(str, Enum):
Queueing = 'Queueing'
Preparing = 'Preparing'
Processing = 'Processing'
@@ -1265,20 +1616,21 @@ class MinimaxTaskResultResponse(BaseModel):
None,
description='After the task status changes to Success, this field returns the file ID corresponding to the generated video.',
)
- status: Status5 = Field(
+ status: Status6 = Field(
...,
description="Task status: 'Queueing' (in queue), 'Preparing' (task is preparing), 'Processing' (generating), 'Success' (task completed successfully), or 'Fail' (task failed).",
)
task_id: str = Field(..., description='The task ID being queried.')
-class Model(str, Enum):
+class MiniMaxModel(str, Enum):
T2V_01_Director = 'T2V-01-Director'
I2V_01_Director = 'I2V-01-Director'
S2V_01 = 'S2V-01'
I2V_01 = 'I2V-01'
I2V_01_live = 'I2V-01-live'
T2V_01 = 'T2V-01'
+ Hailuo_02 = 'MiniMax-Hailuo-02'
class SubjectReferenceItem(BaseModel):
@@ -1300,7 +1652,7 @@ class MinimaxVideoGenerationRequest(BaseModel):
None,
description='URL or base64 encoding of the first frame image. Required when model is I2V-01, I2V-01-Director, or I2V-01-live.',
)
- model: Model = Field(
+ model: MiniMaxModel = Field(
...,
description='Required. ID of model. Options: T2V-01-Director, I2V-01-Director, S2V-01, I2V-01, I2V-01-live, T2V-01',
)
@@ -1317,6 +1669,14 @@ class MinimaxVideoGenerationRequest(BaseModel):
None,
description='Only available when model is S2V-01. The model will generate a video based on the subject uploaded through this parameter.',
)
+ duration: Optional[int] = Field(
+ None,
+ description="The length of the output video in seconds."
+ )
+ resolution: Optional[str] = Field(
+ None,
+ description="The dimensions of the video display. 1080p corresponds to 1920 x 1080 pixels, 768p corresponds to 1366 x 768 pixels."
+ )
class MinimaxVideoGenerationResponse(BaseModel):
@@ -1326,6 +1686,22 @@ class MinimaxVideoGenerationResponse(BaseModel):
)
+class Modality(str, Enum):
+ MODALITY_UNSPECIFIED = 'MODALITY_UNSPECIFIED'
+ TEXT = 'TEXT'
+ IMAGE = 'IMAGE'
+ VIDEO = 'VIDEO'
+ AUDIO = 'AUDIO'
+ DOCUMENT = 'DOCUMENT'
+
+
+class ModalityTokenCount(BaseModel):
+ modality: Optional[Modality] = None
+ tokenCount: Optional[int] = Field(
+ None, description='Number of tokens for the given modality.'
+ )
+
+
class Truncation(str, Enum):
disabled = 'disabled'
auto = 'auto'
@@ -1391,13 +1767,13 @@ class MoonvalleyTextToVideoInferenceParams(BaseModel):
0, description='Index of the conditioning frame'
)
cooldown_steps: Optional[int] = Field(
- None, description='Number of cooldown steps (calculated based on num_frames)'
+ 75, description='Number of cooldown steps (calculated based on num_frames)'
)
fps: Optional[int] = Field(
24, description='Frames per second of the generated video'
)
guidance_scale: Optional[float] = Field(
- 12.5, description='Guidance scale for generation control'
+ 10, description='Guidance scale for generation control'
)
height: Optional[int] = Field(
1080, description='Height of the generated video in pixels'
@@ -1421,7 +1797,7 @@ class MoonvalleyTextToVideoInferenceParams(BaseModel):
True, description='Whether to use timestep transformation'
)
warmup_steps: Optional[int] = Field(
- None, description='Number of warmup steps (calculated based on num_frames)'
+ 0, description='Number of warmup steps (calculated based on num_frames)'
)
width: Optional[int] = Field(
1920, description='Width of the generated video in pixels'
@@ -1463,10 +1839,10 @@ class MoonvalleyVideoToVideoInferenceParams(BaseModel):
0, description='Index of the conditioning frame'
)
cooldown_steps: Optional[int] = Field(
- None, description='Number of cooldown steps (calculated based on num_frames)'
+ 36, description='Number of cooldown steps (calculated based on num_frames)'
)
guidance_scale: Optional[float] = Field(
- 12.5, description='Guidance scale for generation control'
+ 15, description='Guidance scale for generation control'
)
negative_prompt: Optional[str] = Field(None, description='Negative prompt text')
seed: Optional[int] = Field(
@@ -1486,7 +1862,7 @@ class MoonvalleyVideoToVideoInferenceParams(BaseModel):
True, description='Whether to use timestep transformation'
)
warmup_steps: Optional[int] = Field(
- None, description='Number of warmup steps (calculated based on num_frames)'
+ 24, description='Number of warmup steps (calculated based on num_frames)'
)
@@ -1507,6 +1883,34 @@ class MoonvalleyVideoToVideoRequest(BaseModel):
)
+class NodeStatus(str, Enum):
+ NodeStatusActive = 'NodeStatusActive'
+ NodeStatusDeleted = 'NodeStatusDeleted'
+ NodeStatusBanned = 'NodeStatusBanned'
+
+
+class NodeVersionIdentifier(BaseModel):
+ node_id: str = Field(..., description='The unique identifier of the node')
+ version: str = Field(..., description='The version of the node')
+
+
+class NodeVersionStatus(str, Enum):
+ NodeVersionStatusActive = 'NodeVersionStatusActive'
+ NodeVersionStatusDeleted = 'NodeVersionStatusDeleted'
+ NodeVersionStatusBanned = 'NodeVersionStatusBanned'
+ NodeVersionStatusPending = 'NodeVersionStatusPending'
+ NodeVersionStatusFlagged = 'NodeVersionStatusFlagged'
+
+
+class NodeVersionUpdateRequest(BaseModel):
+ changelog: Optional[str] = Field(
+ None, description='The changelog describing the version changes.'
+ )
+ deprecated: Optional[bool] = Field(
+ None, description='Whether the version is deprecated.'
+ )
+
+
class Moderation(str, Enum):
low = 'low'
auto = 'auto'
@@ -1723,38 +2127,57 @@ class Object(str, Enum):
response = 'response'
-class Status6(str, Enum):
+class Status7(str, Enum):
completed = 'completed'
failed = 'failed'
in_progress = 'in_progress'
incomplete = 'incomplete'
-class Type13(str, Enum):
+class Type14(str, Enum):
output_audio = 'output_audio'
class OutputAudioContent(BaseModel):
data: str = Field(..., description='Base64-encoded audio data')
transcript: str = Field(..., description='Transcript of the audio')
- type: Type13 = Field(..., description='The type of output content')
+ type: Type14 = Field(..., description='The type of output content')
class Role4(str, Enum):
assistant = 'assistant'
-class Type14(str, Enum):
+class Type15(str, Enum):
message = 'message'
-class Type15(str, Enum):
+class Type16(str, Enum):
output_text = 'output_text'
class OutputTextContent(BaseModel):
text: str = Field(..., description='The text content')
- type: Type15 = Field(..., description='The type of output content')
+ type: Type16 = Field(..., description='The type of output content')
+
+
+class PersonalAccessToken(BaseModel):
+ createdAt: Optional[datetime] = Field(
+ None, description='[Output Only]The date and time the token was created.'
+ )
+ description: Optional[str] = Field(
+ None,
+ description="Optional. A more detailed description of the token's intended use.",
+ )
+ id: Optional[UUID] = Field(None, description='Unique identifier for the GitCommit')
+ name: Optional[str] = Field(
+ None,
+ description='Required. The name of the token. Can be a simple description.',
+ )
+ token: Optional[str] = Field(
+ None,
+ description='[Output Only]. The personal access token. Only returned during creation.',
+ )
class AspectRatio1(RootModel[float]):
@@ -1961,7 +2384,7 @@ class PixverseVideoResponse(BaseModel):
Resp: Optional[Resp1] = None
-class Status7(int, Enum):
+class Status8(int, Enum):
integer_1 = 1
integer_5 = 5
integer_6 = 6
@@ -1980,7 +2403,7 @@ class Resp2(BaseModel):
resolution_ratio: Optional[int] = None
seed: Optional[int] = None
size: Optional[int] = None
- status: Optional[Status7] = Field(
+ status: Optional[Status8] = Field(
None,
description='Video generation status codes:\n* 1 - Generation successful\n* 5 - Generating\n* 6 - Deleted\n* 7 - Contents moderation failed\n* 8 - Generation failed\n',
)
@@ -1994,6 +2417,17 @@ class PixverseVideoResultResponse(BaseModel):
Resp: Optional[Resp2] = None
+class PublisherStatus(str, Enum):
+ PublisherStatusActive = 'PublisherStatusActive'
+ PublisherStatusBanned = 'PublisherStatusBanned'
+
+
+class PublisherUser(BaseModel):
+ email: Optional[str] = Field(None, description='The email address for this user.')
+ id: Optional[str] = Field(None, description='The unique id for this user.')
+ name: Optional[str] = Field(None, description='The name for this user.')
+
+
class RgbItem(RootModel[int]):
root: int = Field(..., ge=0, le=255)
@@ -2020,13 +2454,13 @@ class ReasoningEffort(str, Enum):
high = 'high'
-class Status8(str, Enum):
+class Status9(str, Enum):
in_progress = 'in_progress'
completed = 'completed'
incomplete = 'incomplete'
-class Type16(str, Enum):
+class Type17(str, Enum):
summary_text = 'summary_text'
@@ -2035,12 +2469,12 @@ class SummaryItem(BaseModel):
...,
description='A short summary of the reasoning used by the model when generating\nthe response.\n',
)
- type: Type16 = Field(
+ type: Type17 = Field(
..., description='The type of the object. Always `summary_text`.\n'
)
-class Type17(str, Enum):
+class Type18(str, Enum):
reasoning = 'reasoning'
@@ -2048,16 +2482,31 @@ class ReasoningItem(BaseModel):
id: str = Field(
..., description='The unique identifier of the reasoning content.\n'
)
- status: Optional[Status8] = Field(
+ status: Optional[Status9] = Field(
None,
description='The status of the item. One of `in_progress`, `completed`, or\n`incomplete`. Populated when items are returned via API.\n',
)
summary: List[SummaryItem] = Field(..., description='Reasoning text contents.\n')
- type: Type17 = Field(
+ type: Type18 = Field(
..., description='The type of the object. Always `reasoning`.\n'
)
+class RecraftImageColor(BaseModel):
+ rgb: Optional[List[int]] = None
+ std: Optional[List[float]] = None
+ weight: Optional[float] = None
+
+
+class RecraftImageFeatures(BaseModel):
+ nsfw_score: Optional[float] = None
+
+
+class RecraftImageFormat(str, Enum):
+ webp = 'webp'
+ png = 'png'
+
+
class Controls(BaseModel):
artistic_level: Optional[int] = Field(
None,
@@ -2111,12 +2560,143 @@ class RecraftImageGenerationResponse(BaseModel):
data: List[Datum3] = Field(..., description='Array of generated image information')
+class RecraftImageStyle(str, Enum):
+ digital_illustration = 'digital_illustration'
+ icon = 'icon'
+ realistic_image = 'realistic_image'
+ vector_illustration = 'vector_illustration'
+
+
+class RecraftImageSubStyle(str, Enum):
+ field_2d_art_poster = '2d_art_poster'
+ field_3d = '3d'
+ field_80s = '80s'
+ glow = 'glow'
+ grain = 'grain'
+ hand_drawn = 'hand_drawn'
+ infantile_sketch = 'infantile_sketch'
+ kawaii = 'kawaii'
+ pixel_art = 'pixel_art'
+ psychedelic = 'psychedelic'
+ seamless = 'seamless'
+ voxel = 'voxel'
+ watercolor = 'watercolor'
+ broken_line = 'broken_line'
+ colored_outline = 'colored_outline'
+ colored_shapes = 'colored_shapes'
+ colored_shapes_gradient = 'colored_shapes_gradient'
+ doodle_fill = 'doodle_fill'
+ doodle_offset_fill = 'doodle_offset_fill'
+ offset_fill = 'offset_fill'
+ outline = 'outline'
+ outline_gradient = 'outline_gradient'
+ uneven_fill = 'uneven_fill'
+ field_70s = '70s'
+ cartoon = 'cartoon'
+ doodle_line_art = 'doodle_line_art'
+ engraving = 'engraving'
+ flat_2 = 'flat_2'
+ kawaii_1 = 'kawaii'
+ line_art = 'line_art'
+ linocut = 'linocut'
+ seamless_1 = 'seamless'
+ b_and_w = 'b_and_w'
+ enterprise = 'enterprise'
+ hard_flash = 'hard_flash'
+ hdr = 'hdr'
+ motion_blur = 'motion_blur'
+ natural_light = 'natural_light'
+ studio_portrait = 'studio_portrait'
+ line_circuit = 'line_circuit'
+ field_2d_art_poster_2 = '2d_art_poster_2'
+ engraving_color = 'engraving_color'
+ flat_air_art = 'flat_air_art'
+ hand_drawn_outline = 'hand_drawn_outline'
+ handmade_3d = 'handmade_3d'
+ stickers_drawings = 'stickers_drawings'
+ plastic = 'plastic'
+ pictogram = 'pictogram'
+
+
+class RecraftResponseFormat(str, Enum):
+ url = 'url'
+ b64_json = 'b64_json'
+
+
+class RecraftTextLayoutItem(BaseModel):
+ bbox: List[List[float]]
+ text: str
+
+
+class RecraftTransformModel(str, Enum):
+ refm1 = 'refm1'
+ recraft20b = 'recraft20b'
+ recraftv2 = 'recraftv2'
+ recraftv3 = 'recraftv3'
+ flux1_1pro = 'flux1_1pro'
+ flux1dev = 'flux1dev'
+ imagen3 = 'imagen3'
+ hidream_i1_dev = 'hidream_i1_dev'
+
+
+class RecraftUserControls(BaseModel):
+ artistic_level: Optional[int] = None
+ background_color: Optional[RecraftImageColor] = None
+ colors: Optional[List[RecraftImageColor]] = None
+ no_text: Optional[bool] = None
+
+
+class Attention(str, Enum):
+ low = 'low'
+ medium = 'medium'
+ high = 'high'
+
+
+class Project(str, Enum):
+ comfyui = 'comfyui'
+ comfyui_frontend = 'comfyui_frontend'
+ desktop = 'desktop'
+
+
+class ReleaseNote(BaseModel):
+ attention: Attention = Field(
+ ..., description='The attention level for this release'
+ )
+ content: str = Field(
+ ..., description='The content of the release note in markdown format'
+ )
+ id: int = Field(..., description='Unique identifier for the release note')
+ project: Project = Field(
+ ..., description='The project this release note belongs to'
+ )
+ published_at: datetime = Field(
+ ..., description='When the release note was published'
+ )
+ version: str = Field(..., description='The version of the release')
+
+
class RenderingSpeed(str, Enum):
BALANCED = 'BALANCED'
TURBO = 'TURBO'
QUALITY = 'QUALITY'
+class Type19(str, Enum):
+ response_completed = 'response.completed'
+
+
+class Type20(str, Enum):
+ response_content_part_added = 'response.content_part.added'
+
+
+class Type21(str, Enum):
+ response_content_part_done = 'response.content_part.done'
+
+
+class Type22(str, Enum):
+ response_created = 'response.created'
+
+
class ResponseErrorCode(str, Enum):
server_error = 'server_error'
rate_limit_exceeded = 'rate_limit_exceeded'
@@ -2138,12 +2718,27 @@ class ResponseErrorCode(str, Enum):
image_file_not_found = 'image_file_not_found'
-class Type18(str, Enum):
+class Type23(str, Enum):
+ error = 'error'
+
+
+class ResponseErrorEvent(BaseModel):
+ code: str = Field(..., description='The error code.\n')
+ message: str = Field(..., description='The error message.\n')
+ param: str = Field(..., description='The error parameter.\n')
+ type: Type23 = Field(..., description='The type of the event. Always `error`.\n')
+
+
+class Type24(str, Enum):
+ response_failed = 'response.failed'
+
+
+class Type25(str, Enum):
json_object = 'json_object'
class ResponseFormatJsonObject(BaseModel):
- type: Type18 = Field(
+ type: Type25 = Field(
...,
description='The type of response format being defined. Always `json_object`.',
)
@@ -2156,16 +2751,32 @@ class ResponseFormatJsonSchemaSchema(BaseModel):
)
-class Type19(str, Enum):
+class Type26(str, Enum):
text = 'text'
class ResponseFormatText(BaseModel):
- type: Type19 = Field(
+ type: Type26 = Field(
..., description='The type of response format being defined. Always `text`.'
)
+class Type27(str, Enum):
+ response_in_progress = 'response.in_progress'
+
+
+class Type28(str, Enum):
+ response_incomplete = 'response.incomplete'
+
+
+class Type29(str, Enum):
+ response_output_item_added = 'response.output_item.added'
+
+
+class Type30(str, Enum):
+ response_output_item_done = 'response.output_item.done'
+
+
class Truncation1(str, Enum):
auto = 'auto'
disabled = 'disabled'
@@ -2200,10 +2811,6 @@ class Rodin3DCheckStatusRequest(BaseModel):
)
-class Rodin3DCheckStatusResponse(BaseModel):
- pass
-
-
class Rodin3DDownloadRequest(BaseModel):
task_uuid: str = Field(..., description='Task UUID')
@@ -2235,6 +2842,13 @@ class RodinResourceItem(BaseModel):
url: Optional[str] = Field(None, description='Download url')
+class RodinStatusOptions(str, Enum):
+ Done = 'Done'
+ Failed = 'Failed'
+ Generating = 'Generating'
+ Waiting = 'Waiting'
+
+
class RodinTierType(str, Enum):
Regular = 'Regular'
Sketch = 'Sketch'
@@ -2325,6 +2939,7 @@ class RunwayTextToImageAspectRatioEnum(str, Enum):
field_1808_768 = '1808:768'
field_2112_912 = '2112:912'
+
class Model4(str, Enum):
gen4_image = 'gen4_image'
@@ -2350,6 +2965,38 @@ class RunwayTextToImageResponse(BaseModel):
id: Optional[str] = Field(None, description='Task ID')
+class Name(str, Enum):
+ content_moderation = 'content_moderation'
+
+
+class StabilityContentModerationResponse(BaseModel):
+ errors: List[str] = Field(
+ ...,
+ description='One or more error messages indicating what went wrong.',
+ examples=[['some-field: is required']],
+ min_length=1,
+ )
+ id: str = Field(
+ ...,
+ description='A unique identifier associated with this error. Please include this in any [support tickets](https://kb.stability.ai/knowledge-base/kb-tickets/new) you file, as it will greatly assist us in diagnosing the root cause of the problem.',
+ examples=['a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4'],
+ min_length=1,
+ )
+ name: Name = Field(
+ ...,
+ description='Our content moderation system has flagged some part of your request and subsequently denied it. You were not charged for this request. While this may at times be frustrating, it is necessary to maintain the integrity of our platform and ensure a safe experience for all users. If you would like to provide feedback, please use the [Support Form](https://kb.stability.ai/knowledge-base/kb-tickets/new).',
+ )
+
+
+class StabilityCreativity(RootModel[float]):
+ root: float = Field(
+ ...,
+ description='Controls the likelihood of creating additional details not heavily conditioned by the init image.',
+ ge=0.2,
+ le=0.5,
+ )
+
+
class StabilityError(BaseModel):
errors: List[str] = Field(
...,
@@ -2371,7 +3018,17 @@ class StabilityError(BaseModel):
)
-class Status9(str, Enum):
+class StabilityGenerationID(RootModel[str]):
+ root: str = Field(
+ ...,
+ description='The `id` of a generation, typically used for async generations, that can be used to check the status of the generation or retrieve the result.',
+ examples=['a6dc6c6e20acda010fe14d71f180658f2896ed9b4ec25aa99a6ff06c796987c4'],
+ max_length=64,
+ min_length=64,
+ )
+
+
+class Status10(str, Enum):
in_progress = 'in-progress'
@@ -2379,10 +3036,860 @@ class StabilityGetResultResponse202(BaseModel):
id: Optional[str] = Field(
None, description='The ID of the generation result.', examples=[1234567890]
)
- status: Optional[Status9] = None
+ status: Optional[Status10] = None
-class Type20(str, Enum):
+class AspectRatio3(str, Enum):
+ field_21_9 = '21:9'
+ field_16_9 = '16:9'
+ field_3_2 = '3:2'
+ field_5_4 = '5:4'
+ field_1_1 = '1:1'
+ field_4_5 = '4:5'
+ field_2_3 = '2:3'
+ field_9_16 = '9:16'
+ field_9_21 = '9:21'
+
+
+class Mode(str, Enum):
+ text_to_image = 'text-to-image'
+ image_to_image = 'image-to-image'
+
+
+class Model5(str, Enum):
+ sd3_5_large = 'sd3.5-large'
+ sd3_5_large_turbo = 'sd3.5-large-turbo'
+ sd3_5_medium = 'sd3.5-medium'
+
+
+class OutputFormat3(str, Enum):
+ png = 'png'
+ jpeg = 'jpeg'
+
+
+class StylePreset(str, Enum):
+ enhance = 'enhance'
+ anime = 'anime'
+ photographic = 'photographic'
+ digital_art = 'digital-art'
+ comic_book = 'comic-book'
+ fantasy_art = 'fantasy-art'
+ line_art = 'line-art'
+ analog_film = 'analog-film'
+ neon_punk = 'neon-punk'
+ isometric = 'isometric'
+ low_poly = 'low-poly'
+ origami = 'origami'
+ modeling_compound = 'modeling-compound'
+ cinematic = 'cinematic'
+ field_3d_model = '3d-model'
+ pixel_art = 'pixel-art'
+ tile_texture = 'tile-texture'
+
+
+class StabilityImageGenerationSD3Request(BaseModel):
+ aspect_ratio: Optional[AspectRatio3] = Field(
+ '1:1',
+ description='Controls the aspect ratio of the generated image. Defaults to 1:1.\n\n> **Important:** This parameter is only valid for **text-to-image** requests.',
+ )
+ cfg_scale: Optional[float] = Field(
+ None,
+ description='How strictly the diffusion process adheres to the prompt text (higher values keep your image closer to your prompt). The _Large_ and _Medium_ models use a default of `4`. The _Turbo_ model uses a default of `1`.',
+ ge=1.0,
+ le=10.0,
+ )
+ image: Optional[StrictBytes] = Field(
+ None,
+ description='The image to use as the starting point for the generation.\n\nSupported formats:\n\n\n\n - jpeg\n - png\n - webp\n\nSupported dimensions:\n\n\n\n - Every side must be at least 64 pixels\n\n> **Important:** This parameter is only valid for **image-to-image** requests.',
+ )
+ mode: Optional[Mode] = Field(
+ 'text-to-image',
+ description='Controls whether this is a text-to-image or image-to-image generation, which affects which parameters are required:\n- **text-to-image** requires only the `prompt` parameter\n- **image-to-image** requires the `prompt`, `image`, and `strength` parameters',
+ title='GenerationMode',
+ )
+ model: Optional[Model5] = Field(
+ 'sd3.5-large',
+ description='The model to use for generation.\n\n- `sd3.5-large` requires 6.5 credits per generation\n- `sd3.5-large-turbo` requires 4 credits per generation\n- `sd3.5-medium` requires 3.5 credits per generation\n- As of the April 17, 2025, `sd3-large`, `sd3-large-turbo` and `sd3-medium`\n\n\n\n are re-routed to their `sd3.5-[model version]` equivalent, at the same price.',
+ )
+ negative_prompt: Optional[str] = Field(
+ None,
+ description='Keywords of what you **do not** wish to see in the output image.\nThis is an advanced feature.',
+ max_length=10000,
+ )
+ output_format: Optional[OutputFormat3] = Field(
+ 'png', description='Dictates the `content-type` of the generated image.'
+ )
+ prompt: str = Field(
+ ...,
+ description='What you wish to see in the output image. A strong, descriptive prompt that clearly defines\nelements, colors, and subjects will lead to better results.',
+ max_length=10000,
+ min_length=1,
+ )
+ seed: Optional[float] = Field(
+ 0,
+ description="A specific value that is used to guide the 'randomness' of the generation. (Omit this parameter or pass `0` to use a random seed.)",
+ ge=0.0,
+ le=4294967294.0,
+ )
+ strength: Optional[float] = Field(
+ None,
+ description='Sometimes referred to as _denoising_, this parameter controls how much influence the\n`image` parameter has on the generated image. A value of 0 would yield an image that\nis identical to the input. A value of 1 would be as if you passed in no image at all.\n\n> **Important:** This parameter is only valid for **image-to-image** requests.',
+ ge=0.0,
+ le=1.0,
+ )
+ style_preset: Optional[StylePreset] = Field(
+ None, description='Guides the image model towards a particular style.'
+ )
+
+
+class FinishReason(str, Enum):
+ SUCCESS = 'SUCCESS'
+ CONTENT_FILTERED = 'CONTENT_FILTERED'
+
+
+class StabilityImageGenrationSD3Response200(BaseModel):
+ finish_reason: FinishReason = Field(
+ ...,
+ description='The reason the generation finished.\n\n- `SUCCESS` = successful generation.\n- `CONTENT_FILTERED` = successful generation, however the output violated our content moderation\npolicy and has been blurred as a result.',
+ examples=['SUCCESS'],
+ )
+ image: str = Field(
+ ...,
+ description='The generated image, encoded to base64.',
+ examples=['AAAAIGZ0eXBpc29tAAACAGlzb21pc28yYXZjMW1...'],
+ )
+ seed: Optional[float] = Field(
+ 0,
+ description='The seed used as random noise for this generation.',
+ examples=[343940597],
+ ge=0.0,
+ le=4294967294.0,
+ )
+
+
+class StabilityImageGenrationSD3Response400(BaseModel):
+ errors: List[str] = Field(
+ ...,
+ description='One or more error messages indicating what went wrong.',
+ examples=[['some-field: is required']],
+ min_length=1,
+ )
+ id: str = Field(
+ ...,
+ description='A unique identifier associated with this error. Please include this in any [support tickets](https://kb.stability.ai/knowledge-base/kb-tickets/new)\nyou file, as it will greatly assist us in diagnosing the root cause of the problem.',
+ examples=['a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4'],
+ min_length=1,
+ )
+ name: str = Field(
+ ...,
+ description='Short-hand name for an error, useful for discriminating between errors with the same status code.',
+ examples=['bad_request'],
+ min_length=1,
+ )
+
+
+class StabilityImageGenrationSD3Response413(BaseModel):
+ errors: List[str] = Field(
+ ...,
+ description='One or more error messages indicating what went wrong.',
+ examples=[['some-field: is required']],
+ min_length=1,
+ )
+ id: str = Field(
+ ...,
+ description='A unique identifier associated with this error. Please include this in any [support tickets](https://kb.stability.ai/knowledge-base/kb-tickets/new)\nyou file, as it will greatly assist us in diagnosing the root cause of the problem.',
+ examples=['a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4'],
+ min_length=1,
+ )
+ name: str = Field(
+ ...,
+ description='Short-hand name for an error, useful for discriminating between errors with the same status code.',
+ examples=['bad_request'],
+ min_length=1,
+ )
+
+
+class StabilityImageGenrationSD3Response422(BaseModel):
+ errors: List[str] = Field(
+ ...,
+ description='One or more error messages indicating what went wrong.',
+ examples=[['some-field: is required']],
+ min_length=1,
+ )
+ id: str = Field(
+ ...,
+ description='A unique identifier associated with this error. Please include this in any [support tickets](https://kb.stability.ai/knowledge-base/kb-tickets/new)\nyou file, as it will greatly assist us in diagnosing the root cause of the problem.',
+ examples=['a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4'],
+ min_length=1,
+ )
+ name: str = Field(
+ ...,
+ description='Short-hand name for an error, useful for discriminating between errors with the same status code.',
+ examples=['bad_request'],
+ min_length=1,
+ )
+
+
+class StabilityImageGenrationSD3Response429(BaseModel):
+ errors: List[str] = Field(
+ ...,
+ description='One or more error messages indicating what went wrong.',
+ examples=[['some-field: is required']],
+ min_length=1,
+ )
+ id: str = Field(
+ ...,
+ description='A unique identifier associated with this error. Please include this in any [support tickets](https://kb.stability.ai/knowledge-base/kb-tickets/new)\nyou file, as it will greatly assist us in diagnosing the root cause of the problem.',
+ examples=['a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4'],
+ min_length=1,
+ )
+ name: str = Field(
+ ...,
+ description='Short-hand name for an error, useful for discriminating between errors with the same status code.',
+ examples=['bad_request'],
+ min_length=1,
+ )
+
+
+class StabilityImageGenrationSD3Response500(BaseModel):
+ errors: List[str] = Field(
+ ...,
+ description='One or more error messages indicating what went wrong.',
+ examples=[['some-field: is required']],
+ min_length=1,
+ )
+ id: str = Field(
+ ...,
+ description='A unique identifier associated with this error. Please include this in any [support tickets](https://kb.stability.ai/knowledge-base/kb-tickets/new)\nyou file, as it will greatly assist us in diagnosing the root cause of the problem.',
+ examples=['a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4'],
+ min_length=1,
+ )
+ name: str = Field(
+ ...,
+ description='Short-hand name for an error, useful for discriminating between errors with the same status code.',
+ examples=['bad_request'],
+ min_length=1,
+ )
+
+
+class OutputFormat4(str, Enum):
+ jpeg = 'jpeg'
+ png = 'png'
+ webp = 'webp'
+
+
+class StabilityImageGenrationUpscaleConservativeRequest(BaseModel):
+ creativity: Optional[StabilityCreativity] = Field(
+ default_factory=lambda: StabilityCreativity.model_validate(0.35)
+ )
+ image: StrictBytes = Field(
+ ...,
+ description='The image you wish to upscale.\n\nSupported Formats:\n- jpeg\n- png\n- webp\n\nValidation Rules:\n- Every side must be at least 64 pixels\n- Total pixel count must be between 4,096 and 9,437,184 pixels\n- The aspect ratio must be between 1:2.5 and 2.5:1',
+ examples=['./some/image.png'],
+ )
+ negative_prompt: Optional[str] = Field(
+ None,
+ description='A blurb of text describing what you **do not** wish to see in the output image.\nThis is an advanced feature.',
+ max_length=10000,
+ )
+ output_format: Optional[OutputFormat4] = Field(
+ 'png', description='Dictates the `content-type` of the generated image.'
+ )
+ prompt: str = Field(
+ ...,
+ description="What you wish to see in the output image. A strong, descriptive prompt that clearly defines\nelements, colors, and subjects will lead to better results.\n\nTo control the weight of a given word use the format `(word:weight)`,\nwhere `word` is the word you'd like to control the weight of and `weight`\nis a value between 0 and 1. For example: `The sky was a crisp (blue:0.3) and (green:0.8)`\nwould convey a sky that was blue and green, but more green than blue.",
+ max_length=10000,
+ min_length=1,
+ )
+ seed: Optional[float] = Field(
+ 0,
+ description="A specific value that is used to guide the 'randomness' of the generation. (Omit this parameter or pass `0` to use a random seed.)",
+ ge=0.0,
+ le=4294967294.0,
+ )
+
+
+class StabilityImageGenrationUpscaleConservativeResponse200(BaseModel):
+ finish_reason: FinishReason = Field(
+ ...,
+ description='The reason the generation finished.\n\n- `SUCCESS` = successful generation.\n- `CONTENT_FILTERED` = successful generation, however the output violated our content moderation\npolicy and has been blurred as a result.',
+ examples=['SUCCESS'],
+ )
+ image: str = Field(
+ ...,
+ description='The generated image, encoded to base64.',
+ examples=['AAAAIGZ0eXBpc29tAAACAGlzb21pc28yYXZjMW1...'],
+ )
+ seed: Optional[float] = Field(
+ 0,
+ description='The seed used as random noise for this generation.',
+ examples=[343940597],
+ ge=0.0,
+ le=4294967294.0,
+ )
+
+
+class StabilityImageGenrationUpscaleConservativeResponse400(BaseModel):
+ errors: List[str] = Field(
+ ...,
+ description='One or more error messages indicating what went wrong.',
+ examples=[['some-field: is required']],
+ min_length=1,
+ )
+ id: str = Field(
+ ...,
+ description='A unique identifier associated with this error. Please include this in any [support tickets](https://kb.stability.ai/knowledge-base/kb-tickets/new)\nyou file, as it will greatly assist us in diagnosing the root cause of the problem.',
+ examples=['a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4'],
+ min_length=1,
+ )
+ name: str = Field(
+ ...,
+ description='Short-hand name for an error, useful for discriminating between errors with the same status code.',
+ examples=['bad_request'],
+ min_length=1,
+ )
+
+
+class StabilityImageGenrationUpscaleConservativeResponse413(BaseModel):
+ errors: List[str] = Field(
+ ...,
+ description='One or more error messages indicating what went wrong.',
+ examples=[['some-field: is required']],
+ min_length=1,
+ )
+ id: str = Field(
+ ...,
+ description='A unique identifier associated with this error. Please include this in any [support tickets](https://kb.stability.ai/knowledge-base/kb-tickets/new)\nyou file, as it will greatly assist us in diagnosing the root cause of the problem.',
+ examples=['a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4'],
+ min_length=1,
+ )
+ name: str = Field(
+ ...,
+ description='Short-hand name for an error, useful for discriminating between errors with the same status code.',
+ examples=['bad_request'],
+ min_length=1,
+ )
+
+
+class StabilityImageGenrationUpscaleConservativeResponse422(BaseModel):
+ errors: List[str] = Field(
+ ...,
+ description='One or more error messages indicating what went wrong.',
+ examples=[['some-field: is required']],
+ min_length=1,
+ )
+ id: str = Field(
+ ...,
+ description='A unique identifier associated with this error. Please include this in any [support tickets](https://kb.stability.ai/knowledge-base/kb-tickets/new)\nyou file, as it will greatly assist us in diagnosing the root cause of the problem.',
+ examples=['a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4'],
+ min_length=1,
+ )
+ name: str = Field(
+ ...,
+ description='Short-hand name for an error, useful for discriminating between errors with the same status code.',
+ examples=['bad_request'],
+ min_length=1,
+ )
+
+
+class StabilityImageGenrationUpscaleConservativeResponse429(BaseModel):
+ errors: List[str] = Field(
+ ...,
+ description='One or more error messages indicating what went wrong.',
+ examples=[['some-field: is required']],
+ min_length=1,
+ )
+ id: str = Field(
+ ...,
+ description='A unique identifier associated with this error. Please include this in any [support tickets](https://kb.stability.ai/knowledge-base/kb-tickets/new)\nyou file, as it will greatly assist us in diagnosing the root cause of the problem.',
+ examples=['a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4'],
+ min_length=1,
+ )
+ name: str = Field(
+ ...,
+ description='Short-hand name for an error, useful for discriminating between errors with the same status code.',
+ examples=['bad_request'],
+ min_length=1,
+ )
+
+
+class StabilityImageGenrationUpscaleConservativeResponse500(BaseModel):
+ errors: List[str] = Field(
+ ...,
+ description='One or more error messages indicating what went wrong.',
+ examples=[['some-field: is required']],
+ min_length=1,
+ )
+ id: str = Field(
+ ...,
+ description='A unique identifier associated with this error. Please include this in any [support tickets](https://kb.stability.ai/knowledge-base/kb-tickets/new)\nyou file, as it will greatly assist us in diagnosing the root cause of the problem.',
+ examples=['a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4'],
+ min_length=1,
+ )
+ name: str = Field(
+ ...,
+ description='Short-hand name for an error, useful for discriminating between errors with the same status code.',
+ examples=['bad_request'],
+ min_length=1,
+ )
+
+
+class StabilityImageGenrationUpscaleCreativeRequest(BaseModel):
+ creativity: Optional[float] = Field(
+ 0.3,
+ description='Indicates how creative the model should be when upscaling an image.\nHigher values will result in more details being added to the image during upscaling.',
+ ge=0.1,
+ le=0.5,
+ )
+ image: StrictBytes = Field(
+ ...,
+ description='The image you wish to upscale.\n\nSupported Formats:\n- jpeg\n- png\n- webp\n\nValidation Rules:\n- Every side must be at least 64 pixels\n- Total pixel count must be between 4,096 and 1,048,576 pixels',
+ examples=['./some/image.png'],
+ )
+ negative_prompt: Optional[str] = Field(
+ None,
+ description='A blurb of text describing what you **do not** wish to see in the output image.\nThis is an advanced feature.',
+ max_length=10000,
+ )
+ output_format: Optional[OutputFormat4] = Field(
+ 'png', description='Dictates the `content-type` of the generated image.'
+ )
+ prompt: str = Field(
+ ...,
+ description="What you wish to see in the output image. A strong, descriptive prompt that clearly defines\nelements, colors, and subjects will lead to better results.\n\nTo control the weight of a given word use the format `(word:weight)`,\nwhere `word` is the word you'd like to control the weight of and `weight`\nis a value between 0 and 1. For example: `The sky was a crisp (blue:0.3) and (green:0.8)`\nwould convey a sky that was blue and green, but more green than blue.",
+ max_length=10000,
+ min_length=1,
+ )
+ seed: Optional[float] = Field(
+ 0,
+ description="A specific value that is used to guide the 'randomness' of the generation. (Omit this parameter or pass `0` to use a random seed.)",
+ ge=0.0,
+ le=4294967294.0,
+ )
+ style_preset: Optional[StylePreset] = Field(
+ None, description='Guides the image model towards a particular style.'
+ )
+
+
+class StabilityImageGenrationUpscaleCreativeResponse200(BaseModel):
+ id: StabilityGenerationID
+
+
+class StabilityImageGenrationUpscaleCreativeResponse400(BaseModel):
+ errors: List[str] = Field(
+ ...,
+ description='One or more error messages indicating what went wrong.',
+ examples=[['some-field: is required']],
+ min_length=1,
+ )
+ id: str = Field(
+ ...,
+ description='A unique identifier associated with this error. Please include this in any [support tickets](https://kb.stability.ai/knowledge-base/kb-tickets/new)\nyou file, as it will greatly assist us in diagnosing the root cause of the problem.',
+ examples=['a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4'],
+ min_length=1,
+ )
+ name: str = Field(
+ ...,
+ description='Short-hand name for an error, useful for discriminating between errors with the same status code.',
+ examples=['bad_request'],
+ min_length=1,
+ )
+
+
+class StabilityImageGenrationUpscaleCreativeResponse413(BaseModel):
+ errors: List[str] = Field(
+ ...,
+ description='One or more error messages indicating what went wrong.',
+ examples=[['some-field: is required']],
+ min_length=1,
+ )
+ id: str = Field(
+ ...,
+ description='A unique identifier associated with this error. Please include this in any [support tickets](https://kb.stability.ai/knowledge-base/kb-tickets/new)\nyou file, as it will greatly assist us in diagnosing the root cause of the problem.',
+ examples=['a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4'],
+ min_length=1,
+ )
+ name: str = Field(
+ ...,
+ description='Short-hand name for an error, useful for discriminating between errors with the same status code.',
+ examples=['bad_request'],
+ min_length=1,
+ )
+
+
+class StabilityImageGenrationUpscaleCreativeResponse422(BaseModel):
+ errors: List[str] = Field(
+ ...,
+ description='One or more error messages indicating what went wrong.',
+ examples=[['some-field: is required']],
+ min_length=1,
+ )
+ id: str = Field(
+ ...,
+ description='A unique identifier associated with this error. Please include this in any [support tickets](https://kb.stability.ai/knowledge-base/kb-tickets/new)\nyou file, as it will greatly assist us in diagnosing the root cause of the problem.',
+ examples=['a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4'],
+ min_length=1,
+ )
+ name: str = Field(
+ ...,
+ description='Short-hand name for an error, useful for discriminating between errors with the same status code.',
+ examples=['bad_request'],
+ min_length=1,
+ )
+
+
+class StabilityImageGenrationUpscaleCreativeResponse429(BaseModel):
+ errors: List[str] = Field(
+ ...,
+ description='One or more error messages indicating what went wrong.',
+ examples=[['some-field: is required']],
+ min_length=1,
+ )
+ id: str = Field(
+ ...,
+ description='A unique identifier associated with this error. Please include this in any [support tickets](https://kb.stability.ai/knowledge-base/kb-tickets/new)\nyou file, as it will greatly assist us in diagnosing the root cause of the problem.',
+ examples=['a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4'],
+ min_length=1,
+ )
+ name: str = Field(
+ ...,
+ description='Short-hand name for an error, useful for discriminating between errors with the same status code.',
+ examples=['bad_request'],
+ min_length=1,
+ )
+
+
+class StabilityImageGenrationUpscaleCreativeResponse500(BaseModel):
+ errors: List[str] = Field(
+ ...,
+ description='One or more error messages indicating what went wrong.',
+ examples=[['some-field: is required']],
+ min_length=1,
+ )
+ id: str = Field(
+ ...,
+ description='A unique identifier associated with this error. Please include this in any [support tickets](https://kb.stability.ai/knowledge-base/kb-tickets/new)\nyou file, as it will greatly assist us in diagnosing the root cause of the problem.',
+ examples=['a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4'],
+ min_length=1,
+ )
+ name: str = Field(
+ ...,
+ description='Short-hand name for an error, useful for discriminating between errors with the same status code.',
+ examples=['bad_request'],
+ min_length=1,
+ )
+
+
+class StabilityImageGenrationUpscaleFastRequest(BaseModel):
+ image: StrictBytes = Field(
+ ...,
+ description='The image you wish to upscale.\n\nSupported Formats:\n- jpeg\n- png\n- webp\n\nValidation Rules:\n- Width must be between 32 and 1,536 pixels\n- Height must be between 32 and 1,536 pixels\n- Total pixel count must be between 1,024 and 1,048,576 pixels',
+ examples=['./some/image.png'],
+ )
+ output_format: Optional[OutputFormat4] = Field(
+ 'png', description='Dictates the `content-type` of the generated image.'
+ )
+
+
+class StabilityImageGenrationUpscaleFastResponse200(BaseModel):
+ finish_reason: FinishReason = Field(
+ ...,
+ description='The reason the generation finished.\n\n- `SUCCESS` = successful generation.\n- `CONTENT_FILTERED` = successful generation, however the output violated our content moderation\npolicy and has been blurred as a result.',
+ examples=['SUCCESS'],
+ )
+ image: str = Field(
+ ...,
+ description='The generated image, encoded to base64.',
+ examples=['AAAAIGZ0eXBpc29tAAACAGlzb21pc28yYXZjMW1...'],
+ )
+ seed: Optional[float] = Field(
+ 0,
+ description='The seed used as random noise for this generation.',
+ examples=[343940597],
+ ge=0.0,
+ le=4294967294.0,
+ )
+
+
+class StabilityImageGenrationUpscaleFastResponse400(BaseModel):
+ errors: List[str] = Field(
+ ...,
+ description='One or more error messages indicating what went wrong.',
+ examples=[['some-field: is required']],
+ min_length=1,
+ )
+ id: str = Field(
+ ...,
+ description='A unique identifier associated with this error. Please include this in any [support tickets](https://kb.stability.ai/knowledge-base/kb-tickets/new)\nyou file, as it will greatly assist us in diagnosing the root cause of the problem.',
+ examples=['a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4'],
+ min_length=1,
+ )
+ name: str = Field(
+ ...,
+ description='Short-hand name for an error, useful for discriminating between errors with the same status code.',
+ examples=['bad_request'],
+ min_length=1,
+ )
+
+
+class StabilityImageGenrationUpscaleFastResponse413(BaseModel):
+ errors: List[str] = Field(
+ ...,
+ description='One or more error messages indicating what went wrong.',
+ examples=[['some-field: is required']],
+ min_length=1,
+ )
+ id: str = Field(
+ ...,
+ description='A unique identifier associated with this error. Please include this in any [support tickets](https://kb.stability.ai/knowledge-base/kb-tickets/new)\nyou file, as it will greatly assist us in diagnosing the root cause of the problem.',
+ examples=['a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4'],
+ min_length=1,
+ )
+ name: str = Field(
+ ...,
+ description='Short-hand name for an error, useful for discriminating between errors with the same status code.',
+ examples=['bad_request'],
+ min_length=1,
+ )
+
+
+class StabilityImageGenrationUpscaleFastResponse422(BaseModel):
+ errors: List[str] = Field(
+ ...,
+ description='One or more error messages indicating what went wrong.',
+ examples=[['some-field: is required']],
+ min_length=1,
+ )
+ id: str = Field(
+ ...,
+ description='A unique identifier associated with this error. Please include this in any [support tickets](https://kb.stability.ai/knowledge-base/kb-tickets/new)\nyou file, as it will greatly assist us in diagnosing the root cause of the problem.',
+ examples=['a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4'],
+ min_length=1,
+ )
+ name: str = Field(
+ ...,
+ description='Short-hand name for an error, useful for discriminating between errors with the same status code.',
+ examples=['bad_request'],
+ min_length=1,
+ )
+
+
+class StabilityImageGenrationUpscaleFastResponse429(BaseModel):
+ errors: List[str] = Field(
+ ...,
+ description='One or more error messages indicating what went wrong.',
+ examples=[['some-field: is required']],
+ min_length=1,
+ )
+ id: str = Field(
+ ...,
+ description='A unique identifier associated with this error. Please include this in any [support tickets](https://kb.stability.ai/knowledge-base/kb-tickets/new)\nyou file, as it will greatly assist us in diagnosing the root cause of the problem.',
+ examples=['a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4'],
+ min_length=1,
+ )
+ name: str = Field(
+ ...,
+ description='Short-hand name for an error, useful for discriminating between errors with the same status code.',
+ examples=['bad_request'],
+ min_length=1,
+ )
+
+
+class StabilityImageGenrationUpscaleFastResponse500(BaseModel):
+ errors: List[str] = Field(
+ ...,
+ description='One or more error messages indicating what went wrong.',
+ examples=[['some-field: is required']],
+ min_length=1,
+ )
+ id: str = Field(
+ ...,
+ description='A unique identifier associated with this error. Please include this in any [support tickets](https://kb.stability.ai/knowledge-base/kb-tickets/new)\nyou file, as it will greatly assist us in diagnosing the root cause of the problem.',
+ examples=['a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4'],
+ min_length=1,
+ )
+ name: str = Field(
+ ...,
+ description='Short-hand name for an error, useful for discriminating between errors with the same status code.',
+ examples=['bad_request'],
+ min_length=1,
+ )
+
+
+class StabilityStabilityClientID(RootModel[str]):
+ root: str = Field(
+ ...,
+ description='The name of your application, used to help us communicate app-specific debugging or moderation issues to you.',
+ examples=['my-awesome-app'],
+ max_length=256,
+ )
+
+
+class StabilityStabilityClientUserID(RootModel[str]):
+ root: str = Field(
+ ...,
+ description='A unique identifier for your end user. Used to help us communicate user-specific debugging or moderation issues to you. Feel free to obfuscate this value to protect user privacy.',
+ examples=['DiscordUser#9999'],
+ max_length=256,
+ )
+
+
+class StabilityStabilityClientVersion(RootModel[str]):
+ root: str = Field(
+ ...,
+ description='The version of your application, used to help us communicate version-specific debugging or moderation issues to you.',
+ examples=['1.2.1'],
+ max_length=256,
+ )
+
+
+class StorageFile(BaseModel):
+ file_path: Optional[str] = Field(None, description='Path to the file in storage')
+ id: Optional[UUID] = Field(
+ None, description='Unique identifier for the storage file'
+ )
+ public_url: Optional[str] = Field(None, description='Public URL')
+
+
+class StripeAddress(BaseModel):
+ city: Optional[str] = None
+ country: Optional[str] = None
+ line1: Optional[str] = None
+ line2: Optional[str] = None
+ postal_code: Optional[str] = None
+ state: Optional[str] = None
+
+
+class StripeAmountDetails(BaseModel):
+ tip: Optional[Dict[str, Any]] = None
+
+
+class StripeBillingDetails(BaseModel):
+ address: Optional[StripeAddress] = None
+ email: Optional[str] = None
+ name: Optional[str] = None
+ phone: Optional[str] = None
+ tax_id: Optional[Any] = None
+
+
+class Checks(BaseModel):
+ address_line1_check: Optional[Any] = None
+ address_postal_code_check: Optional[Any] = None
+ cvc_check: Optional[str] = None
+
+
+class ExtendedAuthorization(BaseModel):
+ status: Optional[str] = None
+
+
+class IncrementalAuthorization(BaseModel):
+ status: Optional[str] = None
+
+
+class Multicapture(BaseModel):
+ status: Optional[str] = None
+
+
+class NetworkToken(BaseModel):
+ used: Optional[bool] = None
+
+
+class Overcapture(BaseModel):
+ maximum_amount_capturable: Optional[int] = None
+ status: Optional[str] = None
+
+
+class StripeCardDetails(BaseModel):
+ amount_authorized: Optional[int] = None
+ authorization_code: Optional[Any] = None
+ brand: Optional[str] = None
+ checks: Optional[Checks] = None
+ country: Optional[str] = None
+ exp_month: Optional[int] = None
+ exp_year: Optional[int] = None
+ extended_authorization: Optional[ExtendedAuthorization] = None
+ fingerprint: Optional[str] = None
+ funding: Optional[str] = None
+ incremental_authorization: Optional[IncrementalAuthorization] = None
+ installments: Optional[Any] = None
+ last4: Optional[str] = None
+ mandate: Optional[Any] = None
+ multicapture: Optional[Multicapture] = None
+ network: Optional[str] = None
+ network_token: Optional[NetworkToken] = None
+ network_transaction_id: Optional[str] = None
+ overcapture: Optional[Overcapture] = None
+ regulated_status: Optional[str] = None
+ three_d_secure: Optional[Any] = None
+ wallet: Optional[Any] = None
+
+
+class Object1(str, Enum):
+ charge = 'charge'
+
+
+class Object2(str, Enum):
+ event = 'event'
+
+
+class Type31(str, Enum):
+ payment_intent_succeeded = 'payment_intent.succeeded'
+
+
+class StripeOutcome(BaseModel):
+ advice_code: Optional[Any] = None
+ network_advice_code: Optional[Any] = None
+ network_decline_code: Optional[Any] = None
+ network_status: Optional[str] = None
+ reason: Optional[Any] = None
+ risk_level: Optional[str] = None
+ risk_score: Optional[int] = None
+ seller_message: Optional[str] = None
+ type: Optional[str] = None
+
+
+class Object3(str, Enum):
+ payment_intent = 'payment_intent'
+
+
+class StripePaymentMethodDetails(BaseModel):
+ card: Optional[StripeCardDetails] = None
+ type: Optional[str] = None
+
+
+class Card(BaseModel):
+ installments: Optional[Any] = None
+ mandate_options: Optional[Any] = None
+ network: Optional[Any] = None
+ request_three_d_secure: Optional[str] = None
+
+
+class StripePaymentMethodOptions(BaseModel):
+ card: Optional[Card] = None
+
+
+class StripeRefundList(BaseModel):
+ data: Optional[List[Dict[str, Any]]] = None
+ has_more: Optional[bool] = None
+ object: Optional[str] = None
+ total_count: Optional[int] = None
+ url: Optional[str] = None
+
+
+class StripeRequestInfo(BaseModel):
+ id: Optional[str] = None
+ idempotency_key: Optional[str] = None
+
+
+class StripeShipping(BaseModel):
+ address: Optional[StripeAddress] = None
+ carrier: Optional[str] = None
+ name: Optional[str] = None
+ phone: Optional[str] = None
+ tracking_number: Optional[str] = None
+
+
+class Type32(str, Enum):
json_schema = 'json_schema'
@@ -2400,19 +3907,19 @@ class TextResponseFormatJsonSchema(BaseModel):
False,
description='Whether to enable strict schema adherence when generating the output.\nIf set to true, the model will always follow the exact schema defined\nin the `schema` field. Only a subset of JSON Schema is supported when\n`strict` is `true`. To learn more, read the [Structured Outputs\nguide](/docs/guides/structured-outputs).\n',
)
- type: Type20 = Field(
+ type: Type32 = Field(
...,
description='The type of response format being defined. Always `json_schema`.',
)
-class Type21(str, Enum):
+class Type33(str, Enum):
function = 'function'
class ToolChoiceFunction(BaseModel):
name: str = Field(..., description='The name of the function to call.')
- type: Type21 = Field(
+ type: Type33 = Field(
..., description='For function calling, the type is always `function`.'
)
@@ -2423,7 +3930,7 @@ class ToolChoiceOptions(str, Enum):
required = 'required'
-class Type22(str, Enum):
+class Type34(str, Enum):
file_search = 'file_search'
web_search_preview = 'web_search_preview'
computer_use_preview = 'computer_use_preview'
@@ -2431,7 +3938,7 @@ class Type22(str, Enum):
class ToolChoiceTypes(BaseModel):
- type: Type22 = Field(
+ type: Type34 = Field(
...,
description='The type of hosted tool the model should to use. Learn more about\n[built-in tools](/docs/guides/tools).\n\nAllowed values are:\n- `file_search`\n- `web_search_preview`\n- `computer_use_preview`\n',
)
@@ -2499,9 +4006,9 @@ class TripoModelStyle(str, Enum):
class TripoModelVersion(str, Enum):
- V2_5 = 'v2.5-20250123'
- V2_0 = 'v2.0-20240919'
- V1_4 = 'v1.4-20240625'
+ v2_5_20250123 = 'v2.5-20250123'
+ v2_0_20240919 = 'v2.0-20240919'
+ v1_4_20240625 = 'v1.4-20240625'
class TripoMultiviewMode(str, Enum):
@@ -2547,13 +4054,13 @@ class Code1(int, Enum):
integer_0 = 0
-class Data8(BaseModel):
+class Data9(BaseModel):
task_id: str = Field(..., description='used for getTask')
class TripoSuccessTask(BaseModel):
code: Code1
- data: Data8
+ data: Data9
class Topology(str, Enum):
@@ -2570,7 +4077,7 @@ class Output(BaseModel):
topology: Optional[Topology] = None
-class Status10(str, Enum):
+class Status11(str, Enum):
queued = 'queued'
running = 'running'
success = 'success'
@@ -2586,7 +4093,7 @@ class TripoTask(BaseModel):
input: Dict[str, Any]
output: Output
progress: int = Field(..., ge=0, le=100)
- status: Status10
+ status: Status11
task_id: str
type: str
@@ -2650,6 +4157,18 @@ class TripoTypeTextureModel(str, Enum):
texture_model = 'texture_model'
+class User(BaseModel):
+ email: Optional[str] = Field(None, description='The email address for this user.')
+ id: Optional[str] = Field(None, description='The unique id for this user.')
+ isAdmin: Optional[bool] = Field(
+ None, description='Indicates if the user has admin privileges.'
+ )
+ isApproved: Optional[bool] = Field(
+ None, description='Indicates if the user is approved.'
+ )
+ name: Optional[str] = Field(None, description='The name for this user.')
+
+
class Veo2GenVidPollRequest(BaseModel):
operationName: str = Field(
...,
@@ -2660,7 +4179,7 @@ class Veo2GenVidPollRequest(BaseModel):
)
-class Error(BaseModel):
+class Error1(BaseModel):
code: Optional[int] = Field(None, description='Error code')
message: Optional[str] = Field(None, description='Error message')
@@ -2692,7 +4211,7 @@ class Response(BaseModel):
class Veo2GenVidPollResponse(BaseModel):
done: Optional[bool] = None
- error: Optional[Error] = Field(
+ error: Optional[Error1] = Field(
None, description='Error details if operation failed'
)
name: Optional[str] = None
@@ -2753,13 +4272,102 @@ class Veo2GenVidResponse(BaseModel):
)
+class VeoGenVidPollRequest(BaseModel):
+ operationName: str = Field(
+ ...,
+ description='Full operation name (from predict response)',
+ examples=[
+ 'projects/PROJECT_ID/locations/us-central1/publishers/google/models/MODEL_ID/operations/OPERATION_ID'
+ ],
+ )
+
+
+class Response1(BaseModel):
+ field_type: Optional[str] = Field(
+ None,
+ alias='@type',
+ examples=[
+ 'type.googleapis.com/cloud.ai.large_models.vision.GenerateVideoResponse'
+ ],
+ )
+ raiMediaFilteredCount: Optional[int] = Field(
+ None, description='Count of media filtered by responsible AI policies'
+ )
+ raiMediaFilteredReasons: Optional[List[str]] = Field(
+ None, description='Reasons why media was filtered by responsible AI policies'
+ )
+ videos: Optional[List[Video]] = None
+
+
+class VeoGenVidPollResponse(BaseModel):
+ done: Optional[bool] = None
+ error: Optional[Error1] = Field(
+ None, description='Error details if operation failed'
+ )
+ name: Optional[str] = None
+ response: Optional[Response1] = Field(
+ None, description='The actual prediction response if done is true'
+ )
+
+
+class Image2(BaseModel):
+ bytesBase64Encoded: str
+ gcsUri: Optional[str] = None
+ mimeType: Optional[str] = None
+
+
+class Image3(BaseModel):
+ bytesBase64Encoded: Optional[str] = None
+ gcsUri: str
+ mimeType: Optional[str] = None
+
+
+class Instance1(BaseModel):
+ image: Optional[Union[Image2, Image3]] = Field(
+ None, description='Optional image to guide video generation'
+ )
+ prompt: str = Field(..., description='Text description of the video')
+
+
+class Parameters1(BaseModel):
+ aspectRatio: Optional[str] = Field(None, examples=['16:9'])
+ durationSeconds: Optional[int] = None
+ enhancePrompt: Optional[bool] = None
+ generateAudio: Optional[bool] = Field(
+ None,
+ description='Generate audio for the video. Only supported by veo 3 models.',
+ )
+ negativePrompt: Optional[str] = None
+ personGeneration: Optional[PersonGeneration1] = None
+ sampleCount: Optional[int] = None
+ seed: Optional[int] = None
+ storageUri: Optional[str] = Field(
+ None, description='Optional Cloud Storage URI to upload the video'
+ )
+
+
+class VeoGenVidRequest(BaseModel):
+ instances: Optional[List[Instance1]] = None
+ parameters: Optional[Parameters1] = None
+
+
+class VeoGenVidResponse(BaseModel):
+ name: str = Field(
+ ...,
+ description='Operation resource name',
+ examples=[
+ 'projects/PROJECT_ID/locations/us-central1/publishers/google/models/MODEL_ID/operations/a1b07c8e-7b5a-4aba-bb34-3e1ccb8afcc8'
+ ],
+ )
+
+
class SearchContextSize(str, Enum):
low = 'low'
medium = 'medium'
high = 'high'
-class Type23(str, Enum):
+class Type35(str, Enum):
web_search_preview = 'web_search_preview'
web_search_preview_2025_03_11 = 'web_search_preview_2025_03_11'
@@ -2775,30 +4383,348 @@ class WebSearchPreviewTool(BaseModel):
)
-class Status11(str, Enum):
+class Status12(str, Enum):
in_progress = 'in_progress'
searching = 'searching'
completed = 'completed'
failed = 'failed'
-class Type24(str, Enum):
+class Type36(str, Enum):
web_search_call = 'web_search_call'
class WebSearchToolCall(BaseModel):
id: str = Field(..., description='The unique ID of the web search tool call.\n')
- status: Status11 = Field(
+ status: Status12 = Field(
..., description='The status of the web search tool call.\n'
)
- type: Type24 = Field(
+ type: Type36 = Field(
...,
description='The type of the web search tool call. Always `web_search_call`.\n',
)
-class CreateModelResponseProperties(ModelResponseProperties):
- pass
+class WorkflowRunStatus(str, Enum):
+ WorkflowRunStatusStarted = 'WorkflowRunStatusStarted'
+ WorkflowRunStatusFailed = 'WorkflowRunStatusFailed'
+ WorkflowRunStatusCompleted = 'WorkflowRunStatusCompleted'
+
+
+class ActionJobResult(BaseModel):
+ action_job_id: Optional[str] = Field(
+ None, description='Identifier of the job this result belongs to'
+ )
+ action_run_id: Optional[str] = Field(
+ None, description='Identifier of the run this result belongs to'
+ )
+ author: Optional[str] = Field(None, description='The author of the commit')
+ avg_vram: Optional[int] = Field(
+ None, description='The average VRAM used by the job'
+ )
+ branch_name: Optional[str] = Field(
+ None, description='Name of the relevant git branch'
+ )
+ comfy_run_flags: Optional[str] = Field(
+ None, description='The comfy run flags. E.g. `--low-vram`'
+ )
+ commit_hash: Optional[str] = Field(None, description='The hash of the commit')
+ commit_id: Optional[str] = Field(None, description='The ID of the commit')
+ commit_message: Optional[str] = Field(None, description='The message of the commit')
+ commit_time: Optional[int] = Field(
+ None, description='The Unix timestamp when the commit was made'
+ )
+ cuda_version: Optional[str] = Field(None, description='CUDA version used')
+ end_time: Optional[int] = Field(
+ None, description='The end time of the job as a Unix timestamp.'
+ )
+ git_repo: Optional[str] = Field(None, description='The repository name')
+ id: Optional[UUID] = Field(None, description='Unique identifier for the job result')
+ job_trigger_user: Optional[str] = Field(
+ None, description='The user who triggered the job.'
+ )
+ machine_stats: Optional[MachineStats] = None
+ operating_system: Optional[str] = Field(None, description='Operating system used')
+ peak_vram: Optional[int] = Field(None, description='The peak VRAM used by the job')
+ pr_number: Optional[str] = Field(None, description='The pull request number')
+ python_version: Optional[str] = Field(None, description='PyTorch version used')
+ pytorch_version: Optional[str] = Field(None, description='PyTorch version used')
+ start_time: Optional[int] = Field(
+ None, description='The start time of the job as a Unix timestamp.'
+ )
+ status: Optional[WorkflowRunStatus] = None
+ storage_file: Optional[StorageFile] = None
+ workflow_name: Optional[str] = Field(None, description='Name of the workflow')
+
+
+class BFLCannyInputs(BaseModel):
+ canny_high_threshold: Optional[CannyHighThreshold] = Field(
+ default_factory=lambda: CannyHighThreshold.model_validate(200),
+ description='High threshold for Canny edge detection',
+ title='Canny High Threshold',
+ )
+ canny_low_threshold: Optional[CannyLowThreshold] = Field(
+ default_factory=lambda: CannyLowThreshold.model_validate(50),
+ description='Low threshold for Canny edge detection',
+ title='Canny Low Threshold',
+ )
+ control_image: Optional[str] = Field(
+ None,
+ description='Base64 encoded image to use as control input if no preprocessed image is provided',
+ title='Control Image',
+ )
+ guidance: Optional[Guidance] = Field(
+ default_factory=lambda: Guidance.model_validate(30),
+ description='Guidance strength for the image generation process',
+ title='Guidance',
+ )
+ output_format: Optional[BFLOutputFormat] = Field(
+ 'jpeg',
+ description="Output format for the generated image. Can be 'jpeg' or 'png'.",
+ )
+ preprocessed_image: Optional[str] = Field(
+ None,
+ description='Optional pre-processed image that will bypass the control preprocessing step',
+ title='Preprocessed Image',
+ )
+ prompt: str = Field(
+ ...,
+ description='Text prompt for image generation',
+ examples=['ein fantastisches bild'],
+ title='Prompt',
+ )
+ prompt_upsampling: Optional[bool] = Field(
+ False,
+ description='Whether to perform upsampling on the prompt',
+ title='Prompt Upsampling',
+ )
+ safety_tolerance: Optional[int] = Field(
+ 2,
+ description='Tolerance level for input and output moderation. Between 0 and 6, 0 being most strict, 6 being least strict.',
+ ge=0,
+ le=6,
+ title='Safety Tolerance',
+ )
+ seed: Optional[int] = Field(
+ None,
+ description='Optional seed for reproducibility',
+ examples=[42],
+ title='Seed',
+ )
+ steps: Optional[Steps] = Field(
+ default_factory=lambda: Steps.model_validate(50),
+ description='Number of steps for the image generation process',
+ title='Steps',
+ )
+ webhook_secret: Optional[str] = Field(
+ None,
+ description='Optional secret for webhook signature verification',
+ title='Webhook Secret',
+ )
+ webhook_url: Optional[WebhookUrl] = Field(
+ None, description='URL to receive webhook notifications', title='Webhook Url'
+ )
+
+
+class BFLDepthInputs(BaseModel):
+ control_image: Optional[str] = Field(
+ None,
+ description='Base64 encoded image to use as control input',
+ title='Control Image',
+ )
+ guidance: Optional[Guidance] = Field(
+ default_factory=lambda: Guidance.model_validate(15),
+ description='Guidance strength for the image generation process',
+ title='Guidance',
+ )
+ output_format: Optional[BFLOutputFormat] = Field(
+ 'jpeg',
+ description="Output format for the generated image. Can be 'jpeg' or 'png'.",
+ )
+ preprocessed_image: Optional[str] = Field(
+ None,
+ description='Optional pre-processed image that will bypass the control preprocessing step',
+ title='Preprocessed Image',
+ )
+ prompt: str = Field(
+ ...,
+ description='Text prompt for image generation',
+ examples=['ein fantastisches bild'],
+ title='Prompt',
+ )
+ prompt_upsampling: Optional[bool] = Field(
+ False,
+ description='Whether to perform upsampling on the prompt',
+ title='Prompt Upsampling',
+ )
+ safety_tolerance: Optional[int] = Field(
+ 2,
+ description='Tolerance level for input and output moderation. Between 0 and 6, 0 being most strict, 6 being least strict.',
+ ge=0,
+ le=6,
+ title='Safety Tolerance',
+ )
+ seed: Optional[int] = Field(
+ None,
+ description='Optional seed for reproducibility',
+ examples=[42],
+ title='Seed',
+ )
+ steps: Optional[Steps] = Field(
+ default_factory=lambda: Steps.model_validate(50),
+ description='Number of steps for the image generation process',
+ title='Steps',
+ )
+ webhook_secret: Optional[str] = Field(
+ None,
+ description='Optional secret for webhook signature verification',
+ title='Webhook Secret',
+ )
+ webhook_url: Optional[WebhookUrl] = Field(
+ None, description='URL to receive webhook notifications', title='Webhook Url'
+ )
+
+
+class BFLFluxProExpandInputs(BaseModel):
+ bottom: Optional[Bottom] = Field(
+ 0,
+ description='Number of pixels to expand at the bottom of the image',
+ title='Bottom',
+ )
+ guidance: Optional[Guidance2] = Field(
+ default_factory=lambda: Guidance2.model_validate(60),
+ description='Guidance strength for the image generation process',
+ title='Guidance',
+ )
+ image: str = Field(
+ ...,
+ description='A Base64-encoded string representing the image you wish to expand.',
+ title='Image',
+ )
+ left: Optional[Left] = Field(
+ 0,
+ description='Number of pixels to expand on the left side of the image',
+ title='Left',
+ )
+ output_format: Optional[BFLOutputFormat] = Field(
+ 'jpeg',
+ description="Output format for the generated image. Can be 'jpeg' or 'png'.",
+ )
+ prompt: Optional[str] = Field(
+ '',
+ description='The description of the changes you want to make. This text guides the expansion process, allowing you to specify features, styles, or modifications for the expanded areas.',
+ examples=['ein fantastisches bild'],
+ title='Prompt',
+ )
+ prompt_upsampling: Optional[bool] = Field(
+ False,
+ description='Whether to perform upsampling on the prompt. If active, automatically modifies the prompt for more creative generation',
+ title='Prompt Upsampling',
+ )
+ right: Optional[Right] = Field(
+ 0,
+ description='Number of pixels to expand on the right side of the image',
+ title='Right',
+ )
+ safety_tolerance: Optional[int] = Field(
+ 2,
+ description='Tolerance level for input and output moderation. Between 0 and 6, 0 being most strict, 6 being least strict.',
+ examples=[2],
+ ge=0,
+ le=6,
+ title='Safety Tolerance',
+ )
+ seed: Optional[int] = Field(
+ None, description='Optional seed for reproducibility', title='Seed'
+ )
+ steps: Optional[Steps2] = Field(
+ default_factory=lambda: Steps2.model_validate(50),
+ description='Number of steps for the image generation process',
+ examples=[50],
+ title='Steps',
+ )
+ top: Optional[Top] = Field(
+ 0, description='Number of pixels to expand at the top of the image', title='Top'
+ )
+ webhook_secret: Optional[str] = Field(
+ None,
+ description='Optional secret for webhook signature verification',
+ title='Webhook Secret',
+ )
+ webhook_url: Optional[WebhookUrl] = Field(
+ None, description='URL to receive webhook notifications', title='Webhook Url'
+ )
+
+
+class BFLFluxProFillInputs(BaseModel):
+ guidance: Optional[Guidance2] = Field(
+ default_factory=lambda: Guidance2.model_validate(60),
+ description='Guidance strength for the image generation process',
+ title='Guidance',
+ )
+ image: str = Field(
+ ...,
+ description='A Base64-encoded string representing the image you wish to modify. Can contain alpha mask if desired.',
+ title='Image',
+ )
+ mask: Optional[str] = Field(
+ None,
+ description='A Base64-encoded string representing a mask for the areas you want to modify in the image. The mask should be the same dimensions as the image and in black and white. Black areas (0%) indicate no modification, while white areas (100%) specify areas for inpainting. Optional if you provide an alpha mask in the original image. Validation: The endpoint verifies that the dimensions of the mask match the original image.',
+ title='Mask',
+ )
+ output_format: Optional[BFLOutputFormat] = Field(
+ 'jpeg',
+ description="Output format for the generated image. Can be 'jpeg' or 'png'.",
+ )
+ prompt: Optional[str] = Field(
+ '',
+ description='The description of the changes you want to make. This text guides the inpainting process, allowing you to specify features, styles, or modifications for the masked area.',
+ examples=['ein fantastisches bild'],
+ title='Prompt',
+ )
+ prompt_upsampling: Optional[bool] = Field(
+ False,
+ description='Whether to perform upsampling on the prompt. If active, automatically modifies the prompt for more creative generation',
+ title='Prompt Upsampling',
+ )
+ safety_tolerance: Optional[int] = Field(
+ 2,
+ description='Tolerance level for input and output moderation. Between 0 and 6, 0 being most strict, 6 being least strict.',
+ examples=[2],
+ ge=0,
+ le=6,
+ title='Safety Tolerance',
+ )
+ seed: Optional[int] = Field(
+ None, description='Optional seed for reproducibility', title='Seed'
+ )
+ steps: Optional[Steps2] = Field(
+ default_factory=lambda: Steps2.model_validate(50),
+ description='Number of steps for the image generation process',
+ examples=[50],
+ title='Steps',
+ )
+ webhook_secret: Optional[str] = Field(
+ None,
+ description='Optional secret for webhook signature verification',
+ title='Webhook Secret',
+ )
+ webhook_url: Optional[WebhookUrl] = Field(
+ None, description='URL to receive webhook notifications', title='Webhook Url'
+ )
+
+
+class BFLHTTPValidationError(BaseModel):
+ detail: Optional[List[BFLValidationError]] = Field(None, title='Detail')
+
+
+class BulkNodeVersionsRequest(BaseModel):
+ node_versions: List[NodeVersionIdentifier] = Field(
+ ..., description='List of node ID and version pairs to retrieve'
+ )
+
+
+CreateModelResponseProperties = ModelResponseProperties
class GeminiInlineData(BaseModel):
@@ -2841,6 +4767,125 @@ class GeminiSystemInstructionContent(BaseModel):
)
+class GeminiUsageMetadata(BaseModel):
+ cachedContentTokenCount: Optional[int] = Field(
+ None,
+ description='Output only. Number of tokens in the cached part in the input (the cached content).',
+ )
+ candidatesTokenCount: Optional[int] = Field(
+ None, description='Number of tokens in the response(s).'
+ )
+ candidatesTokensDetails: Optional[List[ModalityTokenCount]] = Field(
+ None, description='Breakdown of candidate tokens by modality.'
+ )
+ promptTokenCount: Optional[int] = Field(
+ None,
+ description='Number of tokens in the request. When cachedContent is set, this is still the total effective prompt size meaning this includes the number of tokens in the cached content.',
+ )
+ promptTokensDetails: Optional[List[ModalityTokenCount]] = Field(
+ None, description='Breakdown of prompt tokens by modality.'
+ )
+ thoughtsTokenCount: Optional[int] = Field(
+ None, description='Number of tokens present in thoughts output.'
+ )
+ toolUsePromptTokenCount: Optional[int] = Field(
+ None, description='Number of tokens present in tool-use prompt(s).'
+ )
+
+
+class GithubInstallation(BaseModel):
+ access_tokens_url: str = Field(..., description='The API URL for access tokens')
+ account: GithubUser
+ app_id: int = Field(..., description='The GitHub App ID')
+ created_at: datetime = Field(..., description='When the installation was created')
+ events: List[str] = Field(
+ ..., description='The events the installation subscribes to'
+ )
+ html_url: str = Field(..., description='The HTML URL of the installation')
+ id: int = Field(..., description='The installation ID')
+ permissions: Dict[str, Any] = Field(..., description='The installation permissions')
+ repositories_url: str = Field(..., description='The API URL for repositories')
+ repository_selection: RepositorySelection = Field(
+ ..., description='Repository selection for the installation'
+ )
+ single_file_name: Optional[str] = Field(
+ None, description='The single file name if applicable'
+ )
+ target_id: int = Field(..., description='The target ID')
+ target_type: str = Field(..., description='The target type')
+ updated_at: datetime = Field(
+ ..., description='When the installation was last updated'
+ )
+
+
+class GithubReleaseAsset(BaseModel):
+ browser_download_url: str = Field(..., description='The browser download URL')
+ content_type: str = Field(..., description='The content type of the asset')
+ created_at: datetime = Field(..., description='When the asset was created')
+ download_count: int = Field(..., description='The number of downloads')
+ id: int = Field(..., description='The asset ID')
+ label: Optional[str] = Field(None, description='The label of the asset')
+ name: str = Field(..., description='The name of the asset')
+ node_id: str = Field(..., description='The asset node ID')
+ size: int = Field(..., description='The size of the asset in bytes')
+ state: State = Field(..., description='The state of the asset')
+ updated_at: datetime = Field(..., description='When the asset was last updated')
+ uploader: GithubUser
+
+
+class Release(BaseModel):
+ assets: List[GithubReleaseAsset] = Field(..., description='Array of release assets')
+ assets_url: Optional[str] = Field(None, description='The URL to the release assets')
+ author: GithubUser
+ body: Optional[str] = Field(None, description='The release notes/body')
+ created_at: datetime = Field(..., description='When the release was created')
+ draft: bool = Field(..., description='Whether the release is a draft')
+ html_url: str = Field(..., description='The HTML URL of the release')
+ id: int = Field(..., description='The ID of the release')
+ name: Optional[str] = Field(None, description='The name of the release')
+ node_id: str = Field(..., description='The node ID of the release')
+ prerelease: bool = Field(..., description='Whether the release is a prerelease')
+ published_at: Optional[datetime] = Field(
+ None, description='When the release was published'
+ )
+ tag_name: str = Field(..., description='The tag name of the release')
+ tarball_url: str = Field(..., description='URL to the tarball')
+ target_commitish: str = Field(
+ ..., description='The branch or commit the release was created from'
+ )
+ upload_url: Optional[str] = Field(
+ None, description='The URL to upload release assets'
+ )
+ url: str = Field(..., description='The API URL of the release')
+ zipball_url: str = Field(..., description='URL to the zipball')
+
+
+class GithubRepository(BaseModel):
+ clone_url: str = Field(..., description='The clone URL of the repository')
+ created_at: datetime = Field(..., description='When the repository was created')
+ default_branch: str = Field(..., description='The default branch of the repository')
+ description: Optional[str] = Field(None, description='The repository description')
+ fork: bool = Field(..., description='Whether the repository is a fork')
+ full_name: str = Field(
+ ..., description='The full name of the repository (owner/repo)'
+ )
+ git_url: str = Field(..., description='The git URL of the repository')
+ html_url: str = Field(..., description='The HTML URL of the repository')
+ id: int = Field(..., description='The repository ID')
+ name: str = Field(..., description='The name of the repository')
+ node_id: str = Field(..., description='The repository node ID')
+ owner: GithubUser
+ private: bool = Field(..., description='Whether the repository is private')
+ pushed_at: datetime = Field(
+ ..., description='When the repository was last pushed to'
+ )
+ ssh_url: str = Field(..., description='The SSH URL of the repository')
+ updated_at: datetime = Field(
+ ..., description='When the repository was last updated'
+ )
+ url: str = Field(..., description='The API URL of the repository')
+
+
class IdeogramV3EditRequest(BaseModel):
color_palette: Optional[IdeogramColorPalette] = None
image: Optional[StrictBytes] = Field(
@@ -3276,6 +5321,52 @@ class MoonvalleyTextToImageRequest(BaseModel):
webhook_url: Optional[str] = None
+class NodeVersion(BaseModel):
+ changelog: Optional[str] = Field(
+ None, description='Summary of changes made in this version'
+ )
+ comfy_node_extract_status: Optional[str] = Field(
+ None, description='The status of comfy node extraction process.'
+ )
+ createdAt: Optional[datetime] = Field(
+ None, description='The date and time the version was created.'
+ )
+ dependencies: Optional[List[str]] = Field(
+ None, description='A list of pip dependencies required by the node.'
+ )
+ deprecated: Optional[bool] = Field(
+ None, description='Indicates if this version is deprecated.'
+ )
+ downloadUrl: Optional[str] = Field(
+ None, description='[Output Only] URL to download this version of the node'
+ )
+ id: Optional[str] = None
+ node_id: Optional[str] = Field(
+ None, description='The unique identifier of the node.'
+ )
+ status: Optional[NodeVersionStatus] = None
+ status_reason: Optional[str] = Field(
+ None, description='The reason for the status change.'
+ )
+ supported_accelerators: Optional[List[str]] = Field(
+ None,
+ description='List of accelerators (e.g. CUDA, DirectML, ROCm) that this node supports',
+ )
+ supported_comfyui_frontend_version: Optional[str] = Field(
+ None, description='Supported versions of ComfyUI frontend'
+ )
+ supported_comfyui_version: Optional[str] = Field(
+ None, description='Supported versions of ComfyUI'
+ )
+ supported_os: Optional[List[str]] = Field(
+ None, description='List of operating systems that this node supports'
+ )
+ version: Optional[str] = Field(
+ None,
+ description='The version identifier, following semantic versioning. Must be unique for the node.',
+ )
+
+
class OutputContent(RootModel[Union[OutputTextContent, OutputAudioContent]]):
root: Union[OutputTextContent, OutputAudioContent]
@@ -3283,7 +5374,7 @@ class OutputContent(RootModel[Union[OutputTextContent, OutputAudioContent]]):
class OutputMessage(BaseModel):
content: List[OutputContent] = Field(..., description='The content of the message')
role: Role4 = Field(..., description='The role of the message')
- type: Type14 = Field(..., description='The type of output item')
+ type: Type15 = Field(..., description='The type of output item')
class PikaBodyGenerate22I2vGenerate22I2vPost(BaseModel):
@@ -3333,6 +5424,16 @@ class PikaHTTPValidationError(BaseModel):
detail: Optional[List[PikaValidationError]] = Field(None, title='Detail')
+class PublisherMember(BaseModel):
+ id: Optional[str] = Field(
+ None, description='The unique identifier for the publisher member.'
+ )
+ role: Optional[str] = Field(
+ None, description='The role of the user in the publisher.'
+ )
+ user: Optional[PublisherUser] = None
+
+
class Reasoning(BaseModel):
effort: Optional[ReasoningEffort] = 'medium'
generate_summary: Optional[GenerateSummary] = Field(
@@ -3345,13 +5446,88 @@ class Reasoning(BaseModel):
)
+class RecraftImage(BaseModel):
+ b64_json: Optional[str] = None
+ features: Optional[RecraftImageFeatures] = None
+ image_id: UUID
+ revised_prompt: Optional[str] = None
+ url: Optional[str] = None
+
+
+class RecraftProcessImageRequest(BaseModel):
+ image: StrictBytes
+ image_format: Optional[RecraftImageFormat] = None
+ response_format: Optional[RecraftResponseFormat] = None
+
+
+class RecraftProcessImageResponse(BaseModel):
+ created: int
+ credits: int
+ image: RecraftImage
+
+
+class RecraftTextLayout(RootModel[List[RecraftTextLayoutItem]]):
+ root: List[RecraftTextLayoutItem]
+
+
+class RecraftTransformImageWithMaskRequest(BaseModel):
+ block_nsfw: Optional[bool] = None
+ calculate_features: Optional[bool] = None
+ image: StrictBytes
+ image_format: Optional[RecraftImageFormat] = None
+ mask: StrictBytes
+ model: Optional[RecraftTransformModel] = None
+ n: Optional[int] = None
+ negative_prompt: Optional[str] = None
+ prompt: str
+ response_format: Optional[RecraftResponseFormat] = None
+ style: Optional[RecraftImageStyle] = None
+ style_id: Optional[UUID] = None
+ substyle: Optional[RecraftImageSubStyle] = None
+ text_layout: Optional[RecraftTextLayout] = None
+
+
+class ResponseContentPartAddedEvent(BaseModel):
+ content_index: int = Field(
+ ..., description='The index of the content part that was added.'
+ )
+ item_id: str = Field(
+ ..., description='The ID of the output item that the content part was added to.'
+ )
+ output_index: int = Field(
+ ...,
+ description='The index of the output item that the content part was added to.',
+ )
+ part: OutputContent
+ type: Type20 = Field(
+ ..., description='The type of the event. Always `response.content_part.added`.'
+ )
+
+
+class ResponseContentPartDoneEvent(BaseModel):
+ content_index: int = Field(
+ ..., description='The index of the content part that is done.'
+ )
+ item_id: str = Field(
+ ..., description='The ID of the output item that the content part was added to.'
+ )
+ output_index: int = Field(
+ ...,
+ description='The index of the output item that the content part was added to.',
+ )
+ part: OutputContent
+ type: Type21 = Field(
+ ..., description='The type of the event. Always `response.content_part.done`.'
+ )
+
+
class ResponseError(BaseModel):
code: ResponseErrorCode
message: str = Field(..., description='A human-readable description of the error.')
class Rodin3DDownloadResponse(BaseModel):
- list: Optional[RodinResourceItem] = None
+ list: Optional[List[RodinResourceItem]] = None
class Rodin3DGenerateRequest(BaseModel):
@@ -3371,6 +5547,11 @@ class Rodin3DGenerateResponse(BaseModel):
uuid: Optional[str] = Field(None, description='Task UUID')
+class RodinCheckStatusJobItem(BaseModel):
+ status: Optional[RodinStatusOptions] = None
+ uuid: Optional[str] = Field(None, description='sub uuid')
+
+
class RunwayImageToVideoRequest(BaseModel):
duration: RunwayDurationEnum
model: RunwayModelEnum
@@ -3384,6 +5565,109 @@ class RunwayImageToVideoRequest(BaseModel):
)
+class StripeCharge(BaseModel):
+ amount: Optional[int] = None
+ amount_captured: Optional[int] = None
+ amount_refunded: Optional[int] = None
+ application: Optional[str] = None
+ application_fee: Optional[str] = None
+ application_fee_amount: Optional[int] = None
+ balance_transaction: Optional[str] = None
+ billing_details: Optional[StripeBillingDetails] = None
+ calculated_statement_descriptor: Optional[str] = None
+ captured: Optional[bool] = None
+ created: Optional[int] = None
+ currency: Optional[str] = None
+ customer: Optional[str] = None
+ description: Optional[str] = None
+ destination: Optional[Any] = None
+ dispute: Optional[Any] = None
+ disputed: Optional[bool] = None
+ failure_balance_transaction: Optional[Any] = None
+ failure_code: Optional[Any] = None
+ failure_message: Optional[Any] = None
+ fraud_details: Optional[Dict[str, Any]] = None
+ id: Optional[str] = None
+ invoice: Optional[Any] = None
+ livemode: Optional[bool] = None
+ metadata: Optional[Dict[str, Any]] = None
+ object: Optional[Object1] = None
+ on_behalf_of: Optional[Any] = None
+ order: Optional[Any] = None
+ outcome: Optional[StripeOutcome] = None
+ paid: Optional[bool] = None
+ payment_intent: Optional[str] = None
+ payment_method: Optional[str] = None
+ payment_method_details: Optional[StripePaymentMethodDetails] = None
+ radar_options: Optional[Dict[str, Any]] = None
+ receipt_email: Optional[str] = None
+ receipt_number: Optional[str] = None
+ receipt_url: Optional[str] = None
+ refunded: Optional[bool] = None
+ refunds: Optional[StripeRefundList] = None
+ review: Optional[Any] = None
+ shipping: Optional[StripeShipping] = None
+ source: Optional[Any] = None
+ source_transfer: Optional[Any] = None
+ statement_descriptor: Optional[Any] = None
+ statement_descriptor_suffix: Optional[Any] = None
+ status: Optional[str] = None
+ transfer_data: Optional[Any] = None
+ transfer_group: Optional[Any] = None
+
+
+class StripeChargeList(BaseModel):
+ data: Optional[List[StripeCharge]] = None
+ has_more: Optional[bool] = None
+ object: Optional[str] = None
+ total_count: Optional[int] = None
+ url: Optional[str] = None
+
+
+class StripePaymentIntent(BaseModel):
+ amount: Optional[int] = None
+ amount_capturable: Optional[int] = None
+ amount_details: Optional[StripeAmountDetails] = None
+ amount_received: Optional[int] = None
+ application: Optional[str] = None
+ application_fee_amount: Optional[int] = None
+ automatic_payment_methods: Optional[Any] = None
+ canceled_at: Optional[int] = None
+ cancellation_reason: Optional[str] = None
+ capture_method: Optional[str] = None
+ charges: Optional[StripeChargeList] = None
+ client_secret: Optional[str] = None
+ confirmation_method: Optional[str] = None
+ created: Optional[int] = None
+ currency: Optional[str] = None
+ customer: Optional[str] = None
+ description: Optional[str] = None
+ id: Optional[str] = None
+ invoice: Optional[str] = None
+ last_payment_error: Optional[Any] = None
+ latest_charge: Optional[str] = None
+ livemode: Optional[bool] = None
+ metadata: Optional[Dict[str, Any]] = None
+ next_action: Optional[Any] = None
+ object: Optional[Object3] = None
+ on_behalf_of: Optional[Any] = None
+ payment_method: Optional[str] = None
+ payment_method_configuration_details: Optional[Any] = None
+ payment_method_options: Optional[StripePaymentMethodOptions] = None
+ payment_method_types: Optional[List[str]] = None
+ processing: Optional[Any] = None
+ receipt_email: Optional[str] = None
+ review: Optional[Any] = None
+ setup_future_usage: Optional[Any] = None
+ shipping: Optional[StripeShipping] = None
+ source: Optional[Any] = None
+ statement_descriptor: Optional[Any] = None
+ statement_descriptor_suffix: Optional[Any] = None
+ status: Optional[str] = None
+ transfer_data: Optional[Any] = None
+ transfer_group: Optional[Any] = None
+
+
class TextResponseFormatConfiguration(
RootModel[
Union[
@@ -3411,6 +5695,22 @@ class Tool(
] = Field(..., discriminator='type')
+class BulkNodeVersionResult(BaseModel):
+ error_message: Optional[str] = Field(
+ None,
+ description='Error message if retrieval failed (only present if status is error)',
+ )
+ identifier: NodeVersionIdentifier
+ node_version: Optional[NodeVersion] = None
+ status: Status = Field(..., description='Status of the retrieval operation')
+
+
+class BulkNodeVersionsResponse(BaseModel):
+ node_versions: List[BulkNodeVersionResult] = Field(
+ ..., description='List of retrieved node versions with their status'
+ )
+
+
class EasyInputMessage(BaseModel):
content: Union[str, InputMessageContentList] = Field(
...,
@@ -3439,6 +5739,16 @@ class GeminiGenerateContentRequest(BaseModel):
videoMetadata: Optional[GeminiVideoMetadata] = None
+class GithubReleaseWebhook(BaseModel):
+ action: Action = Field(..., description='The action performed on the release')
+ enterprise: Optional[GithubEnterprise] = None
+ installation: Optional[GithubInstallation] = None
+ organization: Optional[GithubOrganization] = None
+ release: Release = Field(..., description='The release object')
+ repository: GithubRepository
+ sender: GithubUser
+
+
class ImagenGenerateImageRequest(BaseModel):
instances: List[ImagenImageGenerationInstance]
parameters: ImagenImageGenerationParameters
@@ -3447,8 +5757,8 @@ class ImagenGenerateImageRequest(BaseModel):
class InputMessage(BaseModel):
content: Optional[InputMessageContentList] = None
role: Optional[Role3] = None
- status: Optional[Status2] = None
- type: Optional[Type9] = None
+ status: Optional[Status3] = None
+ type: Optional[Type10] = None
class Item(
@@ -3519,6 +5829,70 @@ class OutputItem(
]
+class Publisher(BaseModel):
+ createdAt: Optional[datetime] = Field(
+ None, description='The date and time the publisher was created.'
+ )
+ description: Optional[str] = None
+ id: Optional[str] = Field(
+ None,
+ description="The unique identifier for the publisher. It's akin to a username. Should be lowercase.",
+ )
+ logo: Optional[str] = Field(None, description="URL to the publisher's logo.")
+ members: Optional[List[PublisherMember]] = Field(
+ None, description='A list of members in the publisher.'
+ )
+ name: Optional[str] = None
+ source_code_repo: Optional[str] = None
+ status: Optional[PublisherStatus] = None
+ support: Optional[str] = None
+ website: Optional[str] = None
+
+
+class RecraftGenerateImageResponse(BaseModel):
+ created: int
+ credits: int
+ data: List[RecraftImage]
+
+
+class RecraftImageToImageRequest(BaseModel):
+ block_nsfw: Optional[bool] = None
+ calculate_features: Optional[bool] = None
+ controls: Optional[RecraftUserControls] = None
+ image: StrictBytes
+ image_format: Optional[RecraftImageFormat] = None
+ model: Optional[RecraftTransformModel] = None
+ n: Optional[int] = None
+ negative_prompt: Optional[str] = None
+ prompt: str
+ response_format: Optional[RecraftResponseFormat] = None
+ strength: float
+ style: Optional[RecraftImageStyle] = None
+ style_id: Optional[UUID] = None
+ substyle: Optional[RecraftImageSubStyle] = None
+ text_layout: Optional[RecraftTextLayout] = None
+
+
+class ResponseOutputItemAddedEvent(BaseModel):
+ item: OutputItem
+ output_index: int = Field(
+ ..., description='The index of the output item that was added.\n'
+ )
+ type: Type29 = Field(
+ ..., description='The type of the event. Always `response.output_item.added`.\n'
+ )
+
+
+class ResponseOutputItemDoneEvent(BaseModel):
+ item: OutputItem
+ output_index: int = Field(
+ ..., description='The index of the output item that was marked done.\n'
+ )
+ type: Type30 = Field(
+ ..., description='The type of the event. Always `response.output_item.done`.\n'
+ )
+
+
class Text(BaseModel):
format: Optional[TextResponseFormatConfiguration] = None
@@ -3552,6 +5926,28 @@ class ResponseProperties(BaseModel):
)
+class Rodin3DCheckStatusResponse(BaseModel):
+ jobs: Optional[List[RodinCheckStatusJobItem]] = Field(
+ None, description='Details for the generation status.'
+ )
+
+
+class Data8(BaseModel):
+ object: Optional[StripePaymentIntent] = None
+
+
+class StripeEvent(BaseModel):
+ api_version: Optional[str] = None
+ created: Optional[int] = None
+ data: Data8
+ id: str
+ livemode: Optional[bool] = None
+ object: Object2
+ pending_webhooks: Optional[int] = None
+ request: Optional[StripeRequestInfo] = None
+ type: Type31
+
+
class GeminiCandidate(BaseModel):
citationMetadata: Optional[GeminiCitationMetadata] = None
content: Optional[GeminiContent] = None
@@ -3562,12 +5958,67 @@ class GeminiCandidate(BaseModel):
class GeminiGenerateContentResponse(BaseModel):
candidates: Optional[List[GeminiCandidate]] = None
promptFeedback: Optional[GeminiPromptFeedback] = None
+ usageMetadata: Optional[GeminiUsageMetadata] = None
class InputItem(RootModel[Union[EasyInputMessage, Item]]):
root: Union[EasyInputMessage, Item]
+class Node(BaseModel):
+ author: Optional[str] = None
+ banner_url: Optional[str] = Field(None, description="URL to the node's banner.")
+ category: Optional[str] = Field(None, description='The category of the node.')
+ created_at: Optional[datetime] = Field(
+ None, description='The date and time when the node was created'
+ )
+ description: Optional[str] = None
+ downloads: Optional[int] = Field(
+ None, description='The number of downloads of the node.'
+ )
+ github_stars: Optional[int] = Field(
+ None, description='Number of stars on the GitHub repository.'
+ )
+ icon: Optional[str] = Field(None, description="URL to the node's icon.")
+ id: Optional[str] = Field(None, description='The unique identifier of the node.')
+ latest_version: Optional[NodeVersion] = None
+ license: Optional[str] = Field(
+ None, description="The path to the LICENSE file in the node's repository."
+ )
+ name: Optional[str] = Field(None, description='The display name of the node.')
+ preempted_comfy_node_names: Optional[List[str]] = Field(
+ None, description='A list of Comfy node names that are preempted by this node.'
+ )
+ publisher: Optional[Publisher] = None
+ rating: Optional[float] = Field(None, description='The average rating of the node.')
+ repository: Optional[str] = Field(None, description="URL to the node's repository.")
+ search_ranking: Optional[int] = Field(
+ None,
+ description="A numerical value representing the node's search ranking, used for sorting search results.",
+ )
+ status: Optional[NodeStatus] = None
+ status_detail: Optional[str] = Field(
+ None, description='The status detail of the node.'
+ )
+ supported_accelerators: Optional[List[str]] = Field(
+ None,
+ description='List of accelerators (e.g. CUDA, DirectML, ROCm) that this node supports',
+ )
+ supported_comfyui_frontend_version: Optional[str] = Field(
+ None, description='Supported versions of ComfyUI frontend'
+ )
+ supported_comfyui_version: Optional[str] = Field(
+ None, description='Supported versions of ComfyUI'
+ )
+ supported_os: Optional[List[str]] = Field(
+ None, description='List of operating systems that this node supports'
+ )
+ tags: Optional[List[str]] = None
+ translations: Optional[Dict[str, Dict[str, Any]]] = Field(
+ None, description='Translations of node metadata in different languages.'
+ )
+
+
class OpenAICreateResponse(CreateModelResponseProperties, ResponseProperties):
include: Optional[List[Includable]] = Field(
None,
@@ -3615,8 +6066,73 @@ class OpenAIResponse(ModelResponseProperties, ResponseProperties):
parallel_tool_calls: Optional[bool] = Field(
True, description='Whether to allow the model to run tool calls in parallel.\n'
)
- status: Optional[Status6] = Field(
+ status: Optional[Status7] = Field(
None,
description='The status of the response generation. One of `completed`, `failed`, `in_progress`, or `incomplete`.',
)
usage: Optional[ResponseUsage] = None
+
+
+class ResponseCompletedEvent(BaseModel):
+ response: OpenAIResponse
+ type: Type19 = Field(
+ ..., description='The type of the event. Always `response.completed`.'
+ )
+
+
+class ResponseCreatedEvent(BaseModel):
+ response: OpenAIResponse
+ type: Type22 = Field(
+ ..., description='The type of the event. Always `response.created`.'
+ )
+
+
+class ResponseFailedEvent(BaseModel):
+ response: OpenAIResponse
+ type: Type24 = Field(
+ ..., description='The type of the event. Always `response.failed`.\n'
+ )
+
+
+class ResponseInProgressEvent(BaseModel):
+ response: OpenAIResponse
+ type: Type27 = Field(
+ ..., description='The type of the event. Always `response.in_progress`.\n'
+ )
+
+
+class ResponseIncompleteEvent(BaseModel):
+ response: OpenAIResponse
+ type: Type28 = Field(
+ ..., description='The type of the event. Always `response.incomplete`.\n'
+ )
+
+
+class OpenAIResponseStreamEvent(
+ RootModel[
+ Union[
+ ResponseCreatedEvent,
+ ResponseInProgressEvent,
+ ResponseCompletedEvent,
+ ResponseFailedEvent,
+ ResponseIncompleteEvent,
+ ResponseOutputItemAddedEvent,
+ ResponseOutputItemDoneEvent,
+ ResponseContentPartAddedEvent,
+ ResponseContentPartDoneEvent,
+ ResponseErrorEvent,
+ ]
+ ]
+):
+ root: Union[
+ ResponseCreatedEvent,
+ ResponseInProgressEvent,
+ ResponseCompletedEvent,
+ ResponseFailedEvent,
+ ResponseIncompleteEvent,
+ ResponseOutputItemAddedEvent,
+ ResponseOutputItemDoneEvent,
+ ResponseContentPartAddedEvent,
+ ResponseContentPartDoneEvent,
+ ResponseErrorEvent,
+ ] = Field(..., description='Events that can be emitted during response streaming')
diff --git a/comfy_api_nodes/apis/client.py b/comfy_api_nodes/apis/client.py
index 2a4bac88b..4ad0b783b 100644
--- a/comfy_api_nodes/apis/client.py
+++ b/comfy_api_nodes/apis/client.py
@@ -43,7 +43,7 @@ operation = ApiOperation(
endpoint=user_info_endpoint,
request=request
)
-user_profile = operation.execute(client=api_client) # Returns immediately with the result
+user_profile = await operation.execute(client=api_client) # Returns immediately with the result
# Example 2: Asynchronous API Operation with Polling
@@ -87,18 +87,19 @@ operation = PollingOperation(
)
# This will make the initial request and then poll until completion
-result = operation.execute(client=api_client) # Returns the final ImageGenerationResult when done
+result = await operation.execute(client=api_client) # Returns the final ImageGenerationResult when done
"""
from __future__ import annotations
+import aiohttp
+import asyncio
import logging
-import time
import io
import socket
+from aiohttp.client_exceptions import ClientError, ClientResponseError
from typing import Dict, Type, Optional, Any, TypeVar, Generic, Callable, Tuple
from enum import Enum
import json
-import requests
from urllib.parse import urljoin, urlparse
from pydantic import BaseModel, Field
import uuid # For generating unique operation IDs
@@ -174,6 +175,7 @@ class ApiClient:
retry_delay: float = 1.0,
retry_backoff_factor: float = 2.0,
retry_status_codes: Optional[Tuple[int, ...]] = None,
+ session: Optional[aiohttp.ClientSession] = None,
):
self.base_url = base_url
self.auth_token = auth_token
@@ -186,13 +188,16 @@ class ApiClient:
# Default retry status codes: 408 (Request Timeout), 429 (Too Many Requests),
# 500, 502, 503, 504 (Server Errors)
self.retry_status_codes = retry_status_codes or (408, 429, 500, 502, 503, 504)
+ self._session: Optional[aiohttp.ClientSession] = session
+ self._owns_session = session is None # Track if we have to close it
- def _generate_operation_id(self, path: str) -> str:
+ @staticmethod
+ def _generate_operation_id(path: str) -> str:
"""Generates a unique operation ID for logging."""
return f"{path.strip('/').replace('/', '_')}_{uuid.uuid4().hex[:8]}"
+ @staticmethod
def _create_json_payload_args(
- self,
data: Optional[Dict[str, Any]] = None,
headers: Optional[Dict[str, str]] = None,
) -> Dict[str, Any]:
@@ -203,31 +208,53 @@ class ApiClient:
def _create_form_data_args(
self,
- data: Dict[str, Any],
- files: Dict[str, Any],
+ data: Dict[str, Any] | None,
+ files: Dict[str, Any] | None,
headers: Optional[Dict[str, str]] = None,
- multipart_parser = None,
+ multipart_parser: Callable | None = None,
) -> Dict[str, Any]:
if headers and "Content-Type" in headers:
del headers["Content-Type"]
- if multipart_parser:
+ if multipart_parser and data:
data = multipart_parser(data)
- return {
- "data": data,
- "files": files,
- "headers": headers,
- }
+ form = aiohttp.FormData(default_to_multipart=True)
+ if data: # regular text fields
+ for k, v in data.items():
+ if v is None:
+ continue # aiohttp fails to serialize "None" values
+ # aiohttp expects strings or bytes; convert enums etc.
+ form.add_field(k, str(v) if not isinstance(v, (bytes, bytearray)) else v)
+ if files:
+ file_iter = files if isinstance(files, list) else files.items()
+ for field_name, file_obj in file_iter:
+ if file_obj is None:
+ continue # aiohttp fails to serialize "None" values
+ # file_obj can be (filename, bytes/io.BytesIO, content_type) tuple
+ if isinstance(file_obj, tuple):
+ filename, file_value, content_type = self._unpack_tuple(file_obj)
+ else:
+ file_value = file_obj
+ filename = getattr(file_obj, "name", field_name)
+ content_type = "application/octet-stream"
+
+ form.add_field(
+ name=field_name,
+ value=file_value,
+ filename=filename,
+ content_type=content_type,
+ )
+ return {"data": form, "headers": headers or {}}
+
+ @staticmethod
def _create_urlencoded_form_data_args(
- self,
data: Dict[str, Any],
headers: Optional[Dict[str, str]] = None,
) -> Dict[str, Any]:
headers = headers or {}
headers["Content-Type"] = "application/x-www-form-urlencoded"
-
return {
"data": data,
"headers": headers,
@@ -244,7 +271,7 @@ class ApiClient:
return headers
- def _check_connectivity(self, target_url: str) -> Dict[str, bool]:
+ async def _check_connectivity(self, target_url: str) -> Dict[str, bool]:
"""
Check connectivity to determine if network issues are local or server-related.
@@ -258,52 +285,39 @@ class ApiClient:
"internet_accessible": False,
"api_accessible": False,
"is_local_issue": False,
- "is_api_issue": False
+ "is_api_issue": False,
}
+ timeout = aiohttp.ClientTimeout(total=5.0)
+ async with aiohttp.ClientSession(timeout=timeout) as session:
+ try:
+ async with session.get("https://www.google.com", ssl=self.verify_ssl) as resp:
+ results["internet_accessible"] = resp.status < 500
+ except (ClientError, asyncio.TimeoutError, socket.gaierror):
+ results["is_local_issue"] = True
+ return results # cannot reach the internet – early exit
- # First check basic internet connectivity using a reliable external site
- try:
- # Use a reliable external domain for checking basic connectivity
- check_response = requests.get("https://www.google.com",
- timeout=5.0,
- verify=self.verify_ssl)
- if check_response.status_code < 500:
- results["internet_accessible"] = True
- except (requests.RequestException, socket.error):
- results["internet_accessible"] = False
- results["is_local_issue"] = True
- return results
-
- # Now check API server connectivity
- try:
- # Extract domain from the target URL to do a simpler health check
- parsed_url = urlparse(target_url)
- api_base = f"{parsed_url.scheme}://{parsed_url.netloc}"
-
- # Try to reach the API domain
- api_response = requests.get(f"{api_base}/health", timeout=5.0, verify=self.verify_ssl)
- if api_response.status_code < 500:
- results["api_accessible"] = True
- else:
- results["api_accessible"] = False
- results["is_api_issue"] = True
- except requests.RequestException:
- results["api_accessible"] = False
- # If we can reach the internet but not the API, it's an API issue
- results["is_api_issue"] = True
+ # Now check API health endpoint
+ parsed = urlparse(target_url)
+ health_url = f"{parsed.scheme}://{parsed.netloc}/health"
+ try:
+ async with session.get(health_url, ssl=self.verify_ssl) as resp:
+ results["api_accessible"] = resp.status < 500
+ except ClientError:
+ pass # leave as False
+ results["is_api_issue"] = results["internet_accessible"] and not results["api_accessible"]
return results
- def request(
+ async def request(
self,
method: str,
path: str,
params: Optional[Dict[str, Any]] = None,
data: Optional[Dict[str, Any]] = None,
- files: Optional[Dict[str, Any]] = None,
+ files: Optional[Dict[str, Any] | list[tuple[str, Any]]] = None,
headers: Optional[Dict[str, str]] = None,
content_type: str = "application/json",
- multipart_parser: Callable = None,
+ multipart_parser: Callable | None = None,
retry_count: int = 0, # Used internally for tracking retries
) -> Dict[str, Any]:
"""
@@ -327,18 +341,19 @@ class ApiClient:
ApiServerError: If the API server is unreachable but internet is working
Exception: For other request failures
"""
- # Use urljoin but ensure path is relative to avoid absolute path behavior
- relative_path = path.lstrip('/')
+
+ # Build full URL and merge headers
+ relative_path = path.lstrip("/")
url = urljoin(self.base_url, relative_path)
- self.check_auth(self.auth_token, self.comfy_api_key)
- # Combine default headers with any provided headers
+ self._check_auth(self.auth_token, self.comfy_api_key)
+
request_headers = self.get_headers()
if headers:
request_headers.update(headers)
-
- # Let requests handle the content type when files are present.
if files:
- del request_headers["Content-Type"]
+ request_headers.pop("Content-Type", None)
+ if params:
+ params = {k: v for k, v in params.items() if v is not None} # aiohttp fails to serialize None values
logging.debug(f"[DEBUG] Request Headers: {request_headers}")
logging.debug(f"[DEBUG] Files: {files}")
@@ -346,11 +361,9 @@ class ApiClient:
logging.debug(f"[DEBUG] Data: {data}")
if content_type == "application/x-www-form-urlencoded":
- payload_args = self._create_urlencoded_form_data_args(data, request_headers)
+ payload_args = self._create_urlencoded_form_data_args(data or {}, request_headers)
elif content_type == "multipart/form-data":
- payload_args = self._create_form_data_args(
- data, files, request_headers, multipart_parser
- )
+ payload_args = self._create_form_data_args(data, files, request_headers, multipart_parser)
else:
payload_args = self._create_json_payload_args(data, request_headers)
@@ -361,220 +374,67 @@ class ApiClient:
request_url=url,
request_headers=request_headers,
request_params=params,
- request_data=data if content_type == "application/json" else "[form-data or other]"
+ request_data=data if content_type == "application/json" else "[form-data or other]",
)
+ session = await self._get_session()
try:
- response = requests.request(
- method=method,
- url=url,
+ async with session.request(
+ method,
+ url,
params=params,
- timeout=self.timeout,
- verify=self.verify_ssl,
+ ssl=self.verify_ssl,
**payload_args,
- )
+ ) as resp:
+ if resp.status >= 400:
+ try:
+ error_data = await resp.json()
+ except (aiohttp.ContentTypeError, json.JSONDecodeError):
+ error_data = await resp.text()
- # Check if we should retry based on status code
- if (response.status_code in self.retry_status_codes and
- retry_count < self.max_retries):
+ return await self._handle_http_error(
+ ClientResponseError(resp.request_info, resp.history, status=resp.status, message=error_data),
+ operation_id,
+ method,
+ url,
+ params,
+ data,
+ files,
+ headers,
+ content_type,
+ multipart_parser,
+ retry_count=retry_count,
+ response_content=error_data,
+ )
- # Calculate delay with exponential backoff
- delay = self.retry_delay * (self.retry_backoff_factor ** retry_count)
-
- logging.warning(
- f"Request failed with status {response.status_code}. "
- f"Retrying in {delay:.2f}s ({retry_count + 1}/{self.max_retries})"
- )
-
- time.sleep(delay)
- return self.request(
- method=method,
- path=path,
- params=params,
- data=data,
- files=files,
- headers=headers,
- content_type=content_type,
- multipart_parser=multipart_parser,
- retry_count=retry_count + 1,
- )
-
- # Raise exception for error status codes
- response.raise_for_status()
-
- # Log successful response
- response_content_to_log = response.content
- try:
- # Attempt to parse JSON for prettier logging, fallback to raw content
- response_content_to_log = response.json()
- except json.JSONDecodeError:
- pass # Keep as bytes/str if not JSON
-
- request_logger.log_request_response(
- operation_id=operation_id,
- request_method=method, # Pass request details again for context in log
- request_url=url,
- response_status_code=response.status_code,
- response_headers=dict(response.headers),
- response_content=response_content_to_log
- )
-
- except requests.ConnectionError as e:
- error_message = f"ConnectionError: {str(e)}"
- request_logger.log_request_response(
- operation_id=operation_id,
- request_method=method,
- request_url=url,
- error_message=error_message
- )
- # Only perform connectivity check if we've exhausted all retries
- if retry_count >= self.max_retries:
- # Check connectivity to determine if it's a local or API issue
- connectivity = self._check_connectivity(self.base_url)
-
- if connectivity["is_local_issue"]:
- raise LocalNetworkError(
- "Unable to connect to the API server due to local network issues. "
- "Please check your internet connection and try again."
- ) from e
- elif connectivity["is_api_issue"]:
- raise ApiServerError(
- f"The API server at {self.base_url} is currently unreachable. "
- f"The service may be experiencing issues. Please try again later."
- ) from e
-
- # If we haven't exhausted retries yet, retry the request
- if retry_count < self.max_retries:
- delay = self.retry_delay * (self.retry_backoff_factor ** retry_count)
- logging.warning(
- f"Connection error: {str(e)}. "
- f"Retrying in {delay:.2f}s ({retry_count + 1}/{self.max_retries})"
- )
- time.sleep(delay)
- return self.request(
- method=method,
- path=path,
- params=params,
- data=data,
- files=files,
- headers=headers,
- content_type=content_type,
- multipart_parser=multipart_parser,
- retry_count=retry_count + 1,
- )
-
- # If we've exhausted retries and didn't identify the specific issue,
- # raise a generic exception
- final_error_message = (
- f"Unable to connect to the API server after {self.max_retries} attempts. "
- f"Please check your internet connection or try again later."
- )
- request_logger.log_request_response( # Log final failure
- operation_id=operation_id,
- request_method=method, request_url=url,
- error_message=final_error_message
- )
- raise Exception(final_error_message) from e
-
- except requests.Timeout as e:
- error_message = f"Timeout: {str(e)}"
- request_logger.log_request_response(
- operation_id=operation_id,
- request_method=method, request_url=url,
- error_message=error_message
- )
- # Retry timeouts if we haven't exhausted retries
- if retry_count < self.max_retries:
- delay = self.retry_delay * (self.retry_backoff_factor ** retry_count)
- logging.warning(
- f"Request timed out. "
- f"Retrying in {delay:.2f}s ({retry_count + 1}/{self.max_retries})"
- )
- time.sleep(delay)
- return self.request(
- method=method,
- path=path,
- params=params,
- data=data,
- files=files,
- headers=headers,
- content_type=content_type,
- multipart_parser=multipart_parser,
- retry_count=retry_count + 1,
- )
- final_error_message = (
- f"Request timed out after {self.timeout} seconds and {self.max_retries} retry attempts. "
- f"The server might be experiencing high load or the operation is taking longer than expected."
- )
- request_logger.log_request_response( # Log final failure
- operation_id=operation_id,
- request_method=method, request_url=url,
- error_message=final_error_message
- )
- raise Exception(final_error_message) from e
-
- except requests.HTTPError as e:
- status_code = e.response.status_code if hasattr(e, "response") else None
- original_error_message = f"HTTP Error: {str(e)}"
- error_content_for_log = None
- if hasattr(e, "response") and e.response is not None:
- error_content_for_log = e.response.content
+ # Success – parse JSON (safely) and log
try:
- error_content_for_log = e.response.json()
- except json.JSONDecodeError:
- pass
+ payload = await resp.json()
+ response_content_to_log = payload
+ except (aiohttp.ContentTypeError, json.JSONDecodeError):
+ payload = {}
+ response_content_to_log = await resp.text()
-
- # Try to extract detailed error message from JSON response for user display
- # but log the full error content.
- user_display_error_message = original_error_message
-
- try:
- if hasattr(e, "response") and e.response is not None and e.response.content:
- error_json = e.response.json()
- if "error" in error_json and "message" in error_json["error"]:
- user_display_error_message = f"API Error: {error_json['error']['message']}"
- if "type" in error_json["error"]:
- user_display_error_message += f" (Type: {error_json['error']['type']})"
- elif isinstance(error_json, dict): # Handle cases where error is just a JSON dict
- user_display_error_message = f"API Error: {json.dumps(error_json)}"
- else: # Non-dict JSON error
- user_display_error_message = f"API Error: {str(error_json)}"
- except json.JSONDecodeError:
- # If not JSON, use the raw content if it's not too long, or a summary
- if hasattr(e, "response") and e.response is not None and e.response.content:
- raw_content = e.response.content.decode(errors='ignore')
- if len(raw_content) < 200: # Arbitrary limit for display
- user_display_error_message = f"API Error (raw): {raw_content}"
- else:
- user_display_error_message = f"API Error (raw, status {status_code})"
-
- request_logger.log_request_response(
- operation_id=operation_id,
- request_method=method, request_url=url,
- response_status_code=status_code,
- response_headers=dict(e.response.headers) if hasattr(e, "response") and e.response is not None else None,
- response_content=error_content_for_log,
- error_message=original_error_message # Log the original exception string as error
- )
-
- logging.debug(f"[DEBUG] API Error: {user_display_error_message} (Status: {status_code})")
- if hasattr(e, "response") and e.response is not None and e.response.content:
- logging.debug(f"[DEBUG] Response content: {e.response.content}")
-
- # Retry if the status code is in our retry list and we haven't exhausted retries
- if (status_code in self.retry_status_codes and
- retry_count < self.max_retries):
-
- delay = self.retry_delay * (self.retry_backoff_factor ** retry_count)
- logging.warning(
- f"HTTP error {status_code}. "
- f"Retrying in {delay:.2f}s ({retry_count + 1}/{self.max_retries})"
+ request_logger.log_request_response(
+ operation_id=operation_id,
+ request_method=method,
+ request_url=url,
+ response_status_code=resp.status,
+ response_headers=dict(resp.headers),
+ response_content=response_content_to_log,
)
- time.sleep(delay)
- return self.request(
- method=method,
- path=path,
+ return payload
+
+ except (ClientError, asyncio.TimeoutError, socket.gaierror) as e:
+ # Treat as *connection* problem – optionally retry, else escalate
+ if retry_count < self.max_retries:
+ delay = self.retry_delay * (self.retry_backoff_factor ** retry_count)
+ logging.warning("Connection error. Retrying in %.2fs (%s/%s): %s", delay, retry_count + 1,
+ self.max_retries, str(e))
+ await asyncio.sleep(delay)
+ return await self.request(
+ method,
+ path,
params=params,
data=data,
files=files,
@@ -583,40 +443,34 @@ class ApiClient:
multipart_parser=multipart_parser,
retry_count=retry_count + 1,
)
+ # One final connectivity check for diagnostics
+ connectivity = await self._check_connectivity(self.base_url)
+ if connectivity["is_local_issue"]:
+ raise LocalNetworkError(
+ "Unable to connect to the API server due to local network issues. "
+ "Please check your internet connection and try again."
+ ) from e
+ raise ApiServerError(
+ f"The API server at {self.base_url} is currently unreachable. "
+ f"The service may be experiencing issues. Please try again later."
+ ) from e
- # Specific error messages for common status codes for user display
- if status_code == 401:
- user_display_error_message = "Unauthorized: Please login first to use this node."
- elif status_code == 402:
- user_display_error_message = "Payment Required: Please add credits to your account to use this node."
- elif status_code == 409:
- user_display_error_message = "There is a problem with your account. Please contact support@comfy.org."
- elif status_code == 429:
- user_display_error_message = "Rate Limit Exceeded: Please try again later."
- # else, user_display_error_message remains as parsed from response or original HTTPError string
-
- raise Exception(user_display_error_message) # Raise with the user-friendly message
-
- # Parse and return JSON response
- if response.content:
- return response.json()
- return {}
-
- def check_auth(self, auth_token, comfy_api_key):
+ @staticmethod
+ def _check_auth(auth_token, comfy_api_key):
"""Verify that an auth token is present or comfy_api_key is present"""
if auth_token is None and comfy_api_key is None:
raise Exception("Unauthorized: Please login first to use this node.")
return auth_token or comfy_api_key
@staticmethod
- def upload_file(
+ async def upload_file(
upload_url: str,
file: io.BytesIO | str,
content_type: str | None = None,
max_retries: int = 3,
retry_delay: float = 1.0,
retry_backoff_factor: float = 2.0,
- ):
+ ) -> aiohttp.ClientResponse:
"""Upload a file to the API with retry logic.
Args:
@@ -627,112 +481,167 @@ class ApiClient:
retry_delay: Initial delay between retries in seconds
retry_backoff_factor: Multiplier for the delay after each retry
"""
- headers = {}
+ headers: Dict[str, str] = {}
+ skip_auto_headers: set[str] = set()
if content_type:
headers["Content-Type"] = content_type
+ else:
+ # tell aiohttp not to add Content-Type that will break the request signature and result in a 403 status.
+ skip_auto_headers.add("Content-Type")
- # Prepare the file data
+ # Extract file bytes
if isinstance(file, io.BytesIO):
- file.seek(0) # Ensure we're at the start of the file
+ file.seek(0)
data = file.read()
elif isinstance(file, str):
with open(file, "rb") as f:
data = f.read()
else:
- raise ValueError("File must be either a BytesIO object or a file path string")
+ raise ValueError("File must be BytesIO or str path")
- # Try the upload with retries
- last_exception = None
- operation_id = f"upload_{upload_url.split('/')[-1]}_{uuid.uuid4().hex[:8]}" # Simplified ID for uploads
-
- # Log initial attempt (without full file data for brevity)
+ operation_id = f"upload_{upload_url.split('/')[-1]}_{uuid.uuid4().hex[:8]}"
request_logger.log_request_response(
operation_id=operation_id,
request_method="PUT",
request_url=upload_url,
request_headers=headers,
- request_data=f"[File data of type {content_type or 'unknown'}, size {len(data)} bytes]"
+ request_data=f"[File data {len(data)} bytes]",
)
- for retry_attempt in range(max_retries + 1):
+ delay = retry_delay
+ for attempt in range(max_retries + 1):
try:
- response = requests.put(upload_url, data=data, headers=headers)
- response.raise_for_status()
+ timeout = aiohttp.ClientTimeout(total=None) # honour server side timeouts
+ async with aiohttp.ClientSession(timeout=timeout) as session:
+ async with session.put(
+ upload_url, data=data, headers=headers, skip_auto_headers=skip_auto_headers,
+ ) as resp:
+ resp.raise_for_status()
+ request_logger.log_request_response(
+ operation_id=operation_id,
+ request_method="PUT",
+ request_url=upload_url,
+ response_status_code=resp.status,
+ response_headers=dict(resp.headers),
+ response_content="File uploaded successfully.",
+ )
+ return resp
+ except (ClientError, asyncio.TimeoutError) as e:
request_logger.log_request_response(
operation_id=operation_id,
- request_method="PUT", request_url=upload_url, # For context
- response_status_code=response.status_code,
- response_headers=dict(response.headers),
- response_content="File uploaded successfully." # Or response.text if available
+ request_method="PUT",
+ request_url=upload_url,
+ response_status_code=e.status if hasattr(e, "status") else None,
+ response_headers=dict(e.headers) if getattr(e, "headers") else None,
+ response_content=None,
+ error_message=f"{type(e).__name__}: {str(e)}",
)
- return response
-
- except (requests.ConnectionError, requests.Timeout, requests.HTTPError) as e:
- last_exception = e
- error_message_for_log = f"{type(e).__name__}: {str(e)}"
- response_content_for_log = None
- status_code_for_log = None
- headers_for_log = None
-
- if hasattr(e, 'response') and e.response is not None:
- status_code_for_log = e.response.status_code
- headers_for_log = dict(e.response.headers)
- try:
- response_content_for_log = e.response.json()
- except json.JSONDecodeError:
- response_content_for_log = e.response.content
-
-
- request_logger.log_request_response(
- operation_id=operation_id,
- request_method="PUT", request_url=upload_url,
- response_status_code=status_code_for_log,
- response_headers=headers_for_log,
- response_content=response_content_for_log,
- error_message=error_message_for_log
- )
-
- if retry_attempt < max_retries:
- delay = retry_delay * (retry_backoff_factor ** retry_attempt)
+ if attempt < max_retries:
logging.warning(
- f"File upload failed: {str(e)}. "
- f"Retrying in {delay:.2f}s ({retry_attempt + 1}/{max_retries})"
+ "Upload failed (%s/%s). Retrying in %.2fs. %s", attempt + 1, max_retries, delay, str(e)
)
- time.sleep(delay)
+ await asyncio.sleep(delay)
+ delay *= retry_backoff_factor
else:
- break # Max retries reached
+ raise NetworkError(f"Failed to upload file after {max_retries + 1} attempts: {e}") from e
- # If we've exhausted all retries, determine the final error type and raise
- final_error_message = f"Failed to upload file after {max_retries + 1} attempts. Error: {str(last_exception)}"
- try:
- # Check basic internet connectivity
- check_response = requests.get("https://www.google.com", timeout=5.0, verify=True) # Assuming verify=True is desired
- if check_response.status_code >= 500: # Google itself has an issue (rare)
- final_error_message = (f"Failed to upload file. Internet connectivity check to Google failed "
- f"(status {check_response.status_code}). Original error: {str(last_exception)}")
- # Not raising LocalNetworkError here as Google itself might be down.
- # If Google is reachable, the issue is likely with the upload server or a more specific local problem
- # not caught by a simple Google ping (e.g., DNS for the specific upload URL, firewall).
- # The original last_exception is probably most relevant.
+ async def _handle_http_error(
+ self,
+ exc: ClientResponseError,
+ operation_id: str,
+ *req_meta,
+ retry_count: int,
+ response_content: dict | str = "",
+ ) -> Dict[str, Any]:
+ status_code = exc.status
+ if status_code == 401:
+ user_friendly = "Unauthorized: Please login first to use this node."
+ elif status_code == 402:
+ user_friendly = "Payment Required: Please add credits to your account to use this node."
+ elif status_code == 409:
+ user_friendly = "There is a problem with your account. Please contact support@comfy.org."
+ elif status_code == 429:
+ user_friendly = "Rate Limit Exceeded: Please try again later."
+ else:
+ if isinstance(response_content, dict):
+ if "error" in response_content and "message" in response_content["error"]:
+ user_friendly = f"API Error: {response_content['error']['message']}"
+ if "type" in response_content["error"]:
+ user_friendly += f" (Type: {response_content['error']['type']})"
+ else: # Handle cases where error is just a JSON dict with unknown format
+ user_friendly = f"API Error: {json.dumps(response_content)}"
+ else:
+ if len(response_content) < 200: # Arbitrary limit for display
+ user_friendly = f"API Error (raw): {response_content}"
+ else:
+ user_friendly = f"API Error (raw, status {response_content})"
- except (requests.RequestException, socket.error) as conn_check_exc:
- # Could not reach Google, likely a local network issue
- final_error_message = (f"Failed to upload file due to network connectivity issues "
- f"(cannot reach Google: {str(conn_check_exc)}). "
- f"Original upload error: {str(last_exception)}")
- request_logger.log_request_response( # Log final failure reason
- operation_id=operation_id,
- request_method="PUT", request_url=upload_url,
- error_message=final_error_message
- )
- raise LocalNetworkError(final_error_message) from last_exception
-
- request_logger.log_request_response( # Log final failure reason if not LocalNetworkError
+ request_logger.log_request_response(
operation_id=operation_id,
- request_method="PUT", request_url=upload_url,
- error_message=final_error_message
+ request_method=req_meta[0],
+ request_url=req_meta[1],
+ response_status_code=exc.status,
+ response_headers=dict(req_meta[5]) if req_meta[5] else None,
+ response_content=response_content,
+ error_message=f"HTTP Error {exc.status}",
)
- raise Exception(final_error_message) from last_exception
+
+ logging.debug(f"[DEBUG] API Error: {user_friendly} (Status: {status_code})")
+ if response_content:
+ logging.debug(f"[DEBUG] Response content: {response_content}")
+
+ # Retry if eligible
+ if status_code in self.retry_status_codes and retry_count < self.max_retries:
+ delay = self.retry_delay * (self.retry_backoff_factor ** retry_count)
+ logging.warning(
+ "HTTP error %s. Retrying in %.2fs (%s/%s)",
+ status_code,
+ delay,
+ retry_count + 1,
+ self.max_retries,
+ )
+ await asyncio.sleep(delay)
+ return await self.request(
+ req_meta[0], # method
+ req_meta[1].replace(self.base_url, ""), # path
+ params=req_meta[2],
+ data=req_meta[3],
+ files=req_meta[4],
+ headers=req_meta[5],
+ content_type=req_meta[6],
+ multipart_parser=req_meta[7],
+ retry_count=retry_count + 1,
+ )
+
+ raise Exception(user_friendly) from exc
+
+ @staticmethod
+ def _unpack_tuple(t):
+ """Helper to normalise (filename, file, content_type) tuples."""
+ if len(t) == 3:
+ return t
+ elif len(t) == 2:
+ return t[0], t[1], "application/octet-stream"
+ else:
+ raise ValueError("files tuple must be (filename, file[, content_type])")
+
+ async def _get_session(self) -> aiohttp.ClientSession:
+ if self._session is None or self._session.closed:
+ timeout = aiohttp.ClientTimeout(total=self.timeout)
+ self._session = aiohttp.ClientSession(timeout=timeout)
+ self._owns_session = True
+ return self._session
+
+ async def close(self) -> None:
+ if self._owns_session and self._session and not self._session.closed:
+ await self._session.close()
+
+ async def __aenter__(self) -> "ApiClient":
+ """Allow usage as async‑context‑manager – ensures clean teardown"""
+ return self
+
+ async def __aexit__(self, exc_type, exc, tb):
+ await self.close()
class ApiEndpoint(Generic[T, R]):
@@ -763,31 +672,28 @@ class ApiEndpoint(Generic[T, R]):
class SynchronousOperation(Generic[T, R]):
- """
- Represents a single synchronous API operation.
- """
+ """Represents a single synchronous API operation."""
def __init__(
self,
endpoint: ApiEndpoint[T, R],
request: T,
- files: Optional[Dict[str, Any]] = None,
+ files: Optional[Dict[str, Any] | list[tuple[str, Any]]] = None,
api_base: str | None = None,
auth_token: Optional[str] = None,
comfy_api_key: Optional[str] = None,
- auth_kwargs: Optional[Dict[str,str]] = None,
+ auth_kwargs: Optional[Dict[str, str]] = None,
timeout: float = 604800.0,
verify_ssl: bool = True,
content_type: str = "application/json",
- multipart_parser: Callable = None,
+ multipart_parser: Callable | None = None,
max_retries: int = 3,
retry_delay: float = 1.0,
retry_backoff_factor: float = 2.0,
- ):
+ ) -> None:
self.endpoint = endpoint
self.request = request
- self.response = None
- self.error = None
+ self.files = files
self.api_base: str = api_base or args.comfy_api_base
self.auth_token = auth_token
self.comfy_api_key = comfy_api_key
@@ -796,91 +702,64 @@ class SynchronousOperation(Generic[T, R]):
self.comfy_api_key = auth_kwargs.get("comfy_api_key", self.comfy_api_key)
self.timeout = timeout
self.verify_ssl = verify_ssl
- self.files = files
self.content_type = content_type
self.multipart_parser = multipart_parser
self.max_retries = max_retries
self.retry_delay = retry_delay
self.retry_backoff_factor = retry_backoff_factor
- def execute(self, client: Optional[ApiClient] = None) -> R:
- """Execute the API operation using the provided client or create one with retry support"""
- try:
- # Create client if not provided
- if client is None:
- client = ApiClient(
- base_url=self.api_base,
- auth_token=self.auth_token,
- comfy_api_key=self.comfy_api_key,
- timeout=self.timeout,
- verify_ssl=self.verify_ssl,
- max_retries=self.max_retries,
- retry_delay=self.retry_delay,
- retry_backoff_factor=self.retry_backoff_factor,
- )
-
- # Convert request model to dict, but use None for EmptyRequest
- request_dict = (
- None
- if isinstance(self.request, EmptyRequest)
- else self.request.model_dump(exclude_none=True)
+ async def execute(self, client: Optional[ApiClient] = None) -> R:
+ owns_client = client is None
+ if owns_client:
+ client = ApiClient(
+ base_url=self.api_base,
+ auth_token=self.auth_token,
+ comfy_api_key=self.comfy_api_key,
+ timeout=self.timeout,
+ verify_ssl=self.verify_ssl,
+ max_retries=self.max_retries,
+ retry_delay=self.retry_delay,
+ retry_backoff_factor=self.retry_backoff_factor,
)
- if request_dict:
- for key, value in request_dict.items():
- if isinstance(value, Enum):
- request_dict[key] = value.value
- # Debug log for request
+ try:
+ request_dict: Optional[Dict[str, Any]]
+ if isinstance(self.request, EmptyRequest):
+ request_dict = None
+ else:
+ request_dict = self.request.model_dump(exclude_none=True)
+ for k, v in list(request_dict.items()):
+ if isinstance(v, Enum):
+ request_dict[k] = v.value
+
logging.debug(
f"[DEBUG] API Request: {self.endpoint.method.value} {self.endpoint.path}"
)
logging.debug(f"[DEBUG] Request Data: {json.dumps(request_dict, indent=2)}")
logging.debug(f"[DEBUG] Query Params: {self.endpoint.query_params}")
- # Make the request with built-in retry
- resp = client.request(
- method=self.endpoint.method.value,
- path=self.endpoint.path,
- data=request_dict,
+ response_json = await client.request(
+ self.endpoint.method.value,
+ self.endpoint.path,
params=self.endpoint.query_params,
+ data=request_dict,
files=self.files,
content_type=self.content_type,
- multipart_parser=self.multipart_parser
+ multipart_parser=self.multipart_parser,
)
- # Debug log for response
logging.debug("=" * 50)
logging.debug("[DEBUG] RESPONSE DETAILS:")
logging.debug("[DEBUG] Status Code: 200 (Success)")
- logging.debug(f"[DEBUG] Response Body: {json.dumps(resp, indent=2)}")
+ logging.debug(f"[DEBUG] Response Body: {json.dumps(response_json, indent=2)}")
logging.debug("=" * 50)
- # Parse and return the response
- return self._parse_response(resp)
-
- except LocalNetworkError as e:
- # Propagate specific network error types
- logging.error(f"[ERROR] Local network error: {str(e)}")
- raise
-
- except ApiServerError as e:
- # Propagate API server errors
- logging.error(f"[ERROR] API server error: {str(e)}")
- raise
-
- except Exception as e:
- logging.error(f"[ERROR] API Exception: {str(e)}")
- raise Exception(str(e))
-
- def _parse_response(self, resp):
- """Parse response data - can be overridden by subclasses"""
- # The response is already the complete object, don't extract just the "data" field
- # as that would lose the outer structure (created timestamp, etc.)
-
- # Parse response using the provided model
- self.response = self.endpoint.response_model.model_validate(resp)
- logging.debug(f"[DEBUG] Parsed Response: {self.response}")
- return self.response
+ parsed_response = self.endpoint.response_model.model_validate(response_json)
+ logging.debug(f"[DEBUG] Parsed Response: {parsed_response}")
+ return parsed_response
+ finally:
+ if owns_client:
+ await client.close()
class TaskStatus(str, Enum):
@@ -892,23 +771,21 @@ class TaskStatus(str, Enum):
class PollingOperation(Generic[T, R]):
- """
- Represents an asynchronous API operation that requires polling for completion.
- """
+ """Represents an asynchronous API operation that requires polling for completion."""
def __init__(
self,
poll_endpoint: ApiEndpoint[EmptyRequest, R],
- completed_statuses: list,
- failed_statuses: list,
+ completed_statuses: list[str],
+ failed_statuses: list[str],
status_extractor: Callable[[R], str],
- progress_extractor: Callable[[R], float] = None,
- result_url_extractor: Callable[[R], str] = None,
+ progress_extractor: Callable[[R], float] | None = None,
+ result_url_extractor: Callable[[R], str] | None = None,
request: Optional[T] = None,
api_base: str | None = None,
auth_token: Optional[str] = None,
comfy_api_key: Optional[str] = None,
- auth_kwargs: Optional[Dict[str,str]] = None,
+ auth_kwargs: Optional[Dict[str, str]] = None,
poll_interval: float = 5.0,
max_poll_attempts: int = 120, # Default max polling attempts (10 minutes with 5s interval)
max_retries: int = 3, # Max retries per individual API call
@@ -916,7 +793,7 @@ class PollingOperation(Generic[T, R]):
retry_backoff_factor: float = 2.0,
estimated_duration: Optional[float] = None,
node_id: Optional[str] = None,
- ):
+ ) -> None:
self.poll_endpoint = poll_endpoint
self.request = request
self.api_base: str = api_base or args.comfy_api_base
@@ -931,100 +808,73 @@ class PollingOperation(Generic[T, R]):
self.retry_delay = retry_delay
self.retry_backoff_factor = retry_backoff_factor
self.estimated_duration = estimated_duration
-
- # Polling configuration
- self.status_extractor = status_extractor or (
- lambda x: getattr(x, "status", None)
- )
+ self.status_extractor = status_extractor or (lambda x: getattr(x, "status", None))
self.progress_extractor = progress_extractor
self.result_url_extractor = result_url_extractor
self.node_id = node_id
self.completed_statuses = completed_statuses
self.failed_statuses = failed_statuses
+ self.final_response: Optional[R] = None
- # For storing response data
- self.final_response = None
- self.error = None
-
- def execute(self, client: Optional[ApiClient] = None) -> R:
- """Execute the polling operation using the provided client. If failed, raise an exception."""
+ async def execute(self, client: Optional[ApiClient] = None) -> R:
+ owns_client = client is None
+ if owns_client:
+ client = ApiClient(
+ base_url=self.api_base,
+ auth_token=self.auth_token,
+ comfy_api_key=self.comfy_api_key,
+ max_retries=self.max_retries,
+ retry_delay=self.retry_delay,
+ retry_backoff_factor=self.retry_backoff_factor,
+ )
try:
- if client is None:
- client = ApiClient(
- base_url=self.api_base,
- auth_token=self.auth_token,
- comfy_api_key=self.comfy_api_key,
- max_retries=self.max_retries,
- retry_delay=self.retry_delay,
- retry_backoff_factor=self.retry_backoff_factor,
- )
- return self._poll_until_complete(client)
- except LocalNetworkError as e:
- # Provide clear message for local network issues
- raise Exception(
- f"Polling failed due to local network issues. Please check your internet connection. "
- f"Details: {str(e)}"
- ) from e
- except ApiServerError as e:
- # Provide clear message for API server issues
- raise Exception(
- f"Polling failed due to API server issues. The service may be experiencing problems. "
- f"Please try again later. Details: {str(e)}"
- ) from e
- except Exception as e:
- raise Exception(f"Error during polling: {str(e)}")
+ return await self._poll_until_complete(client)
+ finally:
+ if owns_client:
+ await client.close()
def _display_text_on_node(self, text: str):
- """Sends text to the client which will be displayed on the node in the UI"""
if not self.node_id:
return
-
PromptServer.instance.send_progress_text(text, self.node_id)
- def _display_time_progress_on_node(self, time_completed: int):
+ def _display_time_progress_on_node(self, time_completed: int | float):
if not self.node_id:
return
-
if self.estimated_duration is not None:
- estimated_time_remaining = max(
- 0, int(self.estimated_duration) - int(time_completed)
- )
- message = f"Task in progress: {time_completed:.0f}s (~{estimated_time_remaining:.0f}s remaining)"
+ remaining = max(0, int(self.estimated_duration) - time_completed)
+ message = f"Task in progress: {time_completed}s (~{remaining}s remaining)"
else:
- message = f"Task in progress: {time_completed:.0f}s"
+ message = f"Task in progress: {time_completed}s"
self._display_text_on_node(message)
def _check_task_status(self, response: R) -> TaskStatus:
- """Check task status using the status extractor function"""
try:
status = self.status_extractor(response)
if status in self.completed_statuses:
return TaskStatus.COMPLETED
- elif status in self.failed_statuses:
+ if status in self.failed_statuses:
return TaskStatus.FAILED
return TaskStatus.PENDING
except Exception as e:
- logging.error(f"Error extracting status: {e}")
+ logging.error("Error extracting status: %s", e)
return TaskStatus.PENDING
- def _poll_until_complete(self, client: ApiClient) -> R:
+ async def _poll_until_complete(self, client: ApiClient) -> R:
"""Poll until the task is complete"""
- poll_count = 0
consecutive_errors = 0
max_consecutive_errors = min(5, self.max_retries * 2) # Limit consecutive errors
if self.progress_extractor:
progress = utils.ProgressBar(PROGRESS_BAR_MAX)
- while poll_count < self.max_poll_attempts:
+ status = TaskStatus.PENDING
+ for poll_count in range(1, self.max_poll_attempts + 1):
try:
- poll_count += 1
logging.debug(f"[DEBUG] Polling attempt #{poll_count}")
request_dict = (
- self.request.model_dump(exclude_none=True)
- if self.request is not None
- else None
+ None if self.request is None else self.request.model_dump(exclude_none=True)
)
if poll_count == 1:
@@ -1036,18 +886,14 @@ class PollingOperation(Generic[T, R]):
)
# Query task status
- resp = client.request(
- method=self.poll_endpoint.method.value,
- path=self.poll_endpoint.path,
+ resp = await client.request(
+ self.poll_endpoint.method.value,
+ self.poll_endpoint.path,
params=self.poll_endpoint.query_params,
data=request_dict,
)
-
- # Successfully got a response, reset consecutive error count
- consecutive_errors = 0
-
- # Parse response
- response_obj = self.poll_endpoint.response_model.model_validate(resp)
+ consecutive_errors = 0 # reset on success
+ response_obj: R = self.poll_endpoint.response_model.model_validate(resp)
# Check if task is complete
status = self._check_task_status(response_obj)
@@ -1065,45 +911,30 @@ class PollingOperation(Generic[T, R]):
result_url = self.result_url_extractor(response_obj)
if result_url:
message = f"Result URL: {result_url}"
- else:
- message = "Task completed successfully!"
logging.debug(f"[DEBUG] {message}")
self._display_text_on_node(message)
self.final_response = response_obj
if self.progress_extractor:
progress.update(100)
return self.final_response
- elif status == TaskStatus.FAILED:
+ if status == TaskStatus.FAILED:
message = f"Task failed: {json.dumps(resp)}"
logging.error(f"[DEBUG] {message}")
raise Exception(message)
- else:
- logging.debug("[DEBUG] Task still pending, continuing to poll...")
-
- # Wait before polling again
- logging.debug(
- f"[DEBUG] Waiting {self.poll_interval} seconds before next poll"
- )
+ logging.debug("[DEBUG] Task still pending, continuing to poll...")
+ # Task pending – wait
for i in range(int(self.poll_interval)):
- time_completed = (poll_count * self.poll_interval) + i
- self._display_time_progress_on_node(time_completed)
- time.sleep(1)
+ self._display_time_progress_on_node((poll_count - 1) * self.poll_interval + i)
+ await asyncio.sleep(1)
- except (LocalNetworkError, ApiServerError) as e:
- # For network-related errors, increment error count and potentially abort
+ except (LocalNetworkError, ApiServerError, NetworkError) as e:
consecutive_errors += 1
if consecutive_errors >= max_consecutive_errors:
raise Exception(
- f"Polling aborted after {consecutive_errors} consecutive network errors: {str(e)}"
+ f"Polling aborted after {consecutive_errors} network errors: {str(e)}"
) from e
-
- # Log the error but continue polling
- logging.warning(
- f"Network error during polling (attempt {poll_count}/{self.max_poll_attempts}): {str(e)}. "
- f"Will retry in {self.poll_interval} seconds."
- )
- time.sleep(self.poll_interval)
-
+ logging.warning("Network error (%s/%s): %s", consecutive_errors, max_consecutive_errors, str(e))
+ await asyncio.sleep(self.poll_interval)
except Exception as e:
# For other errors, increment count and potentially abort
consecutive_errors += 1
@@ -1117,10 +948,10 @@ class PollingOperation(Generic[T, R]):
f"Error during polling (attempt {poll_count}/{self.max_poll_attempts}): {str(e)}. "
f"Will retry in {self.poll_interval} seconds."
)
- time.sleep(self.poll_interval)
+ await asyncio.sleep(self.poll_interval)
# If we've exhausted all polling attempts
raise Exception(
- f"Polling timed out after {poll_count} attempts ({poll_count * self.poll_interval} seconds). "
- f"The operation may still be running on the server but is taking longer than expected."
+ f"Polling timed out after {self.max_poll_attempts} attempts (" f"{self.max_poll_attempts * self.poll_interval} seconds). "
+ "The operation may still be running on the server but is taking longer than expected."
)
diff --git a/comfy_api_nodes/apis/gemini_api.py b/comfy_api_nodes/apis/gemini_api.py
new file mode 100644
index 000000000..138bf035d
--- /dev/null
+++ b/comfy_api_nodes/apis/gemini_api.py
@@ -0,0 +1,19 @@
+from __future__ import annotations
+
+from typing import List, Optional
+
+from comfy_api_nodes.apis import GeminiGenerationConfig, GeminiContent, GeminiSafetySetting, GeminiSystemInstructionContent, GeminiTool, GeminiVideoMetadata
+from pydantic import BaseModel
+
+
+class GeminiImageGenerationConfig(GeminiGenerationConfig):
+ responseModalities: Optional[List[str]] = None
+
+
+class GeminiImageGenerateContentRequest(BaseModel):
+ contents: List[GeminiContent]
+ generationConfig: Optional[GeminiImageGenerationConfig] = None
+ safetySettings: Optional[List[GeminiSafetySetting]] = None
+ systemInstruction: Optional[GeminiSystemInstructionContent] = None
+ tools: Optional[List[GeminiTool]] = None
+ videoMetadata: Optional[GeminiVideoMetadata] = None
diff --git a/comfy_api_nodes/apis/request_logger.py b/comfy_api_nodes/apis/request_logger.py
index 93517ede9..42901e141 100644
--- a/comfy_api_nodes/apis/request_logger.py
+++ b/comfy_api_nodes/apis/request_logger.py
@@ -1,3 +1,5 @@
+from __future__ import annotations
+
import os
import datetime
import json
diff --git a/comfy_api_nodes/apis/tripo_api.py b/comfy_api_nodes/apis/tripo_api.py
index 626e8d277..9f43d4d09 100644
--- a/comfy_api_nodes/apis/tripo_api.py
+++ b/comfy_api_nodes/apis/tripo_api.py
@@ -127,7 +127,7 @@ class TripoTextToModelRequest(BaseModel):
type: TripoTaskType = Field(TripoTaskType.TEXT_TO_MODEL, description='Type of task')
prompt: str = Field(..., description='The text prompt describing the model to generate', max_length=1024)
negative_prompt: Optional[str] = Field(None, description='The negative text prompt', max_length=1024)
- model_version: Optional[TripoModelVersion] = TripoModelVersion.V2_5
+ model_version: Optional[TripoModelVersion] = TripoModelVersion.v2_5_20250123
face_limit: Optional[int] = Field(None, description='The number of faces to limit the generation to')
texture: Optional[bool] = Field(True, description='Whether to apply texture to the generated model')
pbr: Optional[bool] = Field(True, description='Whether to apply PBR to the generated model')
diff --git a/comfy_api_nodes/nodes_bfl.py b/comfy_api_nodes/nodes_bfl.py
index d93fbd778..c09be8d5b 100644
--- a/comfy_api_nodes/nodes_bfl.py
+++ b/comfy_api_nodes/nodes_bfl.py
@@ -1,3 +1,4 @@
+import asyncio
import io
from inspect import cleandoc
from typing import Union, Optional
@@ -28,7 +29,7 @@ from comfy_api_nodes.apinode_utils import (
import numpy as np
from PIL import Image
-import requests
+import aiohttp
import torch
import base64
import time
@@ -44,18 +45,18 @@ def convert_mask_to_image(mask: torch.Tensor):
return mask
-def handle_bfl_synchronous_operation(
+async def handle_bfl_synchronous_operation(
operation: SynchronousOperation,
timeout_bfl_calls=360,
node_id: Union[str, None] = None,
):
- response_api: BFLFluxProGenerateResponse = operation.execute()
- return _poll_until_generated(
+ response_api: BFLFluxProGenerateResponse = await operation.execute()
+ return await _poll_until_generated(
response_api.polling_url, timeout=timeout_bfl_calls, node_id=node_id
)
-def _poll_until_generated(
+async def _poll_until_generated(
polling_url: str, timeout=360, node_id: Union[str, None] = None
):
# used bfl-comfy-nodes to verify code implementation:
@@ -66,55 +67,56 @@ def _poll_until_generated(
retry_404_seconds = 2
retry_202_seconds = 2
retry_pending_seconds = 1
- request = requests.Request(method=HttpMethod.GET, url=polling_url)
- # NOTE: should True loop be replaced with checking if workflow has been interrupted?
- while True:
- if node_id:
- time_elapsed = time.time() - start_time
- PromptServer.instance.send_progress_text(
- f"Generating ({time_elapsed:.0f}s)", node_id
- )
- response = requests.Session().send(request.prepare())
- if response.status_code == 200:
- result = response.json()
- if result["status"] == BFLStatus.ready:
- img_url = result["result"]["sample"]
- if node_id:
- PromptServer.instance.send_progress_text(
- f"Result URL: {img_url}", node_id
- )
- img_response = requests.get(img_url)
- return process_image_response(img_response)
- elif result["status"] in [
- BFLStatus.request_moderated,
- BFLStatus.content_moderated,
- ]:
- status = result["status"]
- raise Exception(
- f"BFL API did not return an image due to: {status}."
+ async with aiohttp.ClientSession() as session:
+ # NOTE: should True loop be replaced with checking if workflow has been interrupted?
+ while True:
+ if node_id:
+ time_elapsed = time.time() - start_time
+ PromptServer.instance.send_progress_text(
+ f"Generating ({time_elapsed:.0f}s)", node_id
)
- elif result["status"] == BFLStatus.error:
- raise Exception(f"BFL API encountered an error: {result}.")
- elif result["status"] == BFLStatus.pending:
- time.sleep(retry_pending_seconds)
- continue
- elif response.status_code == 404:
- if retries_404 < max_retries_404:
- retries_404 += 1
- time.sleep(retry_404_seconds)
- continue
- raise Exception(
- f"BFL API could not find task after {max_retries_404} tries."
- )
- elif response.status_code == 202:
- time.sleep(retry_202_seconds)
- elif time.time() - start_time > timeout:
- raise Exception(
- f"BFL API experienced a timeout; could not return request under {timeout} seconds."
- )
- else:
- raise Exception(f"BFL API encountered an error: {response.json()}")
+
+ async with session.get(polling_url) as response:
+ if response.status == 200:
+ result = await response.json()
+ if result["status"] == BFLStatus.ready:
+ img_url = result["result"]["sample"]
+ if node_id:
+ PromptServer.instance.send_progress_text(
+ f"Result URL: {img_url}", node_id
+ )
+ async with session.get(img_url) as img_resp:
+ return process_image_response(await img_resp.content.read())
+ elif result["status"] in [
+ BFLStatus.request_moderated,
+ BFLStatus.content_moderated,
+ ]:
+ status = result["status"]
+ raise Exception(
+ f"BFL API did not return an image due to: {status}."
+ )
+ elif result["status"] == BFLStatus.error:
+ raise Exception(f"BFL API encountered an error: {result}.")
+ elif result["status"] == BFLStatus.pending:
+ await asyncio.sleep(retry_pending_seconds)
+ continue
+ elif response.status == 404:
+ if retries_404 < max_retries_404:
+ retries_404 += 1
+ await asyncio.sleep(retry_404_seconds)
+ continue
+ raise Exception(
+ f"BFL API could not find task after {max_retries_404} tries."
+ )
+ elif response.status == 202:
+ await asyncio.sleep(retry_202_seconds)
+ elif time.time() - start_time > timeout:
+ raise Exception(
+ f"BFL API experienced a timeout; could not return request under {timeout} seconds."
+ )
+ else:
+ raise Exception(f"BFL API encountered an error: {response.json()}")
def convert_image_to_base64(image: torch.Tensor):
scaled_image = downscale_image_tensor(image, total_pixels=2048 * 2048)
@@ -222,7 +224,7 @@ class FluxProUltraImageNode(ComfyNodeABC):
API_NODE = True
CATEGORY = "api node/image/BFL"
- def api_call(
+ async def api_call(
self,
prompt: str,
aspect_ratio: str,
@@ -266,7 +268,7 @@ class FluxProUltraImageNode(ComfyNodeABC):
),
auth_kwargs=kwargs,
)
- output_image = handle_bfl_synchronous_operation(operation, node_id=unique_id)
+ output_image = await handle_bfl_synchronous_operation(operation, node_id=unique_id)
return (output_image,)
@@ -354,7 +356,7 @@ class FluxKontextProImageNode(ComfyNodeABC):
BFL_PATH = "/proxy/bfl/flux-kontext-pro/generate"
- def api_call(
+ async def api_call(
self,
prompt: str,
aspect_ratio: str,
@@ -397,7 +399,7 @@ class FluxKontextProImageNode(ComfyNodeABC):
),
auth_kwargs=kwargs,
)
- output_image = handle_bfl_synchronous_operation(operation, node_id=unique_id)
+ output_image = await handle_bfl_synchronous_operation(operation, node_id=unique_id)
return (output_image,)
@@ -489,7 +491,7 @@ class FluxProImageNode(ComfyNodeABC):
API_NODE = True
CATEGORY = "api node/image/BFL"
- def api_call(
+ async def api_call(
self,
prompt: str,
prompt_upsampling,
@@ -524,7 +526,7 @@ class FluxProImageNode(ComfyNodeABC):
),
auth_kwargs=kwargs,
)
- output_image = handle_bfl_synchronous_operation(operation, node_id=unique_id)
+ output_image = await handle_bfl_synchronous_operation(operation, node_id=unique_id)
return (output_image,)
@@ -632,7 +634,7 @@ class FluxProExpandNode(ComfyNodeABC):
API_NODE = True
CATEGORY = "api node/image/BFL"
- def api_call(
+ async def api_call(
self,
image: torch.Tensor,
prompt: str,
@@ -670,7 +672,7 @@ class FluxProExpandNode(ComfyNodeABC):
),
auth_kwargs=kwargs,
)
- output_image = handle_bfl_synchronous_operation(operation, node_id=unique_id)
+ output_image = await handle_bfl_synchronous_operation(operation, node_id=unique_id)
return (output_image,)
@@ -744,7 +746,7 @@ class FluxProFillNode(ComfyNodeABC):
API_NODE = True
CATEGORY = "api node/image/BFL"
- def api_call(
+ async def api_call(
self,
image: torch.Tensor,
mask: torch.Tensor,
@@ -780,7 +782,7 @@ class FluxProFillNode(ComfyNodeABC):
),
auth_kwargs=kwargs,
)
- output_image = handle_bfl_synchronous_operation(operation, node_id=unique_id)
+ output_image = await handle_bfl_synchronous_operation(operation, node_id=unique_id)
return (output_image,)
@@ -879,7 +881,7 @@ class FluxProCannyNode(ComfyNodeABC):
API_NODE = True
CATEGORY = "api node/image/BFL"
- def api_call(
+ async def api_call(
self,
control_image: torch.Tensor,
prompt: str,
@@ -929,7 +931,7 @@ class FluxProCannyNode(ComfyNodeABC):
),
auth_kwargs=kwargs,
)
- output_image = handle_bfl_synchronous_operation(operation, node_id=unique_id)
+ output_image = await handle_bfl_synchronous_operation(operation, node_id=unique_id)
return (output_image,)
@@ -1008,7 +1010,7 @@ class FluxProDepthNode(ComfyNodeABC):
API_NODE = True
CATEGORY = "api node/image/BFL"
- def api_call(
+ async def api_call(
self,
control_image: torch.Tensor,
prompt: str,
@@ -1045,7 +1047,7 @@ class FluxProDepthNode(ComfyNodeABC):
),
auth_kwargs=kwargs,
)
- output_image = handle_bfl_synchronous_operation(operation, node_id=unique_id)
+ output_image = await handle_bfl_synchronous_operation(operation, node_id=unique_id)
return (output_image,)
diff --git a/comfy_api_nodes/nodes_gemini.py b/comfy_api_nodes/nodes_gemini.py
index 5935ab2bb..baa379b75 100644
--- a/comfy_api_nodes/nodes_gemini.py
+++ b/comfy_api_nodes/nodes_gemini.py
@@ -2,8 +2,14 @@
API Nodes for Gemini Multimodal LLM Usage via Remote API
See: https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/inference
"""
+from __future__ import annotations
+import json
+import time
import os
+import uuid
+import base64
+from io import BytesIO
from enum import Enum
from typing import Optional, Literal
@@ -20,6 +26,7 @@ from comfy_api_nodes.apis import (
GeminiPart,
GeminiMimeType,
)
+from comfy_api_nodes.apis.gemini_api import GeminiImageGenerationConfig, GeminiImageGenerateContentRequest
from comfy_api_nodes.apis.client import (
ApiEndpoint,
HttpMethod,
@@ -30,6 +37,7 @@ from comfy_api_nodes.apinode_utils import (
audio_to_base64_string,
video_to_base64_string,
tensor_to_base64_string,
+ bytesio_to_image_tensor,
)
@@ -44,6 +52,16 @@ class GeminiModel(str, Enum):
gemini_2_5_pro_preview_05_06 = "gemini-2.5-pro-preview-05-06"
gemini_2_5_flash_preview_04_17 = "gemini-2.5-flash-preview-04-17"
+ gemini_2_5_pro = "gemini-2.5-pro"
+ gemini_2_5_flash = "gemini-2.5-flash"
+
+
+class GeminiImageModel(str, Enum):
+ """
+ Gemini Image Model Names allowed by comfy-api
+ """
+
+ gemini_2_5_flash_image_preview = "gemini-2.5-flash-image-preview"
def get_gemini_endpoint(
@@ -68,6 +86,135 @@ def get_gemini_endpoint(
)
+def get_gemini_image_endpoint(
+ model: GeminiImageModel,
+) -> ApiEndpoint[GeminiGenerateContentRequest, GeminiGenerateContentResponse]:
+ """
+ Get the API endpoint for a given Gemini model.
+
+ Args:
+ model: The Gemini model to use, either as enum or string value.
+
+ Returns:
+ ApiEndpoint configured for the specific Gemini model.
+ """
+ if isinstance(model, str):
+ model = GeminiImageModel(model)
+ return ApiEndpoint(
+ path=f"{GEMINI_BASE_ENDPOINT}/{model.value}",
+ method=HttpMethod.POST,
+ request_model=GeminiImageGenerateContentRequest,
+ response_model=GeminiGenerateContentResponse,
+ )
+
+
+def create_image_parts(image_input: torch.Tensor) -> list[GeminiPart]:
+ """
+ Convert image tensor input to Gemini API compatible parts.
+
+ Args:
+ image_input: Batch of image tensors from ComfyUI.
+
+ Returns:
+ List of GeminiPart objects containing the encoded images.
+ """
+ image_parts: list[GeminiPart] = []
+ for image_index in range(image_input.shape[0]):
+ image_as_b64 = tensor_to_base64_string(
+ image_input[image_index].unsqueeze(0)
+ )
+ image_parts.append(
+ GeminiPart(
+ inlineData=GeminiInlineData(
+ mimeType=GeminiMimeType.image_png,
+ data=image_as_b64,
+ )
+ )
+ )
+ return image_parts
+
+
+def create_text_part(text: str) -> GeminiPart:
+ """
+ Create a text part for the Gemini API request.
+
+ Args:
+ text: The text content to include in the request.
+
+ Returns:
+ A GeminiPart object with the text content.
+ """
+ return GeminiPart(text=text)
+
+
+def get_parts_from_response(
+ response: GeminiGenerateContentResponse
+) -> list[GeminiPart]:
+ """
+ Extract all parts from the Gemini API response.
+
+ Args:
+ response: The API response from Gemini.
+
+ Returns:
+ List of response parts from the first candidate.
+ """
+ return response.candidates[0].content.parts
+
+
+def get_parts_by_type(
+ response: GeminiGenerateContentResponse, part_type: Literal["text"] | str
+) -> list[GeminiPart]:
+ """
+ Filter response parts by their type.
+
+ Args:
+ response: The API response from Gemini.
+ part_type: Type of parts to extract ("text" or a MIME type).
+
+ Returns:
+ List of response parts matching the requested type.
+ """
+ parts = []
+ for part in get_parts_from_response(response):
+ if part_type == "text" and hasattr(part, "text") and part.text:
+ parts.append(part)
+ elif (
+ hasattr(part, "inlineData")
+ and part.inlineData
+ and part.inlineData.mimeType == part_type
+ ):
+ parts.append(part)
+ # Skip parts that don't match the requested type
+ return parts
+
+
+def get_text_from_response(response: GeminiGenerateContentResponse) -> str:
+ """
+ Extract and concatenate all text parts from the response.
+
+ Args:
+ response: The API response from Gemini.
+
+ Returns:
+ Combined text from all text parts in the response.
+ """
+ parts = get_parts_by_type(response, "text")
+ return "\n".join([part.text for part in parts])
+
+
+def get_image_from_response(response: GeminiGenerateContentResponse) -> torch.Tensor:
+ image_tensors: list[torch.Tensor] = []
+ parts = get_parts_by_type(response, "image/png")
+ for part in parts:
+ image_data = base64.b64decode(part.inlineData.data)
+ returned_image = bytesio_to_image_tensor(BytesIO(image_data))
+ image_tensors.append(returned_image)
+ if len(image_tensors) == 0:
+ return torch.zeros((1,1024,1024,4))
+ return torch.cat(image_tensors, dim=0)
+
+
class GeminiNode(ComfyNodeABC):
"""
Node to generate text responses from a Gemini model.
@@ -95,7 +242,7 @@ class GeminiNode(ComfyNodeABC):
{
"tooltip": "The Gemini model to use for generating responses.",
"options": [model.value for model in GeminiModel],
- "default": GeminiModel.gemini_2_5_pro_preview_05_06.value,
+ "default": GeminiModel.gemini_2_5_pro.value,
},
),
"seed": (
@@ -152,59 +299,6 @@ class GeminiNode(ComfyNodeABC):
CATEGORY = "api node/text/Gemini"
API_NODE = True
- def get_parts_from_response(
- self, response: GeminiGenerateContentResponse
- ) -> list[GeminiPart]:
- """
- Extract all parts from the Gemini API response.
-
- Args:
- response: The API response from Gemini.
-
- Returns:
- List of response parts from the first candidate.
- """
- return response.candidates[0].content.parts
-
- def get_parts_by_type(
- self, response: GeminiGenerateContentResponse, part_type: Literal["text"] | str
- ) -> list[GeminiPart]:
- """
- Filter response parts by their type.
-
- Args:
- response: The API response from Gemini.
- part_type: Type of parts to extract ("text" or a MIME type).
-
- Returns:
- List of response parts matching the requested type.
- """
- parts = []
- for part in self.get_parts_from_response(response):
- if part_type == "text" and hasattr(part, "text") and part.text:
- parts.append(part)
- elif (
- hasattr(part, "inlineData")
- and part.inlineData
- and part.inlineData.mimeType == part_type
- ):
- parts.append(part)
- # Skip parts that don't match the requested type
- return parts
-
- def get_text_from_response(self, response: GeminiGenerateContentResponse) -> str:
- """
- Extract and concatenate all text parts from the response.
-
- Args:
- response: The API response from Gemini.
-
- Returns:
- Combined text from all text parts in the response.
- """
- parts = self.get_parts_by_type(response, "text")
- return "\n".join([part.text for part in parts])
-
def create_video_parts(self, video_input: IO.VIDEO, **kwargs) -> list[GeminiPart]:
"""
Convert video input to Gemini API compatible parts.
@@ -264,44 +358,7 @@ class GeminiNode(ComfyNodeABC):
)
return audio_parts
- def create_image_parts(self, image_input: torch.Tensor) -> list[GeminiPart]:
- """
- Convert image tensor input to Gemini API compatible parts.
-
- Args:
- image_input: Batch of image tensors from ComfyUI.
-
- Returns:
- List of GeminiPart objects containing the encoded images.
- """
- image_parts: list[GeminiPart] = []
- for image_index in range(image_input.shape[0]):
- image_as_b64 = tensor_to_base64_string(
- image_input[image_index].unsqueeze(0)
- )
- image_parts.append(
- GeminiPart(
- inlineData=GeminiInlineData(
- mimeType=GeminiMimeType.image_png,
- data=image_as_b64,
- )
- )
- )
- return image_parts
-
- def create_text_part(self, text: str) -> GeminiPart:
- """
- Create a text part for the Gemini API request.
-
- Args:
- text: The text content to include in the request.
-
- Returns:
- A GeminiPart object with the text content.
- """
- return GeminiPart(text=text)
-
- def api_call(
+ async def api_call(
self,
prompt: str,
model: GeminiModel,
@@ -316,11 +373,11 @@ class GeminiNode(ComfyNodeABC):
validate_string(prompt, strip_whitespace=False)
# Create parts list with text prompt as the first part
- parts: list[GeminiPart] = [self.create_text_part(prompt)]
+ parts: list[GeminiPart] = [create_text_part(prompt)]
# Add other modal parts
if images is not None:
- image_parts = self.create_image_parts(images)
+ image_parts = create_image_parts(images)
parts.extend(image_parts)
if audio is not None:
parts.extend(self.create_audio_parts(audio))
@@ -330,7 +387,7 @@ class GeminiNode(ComfyNodeABC):
parts.extend(files)
# Create response
- response = SynchronousOperation(
+ response = await SynchronousOperation(
endpoint=get_gemini_endpoint(model),
request=GeminiGenerateContentRequest(
contents=[
@@ -344,9 +401,29 @@ class GeminiNode(ComfyNodeABC):
).execute()
# Get result output
- output_text = self.get_text_from_response(response)
+ output_text = get_text_from_response(response)
if unique_id and output_text:
- PromptServer.instance.send_progress_text(output_text, node_id=unique_id)
+ # Not a true chat history like the OpenAI Chat node. It is emulated so the frontend can show a copy button.
+ render_spec = {
+ "node_id": unique_id,
+ "component": "ChatHistoryWidget",
+ "props": {
+ "history": json.dumps(
+ [
+ {
+ "prompt": prompt,
+ "response": output_text,
+ "response_id": str(uuid.uuid4()),
+ "timestamp": time.time(),
+ }
+ ]
+ ),
+ },
+ }
+ PromptServer.instance.send_sync(
+ "display_component",
+ render_spec,
+ )
return (output_text or "Empty response from Gemini model...",)
@@ -435,12 +512,162 @@ class GeminiInputFiles(ComfyNodeABC):
return (files,)
+class GeminiImage(ComfyNodeABC):
+ """
+ Node to generate text and image responses from a Gemini model.
+
+ This node allows users to interact with Google's Gemini AI models, providing
+ multimodal inputs (text, images, files) to generate coherent
+ text and image responses. The node works with the latest Gemini models, handling the
+ API communication and response parsing.
+ """
+ @classmethod
+ def INPUT_TYPES(cls) -> InputTypeDict:
+ return {
+ "required": {
+ "prompt": (
+ IO.STRING,
+ {
+ "multiline": True,
+ "default": "",
+ "tooltip": "Text prompt for generation",
+ },
+ ),
+ "model": (
+ IO.COMBO,
+ {
+ "tooltip": "The Gemini model to use for generating responses.",
+ "options": [model.value for model in GeminiImageModel],
+ "default": GeminiImageModel.gemini_2_5_flash_image_preview.value,
+ },
+ ),
+ "seed": (
+ IO.INT,
+ {
+ "default": 42,
+ "min": 0,
+ "max": 0xFFFFFFFFFFFFFFFF,
+ "control_after_generate": True,
+ "tooltip": "When seed is fixed to a specific value, the model makes a best effort to provide the same response for repeated requests. Deterministic output isn't guaranteed. Also, changing the model or parameter settings, such as the temperature, can cause variations in the response even when you use the same seed value. By default, a random seed value is used.",
+ },
+ ),
+ },
+ "optional": {
+ "images": (
+ IO.IMAGE,
+ {
+ "default": None,
+ "tooltip": "Optional image(s) to use as context for the model. To include multiple images, you can use the Batch Images node.",
+ },
+ ),
+ "files": (
+ "GEMINI_INPUT_FILES",
+ {
+ "default": None,
+ "tooltip": "Optional file(s) to use as context for the model. Accepts inputs from the Gemini Generate Content Input Files node.",
+ },
+ ),
+ # TODO: later we can add this parameter later
+ # "n": (
+ # IO.INT,
+ # {
+ # "default": 1,
+ # "min": 1,
+ # "max": 8,
+ # "step": 1,
+ # "display": "number",
+ # "tooltip": "How many images to generate",
+ # },
+ # ),
+ },
+ "hidden": {
+ "auth_token": "AUTH_TOKEN_COMFY_ORG",
+ "comfy_api_key": "API_KEY_COMFY_ORG",
+ "unique_id": "UNIQUE_ID",
+ },
+ }
+
+ RETURN_TYPES = (IO.IMAGE, IO.STRING)
+ FUNCTION = "api_call"
+ CATEGORY = "api node/image/Gemini"
+ DESCRIPTION = "Edit images synchronously via Google API."
+ API_NODE = True
+
+ async def api_call(
+ self,
+ prompt: str,
+ model: GeminiImageModel,
+ images: Optional[IO.IMAGE] = None,
+ files: Optional[list[GeminiPart]] = None,
+ n=1,
+ unique_id: Optional[str] = None,
+ **kwargs,
+ ):
+ # Validate inputs
+ validate_string(prompt, strip_whitespace=True, min_length=1)
+ # Create parts list with text prompt as the first part
+ parts: list[GeminiPart] = [create_text_part(prompt)]
+
+ # Add other modal parts
+ if images is not None:
+ image_parts = create_image_parts(images)
+ parts.extend(image_parts)
+ if files is not None:
+ parts.extend(files)
+
+ response = await SynchronousOperation(
+ endpoint=get_gemini_image_endpoint(model),
+ request=GeminiImageGenerateContentRequest(
+ contents=[
+ GeminiContent(
+ role="user",
+ parts=parts,
+ ),
+ ],
+ generationConfig=GeminiImageGenerationConfig(
+ responseModalities=["TEXT","IMAGE"]
+ )
+ ),
+ auth_kwargs=kwargs,
+ ).execute()
+
+ output_image = get_image_from_response(response)
+ output_text = get_text_from_response(response)
+ if unique_id and output_text:
+ # Not a true chat history like the OpenAI Chat node. It is emulated so the frontend can show a copy button.
+ render_spec = {
+ "node_id": unique_id,
+ "component": "ChatHistoryWidget",
+ "props": {
+ "history": json.dumps(
+ [
+ {
+ "prompt": prompt,
+ "response": output_text,
+ "response_id": str(uuid.uuid4()),
+ "timestamp": time.time(),
+ }
+ ]
+ ),
+ },
+ }
+ PromptServer.instance.send_sync(
+ "display_component",
+ render_spec,
+ )
+
+ output_text = output_text or "Empty response from Gemini model..."
+ return (output_image, output_text,)
+
+
NODE_CLASS_MAPPINGS = {
"GeminiNode": GeminiNode,
+ "GeminiImageNode": GeminiImage,
"GeminiInputFiles": GeminiInputFiles,
}
NODE_DISPLAY_NAME_MAPPINGS = {
"GeminiNode": "Google Gemini",
+ "GeminiImageNode": "Google Gemini Image",
"GeminiInputFiles": "Gemini Input Files",
}
diff --git a/comfy_api_nodes/nodes_ideogram.py b/comfy_api_nodes/nodes_ideogram.py
index b8487355f..d28895f3e 100644
--- a/comfy_api_nodes/nodes_ideogram.py
+++ b/comfy_api_nodes/nodes_ideogram.py
@@ -1,8 +1,8 @@
-from comfy.comfy_types.node_typing import IO, ComfyNodeABC, InputTypeDict
-from inspect import cleandoc
+from io import BytesIO
+from typing_extensions import override
+from comfy_api.latest import ComfyExtension, io as comfy_io
from PIL import Image
import numpy as np
-import io
import torch
from comfy_api_nodes.apis import (
IdeogramGenerateRequest,
@@ -212,7 +212,7 @@ V3_RESOLUTIONS= [
"1536x640"
]
-def download_and_process_images(image_urls):
+async def download_and_process_images(image_urls):
"""Helper function to download and process multiple images from URLs"""
# Initialize list to store image tensors
@@ -220,7 +220,7 @@ def download_and_process_images(image_urls):
for image_url in image_urls:
# Using functions from apinode_utils.py to handle downloading and processing
- image_bytesio = download_url_to_bytesio(image_url) # Download image content to BytesIO
+ image_bytesio = await download_url_to_bytesio(image_url) # Download image content to BytesIO
img_tensor = bytesio_to_image_tensor(image_bytesio, mode="RGB") # Convert to torch.Tensor with RGB mode
image_tensors.append(img_tensor)
@@ -246,90 +246,81 @@ def display_image_urls_on_node(image_urls, node_id):
PromptServer.instance.send_progress_text(urls_text, node_id)
-class IdeogramV1(ComfyNodeABC):
- """
- Generates images using the Ideogram V1 model.
- """
-
- def __init__(self):
- pass
+class IdeogramV1(comfy_io.ComfyNode):
@classmethod
- def INPUT_TYPES(cls) -> InputTypeDict:
- return {
- "required": {
- "prompt": (
- IO.STRING,
- {
- "multiline": True,
- "default": "",
- "tooltip": "Prompt for the image generation",
- },
+ def define_schema(cls):
+ return comfy_io.Schema(
+ node_id="IdeogramV1",
+ display_name="Ideogram V1",
+ category="api node/image/Ideogram",
+ description="Generates images using the Ideogram V1 model.",
+ inputs=[
+ comfy_io.String.Input(
+ "prompt",
+ multiline=True,
+ default="",
+ tooltip="Prompt for the image generation",
),
- "turbo": (
- IO.BOOLEAN,
- {
- "default": False,
- "tooltip": "Whether to use turbo mode (faster generation, potentially lower quality)",
- }
+ comfy_io.Boolean.Input(
+ "turbo",
+ default=False,
+ tooltip="Whether to use turbo mode (faster generation, potentially lower quality)",
),
- },
- "optional": {
- "aspect_ratio": (
- IO.COMBO,
- {
- "options": list(V1_V2_RATIO_MAP.keys()),
- "default": "1:1",
- "tooltip": "The aspect ratio for image generation.",
- },
+ comfy_io.Combo.Input(
+ "aspect_ratio",
+ options=list(V1_V2_RATIO_MAP.keys()),
+ default="1:1",
+ tooltip="The aspect ratio for image generation.",
+ optional=True,
),
- "magic_prompt_option": (
- IO.COMBO,
- {
- "options": ["AUTO", "ON", "OFF"],
- "default": "AUTO",
- "tooltip": "Determine if MagicPrompt should be used in generation",
- },
+ comfy_io.Combo.Input(
+ "magic_prompt_option",
+ options=["AUTO", "ON", "OFF"],
+ default="AUTO",
+ tooltip="Determine if MagicPrompt should be used in generation",
+ optional=True,
),
- "seed": (
- IO.INT,
- {
- "default": 0,
- "min": 0,
- "max": 2147483647,
- "step": 1,
- "control_after_generate": True,
- "display": "number",
- },
+ comfy_io.Int.Input(
+ "seed",
+ default=0,
+ min=0,
+ max=2147483647,
+ step=1,
+ control_after_generate=True,
+ display_mode=comfy_io.NumberDisplay.number,
+ optional=True,
),
- "negative_prompt": (
- IO.STRING,
- {
- "multiline": True,
- "default": "",
- "tooltip": "Description of what to exclude from the image",
- },
+ comfy_io.String.Input(
+ "negative_prompt",
+ multiline=True,
+ default="",
+ tooltip="Description of what to exclude from the image",
+ optional=True,
),
- "num_images": (
- IO.INT,
- {"default": 1, "min": 1, "max": 8, "step": 1, "display": "number"},
+ comfy_io.Int.Input(
+ "num_images",
+ default=1,
+ min=1,
+ max=8,
+ step=1,
+ display_mode=comfy_io.NumberDisplay.number,
+ optional=True,
),
- },
- "hidden": {
- "auth_token": "AUTH_TOKEN_COMFY_ORG",
- "comfy_api_key": "API_KEY_COMFY_ORG",
- "unique_id": "UNIQUE_ID",
- },
- }
+ ],
+ outputs=[
+ comfy_io.Image.Output(),
+ ],
+ hidden=[
+ comfy_io.Hidden.auth_token_comfy_org,
+ comfy_io.Hidden.api_key_comfy_org,
+ comfy_io.Hidden.unique_id,
+ ],
+ )
- RETURN_TYPES = (IO.IMAGE,)
- FUNCTION = "api_call"
- CATEGORY = "api node/image/Ideogram"
- DESCRIPTION = cleandoc(__doc__ or "")
- API_NODE = True
-
- def api_call(
- self,
+ @classmethod
+ async def execute(
+ cls,
prompt,
turbo=False,
aspect_ratio="1:1",
@@ -337,13 +328,15 @@ class IdeogramV1(ComfyNodeABC):
seed=0,
negative_prompt="",
num_images=1,
- unique_id=None,
- **kwargs,
):
# Determine the model based on turbo setting
aspect_ratio = V1_V2_RATIO_MAP.get(aspect_ratio, None)
model = "V_1_TURBO" if turbo else "V_1"
+ auth = {
+ "auth_token": cls.hidden.auth_token_comfy_org,
+ "comfy_api_key": cls.hidden.api_key_comfy_org,
+ }
operation = SynchronousOperation(
endpoint=ApiEndpoint(
path="/proxy/ideogram/generate",
@@ -364,10 +357,10 @@ class IdeogramV1(ComfyNodeABC):
negative_prompt=negative_prompt if negative_prompt else None,
)
),
- auth_kwargs=kwargs,
+ auth_kwargs=auth,
)
- response = operation.execute()
+ response = await operation.execute()
if not response.data or len(response.data) == 0:
raise Exception("No images were generated in the response")
@@ -377,93 +370,85 @@ class IdeogramV1(ComfyNodeABC):
if not image_urls:
raise Exception("No image URLs were generated in the response")
- display_image_urls_on_node(image_urls, unique_id)
- return (download_and_process_images(image_urls),)
+ display_image_urls_on_node(image_urls, cls.hidden.unique_id)
+ return comfy_io.NodeOutput(await download_and_process_images(image_urls))
-class IdeogramV2(ComfyNodeABC):
- """
- Generates images using the Ideogram V2 model.
- """
-
- def __init__(self):
- pass
+class IdeogramV2(comfy_io.ComfyNode):
@classmethod
- def INPUT_TYPES(cls) -> InputTypeDict:
- return {
- "required": {
- "prompt": (
- IO.STRING,
- {
- "multiline": True,
- "default": "",
- "tooltip": "Prompt for the image generation",
- },
+ def define_schema(cls):
+ return comfy_io.Schema(
+ node_id="IdeogramV2",
+ display_name="Ideogram V2",
+ category="api node/image/Ideogram",
+ description="Generates images using the Ideogram V2 model.",
+ inputs=[
+ comfy_io.String.Input(
+ "prompt",
+ multiline=True,
+ default="",
+ tooltip="Prompt for the image generation",
),
- "turbo": (
- IO.BOOLEAN,
- {
- "default": False,
- "tooltip": "Whether to use turbo mode (faster generation, potentially lower quality)",
- }
+ comfy_io.Boolean.Input(
+ "turbo",
+ default=False,
+ tooltip="Whether to use turbo mode (faster generation, potentially lower quality)",
),
- },
- "optional": {
- "aspect_ratio": (
- IO.COMBO,
- {
- "options": list(V1_V2_RATIO_MAP.keys()),
- "default": "1:1",
- "tooltip": "The aspect ratio for image generation. Ignored if resolution is not set to AUTO.",
- },
+ comfy_io.Combo.Input(
+ "aspect_ratio",
+ options=list(V1_V2_RATIO_MAP.keys()),
+ default="1:1",
+ tooltip="The aspect ratio for image generation. Ignored if resolution is not set to AUTO.",
+ optional=True,
),
- "resolution": (
- IO.COMBO,
- {
- "options": list(V1_V1_RES_MAP.keys()),
- "default": "Auto",
- "tooltip": "The resolution for image generation. If not set to AUTO, this overrides the aspect_ratio setting.",
- },
+ comfy_io.Combo.Input(
+ "resolution",
+ options=list(V1_V1_RES_MAP.keys()),
+ default="Auto",
+ tooltip="The resolution for image generation. "
+ "If not set to AUTO, this overrides the aspect_ratio setting.",
+ optional=True,
),
- "magic_prompt_option": (
- IO.COMBO,
- {
- "options": ["AUTO", "ON", "OFF"],
- "default": "AUTO",
- "tooltip": "Determine if MagicPrompt should be used in generation",
- },
+ comfy_io.Combo.Input(
+ "magic_prompt_option",
+ options=["AUTO", "ON", "OFF"],
+ default="AUTO",
+ tooltip="Determine if MagicPrompt should be used in generation",
+ optional=True,
),
- "seed": (
- IO.INT,
- {
- "default": 0,
- "min": 0,
- "max": 2147483647,
- "step": 1,
- "control_after_generate": True,
- "display": "number",
- },
+ comfy_io.Int.Input(
+ "seed",
+ default=0,
+ min=0,
+ max=2147483647,
+ step=1,
+ control_after_generate=True,
+ display_mode=comfy_io.NumberDisplay.number,
+ optional=True,
),
- "style_type": (
- IO.COMBO,
- {
- "options": ["AUTO", "GENERAL", "REALISTIC", "DESIGN", "RENDER_3D", "ANIME"],
- "default": "NONE",
- "tooltip": "Style type for generation (V2 only)",
- },
+ comfy_io.Combo.Input(
+ "style_type",
+ options=["AUTO", "GENERAL", "REALISTIC", "DESIGN", "RENDER_3D", "ANIME"],
+ default="NONE",
+ tooltip="Style type for generation (V2 only)",
+ optional=True,
),
- "negative_prompt": (
- IO.STRING,
- {
- "multiline": True,
- "default": "",
- "tooltip": "Description of what to exclude from the image",
- },
+ comfy_io.String.Input(
+ "negative_prompt",
+ multiline=True,
+ default="",
+ tooltip="Description of what to exclude from the image",
+ optional=True,
),
- "num_images": (
- IO.INT,
- {"default": 1, "min": 1, "max": 8, "step": 1, "display": "number"},
+ comfy_io.Int.Input(
+ "num_images",
+ default=1,
+ min=1,
+ max=8,
+ step=1,
+ display_mode=comfy_io.NumberDisplay.number,
+ optional=True,
),
#"color_palette": (
# IO.STRING,
@@ -473,22 +458,20 @@ class IdeogramV2(ComfyNodeABC):
# "tooltip": "Color palette preset name or hex colors with weights",
# },
#),
- },
- "hidden": {
- "auth_token": "AUTH_TOKEN_COMFY_ORG",
- "comfy_api_key": "API_KEY_COMFY_ORG",
- "unique_id": "UNIQUE_ID",
- },
- }
+ ],
+ outputs=[
+ comfy_io.Image.Output(),
+ ],
+ hidden=[
+ comfy_io.Hidden.auth_token_comfy_org,
+ comfy_io.Hidden.api_key_comfy_org,
+ comfy_io.Hidden.unique_id,
+ ],
+ )
- RETURN_TYPES = (IO.IMAGE,)
- FUNCTION = "api_call"
- CATEGORY = "api node/image/Ideogram"
- DESCRIPTION = cleandoc(__doc__ or "")
- API_NODE = True
-
- def api_call(
- self,
+ @classmethod
+ async def execute(
+ cls,
prompt,
turbo=False,
aspect_ratio="1:1",
@@ -499,8 +482,6 @@ class IdeogramV2(ComfyNodeABC):
negative_prompt="",
num_images=1,
color_palette="",
- unique_id=None,
- **kwargs,
):
aspect_ratio = V1_V2_RATIO_MAP.get(aspect_ratio, None)
resolution = V1_V1_RES_MAP.get(resolution, None)
@@ -517,6 +498,10 @@ class IdeogramV2(ComfyNodeABC):
else:
final_aspect_ratio = aspect_ratio if aspect_ratio != "ASPECT_1_1" else None
+ auth = {
+ "auth_token": cls.hidden.auth_token_comfy_org,
+ "comfy_api_key": cls.hidden.api_key_comfy_org,
+ }
operation = SynchronousOperation(
endpoint=ApiEndpoint(
path="/proxy/ideogram/generate",
@@ -540,10 +525,10 @@ class IdeogramV2(ComfyNodeABC):
color_palette=color_palette if color_palette else None,
)
),
- auth_kwargs=kwargs,
+ auth_kwargs=auth,
)
- response = operation.execute()
+ response = await operation.execute()
if not response.data or len(response.data) == 0:
raise Exception("No images were generated in the response")
@@ -553,108 +538,99 @@ class IdeogramV2(ComfyNodeABC):
if not image_urls:
raise Exception("No image URLs were generated in the response")
- display_image_urls_on_node(image_urls, unique_id)
- return (download_and_process_images(image_urls),)
+ display_image_urls_on_node(image_urls, cls.hidden.unique_id)
+ return comfy_io.NodeOutput(await download_and_process_images(image_urls))
-class IdeogramV3(ComfyNodeABC):
- """
- Generates images using the Ideogram V3 model. Supports both regular image generation from text prompts and image editing with mask.
- """
- def __init__(self):
- pass
+class IdeogramV3(comfy_io.ComfyNode):
@classmethod
- def INPUT_TYPES(cls) -> InputTypeDict:
- return {
- "required": {
- "prompt": (
- IO.STRING,
- {
- "multiline": True,
- "default": "",
- "tooltip": "Prompt for the image generation or editing",
- },
+ def define_schema(cls):
+ return comfy_io.Schema(
+ node_id="IdeogramV3",
+ display_name="Ideogram V3",
+ category="api node/image/Ideogram",
+ description="Generates images using the Ideogram V3 model. "
+ "Supports both regular image generation from text prompts and image editing with mask.",
+ inputs=[
+ comfy_io.String.Input(
+ "prompt",
+ multiline=True,
+ default="",
+ tooltip="Prompt for the image generation or editing",
),
- },
- "optional": {
- "image": (
- IO.IMAGE,
- {
- "default": None,
- "tooltip": "Optional reference image for image editing.",
- },
+ comfy_io.Image.Input(
+ "image",
+ tooltip="Optional reference image for image editing.",
+ optional=True,
),
- "mask": (
- IO.MASK,
- {
- "default": None,
- "tooltip": "Optional mask for inpainting (white areas will be replaced)",
- },
+ comfy_io.Mask.Input(
+ "mask",
+ tooltip="Optional mask for inpainting (white areas will be replaced)",
+ optional=True,
),
- "aspect_ratio": (
- IO.COMBO,
- {
- "options": list(V3_RATIO_MAP.keys()),
- "default": "1:1",
- "tooltip": "The aspect ratio for image generation. Ignored if resolution is not set to Auto.",
- },
+ comfy_io.Combo.Input(
+ "aspect_ratio",
+ options=list(V3_RATIO_MAP.keys()),
+ default="1:1",
+ tooltip="The aspect ratio for image generation. Ignored if resolution is not set to Auto.",
+ optional=True,
),
- "resolution": (
- IO.COMBO,
- {
- "options": V3_RESOLUTIONS,
- "default": "Auto",
- "tooltip": "The resolution for image generation. If not set to Auto, this overrides the aspect_ratio setting.",
- },
+ comfy_io.Combo.Input(
+ "resolution",
+ options=V3_RESOLUTIONS,
+ default="Auto",
+ tooltip="The resolution for image generation. "
+ "If not set to Auto, this overrides the aspect_ratio setting.",
+ optional=True,
),
- "magic_prompt_option": (
- IO.COMBO,
- {
- "options": ["AUTO", "ON", "OFF"],
- "default": "AUTO",
- "tooltip": "Determine if MagicPrompt should be used in generation",
- },
+ comfy_io.Combo.Input(
+ "magic_prompt_option",
+ options=["AUTO", "ON", "OFF"],
+ default="AUTO",
+ tooltip="Determine if MagicPrompt should be used in generation",
+ optional=True,
),
- "seed": (
- IO.INT,
- {
- "default": 0,
- "min": 0,
- "max": 2147483647,
- "step": 1,
- "control_after_generate": True,
- "display": "number",
- },
+ comfy_io.Int.Input(
+ "seed",
+ default=0,
+ min=0,
+ max=2147483647,
+ step=1,
+ control_after_generate=True,
+ display_mode=comfy_io.NumberDisplay.number,
+ optional=True,
),
- "num_images": (
- IO.INT,
- {"default": 1, "min": 1, "max": 8, "step": 1, "display": "number"},
+ comfy_io.Int.Input(
+ "num_images",
+ default=1,
+ min=1,
+ max=8,
+ step=1,
+ display_mode=comfy_io.NumberDisplay.number,
+ optional=True,
),
- "rendering_speed": (
- IO.COMBO,
- {
- "options": ["BALANCED", "TURBO", "QUALITY"],
- "default": "BALANCED",
- "tooltip": "Controls the trade-off between generation speed and quality",
- },
+ comfy_io.Combo.Input(
+ "rendering_speed",
+ options=["BALANCED", "TURBO", "QUALITY"],
+ default="BALANCED",
+ tooltip="Controls the trade-off between generation speed and quality",
+ optional=True,
),
- },
- "hidden": {
- "auth_token": "AUTH_TOKEN_COMFY_ORG",
- "comfy_api_key": "API_KEY_COMFY_ORG",
- "unique_id": "UNIQUE_ID",
- },
- }
+ ],
+ outputs=[
+ comfy_io.Image.Output(),
+ ],
+ hidden=[
+ comfy_io.Hidden.auth_token_comfy_org,
+ comfy_io.Hidden.api_key_comfy_org,
+ comfy_io.Hidden.unique_id,
+ ],
+ )
- RETURN_TYPES = (IO.IMAGE,)
- FUNCTION = "api_call"
- CATEGORY = "api node/image/Ideogram"
- DESCRIPTION = cleandoc(__doc__ or "")
- API_NODE = True
-
- def api_call(
- self,
+ @classmethod
+ async def execute(
+ cls,
prompt,
image=None,
mask=None,
@@ -664,9 +640,11 @@ class IdeogramV3(ComfyNodeABC):
seed=0,
num_images=1,
rendering_speed="BALANCED",
- unique_id=None,
- **kwargs,
):
+ auth = {
+ "auth_token": cls.hidden.auth_token_comfy_org,
+ "comfy_api_key": cls.hidden.api_key_comfy_org,
+ }
# Check if both image and mask are provided for editing mode
if image is not None and mask is not None:
# Edit mode
@@ -686,7 +664,7 @@ class IdeogramV3(ComfyNodeABC):
# Process image
img_np = (input_tensor.numpy() * 255).astype(np.uint8)
img = Image.fromarray(img_np)
- img_byte_arr = io.BytesIO()
+ img_byte_arr = BytesIO()
img.save(img_byte_arr, format="PNG")
img_byte_arr.seek(0)
img_binary = img_byte_arr
@@ -695,7 +673,7 @@ class IdeogramV3(ComfyNodeABC):
# Process mask - white areas will be replaced
mask_np = (mask.squeeze().cpu().numpy() * 255).astype(np.uint8)
mask_img = Image.fromarray(mask_np)
- mask_byte_arr = io.BytesIO()
+ mask_byte_arr = BytesIO()
mask_img.save(mask_byte_arr, format="PNG")
mask_byte_arr.seek(0)
mask_binary = mask_byte_arr
@@ -729,7 +707,7 @@ class IdeogramV3(ComfyNodeABC):
"mask": mask_binary,
},
content_type="multipart/form-data",
- auth_kwargs=kwargs,
+ auth_kwargs=auth,
)
elif image is not None or mask is not None:
@@ -770,11 +748,11 @@ class IdeogramV3(ComfyNodeABC):
response_model=IdeogramGenerateResponse,
),
request=gen_request,
- auth_kwargs=kwargs,
+ auth_kwargs=auth,
)
# Execute the operation and process response
- response = operation.execute()
+ response = await operation.execute()
if not response.data or len(response.data) == 0:
raise Exception("No images were generated in the response")
@@ -784,18 +762,18 @@ class IdeogramV3(ComfyNodeABC):
if not image_urls:
raise Exception("No image URLs were generated in the response")
- display_image_urls_on_node(image_urls, unique_id)
- return (download_and_process_images(image_urls),)
+ display_image_urls_on_node(image_urls, cls.hidden.unique_id)
+ return comfy_io.NodeOutput(await download_and_process_images(image_urls))
-NODE_CLASS_MAPPINGS = {
- "IdeogramV1": IdeogramV1,
- "IdeogramV2": IdeogramV2,
- "IdeogramV3": IdeogramV3,
-}
+class IdeogramExtension(ComfyExtension):
+ @override
+ async def get_node_list(self) -> list[type[comfy_io.ComfyNode]]:
+ return [
+ IdeogramV1,
+ IdeogramV2,
+ IdeogramV3,
+ ]
-NODE_DISPLAY_NAME_MAPPINGS = {
- "IdeogramV1": "Ideogram V1",
- "IdeogramV2": "Ideogram V2",
- "IdeogramV3": "Ideogram V3",
-}
+async def comfy_entrypoint() -> IdeogramExtension:
+ return IdeogramExtension()
diff --git a/comfy_api_nodes/nodes_kling.py b/comfy_api_nodes/nodes_kling.py
index 69e9e5cf0..9fa390985 100644
--- a/comfy_api_nodes/nodes_kling.py
+++ b/comfy_api_nodes/nodes_kling.py
@@ -109,7 +109,7 @@ class KlingApiError(Exception):
pass
-def poll_until_finished(
+async def poll_until_finished(
auth_kwargs: dict[str, str],
api_endpoint: ApiEndpoint[Any, R],
result_url_extractor: Optional[Callable[[R], str]] = None,
@@ -117,7 +117,7 @@ def poll_until_finished(
node_id: Optional[str] = None,
) -> R:
"""Polls the Kling API endpoint until the task reaches a terminal state, then returns the response."""
- return PollingOperation(
+ return await PollingOperation(
poll_endpoint=api_endpoint,
completed_statuses=[
KlingTaskStatus.succeed.value,
@@ -278,18 +278,18 @@ def get_images_urls_from_response(response) -> Optional[str]:
return None
-def video_result_to_node_output(
+async def video_result_to_node_output(
video: KlingVideoResult,
) -> tuple[VideoFromFile, str, str]:
"""Converts a KlingVideoResult to a tuple of (VideoFromFile, str, str) to be used as a ComfyUI node output."""
return (
- download_url_to_video_output(video.url),
+ await download_url_to_video_output(str(video.url)),
str(video.id),
str(video.duration),
)
-def image_result_to_node_output(
+async def image_result_to_node_output(
images: list[KlingImageResult],
) -> torch.Tensor:
"""
@@ -297,9 +297,9 @@ def image_result_to_node_output(
If multiple images are returned, they will be stacked along the batch dimension.
"""
if len(images) == 1:
- return download_url_to_image_tensor(images[0].url)
+ return await download_url_to_image_tensor(str(images[0].url))
else:
- return torch.cat([download_url_to_image_tensor(image.url) for image in images])
+ return torch.cat([await download_url_to_image_tensor(str(image.url)) for image in images])
class KlingNodeBase(ComfyNodeABC):
@@ -421,6 +421,8 @@ class KlingTextToVideoNode(KlingNodeBase):
"pro mode / 10s duration / kling-v2-master": ("pro", "10", "kling-v2-master"),
"standard mode / 5s duration / kling-v2-master": ("std", "5", "kling-v2-master"),
"standard mode / 10s duration / kling-v2-master": ("std", "10", "kling-v2-master"),
+ "pro mode / 5s duration / kling-v2-1-master": ("pro", "5", "kling-v2-1-master"),
+ "pro mode / 10s duration / kling-v2-1-master": ("pro", "10", "kling-v2-1-master"),
}
@classmethod
@@ -467,10 +469,10 @@ class KlingTextToVideoNode(KlingNodeBase):
RETURN_NAMES = ("VIDEO", "video_id", "duration")
DESCRIPTION = "Kling Text to Video Node"
- def get_response(
+ async def get_response(
self, task_id: str, auth_kwargs: dict[str, str], node_id: Optional[str] = None
) -> KlingText2VideoResponse:
- return poll_until_finished(
+ return await poll_until_finished(
auth_kwargs,
ApiEndpoint(
path=f"{PATH_TEXT_TO_VIDEO}/{task_id}",
@@ -483,7 +485,7 @@ class KlingTextToVideoNode(KlingNodeBase):
node_id=node_id,
)
- def api_call(
+ async def api_call(
self,
prompt: str,
negative_prompt: str,
@@ -519,17 +521,17 @@ class KlingTextToVideoNode(KlingNodeBase):
auth_kwargs=kwargs,
)
- task_creation_response = initial_operation.execute()
+ task_creation_response = await initial_operation.execute()
validate_task_creation_response(task_creation_response)
task_id = task_creation_response.data.task_id
- final_response = self.get_response(
+ final_response = await self.get_response(
task_id, auth_kwargs=kwargs, node_id=unique_id
)
validate_video_result_response(final_response)
video = get_video_from_response(final_response)
- return video_result_to_node_output(video)
+ return await video_result_to_node_output(video)
class KlingCameraControlT2VNode(KlingTextToVideoNode):
@@ -581,7 +583,7 @@ class KlingCameraControlT2VNode(KlingTextToVideoNode):
DESCRIPTION = "Transform text into cinematic videos with professional camera movements that simulate real-world cinematography. Control virtual camera actions including zoom, rotation, pan, tilt, and first-person view, while maintaining focus on your original text."
- def api_call(
+ async def api_call(
self,
prompt: str,
negative_prompt: str,
@@ -591,7 +593,7 @@ class KlingCameraControlT2VNode(KlingTextToVideoNode):
unique_id: Optional[str] = None,
**kwargs,
):
- return super().api_call(
+ return await super().api_call(
model_name=KlingVideoGenModelName.kling_v1,
cfg_scale=cfg_scale,
mode=KlingVideoGenMode.std,
@@ -670,10 +672,10 @@ class KlingImage2VideoNode(KlingNodeBase):
RETURN_NAMES = ("VIDEO", "video_id", "duration")
DESCRIPTION = "Kling Image to Video Node"
- def get_response(
+ async def get_response(
self, task_id: str, auth_kwargs: dict[str, str], node_id: Optional[str] = None
) -> KlingImage2VideoResponse:
- return poll_until_finished(
+ return await poll_until_finished(
auth_kwargs,
ApiEndpoint(
path=f"{PATH_IMAGE_TO_VIDEO}/{task_id}",
@@ -686,7 +688,7 @@ class KlingImage2VideoNode(KlingNodeBase):
node_id=node_id,
)
- def api_call(
+ async def api_call(
self,
start_frame: torch.Tensor,
prompt: str,
@@ -733,17 +735,17 @@ class KlingImage2VideoNode(KlingNodeBase):
auth_kwargs=kwargs,
)
- task_creation_response = initial_operation.execute()
+ task_creation_response = await initial_operation.execute()
validate_task_creation_response(task_creation_response)
task_id = task_creation_response.data.task_id
- final_response = self.get_response(
+ final_response = await self.get_response(
task_id, auth_kwargs=kwargs, node_id=unique_id
)
validate_video_result_response(final_response)
video = get_video_from_response(final_response)
- return video_result_to_node_output(video)
+ return await video_result_to_node_output(video)
class KlingCameraControlI2VNode(KlingImage2VideoNode):
@@ -798,7 +800,7 @@ class KlingCameraControlI2VNode(KlingImage2VideoNode):
DESCRIPTION = "Transform still images into cinematic videos with professional camera movements that simulate real-world cinematography. Control virtual camera actions including zoom, rotation, pan, tilt, and first-person view, while maintaining focus on your original image."
- def api_call(
+ async def api_call(
self,
start_frame: torch.Tensor,
prompt: str,
@@ -809,7 +811,7 @@ class KlingCameraControlI2VNode(KlingImage2VideoNode):
unique_id: Optional[str] = None,
**kwargs,
):
- return super().api_call(
+ return await super().api_call(
model_name=KlingVideoGenModelName.kling_v1_5,
start_frame=start_frame,
cfg_scale=cfg_scale,
@@ -897,7 +899,7 @@ class KlingStartEndFrameNode(KlingImage2VideoNode):
DESCRIPTION = "Generate a video sequence that transitions between your provided start and end images. The node creates all frames in between, producing a smooth transformation from the first frame to the last."
- def api_call(
+ async def api_call(
self,
start_frame: torch.Tensor,
end_frame: torch.Tensor,
@@ -912,7 +914,7 @@ class KlingStartEndFrameNode(KlingImage2VideoNode):
mode, duration, model_name = KlingStartEndFrameNode.get_mode_string_mapping()[
mode
]
- return super().api_call(
+ return await super().api_call(
prompt=prompt,
negative_prompt=negative_prompt,
model_name=model_name,
@@ -964,10 +966,10 @@ class KlingVideoExtendNode(KlingNodeBase):
RETURN_NAMES = ("VIDEO", "video_id", "duration")
DESCRIPTION = "Kling Video Extend Node. Extend videos made by other Kling nodes. The video_id is created by using other Kling Nodes."
- def get_response(
+ async def get_response(
self, task_id: str, auth_kwargs: dict[str, str], node_id: Optional[str] = None
) -> KlingVideoExtendResponse:
- return poll_until_finished(
+ return await poll_until_finished(
auth_kwargs,
ApiEndpoint(
path=f"{PATH_VIDEO_EXTEND}/{task_id}",
@@ -980,7 +982,7 @@ class KlingVideoExtendNode(KlingNodeBase):
node_id=node_id,
)
- def api_call(
+ async def api_call(
self,
prompt: str,
negative_prompt: str,
@@ -1006,17 +1008,17 @@ class KlingVideoExtendNode(KlingNodeBase):
auth_kwargs=kwargs,
)
- task_creation_response = initial_operation.execute()
+ task_creation_response = await initial_operation.execute()
validate_task_creation_response(task_creation_response)
task_id = task_creation_response.data.task_id
- final_response = self.get_response(
+ final_response = await self.get_response(
task_id, auth_kwargs=kwargs, node_id=unique_id
)
validate_video_result_response(final_response)
video = get_video_from_response(final_response)
- return video_result_to_node_output(video)
+ return await video_result_to_node_output(video)
class KlingVideoEffectsBase(KlingNodeBase):
@@ -1025,10 +1027,10 @@ class KlingVideoEffectsBase(KlingNodeBase):
RETURN_TYPES = ("VIDEO", "STRING", "STRING")
RETURN_NAMES = ("VIDEO", "video_id", "duration")
- def get_response(
+ async def get_response(
self, task_id: str, auth_kwargs: dict[str, str], node_id: Optional[str] = None
) -> KlingVideoEffectsResponse:
- return poll_until_finished(
+ return await poll_until_finished(
auth_kwargs,
ApiEndpoint(
path=f"{PATH_VIDEO_EFFECTS}/{task_id}",
@@ -1041,7 +1043,7 @@ class KlingVideoEffectsBase(KlingNodeBase):
node_id=node_id,
)
- def api_call(
+ async def api_call(
self,
dual_character: bool,
effect_scene: KlingDualCharacterEffectsScene | KlingSingleImageEffectsScene,
@@ -1084,17 +1086,17 @@ class KlingVideoEffectsBase(KlingNodeBase):
auth_kwargs=kwargs,
)
- task_creation_response = initial_operation.execute()
+ task_creation_response = await initial_operation.execute()
validate_task_creation_response(task_creation_response)
task_id = task_creation_response.data.task_id
- final_response = self.get_response(
+ final_response = await self.get_response(
task_id, auth_kwargs=kwargs, node_id=unique_id
)
validate_video_result_response(final_response)
video = get_video_from_response(final_response)
- return video_result_to_node_output(video)
+ return await video_result_to_node_output(video)
class KlingDualCharacterVideoEffectNode(KlingVideoEffectsBase):
@@ -1142,7 +1144,7 @@ class KlingDualCharacterVideoEffectNode(KlingVideoEffectsBase):
RETURN_TYPES = ("VIDEO", "STRING")
RETURN_NAMES = ("VIDEO", "duration")
- def api_call(
+ async def api_call(
self,
image_left: torch.Tensor,
image_right: torch.Tensor,
@@ -1153,7 +1155,7 @@ class KlingDualCharacterVideoEffectNode(KlingVideoEffectsBase):
unique_id: Optional[str] = None,
**kwargs,
):
- video, _, duration = super().api_call(
+ video, _, duration = await super().api_call(
dual_character=True,
effect_scene=effect_scene,
model_name=model_name,
@@ -1208,7 +1210,7 @@ class KlingSingleImageVideoEffectNode(KlingVideoEffectsBase):
DESCRIPTION = "Achieve different special effects when generating a video based on the effect_scene."
- def api_call(
+ async def api_call(
self,
image: torch.Tensor,
effect_scene: KlingSingleImageEffectsScene,
@@ -1217,7 +1219,7 @@ class KlingSingleImageVideoEffectNode(KlingVideoEffectsBase):
unique_id: Optional[str] = None,
**kwargs,
):
- return super().api_call(
+ return await super().api_call(
dual_character=False,
effect_scene=effect_scene,
model_name=model_name,
@@ -1253,11 +1255,11 @@ class KlingLipSyncBase(KlingNodeBase):
f"Text is too long. Maximum length is {MAX_PROMPT_LENGTH_LIP_SYNC} characters."
)
- def get_response(
+ async def get_response(
self, task_id: str, auth_kwargs: dict[str, str], node_id: Optional[str] = None
) -> KlingLipSyncResponse:
"""Polls the Kling API endpoint until the task reaches a terminal state."""
- return poll_until_finished(
+ return await poll_until_finished(
auth_kwargs,
ApiEndpoint(
path=f"{PATH_LIP_SYNC}/{task_id}",
@@ -1270,7 +1272,7 @@ class KlingLipSyncBase(KlingNodeBase):
node_id=node_id,
)
- def api_call(
+ async def api_call(
self,
video: VideoInput,
audio: Optional[AudioInput] = None,
@@ -1287,12 +1289,12 @@ class KlingLipSyncBase(KlingNodeBase):
self.validate_lip_sync_video(video)
# Upload video to Comfy API and get download URL
- video_url = upload_video_to_comfyapi(video, auth_kwargs=kwargs)
+ video_url = await upload_video_to_comfyapi(video, auth_kwargs=kwargs)
logging.info("Uploaded video to Comfy API. URL: %s", video_url)
# Upload the audio file to Comfy API and get download URL
if audio:
- audio_url = upload_audio_to_comfyapi(audio, auth_kwargs=kwargs)
+ audio_url = await upload_audio_to_comfyapi(audio, auth_kwargs=kwargs)
logging.info("Uploaded audio to Comfy API. URL: %s", audio_url)
else:
audio_url = None
@@ -1319,17 +1321,17 @@ class KlingLipSyncBase(KlingNodeBase):
auth_kwargs=kwargs,
)
- task_creation_response = initial_operation.execute()
+ task_creation_response = await initial_operation.execute()
validate_task_creation_response(task_creation_response)
task_id = task_creation_response.data.task_id
- final_response = self.get_response(
+ final_response = await self.get_response(
task_id, auth_kwargs=kwargs, node_id=unique_id
)
validate_video_result_response(final_response)
video = get_video_from_response(final_response)
- return video_result_to_node_output(video)
+ return await video_result_to_node_output(video)
class KlingLipSyncAudioToVideoNode(KlingLipSyncBase):
@@ -1357,7 +1359,7 @@ class KlingLipSyncAudioToVideoNode(KlingLipSyncBase):
DESCRIPTION = "Kling Lip Sync Audio to Video Node. Syncs mouth movements in a video file to the audio content of an audio file. When using, ensure that the audio contains clearly distinguishable vocals and that the video contains a distinct face. The audio file should not be larger than 5MB. The video file should not be larger than 100MB, should have height/width between 720px and 1920px, and should be between 2s and 10s in length."
- def api_call(
+ async def api_call(
self,
video: VideoInput,
audio: AudioInput,
@@ -1365,7 +1367,7 @@ class KlingLipSyncAudioToVideoNode(KlingLipSyncBase):
unique_id: Optional[str] = None,
**kwargs,
):
- return super().api_call(
+ return await super().api_call(
video=video,
audio=audio,
voice_language=voice_language,
@@ -1469,7 +1471,7 @@ class KlingLipSyncTextToVideoNode(KlingLipSyncBase):
DESCRIPTION = "Kling Lip Sync Text to Video Node. Syncs mouth movements in a video file to a text prompt. The video file should not be larger than 100MB, should have height/width between 720px and 1920px, and should be between 2s and 10s in length."
- def api_call(
+ async def api_call(
self,
video: VideoInput,
text: str,
@@ -1479,7 +1481,7 @@ class KlingLipSyncTextToVideoNode(KlingLipSyncBase):
**kwargs,
):
voice_id, voice_language = KlingLipSyncTextToVideoNode.get_voice_config()[voice]
- return super().api_call(
+ return await super().api_call(
video=video,
text=text,
voice_language=voice_language,
@@ -1533,10 +1535,10 @@ class KlingVirtualTryOnNode(KlingImageGenerationBase):
DESCRIPTION = "Kling Virtual Try On Node. Input a human image and a cloth image to try on the cloth on the human. You can merge multiple clothing item pictures into one image with a white background."
- def get_response(
+ async def get_response(
self, task_id: str, auth_kwargs: dict[str, str], node_id: Optional[str] = None
) -> KlingVirtualTryOnResponse:
- return poll_until_finished(
+ return await poll_until_finished(
auth_kwargs,
ApiEndpoint(
path=f"{PATH_VIRTUAL_TRY_ON}/{task_id}",
@@ -1549,7 +1551,7 @@ class KlingVirtualTryOnNode(KlingImageGenerationBase):
node_id=node_id,
)
- def api_call(
+ async def api_call(
self,
human_image: torch.Tensor,
cloth_image: torch.Tensor,
@@ -1572,17 +1574,17 @@ class KlingVirtualTryOnNode(KlingImageGenerationBase):
auth_kwargs=kwargs,
)
- task_creation_response = initial_operation.execute()
+ task_creation_response = await initial_operation.execute()
validate_task_creation_response(task_creation_response)
task_id = task_creation_response.data.task_id
- final_response = self.get_response(
+ final_response = await self.get_response(
task_id, auth_kwargs=kwargs, node_id=unique_id
)
validate_image_result_response(final_response)
images = get_images_from_response(final_response)
- return (image_result_to_node_output(images),)
+ return (await image_result_to_node_output(images),)
class KlingImageGenerationNode(KlingImageGenerationBase):
@@ -1655,13 +1657,13 @@ class KlingImageGenerationNode(KlingImageGenerationBase):
DESCRIPTION = "Kling Image Generation Node. Generate an image from a text prompt with an optional reference image."
- def get_response(
+ async def get_response(
self,
task_id: str,
auth_kwargs: Optional[dict[str, str]],
node_id: Optional[str] = None,
) -> KlingImageGenerationsResponse:
- return poll_until_finished(
+ return await poll_until_finished(
auth_kwargs,
ApiEndpoint(
path=f"{PATH_IMAGE_GENERATIONS}/{task_id}",
@@ -1674,7 +1676,7 @@ class KlingImageGenerationNode(KlingImageGenerationBase):
node_id=node_id,
)
- def api_call(
+ async def api_call(
self,
model_name: KlingImageGenModelName,
prompt: str,
@@ -1690,7 +1692,11 @@ class KlingImageGenerationNode(KlingImageGenerationBase):
):
self.validate_prompt(prompt, negative_prompt)
- if image is not None:
+ if image is None:
+ image_type = None
+ elif model_name == KlingImageGenModelName.kling_v1:
+ raise ValueError(f"The model {KlingImageGenModelName.kling_v1.value} does not support reference images.")
+ else:
image = tensor_to_base64_string(image)
initial_operation = SynchronousOperation(
@@ -1714,17 +1720,17 @@ class KlingImageGenerationNode(KlingImageGenerationBase):
auth_kwargs=kwargs,
)
- task_creation_response = initial_operation.execute()
+ task_creation_response = await initial_operation.execute()
validate_task_creation_response(task_creation_response)
task_id = task_creation_response.data.task_id
- final_response = self.get_response(
+ final_response = await self.get_response(
task_id, auth_kwargs=kwargs, node_id=unique_id
)
validate_image_result_response(final_response)
images = get_images_from_response(final_response)
- return (image_result_to_node_output(images),)
+ return (await image_result_to_node_output(images),)
NODE_CLASS_MAPPINGS = {
diff --git a/comfy_api_nodes/nodes_luma.py b/comfy_api_nodes/nodes_luma.py
index 525dc38e6..b3c32bed5 100644
--- a/comfy_api_nodes/nodes_luma.py
+++ b/comfy_api_nodes/nodes_luma.py
@@ -38,7 +38,7 @@ from comfy_api_nodes.apinode_utils import (
)
from server import PromptServer
-import requests
+import aiohttp
import torch
from io import BytesIO
@@ -217,7 +217,7 @@ class LumaImageGenerationNode(ComfyNodeABC):
},
}
- def api_call(
+ async def api_call(
self,
prompt: str,
model: str,
@@ -234,19 +234,19 @@ class LumaImageGenerationNode(ComfyNodeABC):
# handle image_luma_ref
api_image_ref = None
if image_luma_ref is not None:
- api_image_ref = self._convert_luma_refs(
+ api_image_ref = await self._convert_luma_refs(
image_luma_ref, max_refs=4, auth_kwargs=kwargs,
)
# handle style_luma_ref
api_style_ref = None
if style_image is not None:
- api_style_ref = self._convert_style_image(
+ api_style_ref = await self._convert_style_image(
style_image, weight=style_image_weight, auth_kwargs=kwargs,
)
# handle character_ref images
character_ref = None
if character_image is not None:
- download_urls = upload_images_to_comfyapi(
+ download_urls = await upload_images_to_comfyapi(
character_image, max_images=4, auth_kwargs=kwargs,
)
character_ref = LumaCharacterRef(
@@ -270,7 +270,7 @@ class LumaImageGenerationNode(ComfyNodeABC):
),
auth_kwargs=kwargs,
)
- response_api: LumaGeneration = operation.execute()
+ response_api: LumaGeneration = await operation.execute()
operation = PollingOperation(
poll_endpoint=ApiEndpoint(
@@ -286,19 +286,20 @@ class LumaImageGenerationNode(ComfyNodeABC):
node_id=unique_id,
auth_kwargs=kwargs,
)
- response_poll = operation.execute()
+ response_poll = await operation.execute()
- img_response = requests.get(response_poll.assets.image)
- img = process_image_response(img_response)
+ async with aiohttp.ClientSession() as session:
+ async with session.get(response_poll.assets.image) as img_response:
+ img = process_image_response(await img_response.content.read())
return (img,)
- def _convert_luma_refs(
+ async def _convert_luma_refs(
self, luma_ref: LumaReferenceChain, max_refs: int, auth_kwargs: Optional[dict[str,str]] = None
):
luma_urls = []
ref_count = 0
for ref in luma_ref.refs:
- download_urls = upload_images_to_comfyapi(
+ download_urls = await upload_images_to_comfyapi(
ref.image, max_images=1, auth_kwargs=auth_kwargs
)
luma_urls.append(download_urls[0])
@@ -307,13 +308,13 @@ class LumaImageGenerationNode(ComfyNodeABC):
break
return luma_ref.create_api_model(download_urls=luma_urls, max_refs=max_refs)
- def _convert_style_image(
+ async def _convert_style_image(
self, style_image: torch.Tensor, weight: float, auth_kwargs: Optional[dict[str,str]] = None
):
chain = LumaReferenceChain(
first_ref=LumaReference(image=style_image, weight=weight)
)
- return self._convert_luma_refs(chain, max_refs=1, auth_kwargs=auth_kwargs)
+ return await self._convert_luma_refs(chain, max_refs=1, auth_kwargs=auth_kwargs)
class LumaImageModifyNode(ComfyNodeABC):
@@ -370,7 +371,7 @@ class LumaImageModifyNode(ComfyNodeABC):
},
}
- def api_call(
+ async def api_call(
self,
prompt: str,
model: str,
@@ -381,7 +382,7 @@ class LumaImageModifyNode(ComfyNodeABC):
**kwargs,
):
# first, upload image
- download_urls = upload_images_to_comfyapi(
+ download_urls = await upload_images_to_comfyapi(
image, max_images=1, auth_kwargs=kwargs,
)
image_url = download_urls[0]
@@ -402,7 +403,7 @@ class LumaImageModifyNode(ComfyNodeABC):
),
auth_kwargs=kwargs,
)
- response_api: LumaGeneration = operation.execute()
+ response_api: LumaGeneration = await operation.execute()
operation = PollingOperation(
poll_endpoint=ApiEndpoint(
@@ -418,10 +419,11 @@ class LumaImageModifyNode(ComfyNodeABC):
node_id=unique_id,
auth_kwargs=kwargs,
)
- response_poll = operation.execute()
+ response_poll = await operation.execute()
- img_response = requests.get(response_poll.assets.image)
- img = process_image_response(img_response)
+ async with aiohttp.ClientSession() as session:
+ async with session.get(response_poll.assets.image) as img_response:
+ img = process_image_response(await img_response.content.read())
return (img,)
@@ -494,7 +496,7 @@ class LumaTextToVideoGenerationNode(ComfyNodeABC):
},
}
- def api_call(
+ async def api_call(
self,
prompt: str,
model: str,
@@ -529,7 +531,7 @@ class LumaTextToVideoGenerationNode(ComfyNodeABC):
),
auth_kwargs=kwargs,
)
- response_api: LumaGeneration = operation.execute()
+ response_api: LumaGeneration = await operation.execute()
if unique_id:
PromptServer.instance.send_progress_text(f"Luma video generation started: {response_api.id}", unique_id)
@@ -549,10 +551,11 @@ class LumaTextToVideoGenerationNode(ComfyNodeABC):
estimated_duration=LUMA_T2V_AVERAGE_DURATION,
auth_kwargs=kwargs,
)
- response_poll = operation.execute()
+ response_poll = await operation.execute()
- vid_response = requests.get(response_poll.assets.video)
- return (VideoFromFile(BytesIO(vid_response.content)),)
+ async with aiohttp.ClientSession() as session:
+ async with session.get(response_poll.assets.video) as vid_response:
+ return (VideoFromFile(BytesIO(await vid_response.content.read())),)
class LumaImageToVideoGenerationNode(ComfyNodeABC):
@@ -626,7 +629,7 @@ class LumaImageToVideoGenerationNode(ComfyNodeABC):
},
}
- def api_call(
+ async def api_call(
self,
prompt: str,
model: str,
@@ -644,7 +647,7 @@ class LumaImageToVideoGenerationNode(ComfyNodeABC):
raise Exception(
"At least one of first_image and last_image requires an input."
)
- keyframes = self._convert_to_keyframes(first_image, last_image, auth_kwargs=kwargs)
+ keyframes = await self._convert_to_keyframes(first_image, last_image, auth_kwargs=kwargs)
duration = duration if model != LumaVideoModel.ray_1_6 else None
resolution = resolution if model != LumaVideoModel.ray_1_6 else None
@@ -667,7 +670,7 @@ class LumaImageToVideoGenerationNode(ComfyNodeABC):
),
auth_kwargs=kwargs,
)
- response_api: LumaGeneration = operation.execute()
+ response_api: LumaGeneration = await operation.execute()
if unique_id:
PromptServer.instance.send_progress_text(f"Luma video generation started: {response_api.id}", unique_id)
@@ -687,12 +690,13 @@ class LumaImageToVideoGenerationNode(ComfyNodeABC):
estimated_duration=LUMA_I2V_AVERAGE_DURATION,
auth_kwargs=kwargs,
)
- response_poll = operation.execute()
+ response_poll = await operation.execute()
- vid_response = requests.get(response_poll.assets.video)
- return (VideoFromFile(BytesIO(vid_response.content)),)
+ async with aiohttp.ClientSession() as session:
+ async with session.get(response_poll.assets.video) as vid_response:
+ return (VideoFromFile(BytesIO(await vid_response.content.read())),)
- def _convert_to_keyframes(
+ async def _convert_to_keyframes(
self,
first_image: torch.Tensor = None,
last_image: torch.Tensor = None,
@@ -703,12 +707,12 @@ class LumaImageToVideoGenerationNode(ComfyNodeABC):
frame0 = None
frame1 = None
if first_image is not None:
- download_urls = upload_images_to_comfyapi(
+ download_urls = await upload_images_to_comfyapi(
first_image, max_images=1, auth_kwargs=auth_kwargs,
)
frame0 = LumaImageReference(type="image", url=download_urls[0])
if last_image is not None:
- download_urls = upload_images_to_comfyapi(
+ download_urls = await upload_images_to_comfyapi(
last_image, max_images=1, auth_kwargs=auth_kwargs,
)
frame1 = LumaImageReference(type="image", url=download_urls[0])
diff --git a/comfy_api_nodes/nodes_minimax.py b/comfy_api_nodes/nodes_minimax.py
index 9b46636db..bb3c9e710 100644
--- a/comfy_api_nodes/nodes_minimax.py
+++ b/comfy_api_nodes/nodes_minimax.py
@@ -1,3 +1,4 @@
+from inspect import cleandoc
from typing import Union
import logging
import torch
@@ -10,7 +11,7 @@ from comfy_api_nodes.apis import (
MinimaxFileRetrieveResponse,
MinimaxTaskResultResponse,
SubjectReferenceItem,
- Model
+ MiniMaxModel
)
from comfy_api_nodes.apis.client import (
ApiEndpoint,
@@ -84,9 +85,8 @@ class MinimaxTextToVideoNode:
FUNCTION = "generate_video"
CATEGORY = "api node/video/MiniMax"
API_NODE = True
- OUTPUT_NODE = True
- def generate_video(
+ async def generate_video(
self,
prompt_text,
seed=0,
@@ -104,12 +104,12 @@ class MinimaxTextToVideoNode:
# upload image, if passed in
image_url = None
if image is not None:
- image_url = upload_images_to_comfyapi(image, max_images=1, auth_kwargs=kwargs)[0]
+ image_url = (await upload_images_to_comfyapi(image, max_images=1, auth_kwargs=kwargs))[0]
# TODO: figure out how to deal with subject properly, API returns invalid params when using S2V-01 model
subject_reference = None
if subject is not None:
- subject_url = upload_images_to_comfyapi(subject, max_images=1, auth_kwargs=kwargs)[0]
+ subject_url = (await upload_images_to_comfyapi(subject, max_images=1, auth_kwargs=kwargs))[0]
subject_reference = [SubjectReferenceItem(image=subject_url)]
@@ -121,7 +121,7 @@ class MinimaxTextToVideoNode:
response_model=MinimaxVideoGenerationResponse,
),
request=MinimaxVideoGenerationRequest(
- model=Model(model),
+ model=MiniMaxModel(model),
prompt=prompt_text,
callback_url=None,
first_frame_image=image_url,
@@ -130,7 +130,7 @@ class MinimaxTextToVideoNode:
),
auth_kwargs=kwargs,
)
- response = video_generate_operation.execute()
+ response = await video_generate_operation.execute()
task_id = response.task_id
if not task_id:
@@ -151,7 +151,7 @@ class MinimaxTextToVideoNode:
node_id=unique_id,
auth_kwargs=kwargs,
)
- task_result = video_generate_operation.execute()
+ task_result = await video_generate_operation.execute()
file_id = task_result.file_id
if file_id is None:
@@ -167,7 +167,7 @@ class MinimaxTextToVideoNode:
request=EmptyRequest(),
auth_kwargs=kwargs,
)
- file_result = file_retrieve_operation.execute()
+ file_result = await file_retrieve_operation.execute()
file_url = file_result.file.download_url
if file_url is None:
@@ -182,7 +182,7 @@ class MinimaxTextToVideoNode:
message = f"Result URL: {file_url}"
PromptServer.instance.send_progress_text(message, unique_id)
- video_io = download_url_to_bytesio(file_url)
+ video_io = await download_url_to_bytesio(file_url)
if video_io is None:
error_msg = f"Failed to download video from {file_url}"
logging.error(error_msg)
@@ -251,7 +251,6 @@ class MinimaxImageToVideoNode(MinimaxTextToVideoNode):
FUNCTION = "generate_video"
CATEGORY = "api node/video/MiniMax"
API_NODE = True
- OUTPUT_NODE = True
class MinimaxSubjectToVideoNode(MinimaxTextToVideoNode):
@@ -313,7 +312,181 @@ class MinimaxSubjectToVideoNode(MinimaxTextToVideoNode):
FUNCTION = "generate_video"
CATEGORY = "api node/video/MiniMax"
API_NODE = True
- OUTPUT_NODE = True
+
+
+class MinimaxHailuoVideoNode:
+ """Generates videos from prompt, with optional start frame using the new MiniMax Hailuo-02 model."""
+
+ @classmethod
+ def INPUT_TYPES(s):
+ return {
+ "required": {
+ "prompt_text": (
+ "STRING",
+ {
+ "multiline": True,
+ "default": "",
+ "tooltip": "Text prompt to guide the video generation.",
+ },
+ ),
+ },
+ "optional": {
+ "seed": (
+ IO.INT,
+ {
+ "default": 0,
+ "min": 0,
+ "max": 0xFFFFFFFFFFFFFFFF,
+ "control_after_generate": True,
+ "tooltip": "The random seed used for creating the noise.",
+ },
+ ),
+ "first_frame_image": (
+ IO.IMAGE,
+ {
+ "tooltip": "Optional image to use as the first frame to generate a video."
+ },
+ ),
+ "prompt_optimizer": (
+ IO.BOOLEAN,
+ {
+ "tooltip": "Optimize prompt to improve generation quality when needed.",
+ "default": True,
+ },
+ ),
+ "duration": (
+ IO.COMBO,
+ {
+ "tooltip": "The length of the output video in seconds.",
+ "default": 6,
+ "options": [6, 10],
+ },
+ ),
+ "resolution": (
+ IO.COMBO,
+ {
+ "tooltip": "The dimensions of the video display. "
+ "1080p corresponds to 1920 x 1080 pixels, 768p corresponds to 1366 x 768 pixels.",
+ "default": "768P",
+ "options": ["768P", "1080P"],
+ },
+ ),
+ },
+ "hidden": {
+ "auth_token": "AUTH_TOKEN_COMFY_ORG",
+ "comfy_api_key": "API_KEY_COMFY_ORG",
+ "unique_id": "UNIQUE_ID",
+ },
+ }
+
+ RETURN_TYPES = ("VIDEO",)
+ DESCRIPTION = cleandoc(__doc__ or "")
+ FUNCTION = "generate_video"
+ CATEGORY = "api node/video/MiniMax"
+ API_NODE = True
+
+ async def generate_video(
+ self,
+ prompt_text,
+ seed=0,
+ first_frame_image: torch.Tensor=None, # used for ImageToVideo
+ prompt_optimizer=True,
+ duration=6,
+ resolution="768P",
+ model="MiniMax-Hailuo-02",
+ unique_id: Union[str, None]=None,
+ **kwargs,
+ ):
+ if first_frame_image is None:
+ validate_string(prompt_text, field_name="prompt_text")
+
+ if model == "MiniMax-Hailuo-02" and resolution.upper() == "1080P" and duration != 6:
+ raise Exception(
+ "When model is MiniMax-Hailuo-02 and resolution is 1080P, duration is limited to 6 seconds."
+ )
+
+ # upload image, if passed in
+ image_url = None
+ if first_frame_image is not None:
+ image_url = (await upload_images_to_comfyapi(first_frame_image, max_images=1, auth_kwargs=kwargs))[0]
+
+ video_generate_operation = SynchronousOperation(
+ endpoint=ApiEndpoint(
+ path="/proxy/minimax/video_generation",
+ method=HttpMethod.POST,
+ request_model=MinimaxVideoGenerationRequest,
+ response_model=MinimaxVideoGenerationResponse,
+ ),
+ request=MinimaxVideoGenerationRequest(
+ model=MiniMaxModel(model),
+ prompt=prompt_text,
+ callback_url=None,
+ first_frame_image=image_url,
+ prompt_optimizer=prompt_optimizer,
+ duration=duration,
+ resolution=resolution,
+ ),
+ auth_kwargs=kwargs,
+ )
+ response = await video_generate_operation.execute()
+
+ task_id = response.task_id
+ if not task_id:
+ raise Exception(f"MiniMax generation failed: {response.base_resp}")
+
+ average_duration = 120 if resolution == "768P" else 240
+ video_generate_operation = PollingOperation(
+ poll_endpoint=ApiEndpoint(
+ path="/proxy/minimax/query/video_generation",
+ method=HttpMethod.GET,
+ request_model=EmptyRequest,
+ response_model=MinimaxTaskResultResponse,
+ query_params={"task_id": task_id},
+ ),
+ completed_statuses=["Success"],
+ failed_statuses=["Fail"],
+ status_extractor=lambda x: x.status.value,
+ estimated_duration=average_duration,
+ node_id=unique_id,
+ auth_kwargs=kwargs,
+ )
+ task_result = await video_generate_operation.execute()
+
+ file_id = task_result.file_id
+ if file_id is None:
+ raise Exception("Request was not successful. Missing file ID.")
+ file_retrieve_operation = SynchronousOperation(
+ endpoint=ApiEndpoint(
+ path="/proxy/minimax/files/retrieve",
+ method=HttpMethod.GET,
+ request_model=EmptyRequest,
+ response_model=MinimaxFileRetrieveResponse,
+ query_params={"file_id": int(file_id)},
+ ),
+ request=EmptyRequest(),
+ auth_kwargs=kwargs,
+ )
+ file_result = await file_retrieve_operation.execute()
+
+ file_url = file_result.file.download_url
+ if file_url is None:
+ raise Exception(
+ f"No video was found in the response. Full response: {file_result.model_dump()}"
+ )
+ logging.info(f"Generated video URL: {file_url}")
+ if unique_id:
+ if hasattr(file_result.file, "backup_download_url"):
+ message = f"Result URL: {file_url}\nBackup URL: {file_result.file.backup_download_url}"
+ else:
+ message = f"Result URL: {file_url}"
+ PromptServer.instance.send_progress_text(message, unique_id)
+
+ video_io = await download_url_to_bytesio(file_url)
+ if video_io is None:
+ error_msg = f"Failed to download video from {file_url}"
+ logging.error(error_msg)
+ raise Exception(error_msg)
+ return (VideoFromFile(video_io),)
# A dictionary that contains all nodes you want to export with their names
@@ -322,6 +495,7 @@ NODE_CLASS_MAPPINGS = {
"MinimaxTextToVideoNode": MinimaxTextToVideoNode,
"MinimaxImageToVideoNode": MinimaxImageToVideoNode,
# "MinimaxSubjectToVideoNode": MinimaxSubjectToVideoNode,
+ "MinimaxHailuoVideoNode": MinimaxHailuoVideoNode,
}
# A dictionary that contains the friendly/humanly readable titles for the nodes
@@ -329,4 +503,5 @@ NODE_DISPLAY_NAME_MAPPINGS = {
"MinimaxTextToVideoNode": "MiniMax Text to Video",
"MinimaxImageToVideoNode": "MiniMax Image to Video",
"MinimaxSubjectToVideoNode": "MiniMax Subject to Video",
+ "MinimaxHailuoVideoNode": "MiniMax Hailuo Video",
}
diff --git a/comfy_api_nodes/nodes_moonvalley.py b/comfy_api_nodes/nodes_moonvalley.py
index 789fcef02..806a70e06 100644
--- a/comfy_api_nodes/nodes_moonvalley.py
+++ b/comfy_api_nodes/nodes_moonvalley.py
@@ -1,6 +1,5 @@
import logging
from typing import Any, Callable, Optional, TypeVar
-import random
import torch
from comfy_api_nodes.util.validation_utils import (
get_image_dimensions,
@@ -95,14 +94,14 @@ def get_video_url_from_response(response) -> Optional[str]:
return None
-def poll_until_finished(
+async def poll_until_finished(
auth_kwargs: dict[str, str],
api_endpoint: ApiEndpoint[Any, R],
result_url_extractor: Optional[Callable[[R], str]] = None,
node_id: Optional[str] = None,
) -> R:
"""Polls the Moonvalley API endpoint until the task reaches a terminal state, then returns the response."""
- return PollingOperation(
+ return await PollingOperation(
poll_endpoint=api_endpoint,
completed_statuses=[
"completed",
@@ -208,20 +207,29 @@ def _get_video_dimensions(video: VideoInput) -> tuple[int, int]:
def _validate_video_dimensions(width: int, height: int) -> None:
"""Validates video dimensions meet Moonvalley V2V requirements."""
supported_resolutions = {
- (1920, 1080), (1080, 1920), (1152, 1152),
- (1536, 1152), (1152, 1536)
+ (1920, 1080),
+ (1080, 1920),
+ (1152, 1152),
+ (1536, 1152),
+ (1152, 1536),
}
if (width, height) not in supported_resolutions:
- supported_list = ', '.join([f'{w}x{h}' for w, h in sorted(supported_resolutions)])
- raise ValueError(f"Resolution {width}x{height} not supported. Supported: {supported_list}")
+ supported_list = ", ".join(
+ [f"{w}x{h}" for w, h in sorted(supported_resolutions)]
+ )
+ raise ValueError(
+ f"Resolution {width}x{height} not supported. Supported: {supported_list}"
+ )
def _validate_container_format(video: VideoInput) -> None:
"""Validates video container format is MP4."""
container_format = video.get_container_format()
- if container_format not in ['mp4', 'mov,mp4,m4a,3gp,3g2,mj2']:
- raise ValueError(f"Only MP4 container format supported. Got: {container_format}")
+ if container_format not in ["mp4", "mov,mp4,m4a,3gp,3g2,mj2"]:
+ raise ValueError(
+ f"Only MP4 container format supported. Got: {container_format}"
+ )
def _validate_and_trim_duration(video: VideoInput) -> VideoInput:
@@ -244,7 +252,6 @@ def _trim_if_too_long(video: VideoInput, duration: float) -> VideoInput:
return video
-
def trim_video(video: VideoInput, duration_sec: float) -> VideoInput:
"""
Returns a new VideoInput object trimmed from the beginning to the specified duration,
@@ -302,7 +309,9 @@ def trim_video(video: VideoInput, duration_sec: float) -> VideoInput:
# Calculate target frame count that's divisible by 16
fps = input_container.streams.video[0].average_rate
estimated_frames = int(duration_sec * fps)
- target_frames = (estimated_frames // 16) * 16 # Round down to nearest multiple of 16
+ target_frames = (
+ estimated_frames // 16
+ ) * 16 # Round down to nearest multiple of 16
if target_frames == 0:
raise ValueError("Video too short: need at least 16 frames for Moonvalley")
@@ -394,10 +403,10 @@ class BaseMoonvalleyVideoNode:
else:
return control_map["Motion Transfer"]
- def get_response(
+ async def get_response(
self, task_id: str, auth_kwargs: dict[str, str], node_id: Optional[str] = None
) -> MoonvalleyPromptResponse:
- return poll_until_finished(
+ return await poll_until_finished(
auth_kwargs,
ApiEndpoint(
path=f"{API_PROMPTS_ENDPOINT}/{task_id}",
@@ -424,7 +433,7 @@ class BaseMoonvalleyVideoNode:
MoonvalleyTextToVideoInferenceParams,
"negative_prompt",
multiline=True,
- default="low-poly, flat shader, bad rigging, stiff animation, uncanny eyes, low-quality textures, looping glitch, cheap effect, overbloom, bloom spam, default lighting, game asset, stiff face, ugly specular, AI artifacts",
+ default=" gopro, bright, contrast, static, overexposed, vignette, artifacts, still, noise, texture, scanlines, videogame, 360 camera, VR, transition, flare, saturation, distorted, warped, wide angle, saturated, vibrant, glowing, cross dissolve, cheesy, ugly hands, mutated hands, mutant, disfigured, extra fingers, blown out, horrible, blurry, worst quality, bad, dissolve, melt, fade in, fade out, wobbly, weird, low quality, plastic, stock footage, video camera, boring",
),
"resolution": (
IO.COMBO,
@@ -441,12 +450,11 @@ class BaseMoonvalleyVideoNode:
"tooltip": "Resolution of the output video",
},
),
- # "length": (IO.COMBO,{"options":['5s','10s'], "default": '5s'}),
"prompt_adherence": model_field_to_node_input(
IO.FLOAT,
MoonvalleyTextToVideoInferenceParams,
"guidance_scale",
- default=7.0,
+ default=10.0,
step=1,
min=1,
max=20,
@@ -455,13 +463,12 @@ class BaseMoonvalleyVideoNode:
IO.INT,
MoonvalleyTextToVideoInferenceParams,
"seed",
- default=random.randint(0, 2**32 - 1),
+ default=9,
min=0,
max=4294967295,
step=1,
display="number",
tooltip="Random seed value",
- control_after_generate=True,
),
"steps": model_field_to_node_input(
IO.INT,
@@ -507,7 +514,7 @@ class MoonvalleyImg2VideoNode(BaseMoonvalleyVideoNode):
RETURN_NAMES = ("video",)
DESCRIPTION = "Moonvalley Marey Image to Video Node"
- def generate(
+ async def generate(
self, prompt, negative_prompt, unique_id: Optional[str] = None, **kwargs
):
image = kwargs.get("image", None)
@@ -532,8 +539,10 @@ class MoonvalleyImg2VideoNode(BaseMoonvalleyVideoNode):
# Get MIME type from tensor - assuming PNG format for image tensors
mime_type = "image/png"
- image_url = upload_images_to_comfyapi(
- image, max_images=1, auth_kwargs=kwargs, mime_type=mime_type
+ image_url = (
+ await upload_images_to_comfyapi(
+ image, max_images=1, auth_kwargs=kwargs, mime_type=mime_type
+ )
)[0]
request = MoonvalleyTextToVideoRequest(
@@ -549,14 +558,14 @@ class MoonvalleyImg2VideoNode(BaseMoonvalleyVideoNode):
request=request,
auth_kwargs=kwargs,
)
- task_creation_response = initial_operation.execute()
+ task_creation_response = await initial_operation.execute()
validate_task_creation_response(task_creation_response)
task_id = task_creation_response.id
- final_response = self.get_response(
+ final_response = await self.get_response(
task_id, auth_kwargs=kwargs, node_id=unique_id
)
- video = download_url_to_video_output(final_response.output_url)
+ video = await download_url_to_video_output(final_response.output_url)
return (video,)
@@ -570,17 +579,39 @@ class MoonvalleyVideo2VideoNode(BaseMoonvalleyVideoNode):
return {
"required": {
"prompt": model_field_to_node_input(
- IO.STRING, MoonvalleyVideoToVideoRequest, "prompt_text",
- multiline=True
+ IO.STRING,
+ MoonvalleyVideoToVideoRequest,
+ "prompt_text",
+ multiline=True,
),
"negative_prompt": model_field_to_node_input(
IO.STRING,
MoonvalleyVideoToVideoInferenceParams,
"negative_prompt",
multiline=True,
- default="low-poly, flat shader, bad rigging, stiff animation, uncanny eyes, low-quality textures, looping glitch, cheap effect, overbloom, bloom spam, default lighting, game asset, stiff face, ugly specular, AI artifacts"
+ default=" gopro, bright, contrast, static, overexposed, vignette, artifacts, still, noise, texture, scanlines, videogame, 360 camera, VR, transition, flare, saturation, distorted, warped, wide angle, saturated, vibrant, glowing, cross dissolve, cheesy, ugly hands, mutated hands, mutant, disfigured, extra fingers, blown out, horrible, blurry, worst quality, bad, dissolve, melt, fade in, fade out, wobbly, weird, low quality, plastic, stock footage, video camera, boring",
+ ),
+ "seed": model_field_to_node_input(
+ IO.INT,
+ MoonvalleyVideoToVideoInferenceParams,
+ "seed",
+ default=9,
+ min=0,
+ max=4294967295,
+ step=1,
+ display="number",
+ tooltip="Random seed value",
+ control_after_generate=False,
+ ),
+ "prompt_adherence": model_field_to_node_input(
+ IO.FLOAT,
+ MoonvalleyVideoToVideoInferenceParams,
+ "guidance_scale",
+ default=10.0,
+ step=1,
+ min=1,
+ max=20,
),
- "seed": model_field_to_node_input(IO.INT,MoonvalleyVideoToVideoInferenceParams, "seed", default=random.randint(0, 2**32 - 1), min=0, max=4294967295, step=1, display="number", tooltip="Random seed value", control_after_generate=True),
},
"hidden": {
"auth_token": "AUTH_TOKEN_COMFY_ORG",
@@ -588,7 +619,14 @@ class MoonvalleyVideo2VideoNode(BaseMoonvalleyVideoNode):
"unique_id": "UNIQUE_ID",
},
"optional": {
- "video": (IO.VIDEO, {"default": "", "multiline": False, "tooltip": "The reference video used to generate the output video. Must be at least 5 seconds long. Videos longer than 5s will be automatically trimmed. Only MP4 format supported."}),
+ "video": (
+ IO.VIDEO,
+ {
+ "default": "",
+ "multiline": False,
+ "tooltip": "The reference video used to generate the output video. Must be at least 5 seconds long. Videos longer than 5s will be automatically trimmed. Only MP4 format supported.",
+ },
+ ),
"control_type": (
["Motion Transfer", "Pose Transfer"],
{"default": "Motion Transfer"},
@@ -602,17 +640,24 @@ class MoonvalleyVideo2VideoNode(BaseMoonvalleyVideoNode):
"max": 100,
"tooltip": "Only used if control_type is 'Motion Transfer'",
},
- )
- }
+ ),
+ "image": model_field_to_node_input(
+ IO.IMAGE,
+ MoonvalleyTextToVideoRequest,
+ "image_url",
+ tooltip="The reference image used to generate the video",
+ ),
+ },
}
RETURN_TYPES = ("VIDEO",)
RETURN_NAMES = ("video",)
- def generate(
+ async def generate(
self, prompt, negative_prompt, unique_id: Optional[str] = None, **kwargs
):
video = kwargs.get("video")
+ image = kwargs.get("image", None)
if not video:
raise MoonvalleyApiError("video is required")
@@ -620,8 +665,16 @@ class MoonvalleyVideo2VideoNode(BaseMoonvalleyVideoNode):
video_url = ""
if video:
validated_video = validate_video_to_video_input(video)
- video_url = upload_video_to_comfyapi(validated_video, auth_kwargs=kwargs)
+ video_url = await upload_video_to_comfyapi(
+ validated_video, auth_kwargs=kwargs
+ )
+ mime_type = "image/png"
+ if not image is None:
+ validate_input_image(image, with_frame_conditioning=True)
+ image_url = await upload_images_to_comfyapi(
+ image=image, auth_kwargs=kwargs, max_images=1, mime_type=mime_type
+ )
control_type = kwargs.get("control_type")
motion_intensity = kwargs.get("motion_intensity")
@@ -631,12 +684,12 @@ class MoonvalleyVideo2VideoNode(BaseMoonvalleyVideoNode):
# Only include motion_intensity for Motion Transfer
control_params = {}
if control_type == "Motion Transfer" and motion_intensity is not None:
- control_params['motion_intensity'] = motion_intensity
+ control_params["motion_intensity"] = motion_intensity
- inference_params=MoonvalleyVideoToVideoInferenceParams(
+ inference_params = MoonvalleyVideoToVideoInferenceParams(
negative_prompt=negative_prompt,
seed=kwargs.get("seed"),
- control_params=control_params
+ control_params=control_params,
)
control = self.parseControlParameter(control_type)
@@ -647,6 +700,7 @@ class MoonvalleyVideo2VideoNode(BaseMoonvalleyVideoNode):
prompt_text=prompt,
inference_params=inference_params,
)
+ request.image_url = image_url if not image is None else None
initial_operation = SynchronousOperation(
endpoint=ApiEndpoint(
@@ -658,15 +712,15 @@ class MoonvalleyVideo2VideoNode(BaseMoonvalleyVideoNode):
request=request,
auth_kwargs=kwargs,
)
- task_creation_response = initial_operation.execute()
+ task_creation_response = await initial_operation.execute()
validate_task_creation_response(task_creation_response)
task_id = task_creation_response.id
- final_response = self.get_response(
+ final_response = await self.get_response(
task_id, auth_kwargs=kwargs, node_id=unique_id
)
- video = download_url_to_video_output(final_response.output_url)
+ video = await download_url_to_video_output(final_response.output_url)
return (video,)
@@ -688,21 +742,21 @@ class MoonvalleyTxt2VideoNode(BaseMoonvalleyVideoNode):
del input_types["optional"][param]
return input_types
- def generate(
+ async def generate(
self, prompt, negative_prompt, unique_id: Optional[str] = None, **kwargs
):
validate_prompts(prompt, negative_prompt, MOONVALLEY_MAREY_MAX_PROMPT_LENGTH)
width_height = self.parseWidthHeightFromRes(kwargs.get("resolution"))
- inference_params=MoonvalleyTextToVideoInferenceParams(
- negative_prompt=negative_prompt,
- steps=kwargs.get("steps"),
- seed=kwargs.get("seed"),
- guidance_scale=kwargs.get("prompt_adherence"),
- num_frames=128,
- width=width_height.get("width"),
- height=width_height.get("height"),
- )
+ inference_params = MoonvalleyTextToVideoInferenceParams(
+ negative_prompt=negative_prompt,
+ steps=kwargs.get("steps"),
+ seed=kwargs.get("seed"),
+ guidance_scale=kwargs.get("prompt_adherence"),
+ num_frames=128,
+ width=width_height.get("width"),
+ height=width_height.get("height"),
+ )
request = MoonvalleyTextToVideoRequest(
prompt_text=prompt, inference_params=inference_params
)
@@ -717,15 +771,15 @@ class MoonvalleyTxt2VideoNode(BaseMoonvalleyVideoNode):
request=request,
auth_kwargs=kwargs,
)
- task_creation_response = initial_operation.execute()
+ task_creation_response = await initial_operation.execute()
validate_task_creation_response(task_creation_response)
task_id = task_creation_response.id
- final_response = self.get_response(
+ final_response = await self.get_response(
task_id, auth_kwargs=kwargs, node_id=unique_id
)
- video = download_url_to_video_output(final_response.output_url)
+ video = await download_url_to_video_output(final_response.output_url)
return (video,)
diff --git a/comfy_api_nodes/nodes_openai.py b/comfy_api_nodes/nodes_openai.py
index be1d2de4a..e3b81de75 100644
--- a/comfy_api_nodes/nodes_openai.py
+++ b/comfy_api_nodes/nodes_openai.py
@@ -80,6 +80,9 @@ class SupportedOpenAIModel(str, Enum):
gpt_4_1 = "gpt-4.1"
gpt_4_1_mini = "gpt-4.1-mini"
gpt_4_1_nano = "gpt-4.1-nano"
+ gpt_5 = "gpt-5"
+ gpt_5_mini = "gpt-5-mini"
+ gpt_5_nano = "gpt-5-nano"
class OpenAIDalle2(ComfyNodeABC):
@@ -163,7 +166,7 @@ class OpenAIDalle2(ComfyNodeABC):
DESCRIPTION = cleandoc(__doc__ or "")
API_NODE = True
- def api_call(
+ async def api_call(
self,
prompt,
seed=0,
@@ -233,9 +236,9 @@ class OpenAIDalle2(ComfyNodeABC):
auth_kwargs=kwargs,
)
- response = operation.execute()
+ response = await operation.execute()
- img_tensor = validate_and_cast_response(response, node_id=unique_id)
+ img_tensor = await validate_and_cast_response(response, node_id=unique_id)
return (img_tensor,)
@@ -311,7 +314,7 @@ class OpenAIDalle3(ComfyNodeABC):
DESCRIPTION = cleandoc(__doc__ or "")
API_NODE = True
- def api_call(
+ async def api_call(
self,
prompt,
seed=0,
@@ -343,9 +346,9 @@ class OpenAIDalle3(ComfyNodeABC):
auth_kwargs=kwargs,
)
- response = operation.execute()
+ response = await operation.execute()
- img_tensor = validate_and_cast_response(response, node_id=unique_id)
+ img_tensor = await validate_and_cast_response(response, node_id=unique_id)
return (img_tensor,)
@@ -446,7 +449,7 @@ class OpenAIGPTImage1(ComfyNodeABC):
DESCRIPTION = cleandoc(__doc__ or "")
API_NODE = True
- def api_call(
+ async def api_call(
self,
prompt,
seed=0,
@@ -464,8 +467,6 @@ class OpenAIGPTImage1(ComfyNodeABC):
path = "/proxy/openai/images/generations"
content_type = "application/json"
request_class = OpenAIImageGenerationRequest
- img_binaries = []
- mask_binary = None
files = []
if image is not None:
@@ -484,14 +485,11 @@ class OpenAIGPTImage1(ComfyNodeABC):
img_byte_arr = io.BytesIO()
img.save(img_byte_arr, format="PNG")
img_byte_arr.seek(0)
- img_binary = img_byte_arr
- img_binary.name = f"image_{i}.png"
- img_binaries.append(img_binary)
if batch_size == 1:
- files.append(("image", img_binary))
+ files.append(("image", (f"image_{i}.png", img_byte_arr, "image/png")))
else:
- files.append(("image[]", img_binary))
+ files.append(("image[]", (f"image_{i}.png", img_byte_arr, "image/png")))
if mask is not None:
if image is None:
@@ -511,9 +509,7 @@ class OpenAIGPTImage1(ComfyNodeABC):
mask_img_byte_arr = io.BytesIO()
mask_img.save(mask_img_byte_arr, format="PNG")
mask_img_byte_arr.seek(0)
- mask_binary = mask_img_byte_arr
- mask_binary.name = "mask.png"
- files.append(("mask", mask_binary))
+ files.append(("mask", ("mask.png", mask_img_byte_arr, "image/png")))
# Build the operation
operation = SynchronousOperation(
@@ -537,9 +533,9 @@ class OpenAIGPTImage1(ComfyNodeABC):
auth_kwargs=kwargs,
)
- response = operation.execute()
+ response = await operation.execute()
- img_tensor = validate_and_cast_response(response, node_id=unique_id)
+ img_tensor = await validate_and_cast_response(response, node_id=unique_id)
return (img_tensor,)
@@ -623,7 +619,7 @@ class OpenAIChatNode(OpenAITextNode):
DESCRIPTION = "Generate text responses from an OpenAI model."
- def get_result_response(
+ async def get_result_response(
self,
response_id: str,
include: Optional[list[Includable]] = None,
@@ -639,7 +635,7 @@ class OpenAIChatNode(OpenAITextNode):
creation above for more information.
"""
- return PollingOperation(
+ return await PollingOperation(
poll_endpoint=ApiEndpoint(
path=f"{RESPONSES_ENDPOINT}/{response_id}",
method=HttpMethod.GET,
@@ -784,7 +780,7 @@ class OpenAIChatNode(OpenAITextNode):
self.history[session_id] = new_history
- def api_call(
+ async def api_call(
self,
prompt: str,
persist_context: bool,
@@ -815,7 +811,7 @@ class OpenAIChatNode(OpenAITextNode):
previous_response_id = None
# Create response
- create_response = SynchronousOperation(
+ create_response = await SynchronousOperation(
endpoint=ApiEndpoint(
path=RESPONSES_ENDPOINT,
method=HttpMethod.POST,
@@ -848,7 +844,7 @@ class OpenAIChatNode(OpenAITextNode):
response_id = create_response.id
# Get result output
- result_response = self.get_result_response(response_id, auth_kwargs=kwargs)
+ result_response = await self.get_result_response(response_id, auth_kwargs=kwargs)
output_text = self.parse_output_text_from_response(result_response)
# Update history
@@ -1002,7 +998,7 @@ NODE_DISPLAY_NAME_MAPPINGS = {
"OpenAIDalle2": "OpenAI DALL·E 2",
"OpenAIDalle3": "OpenAI DALL·E 3",
"OpenAIGPTImage1": "OpenAI GPT Image 1",
- "OpenAIChatNode": "OpenAI Chat",
- "OpenAIInputFiles": "OpenAI Chat Input Files",
- "OpenAIChatConfig": "OpenAI Chat Advanced Options",
+ "OpenAIChatNode": "OpenAI ChatGPT",
+ "OpenAIInputFiles": "OpenAI ChatGPT Input Files",
+ "OpenAIChatConfig": "OpenAI ChatGPT Advanced Options",
}
diff --git a/comfy_api_nodes/nodes_pika.py b/comfy_api_nodes/nodes_pika.py
index 1cc708564..a8dc43cb3 100644
--- a/comfy_api_nodes/nodes_pika.py
+++ b/comfy_api_nodes/nodes_pika.py
@@ -122,7 +122,7 @@ class PikaNodeBase(ComfyNodeABC):
FUNCTION = "api_call"
RETURN_TYPES = ("VIDEO",)
- def poll_for_task_status(
+ async def poll_for_task_status(
self,
task_id: str,
auth_kwargs: Optional[dict[str, str]] = None,
@@ -152,9 +152,9 @@ class PikaNodeBase(ComfyNodeABC):
node_id=node_id,
estimated_duration=60
)
- return polling_operation.execute()
+ return await polling_operation.execute()
- def execute_task(
+ async def execute_task(
self,
initial_operation: SynchronousOperation[R, PikaGenerateResponse],
auth_kwargs: Optional[dict[str, str]] = None,
@@ -169,14 +169,14 @@ class PikaNodeBase(ComfyNodeABC):
Returns:
A tuple containing the video file as a VIDEO output.
"""
- initial_response = initial_operation.execute()
+ initial_response = await initial_operation.execute()
if not is_valid_initial_response(initial_response):
error_msg = f"Pika initial request failed. Code: {initial_response.code}, Message: {initial_response.message}, Data: {initial_response.data}"
logging.error(error_msg)
raise PikaApiError(error_msg)
task_id = initial_response.video_id
- final_response = self.poll_for_task_status(task_id, auth_kwargs)
+ final_response = await self.poll_for_task_status(task_id, auth_kwargs)
if not is_valid_video_response(final_response):
error_msg = (
f"Pika task {task_id} succeeded but no video data found in response."
@@ -187,7 +187,7 @@ class PikaNodeBase(ComfyNodeABC):
video_url = str(final_response.url)
logging.info("Pika task %s succeeded. Video URL: %s", task_id, video_url)
- return (download_url_to_video_output(video_url),)
+ return (await download_url_to_video_output(video_url),)
class PikaImageToVideoV2_2(PikaNodeBase):
@@ -212,7 +212,7 @@ class PikaImageToVideoV2_2(PikaNodeBase):
DESCRIPTION = "Sends an image and prompt to the Pika API v2.2 to generate a video."
- def api_call(
+ async def api_call(
self,
image: torch.Tensor,
prompt_text: str,
@@ -251,7 +251,7 @@ class PikaImageToVideoV2_2(PikaNodeBase):
auth_kwargs=kwargs,
)
- return self.execute_task(initial_operation, auth_kwargs=kwargs, node_id=unique_id)
+ return await self.execute_task(initial_operation, auth_kwargs=kwargs, node_id=unique_id)
class PikaTextToVideoNodeV2_2(PikaNodeBase):
@@ -281,7 +281,7 @@ class PikaTextToVideoNodeV2_2(PikaNodeBase):
DESCRIPTION = "Sends a text prompt to the Pika API v2.2 to generate a video."
- def api_call(
+ async def api_call(
self,
prompt_text: str,
negative_prompt: str,
@@ -311,7 +311,7 @@ class PikaTextToVideoNodeV2_2(PikaNodeBase):
content_type="application/x-www-form-urlencoded",
)
- return self.execute_task(initial_operation, auth_kwargs=kwargs, node_id=unique_id)
+ return await self.execute_task(initial_operation, auth_kwargs=kwargs, node_id=unique_id)
class PikaScenesV2_2(PikaNodeBase):
@@ -361,7 +361,7 @@ class PikaScenesV2_2(PikaNodeBase):
DESCRIPTION = "Combine your images to create a video with the objects in them. Upload multiple images as ingredients and generate a high-quality video that incorporates all of them."
- def api_call(
+ async def api_call(
self,
prompt_text: str,
negative_prompt: str,
@@ -420,7 +420,7 @@ class PikaScenesV2_2(PikaNodeBase):
auth_kwargs=kwargs,
)
- return self.execute_task(initial_operation, auth_kwargs=kwargs, node_id=unique_id)
+ return await self.execute_task(initial_operation, auth_kwargs=kwargs, node_id=unique_id)
class PikAdditionsNode(PikaNodeBase):
@@ -462,7 +462,7 @@ class PikAdditionsNode(PikaNodeBase):
DESCRIPTION = "Add any object or image into your video. Upload a video and specify what you'd like to add to create a seamlessly integrated result."
- def api_call(
+ async def api_call(
self,
video: VideoInput,
image: torch.Tensor,
@@ -481,10 +481,10 @@ class PikAdditionsNode(PikaNodeBase):
image_bytes_io = tensor_to_bytesio(image)
image_bytes_io.seek(0)
- pika_files = [
- ("video", ("video.mp4", video_bytes_io, "video/mp4")),
- ("image", ("image.png", image_bytes_io, "image/png")),
- ]
+ pika_files = {
+ "video": ("video.mp4", video_bytes_io, "video/mp4"),
+ "image": ("image.png", image_bytes_io, "image/png"),
+ }
# Prepare non-file data
pika_request_data = PikaBodyGeneratePikadditionsGeneratePikadditionsPost(
@@ -506,7 +506,7 @@ class PikAdditionsNode(PikaNodeBase):
auth_kwargs=kwargs,
)
- return self.execute_task(initial_operation, auth_kwargs=kwargs, node_id=unique_id)
+ return await self.execute_task(initial_operation, auth_kwargs=kwargs, node_id=unique_id)
class PikaSwapsNode(PikaNodeBase):
@@ -558,7 +558,7 @@ class PikaSwapsNode(PikaNodeBase):
DESCRIPTION = "Swap out any object or region of your video with a new image or object. Define areas to replace either with a mask or coordinates."
RETURN_TYPES = ("VIDEO",)
- def api_call(
+ async def api_call(
self,
video: VideoInput,
image: torch.Tensor,
@@ -587,11 +587,11 @@ class PikaSwapsNode(PikaNodeBase):
image_bytes_io = tensor_to_bytesio(image)
image_bytes_io.seek(0)
- pika_files = [
- ("video", ("video.mp4", video_bytes_io, "video/mp4")),
- ("image", ("image.png", image_bytes_io, "image/png")),
- ("modifyRegionMask", ("mask.png", mask_bytes_io, "image/png")),
- ]
+ pika_files = {
+ "video": ("video.mp4", video_bytes_io, "video/mp4"),
+ "image": ("image.png", image_bytes_io, "image/png"),
+ "modifyRegionMask": ("mask.png", mask_bytes_io, "image/png"),
+ }
# Prepare non-file data
pika_request_data = PikaBodyGeneratePikaswapsGeneratePikaswapsPost(
@@ -613,7 +613,7 @@ class PikaSwapsNode(PikaNodeBase):
auth_kwargs=kwargs,
)
- return self.execute_task(initial_operation, auth_kwargs=kwargs, node_id=unique_id)
+ return await self.execute_task(initial_operation, auth_kwargs=kwargs, node_id=unique_id)
class PikaffectsNode(PikaNodeBase):
@@ -664,7 +664,7 @@ class PikaffectsNode(PikaNodeBase):
DESCRIPTION = "Generate a video with a specific Pikaffect. Supported Pikaffects: Cake-ify, Crumble, Crush, Decapitate, Deflate, Dissolve, Explode, Eye-pop, Inflate, Levitate, Melt, Peel, Poke, Squish, Ta-da, Tear"
- def api_call(
+ async def api_call(
self,
image: torch.Tensor,
pikaffect: str,
@@ -693,7 +693,7 @@ class PikaffectsNode(PikaNodeBase):
auth_kwargs=kwargs,
)
- return self.execute_task(initial_operation, auth_kwargs=kwargs, node_id=unique_id)
+ return await self.execute_task(initial_operation, auth_kwargs=kwargs, node_id=unique_id)
class PikaStartEndFrameNode2_2(PikaNodeBase):
@@ -718,7 +718,7 @@ class PikaStartEndFrameNode2_2(PikaNodeBase):
DESCRIPTION = "Generate a video by combining your first and last frame. Upload two images to define the start and end points, and let the AI create a smooth transition between them."
- def api_call(
+ async def api_call(
self,
image_start: torch.Tensor,
image_end: torch.Tensor,
@@ -732,10 +732,7 @@ class PikaStartEndFrameNode2_2(PikaNodeBase):
) -> tuple[VideoFromFile]:
pika_files = [
- (
- "keyFrames",
- ("image_start.png", tensor_to_bytesio(image_start), "image/png"),
- ),
+ ("keyFrames", ("image_start.png", tensor_to_bytesio(image_start), "image/png")),
("keyFrames", ("image_end.png", tensor_to_bytesio(image_end), "image/png")),
]
@@ -758,7 +755,7 @@ class PikaStartEndFrameNode2_2(PikaNodeBase):
auth_kwargs=kwargs,
)
- return self.execute_task(initial_operation, auth_kwargs=kwargs, node_id=unique_id)
+ return await self.execute_task(initial_operation, auth_kwargs=kwargs, node_id=unique_id)
NODE_CLASS_MAPPINGS = {
diff --git a/comfy_api_nodes/nodes_pixverse.py b/comfy_api_nodes/nodes_pixverse.py
index ef4a9a802..7c5a52feb 100644
--- a/comfy_api_nodes/nodes_pixverse.py
+++ b/comfy_api_nodes/nodes_pixverse.py
@@ -30,7 +30,7 @@ from comfy.comfy_types.node_typing import IO, ComfyNodeABC
from comfy_api.input_impl import VideoFromFile
import torch
-import requests
+import aiohttp
from io import BytesIO
@@ -47,7 +47,7 @@ def get_video_url_from_response(
return str(response.Resp.url)
-def upload_image_to_pixverse(image: torch.Tensor, auth_kwargs=None):
+async def upload_image_to_pixverse(image: torch.Tensor, auth_kwargs=None):
# first, upload image to Pixverse and get image id to use in actual generation call
files = {"image": tensor_to_bytesio(image)}
operation = SynchronousOperation(
@@ -62,7 +62,7 @@ def upload_image_to_pixverse(image: torch.Tensor, auth_kwargs=None):
content_type="multipart/form-data",
auth_kwargs=auth_kwargs,
)
- response_upload: PixverseImageUploadResponse = operation.execute()
+ response_upload: PixverseImageUploadResponse = await operation.execute()
if response_upload.Resp is None:
raise Exception(
@@ -164,7 +164,7 @@ class PixverseTextToVideoNode(ComfyNodeABC):
},
}
- def api_call(
+ async def api_call(
self,
prompt: str,
aspect_ratio: str,
@@ -205,7 +205,7 @@ class PixverseTextToVideoNode(ComfyNodeABC):
),
auth_kwargs=kwargs,
)
- response_api = operation.execute()
+ response_api = await operation.execute()
if response_api.Resp is None:
raise Exception(f"PixVerse request failed: '{response_api.ErrMsg}'")
@@ -229,11 +229,11 @@ class PixverseTextToVideoNode(ComfyNodeABC):
result_url_extractor=get_video_url_from_response,
estimated_duration=AVERAGE_DURATION_T2V,
)
- response_poll = operation.execute()
+ response_poll = await operation.execute()
- vid_response = requests.get(response_poll.Resp.url)
-
- return (VideoFromFile(BytesIO(vid_response.content)),)
+ async with aiohttp.ClientSession() as session:
+ async with session.get(response_poll.Resp.url) as vid_response:
+ return (VideoFromFile(BytesIO(await vid_response.content.read())),)
class PixverseImageToVideoNode(ComfyNodeABC):
@@ -302,7 +302,7 @@ class PixverseImageToVideoNode(ComfyNodeABC):
},
}
- def api_call(
+ async def api_call(
self,
image: torch.Tensor,
prompt: str,
@@ -316,7 +316,7 @@ class PixverseImageToVideoNode(ComfyNodeABC):
**kwargs,
):
validate_string(prompt, strip_whitespace=False)
- img_id = upload_image_to_pixverse(image, auth_kwargs=kwargs)
+ img_id = await upload_image_to_pixverse(image, auth_kwargs=kwargs)
# 1080p is limited to 5 seconds duration
# only normal motion_mode supported for 1080p or for non-5 second duration
@@ -345,7 +345,7 @@ class PixverseImageToVideoNode(ComfyNodeABC):
),
auth_kwargs=kwargs,
)
- response_api = operation.execute()
+ response_api = await operation.execute()
if response_api.Resp is None:
raise Exception(f"PixVerse request failed: '{response_api.ErrMsg}'")
@@ -369,10 +369,11 @@ class PixverseImageToVideoNode(ComfyNodeABC):
result_url_extractor=get_video_url_from_response,
estimated_duration=AVERAGE_DURATION_I2V,
)
- response_poll = operation.execute()
+ response_poll = await operation.execute()
- vid_response = requests.get(response_poll.Resp.url)
- return (VideoFromFile(BytesIO(vid_response.content)),)
+ async with aiohttp.ClientSession() as session:
+ async with session.get(response_poll.Resp.url) as vid_response:
+ return (VideoFromFile(BytesIO(await vid_response.content.read())),)
class PixverseTransitionVideoNode(ComfyNodeABC):
@@ -436,7 +437,7 @@ class PixverseTransitionVideoNode(ComfyNodeABC):
},
}
- def api_call(
+ async def api_call(
self,
first_frame: torch.Tensor,
last_frame: torch.Tensor,
@@ -450,8 +451,8 @@ class PixverseTransitionVideoNode(ComfyNodeABC):
**kwargs,
):
validate_string(prompt, strip_whitespace=False)
- first_frame_id = upload_image_to_pixverse(first_frame, auth_kwargs=kwargs)
- last_frame_id = upload_image_to_pixverse(last_frame, auth_kwargs=kwargs)
+ first_frame_id = await upload_image_to_pixverse(first_frame, auth_kwargs=kwargs)
+ last_frame_id = await upload_image_to_pixverse(last_frame, auth_kwargs=kwargs)
# 1080p is limited to 5 seconds duration
# only normal motion_mode supported for 1080p or for non-5 second duration
@@ -480,7 +481,7 @@ class PixverseTransitionVideoNode(ComfyNodeABC):
),
auth_kwargs=kwargs,
)
- response_api = operation.execute()
+ response_api = await operation.execute()
if response_api.Resp is None:
raise Exception(f"PixVerse request failed: '{response_api.ErrMsg}'")
@@ -504,10 +505,11 @@ class PixverseTransitionVideoNode(ComfyNodeABC):
result_url_extractor=get_video_url_from_response,
estimated_duration=AVERAGE_DURATION_T2V,
)
- response_poll = operation.execute()
+ response_poll = await operation.execute()
- vid_response = requests.get(response_poll.Resp.url)
- return (VideoFromFile(BytesIO(vid_response.content)),)
+ async with aiohttp.ClientSession() as session:
+ async with session.get(response_poll.Resp.url) as vid_response:
+ return (VideoFromFile(BytesIO(await vid_response.content.read())),)
NODE_CLASS_MAPPINGS = {
diff --git a/comfy_api_nodes/nodes_recraft.py b/comfy_api_nodes/nodes_recraft.py
index e369c4b7e..c8516b368 100644
--- a/comfy_api_nodes/nodes_recraft.py
+++ b/comfy_api_nodes/nodes_recraft.py
@@ -37,7 +37,7 @@ from io import BytesIO
from PIL import UnidentifiedImageError
-def handle_recraft_file_request(
+async def handle_recraft_file_request(
image: torch.Tensor,
path: str,
mask: torch.Tensor=None,
@@ -71,13 +71,13 @@ def handle_recraft_file_request(
auth_kwargs=auth_kwargs,
multipart_parser=recraft_multipart_parser,
)
- response: RecraftImageGenerationResponse = operation.execute()
+ response: RecraftImageGenerationResponse = await operation.execute()
all_bytesio = []
if response.image is not None:
- all_bytesio.append(download_url_to_bytesio(response.image.url, timeout=timeout))
+ all_bytesio.append(await download_url_to_bytesio(response.image.url, timeout=timeout))
else:
for data in response.data:
- all_bytesio.append(download_url_to_bytesio(data.url, timeout=timeout))
+ all_bytesio.append(await download_url_to_bytesio(data.url, timeout=timeout))
return all_bytesio
@@ -395,7 +395,7 @@ class RecraftTextToImageNode:
},
}
- def api_call(
+ async def api_call(
self,
prompt: str,
size: str,
@@ -439,7 +439,7 @@ class RecraftTextToImageNode:
),
auth_kwargs=kwargs,
)
- response: RecraftImageGenerationResponse = operation.execute()
+ response: RecraftImageGenerationResponse = await operation.execute()
images = []
urls = []
for data in response.data:
@@ -451,7 +451,7 @@ class RecraftTextToImageNode:
f"Result URL: {urls_string}", unique_id
)
image = bytesio_to_image_tensor(
- download_url_to_bytesio(data.url, timeout=1024)
+ await download_url_to_bytesio(data.url, timeout=1024)
)
if len(image.shape) < 4:
image = image.unsqueeze(0)
@@ -538,7 +538,7 @@ class RecraftImageToImageNode:
},
}
- def api_call(
+ async def api_call(
self,
image: torch.Tensor,
prompt: str,
@@ -578,7 +578,7 @@ class RecraftImageToImageNode:
total = image.shape[0]
pbar = ProgressBar(total)
for i in range(total):
- sub_bytes = handle_recraft_file_request(
+ sub_bytes = await handle_recraft_file_request(
image=image[i],
path="/proxy/recraft/images/imageToImage",
request=request,
@@ -654,7 +654,7 @@ class RecraftImageInpaintingNode:
},
}
- def api_call(
+ async def api_call(
self,
image: torch.Tensor,
mask: torch.Tensor,
@@ -690,7 +690,7 @@ class RecraftImageInpaintingNode:
total = image.shape[0]
pbar = ProgressBar(total)
for i in range(total):
- sub_bytes = handle_recraft_file_request(
+ sub_bytes = await handle_recraft_file_request(
image=image[i],
mask=mask[i:i+1],
path="/proxy/recraft/images/inpaint",
@@ -779,7 +779,7 @@ class RecraftTextToVectorNode:
},
}
- def api_call(
+ async def api_call(
self,
prompt: str,
substyle: str,
@@ -821,7 +821,7 @@ class RecraftTextToVectorNode:
),
auth_kwargs=kwargs,
)
- response: RecraftImageGenerationResponse = operation.execute()
+ response: RecraftImageGenerationResponse = await operation.execute()
svg_data = []
urls = []
for data in response.data:
@@ -831,7 +831,7 @@ class RecraftTextToVectorNode:
PromptServer.instance.send_progress_text(
f"Result URL: {' '.join(urls)}", unique_id
)
- svg_data.append(download_url_to_bytesio(data.url, timeout=1024))
+ svg_data.append(await download_url_to_bytesio(data.url, timeout=1024))
return (SVG(svg_data),)
@@ -861,7 +861,7 @@ class RecraftVectorizeImageNode:
},
}
- def api_call(
+ async def api_call(
self,
image: torch.Tensor,
**kwargs,
@@ -870,7 +870,7 @@ class RecraftVectorizeImageNode:
total = image.shape[0]
pbar = ProgressBar(total)
for i in range(total):
- sub_bytes = handle_recraft_file_request(
+ sub_bytes = await handle_recraft_file_request(
image=image[i],
path="/proxy/recraft/images/vectorize",
auth_kwargs=kwargs,
@@ -942,7 +942,7 @@ class RecraftReplaceBackgroundNode:
},
}
- def api_call(
+ async def api_call(
self,
image: torch.Tensor,
prompt: str,
@@ -973,7 +973,7 @@ class RecraftReplaceBackgroundNode:
total = image.shape[0]
pbar = ProgressBar(total)
for i in range(total):
- sub_bytes = handle_recraft_file_request(
+ sub_bytes = await handle_recraft_file_request(
image=image[i],
path="/proxy/recraft/images/replaceBackground",
request=request,
@@ -1011,7 +1011,7 @@ class RecraftRemoveBackgroundNode:
},
}
- def api_call(
+ async def api_call(
self,
image: torch.Tensor,
**kwargs,
@@ -1020,7 +1020,7 @@ class RecraftRemoveBackgroundNode:
total = image.shape[0]
pbar = ProgressBar(total)
for i in range(total):
- sub_bytes = handle_recraft_file_request(
+ sub_bytes = await handle_recraft_file_request(
image=image[i],
path="/proxy/recraft/images/removeBackground",
auth_kwargs=kwargs,
@@ -1062,7 +1062,7 @@ class RecraftCrispUpscaleNode:
},
}
- def api_call(
+ async def api_call(
self,
image: torch.Tensor,
**kwargs,
@@ -1071,7 +1071,7 @@ class RecraftCrispUpscaleNode:
total = image.shape[0]
pbar = ProgressBar(total)
for i in range(total):
- sub_bytes = handle_recraft_file_request(
+ sub_bytes = await handle_recraft_file_request(
image=image[i],
path=self.RECRAFT_PATH,
auth_kwargs=kwargs,
diff --git a/comfy_api_nodes/nodes_rodin.py b/comfy_api_nodes/nodes_rodin.py
index 67f90478c..c89d087e5 100644
--- a/comfy_api_nodes/nodes_rodin.py
+++ b/comfy_api_nodes/nodes_rodin.py
@@ -9,11 +9,10 @@ from __future__ import annotations
from inspect import cleandoc
from comfy.comfy_types.node_typing import IO
import folder_paths as comfy_paths
-import requests
+import aiohttp
import os
import datetime
-import shutil
-import time
+import asyncio
import io
import logging
import math
@@ -66,7 +65,6 @@ def create_task_error(response: Rodin3DGenerateResponse):
return hasattr(response, "error")
-
class Rodin3DAPI:
"""
Generate 3D Assets using Rodin API
@@ -123,8 +121,8 @@ class Rodin3DAPI:
else:
return "Generating"
- def CreateGenerateTask(self, images=None, seed=1, material="PBR", quality="medium", tier="Regular", mesh_mode="Quad", **kwargs):
- if images == None:
+ async def create_generate_task(self, images=None, seed=1, material="PBR", quality="medium", tier="Regular", mesh_mode="Quad", **kwargs):
+ if images is None:
raise Exception("Rodin 3D generate requires at least 1 image.")
if len(images) >= 5:
raise Exception("Rodin 3D generate requires up to 5 image.")
@@ -155,7 +153,7 @@ class Rodin3DAPI:
auth_kwargs=kwargs,
)
- response = operation.execute()
+ response = await operation.execute()
if create_task_error(response):
error_message = f"Rodin3D Create 3D generate Task Failed. Message: {response.message}, error: {response.error}"
@@ -168,7 +166,7 @@ class Rodin3DAPI:
logging.info(f"[ Rodin3D API - Submit Jobs ] UUID: {task_uuid}")
return task_uuid, subscription_key
- def poll_for_task_status(self, subscription_key, **kwargs) -> Rodin3DCheckStatusResponse:
+ async def poll_for_task_status(self, subscription_key, **kwargs) -> Rodin3DCheckStatusResponse:
path = "/proxy/rodin/api/v2/status"
@@ -191,11 +189,9 @@ class Rodin3DAPI:
logging.info("[ Rodin3D API - CheckStatus ] Generate Start!")
- return poll_operation.execute()
+ return await poll_operation.execute()
-
-
- def GetRodinDownloadList(self, uuid, **kwargs) -> Rodin3DDownloadResponse:
+ async def get_rodin_download_list(self, uuid, **kwargs) -> Rodin3DDownloadResponse:
logging.info("[ Rodin3D API - Downloading ] Generate Successfully!")
path = "/proxy/rodin/api/v2/download"
@@ -212,53 +208,59 @@ class Rodin3DAPI:
auth_kwargs=kwargs
)
- return operation.execute()
+ return await operation.execute()
- def GetQualityAndMode(self, PolyCount):
- if PolyCount == "200K-Triangle":
+ def get_quality_mode(self, poly_count):
+ if poly_count == "200K-Triangle":
mesh_mode = "Raw"
quality = "medium"
else:
mesh_mode = "Quad"
- if PolyCount == "4K-Quad":
+ if poly_count == "4K-Quad":
quality = "extra-low"
- elif PolyCount == "8K-Quad":
+ elif poly_count == "8K-Quad":
quality = "low"
- elif PolyCount == "18K-Quad":
+ elif poly_count == "18K-Quad":
quality = "medium"
- elif PolyCount == "50K-Quad":
+ elif poly_count == "50K-Quad":
quality = "high"
else:
quality = "medium"
return mesh_mode, quality
- def DownLoadFiles(self, Url_List):
- Save_path = os.path.join(comfy_paths.get_output_directory(), "Rodin3D", datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S"))
- os.makedirs(Save_path, exist_ok=True)
+ async def download_files(self, url_list):
+ save_path = os.path.join(comfy_paths.get_output_directory(), "Rodin3D", datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S"))
+ os.makedirs(save_path, exist_ok=True)
model_file_path = None
- for Item in Url_List.list:
- url = Item.url
- file_name = Item.name
- file_path = os.path.join(Save_path, file_name)
- if file_path.endswith(".glb"):
- model_file_path = file_path
- logging.info(f"[ Rodin3D API - download_files ] Downloading file: {file_path}")
- max_retries = 5
- for attempt in range(max_retries):
- try:
- with requests.get(url, stream=True) as r:
- r.raise_for_status()
- with open(file_path, "wb") as f:
- shutil.copyfileobj(r.raw, f)
- break
- except Exception as e:
- logging.info(f"[ Rodin3D API - download_files ] Error downloading {file_path}:{e}")
- if attempt < max_retries - 1:
- logging.info("Retrying...")
- time.sleep(2)
- else:
- logging.info(f"[ Rodin3D API - download_files ] Failed to download {file_path} after {max_retries} attempts.")
+ async with aiohttp.ClientSession() as session:
+ for i in url_list.list:
+ url = i.url
+ file_name = i.name
+ file_path = os.path.join(save_path, file_name)
+ if file_path.endswith(".glb"):
+ model_file_path = file_path
+ logging.info(f"[ Rodin3D API - download_files ] Downloading file: {file_path}")
+ max_retries = 5
+ for attempt in range(max_retries):
+ try:
+ async with session.get(url) as resp:
+ resp.raise_for_status()
+ with open(file_path, "wb") as f:
+ async for chunk in resp.content.iter_chunked(32 * 1024):
+ f.write(chunk)
+ break
+ except Exception as e:
+ logging.info(f"[ Rodin3D API - download_files ] Error downloading {file_path}:{e}")
+ if attempt < max_retries - 1:
+ logging.info("Retrying...")
+ await asyncio.sleep(2)
+ else:
+ logging.info(
+ "[ Rodin3D API - download_files ] Failed to download %s after %s attempts.",
+ file_path,
+ max_retries,
+ )
return model_file_path
@@ -285,7 +287,7 @@ class Rodin3D_Regular(Rodin3DAPI):
},
}
- def api_call(
+ async def api_call(
self,
Images,
Seed,
@@ -298,14 +300,17 @@ class Rodin3D_Regular(Rodin3DAPI):
m_images = []
for i in range(num_images):
m_images.append(Images[i])
- mesh_mode, quality = self.GetQualityAndMode(Polygon_count)
- task_uuid, subscription_key = self.CreateGenerateTask(images=m_images, seed=Seed, material=Material_Type, quality=quality, tier=tier, mesh_mode=mesh_mode, **kwargs)
- self.poll_for_task_status(subscription_key, **kwargs)
- Download_List = self.GetRodinDownloadList(task_uuid, **kwargs)
- model = self.DownLoadFiles(Download_List)
+ mesh_mode, quality = self.get_quality_mode(Polygon_count)
+ task_uuid, subscription_key = await self.create_generate_task(images=m_images, seed=Seed, material=Material_Type,
+ quality=quality, tier=tier, mesh_mode=mesh_mode,
+ **kwargs)
+ await self.poll_for_task_status(subscription_key, **kwargs)
+ download_list = await self.get_rodin_download_list(task_uuid, **kwargs)
+ model = await self.download_files(download_list)
return (model,)
+
class Rodin3D_Detail(Rodin3DAPI):
@classmethod
def INPUT_TYPES(s):
@@ -328,7 +333,7 @@ class Rodin3D_Detail(Rodin3DAPI):
},
}
- def api_call(
+ async def api_call(
self,
Images,
Seed,
@@ -341,14 +346,17 @@ class Rodin3D_Detail(Rodin3DAPI):
m_images = []
for i in range(num_images):
m_images.append(Images[i])
- mesh_mode, quality = self.GetQualityAndMode(Polygon_count)
- task_uuid, subscription_key = self.CreateGenerateTask(images=m_images, seed=Seed, material=Material_Type, quality=quality, tier=tier, mesh_mode=mesh_mode, **kwargs)
- self.poll_for_task_status(subscription_key, **kwargs)
- Download_List = self.GetRodinDownloadList(task_uuid, **kwargs)
- model = self.DownLoadFiles(Download_List)
+ mesh_mode, quality = self.get_quality_mode(Polygon_count)
+ task_uuid, subscription_key = await self.create_generate_task(images=m_images, seed=Seed, material=Material_Type,
+ quality=quality, tier=tier, mesh_mode=mesh_mode,
+ **kwargs)
+ await self.poll_for_task_status(subscription_key, **kwargs)
+ download_list = await self.get_rodin_download_list(task_uuid, **kwargs)
+ model = await self.download_files(download_list)
return (model,)
+
class Rodin3D_Smooth(Rodin3DAPI):
@classmethod
def INPUT_TYPES(s):
@@ -371,7 +379,7 @@ class Rodin3D_Smooth(Rodin3DAPI):
},
}
- def api_call(
+ async def api_call(
self,
Images,
Seed,
@@ -384,14 +392,17 @@ class Rodin3D_Smooth(Rodin3DAPI):
m_images = []
for i in range(num_images):
m_images.append(Images[i])
- mesh_mode, quality = self.GetQualityAndMode(Polygon_count)
- task_uuid, subscription_key = self.CreateGenerateTask(images=m_images, seed=Seed, material=Material_Type, quality=quality, tier=tier, mesh_mode=mesh_mode, **kwargs)
- self.poll_for_task_status(subscription_key, **kwargs)
- Download_List = self.GetRodinDownloadList(task_uuid, **kwargs)
- model = self.DownLoadFiles(Download_List)
+ mesh_mode, quality = self.get_quality_mode(Polygon_count)
+ task_uuid, subscription_key = await self.create_generate_task(images=m_images, seed=Seed, material=Material_Type,
+ quality=quality, tier=tier, mesh_mode=mesh_mode,
+ **kwargs)
+ await self.poll_for_task_status(subscription_key, **kwargs)
+ download_list = await self.get_rodin_download_list(task_uuid, **kwargs)
+ model = await self.download_files(download_list)
return (model,)
+
class Rodin3D_Sketch(Rodin3DAPI):
@classmethod
def INPUT_TYPES(s):
@@ -423,7 +434,7 @@ class Rodin3D_Sketch(Rodin3DAPI):
},
}
- def api_call(
+ async def api_call(
self,
Images,
Seed,
@@ -437,10 +448,12 @@ class Rodin3D_Sketch(Rodin3DAPI):
material_type = "PBR"
quality = "medium"
mesh_mode = "Quad"
- task_uuid, subscription_key = self.CreateGenerateTask(images=m_images, seed=Seed, material=material_type, quality=quality, tier=tier, mesh_mode=mesh_mode, **kwargs)
- self.poll_for_task_status(subscription_key, **kwargs)
- Download_List = self.GetRodinDownloadList(task_uuid, **kwargs)
- model = self.DownLoadFiles(Download_List)
+ task_uuid, subscription_key = await self.create_generate_task(
+ images=m_images, seed=Seed, material=material_type, quality=quality, tier=tier, mesh_mode=mesh_mode, **kwargs
+ )
+ await self.poll_for_task_status(subscription_key, **kwargs)
+ download_list = await self.get_rodin_download_list(task_uuid, **kwargs)
+ model = await self.download_files(download_list)
return (model,)
diff --git a/comfy_api_nodes/nodes_runway.py b/comfy_api_nodes/nodes_runway.py
index af4b321f9..98024a9fa 100644
--- a/comfy_api_nodes/nodes_runway.py
+++ b/comfy_api_nodes/nodes_runway.py
@@ -99,14 +99,14 @@ def validate_input_image(image: torch.Tensor) -> bool:
return image.shape[2] < 8000 and image.shape[1] < 8000
-def poll_until_finished(
+async def poll_until_finished(
auth_kwargs: dict[str, str],
api_endpoint: ApiEndpoint[Any, TaskStatusResponse],
estimated_duration: Optional[int] = None,
node_id: Optional[str] = None,
) -> TaskStatusResponse:
"""Polls the Runway API endpoint until the task reaches a terminal state, then returns the response."""
- return PollingOperation(
+ return await PollingOperation(
poll_endpoint=api_endpoint,
completed_statuses=[
TaskStatus.SUCCEEDED.value,
@@ -115,7 +115,7 @@ def poll_until_finished(
TaskStatus.FAILED.value,
TaskStatus.CANCELLED.value,
],
- status_extractor=lambda response: (response.status.value),
+ status_extractor=lambda response: response.status.value,
auth_kwargs=auth_kwargs,
result_url_extractor=get_video_url_from_task_status,
estimated_duration=estimated_duration,
@@ -167,11 +167,11 @@ class RunwayVideoGenNode(ComfyNodeABC):
)
return True
- def get_response(
+ async def get_response(
self, task_id: str, auth_kwargs: dict[str, str], node_id: Optional[str] = None
) -> RunwayImageToVideoResponse:
"""Poll the task status until it is finished then get the response."""
- return poll_until_finished(
+ return await poll_until_finished(
auth_kwargs,
ApiEndpoint(
path=f"{PATH_GET_TASK_STATUS}/{task_id}",
@@ -183,7 +183,7 @@ class RunwayVideoGenNode(ComfyNodeABC):
node_id=node_id,
)
- def generate_video(
+ async def generate_video(
self,
request: RunwayImageToVideoRequest,
auth_kwargs: dict[str, str],
@@ -200,15 +200,15 @@ class RunwayVideoGenNode(ComfyNodeABC):
auth_kwargs=auth_kwargs,
)
- initial_response = initial_operation.execute()
+ initial_response = await initial_operation.execute()
self.validate_task_created(initial_response)
task_id = initial_response.id
- final_response = self.get_response(task_id, auth_kwargs, node_id)
+ final_response = await self.get_response(task_id, auth_kwargs, node_id)
self.validate_response(final_response)
video_url = get_video_url_from_task_status(final_response)
- return (download_url_to_video_output(video_url),)
+ return (await download_url_to_video_output(video_url),)
class RunwayImageToVideoNodeGen3a(RunwayVideoGenNode):
@@ -250,7 +250,7 @@ class RunwayImageToVideoNodeGen3a(RunwayVideoGenNode):
},
}
- def api_call(
+ async def api_call(
self,
prompt: str,
start_frame: torch.Tensor,
@@ -265,7 +265,7 @@ class RunwayImageToVideoNodeGen3a(RunwayVideoGenNode):
validate_input_image(start_frame)
# Upload image
- download_urls = upload_images_to_comfyapi(
+ download_urls = await upload_images_to_comfyapi(
start_frame,
max_images=1,
mime_type="image/png",
@@ -274,7 +274,7 @@ class RunwayImageToVideoNodeGen3a(RunwayVideoGenNode):
if len(download_urls) != 1:
raise RunwayApiError("Failed to upload one or more images to comfy api.")
- return self.generate_video(
+ return await self.generate_video(
RunwayImageToVideoRequest(
promptText=prompt,
seed=seed,
@@ -333,7 +333,7 @@ class RunwayImageToVideoNodeGen4(RunwayVideoGenNode):
},
}
- def api_call(
+ async def api_call(
self,
prompt: str,
start_frame: torch.Tensor,
@@ -348,7 +348,7 @@ class RunwayImageToVideoNodeGen4(RunwayVideoGenNode):
validate_input_image(start_frame)
# Upload image
- download_urls = upload_images_to_comfyapi(
+ download_urls = await upload_images_to_comfyapi(
start_frame,
max_images=1,
mime_type="image/png",
@@ -357,7 +357,7 @@ class RunwayImageToVideoNodeGen4(RunwayVideoGenNode):
if len(download_urls) != 1:
raise RunwayApiError("Failed to upload one or more images to comfy api.")
- return self.generate_video(
+ return await self.generate_video(
RunwayImageToVideoRequest(
promptText=prompt,
seed=seed,
@@ -382,10 +382,10 @@ class RunwayFirstLastFrameNode(RunwayVideoGenNode):
DESCRIPTION = "Upload first and last keyframes, draft a prompt, and generate a video. More complex transitions, such as cases where the Last frame is completely different from the First frame, may benefit from the longer 10s duration. This would give the generation more time to smoothly transition between the two inputs. Before diving in, review these best practices to ensure that your input selections will set your generation up for success: https://help.runwayml.com/hc/en-us/articles/34170748696595-Creating-with-Keyframes-on-Gen-3."
- def get_response(
+ async def get_response(
self, task_id: str, auth_kwargs: dict[str, str], node_id: Optional[str] = None
) -> RunwayImageToVideoResponse:
- return poll_until_finished(
+ return await poll_until_finished(
auth_kwargs,
ApiEndpoint(
path=f"{PATH_GET_TASK_STATUS}/{task_id}",
@@ -437,7 +437,7 @@ class RunwayFirstLastFrameNode(RunwayVideoGenNode):
},
}
- def api_call(
+ async def api_call(
self,
prompt: str,
start_frame: torch.Tensor,
@@ -455,7 +455,7 @@ class RunwayFirstLastFrameNode(RunwayVideoGenNode):
# Upload images
stacked_input_images = image_tensor_pair_to_batch(start_frame, end_frame)
- download_urls = upload_images_to_comfyapi(
+ download_urls = await upload_images_to_comfyapi(
stacked_input_images,
max_images=2,
mime_type="image/png",
@@ -464,7 +464,7 @@ class RunwayFirstLastFrameNode(RunwayVideoGenNode):
if len(download_urls) != 2:
raise RunwayApiError("Failed to upload one or more images to comfy api.")
- return self.generate_video(
+ return await self.generate_video(
RunwayImageToVideoRequest(
promptText=prompt,
seed=seed,
@@ -543,11 +543,11 @@ class RunwayTextToImageNode(ComfyNodeABC):
)
return True
- def get_response(
+ async def get_response(
self, task_id: str, auth_kwargs: dict[str, str], node_id: Optional[str] = None
) -> TaskStatusResponse:
"""Poll the task status until it is finished then get the response."""
- return poll_until_finished(
+ return await poll_until_finished(
auth_kwargs,
ApiEndpoint(
path=f"{PATH_GET_TASK_STATUS}/{task_id}",
@@ -559,7 +559,7 @@ class RunwayTextToImageNode(ComfyNodeABC):
node_id=node_id,
)
- def api_call(
+ async def api_call(
self,
prompt: str,
ratio: str,
@@ -574,7 +574,7 @@ class RunwayTextToImageNode(ComfyNodeABC):
reference_images = None
if reference_image is not None:
validate_input_image(reference_image)
- download_urls = upload_images_to_comfyapi(
+ download_urls = await upload_images_to_comfyapi(
reference_image,
max_images=1,
mime_type="image/png",
@@ -605,19 +605,19 @@ class RunwayTextToImageNode(ComfyNodeABC):
auth_kwargs=kwargs,
)
- initial_response = initial_operation.execute()
+ initial_response = await initial_operation.execute()
self.validate_task_created(initial_response)
task_id = initial_response.id
# Poll for completion
- final_response = self.get_response(
+ final_response = await self.get_response(
task_id, auth_kwargs=kwargs, node_id=unique_id
)
self.validate_response(final_response)
# Download and return image
image_url = get_image_url_from_task_status(final_response)
- return (download_url_to_image_tensor(image_url),)
+ return (await download_url_to_image_tensor(image_url),)
NODE_CLASS_MAPPINGS = {
diff --git a/comfy_api_nodes/nodes_stability.py b/comfy_api_nodes/nodes_stability.py
index 02e421678..e05cb6bb2 100644
--- a/comfy_api_nodes/nodes_stability.py
+++ b/comfy_api_nodes/nodes_stability.py
@@ -1,5 +1,8 @@
from inspect import cleandoc
-from comfy.comfy_types.node_typing import IO
+from typing import Optional
+from typing_extensions import override
+
+from comfy_api.latest import ComfyExtension, io as comfy_io
from comfy_api_nodes.apis.stability_api import (
StabilityUpscaleConservativeRequest,
StabilityUpscaleCreativeRequest,
@@ -46,87 +49,94 @@ def get_async_dummy_status(x: StabilityResultsGetResponse):
return StabilityPollStatus.in_progress
-class StabilityStableImageUltraNode:
+class StabilityStableImageUltraNode(comfy_io.ComfyNode):
"""
Generates images synchronously based on prompt and resolution.
"""
- RETURN_TYPES = (IO.IMAGE,)
- DESCRIPTION = cleandoc(__doc__ or "") # Handle potential None value
- FUNCTION = "api_call"
- API_NODE = True
- CATEGORY = "api node/image/Stability AI"
-
@classmethod
- def INPUT_TYPES(s):
- return {
- "required": {
- "prompt": (
- IO.STRING,
- {
- "multiline": True,
- "default": "",
- "tooltip": "What you wish to see in the output image. A strong, descriptive prompt that clearly defines" +
- "What you wish to see in the output image. A strong, descriptive prompt that clearly defines" +
+ def define_schema(cls):
+ return comfy_io.Schema(
+ node_id="StabilityStableImageUltraNode",
+ display_name="Stability AI Stable Image Ultra",
+ category="api node/image/Stability AI",
+ description=cleandoc(cls.__doc__ or ""),
+ inputs=[
+ comfy_io.String.Input(
+ "prompt",
+ multiline=True,
+ default="",
+ tooltip="What you wish to see in the output image. A strong, descriptive prompt that clearly defines" +
"elements, colors, and subjects will lead to better results. " +
"To control the weight of a given word use the format `(word:weight)`," +
"where `word` is the word you'd like to control the weight of and `weight`" +
"is a value between 0 and 1. For example: `The sky was a crisp (blue:0.3) and (green:0.8)`" +
- "would convey a sky that was blue and green, but more green than blue."
- },
+ "would convey a sky that was blue and green, but more green than blue.",
),
- "aspect_ratio": ([x.value for x in StabilityAspectRatio],
- {
- "default": StabilityAspectRatio.ratio_1_1,
- "tooltip": "Aspect ratio of generated image.",
- },
+ comfy_io.Combo.Input(
+ "aspect_ratio",
+ options=[x.value for x in StabilityAspectRatio],
+ default=StabilityAspectRatio.ratio_1_1.value,
+ tooltip="Aspect ratio of generated image.",
),
- "style_preset": (get_stability_style_presets(),
- {
- "tooltip": "Optional desired style of generated image.",
- },
+ comfy_io.Combo.Input(
+ "style_preset",
+ options=get_stability_style_presets(),
+ tooltip="Optional desired style of generated image.",
),
- "seed": (
- IO.INT,
- {
- "default": 0,
- "min": 0,
- "max": 4294967294,
- "control_after_generate": True,
- "tooltip": "The random seed used for creating the noise.",
- },
+ comfy_io.Int.Input(
+ "seed",
+ default=0,
+ min=0,
+ max=4294967294,
+ step=1,
+ display_mode=comfy_io.NumberDisplay.number,
+ control_after_generate=True,
+ tooltip="The random seed used for creating the noise.",
),
- },
- "optional": {
- "image": (IO.IMAGE,),
- "negative_prompt": (
- IO.STRING,
- {
- "default": "",
- "forceInput": True,
- "tooltip": "A blurb of text describing what you do not wish to see in the output image. This is an advanced feature."
- },
+ comfy_io.Image.Input(
+ "image",
+ optional=True,
),
- "image_denoise": (
- IO.FLOAT,
- {
- "default": 0.5,
- "min": 0.0,
- "max": 1.0,
- "step": 0.01,
- "tooltip": "Denoise of input image; 0.0 yields image identical to input, 1.0 is as if no image was provided at all.",
- },
+ comfy_io.String.Input(
+ "negative_prompt",
+ default="",
+ tooltip="A blurb of text describing what you do not wish to see in the output image. This is an advanced feature.",
+ force_input=True,
+ optional=True,
),
- },
- "hidden": {
- "auth_token": "AUTH_TOKEN_COMFY_ORG",
- "comfy_api_key": "API_KEY_COMFY_ORG",
- },
- }
+ comfy_io.Float.Input(
+ "image_denoise",
+ default=0.5,
+ min=0.0,
+ max=1.0,
+ step=0.01,
+ tooltip="Denoise of input image; 0.0 yields image identical to input, 1.0 is as if no image was provided at all.",
+ optional=True,
+ ),
+ ],
+ outputs=[
+ comfy_io.Image.Output(),
+ ],
+ hidden=[
+ comfy_io.Hidden.auth_token_comfy_org,
+ comfy_io.Hidden.api_key_comfy_org,
+ comfy_io.Hidden.unique_id,
+ ],
+ is_api_node=True,
+ )
- def api_call(self, prompt: str, aspect_ratio: str, style_preset: str, seed: int,
- negative_prompt: str=None, image: torch.Tensor = None, image_denoise: float=None,
- **kwargs):
+ @classmethod
+ async def execute(
+ cls,
+ prompt: str,
+ aspect_ratio: str,
+ style_preset: str,
+ seed: int,
+ image: Optional[torch.Tensor] = None,
+ negative_prompt: str = "",
+ image_denoise: Optional[float] = 0.5,
+ ) -> comfy_io.NodeOutput:
validate_string(prompt, strip_whitespace=False)
# prepare image binary if image present
image_binary = None
@@ -144,6 +154,11 @@ class StabilityStableImageUltraNode:
"image": image_binary
}
+ auth = {
+ "auth_token": cls.hidden.auth_token_comfy_org,
+ "comfy_api_key": cls.hidden.api_key_comfy_org,
+ }
+
operation = SynchronousOperation(
endpoint=ApiEndpoint(
path="/proxy/stability/v2beta/stable-image/generate/ultra",
@@ -161,9 +176,9 @@ class StabilityStableImageUltraNode:
),
files=files,
content_type="multipart/form-data",
- auth_kwargs=kwargs,
+ auth_kwargs=auth,
)
- response_api = operation.execute()
+ response_api = await operation.execute()
if response_api.finish_reason != "SUCCESS":
raise Exception(f"Stable Image Ultra generation failed: {response_api.finish_reason}.")
@@ -171,95 +186,106 @@ class StabilityStableImageUltraNode:
image_data = base64.b64decode(response_api.image)
returned_image = bytesio_to_image_tensor(BytesIO(image_data))
- return (returned_image,)
+ return comfy_io.NodeOutput(returned_image)
-class StabilityStableImageSD_3_5Node:
+class StabilityStableImageSD_3_5Node(comfy_io.ComfyNode):
"""
Generates images synchronously based on prompt and resolution.
"""
- RETURN_TYPES = (IO.IMAGE,)
- DESCRIPTION = cleandoc(__doc__ or "") # Handle potential None value
- FUNCTION = "api_call"
- API_NODE = True
- CATEGORY = "api node/image/Stability AI"
+ @classmethod
+ def define_schema(cls):
+ return comfy_io.Schema(
+ node_id="StabilityStableImageSD_3_5Node",
+ display_name="Stability AI Stable Diffusion 3.5 Image",
+ category="api node/image/Stability AI",
+ description=cleandoc(cls.__doc__ or ""),
+ inputs=[
+ comfy_io.String.Input(
+ "prompt",
+ multiline=True,
+ default="",
+ tooltip="What you wish to see in the output image. A strong, descriptive prompt that clearly defines elements, colors, and subjects will lead to better results.",
+ ),
+ comfy_io.Combo.Input(
+ "model",
+ options=[x.value for x in Stability_SD3_5_Model],
+ ),
+ comfy_io.Combo.Input(
+ "aspect_ratio",
+ options=[x.value for x in StabilityAspectRatio],
+ default=StabilityAspectRatio.ratio_1_1.value,
+ tooltip="Aspect ratio of generated image.",
+ ),
+ comfy_io.Combo.Input(
+ "style_preset",
+ options=get_stability_style_presets(),
+ tooltip="Optional desired style of generated image.",
+ ),
+ comfy_io.Float.Input(
+ "cfg_scale",
+ default=4.0,
+ min=1.0,
+ max=10.0,
+ step=0.1,
+ tooltip="How strictly the diffusion process adheres to the prompt text (higher values keep your image closer to your prompt)",
+ ),
+ comfy_io.Int.Input(
+ "seed",
+ default=0,
+ min=0,
+ max=4294967294,
+ step=1,
+ display_mode=comfy_io.NumberDisplay.number,
+ control_after_generate=True,
+ tooltip="The random seed used for creating the noise.",
+ ),
+ comfy_io.Image.Input(
+ "image",
+ optional=True,
+ ),
+ comfy_io.String.Input(
+ "negative_prompt",
+ default="",
+ tooltip="Keywords of what you do not wish to see in the output image. This is an advanced feature.",
+ force_input=True,
+ optional=True,
+ ),
+ comfy_io.Float.Input(
+ "image_denoise",
+ default=0.5,
+ min=0.0,
+ max=1.0,
+ step=0.01,
+ tooltip="Denoise of input image; 0.0 yields image identical to input, 1.0 is as if no image was provided at all.",
+ optional=True,
+ ),
+ ],
+ outputs=[
+ comfy_io.Image.Output(),
+ ],
+ hidden=[
+ comfy_io.Hidden.auth_token_comfy_org,
+ comfy_io.Hidden.api_key_comfy_org,
+ comfy_io.Hidden.unique_id,
+ ],
+ is_api_node=True,
+ )
@classmethod
- def INPUT_TYPES(s):
- return {
- "required": {
- "prompt": (
- IO.STRING,
- {
- "multiline": True,
- "default": "",
- "tooltip": "What you wish to see in the output image. A strong, descriptive prompt that clearly defines elements, colors, and subjects will lead to better results."
- },
- ),
- "model": ([x.value for x in Stability_SD3_5_Model],),
- "aspect_ratio": ([x.value for x in StabilityAspectRatio],
- {
- "default": StabilityAspectRatio.ratio_1_1,
- "tooltip": "Aspect ratio of generated image.",
- },
- ),
- "style_preset": (get_stability_style_presets(),
- {
- "tooltip": "Optional desired style of generated image.",
- },
- ),
- "cfg_scale": (
- IO.FLOAT,
- {
- "default": 4.0,
- "min": 1.0,
- "max": 10.0,
- "step": 0.1,
- "tooltip": "How strictly the diffusion process adheres to the prompt text (higher values keep your image closer to your prompt)",
- },
- ),
- "seed": (
- IO.INT,
- {
- "default": 0,
- "min": 0,
- "max": 4294967294,
- "control_after_generate": True,
- "tooltip": "The random seed used for creating the noise.",
- },
- ),
- },
- "optional": {
- "image": (IO.IMAGE,),
- "negative_prompt": (
- IO.STRING,
- {
- "default": "",
- "forceInput": True,
- "tooltip": "Keywords of what you do not wish to see in the output image. This is an advanced feature."
- },
- ),
- "image_denoise": (
- IO.FLOAT,
- {
- "default": 0.5,
- "min": 0.0,
- "max": 1.0,
- "step": 0.01,
- "tooltip": "Denoise of input image; 0.0 yields image identical to input, 1.0 is as if no image was provided at all.",
- },
- ),
- },
- "hidden": {
- "auth_token": "AUTH_TOKEN_COMFY_ORG",
- "comfy_api_key": "API_KEY_COMFY_ORG",
- },
- }
-
- def api_call(self, model: str, prompt: str, aspect_ratio: str, style_preset: str, seed: int, cfg_scale: float,
- negative_prompt: str=None, image: torch.Tensor = None, image_denoise: float=None,
- **kwargs):
+ async def execute(
+ cls,
+ model: str,
+ prompt: str,
+ aspect_ratio: str,
+ style_preset: str,
+ seed: int,
+ cfg_scale: float,
+ image: Optional[torch.Tensor] = None,
+ negative_prompt: str = "",
+ image_denoise: Optional[float] = 0.5,
+ ) -> comfy_io.NodeOutput:
validate_string(prompt, strip_whitespace=False)
# prepare image binary if image present
image_binary = None
@@ -280,6 +306,11 @@ class StabilityStableImageSD_3_5Node:
"image": image_binary
}
+ auth = {
+ "auth_token": cls.hidden.auth_token_comfy_org,
+ "comfy_api_key": cls.hidden.api_key_comfy_org,
+ }
+
operation = SynchronousOperation(
endpoint=ApiEndpoint(
path="/proxy/stability/v2beta/stable-image/generate/sd3",
@@ -300,9 +331,9 @@ class StabilityStableImageSD_3_5Node:
),
files=files,
content_type="multipart/form-data",
- auth_kwargs=kwargs,
+ auth_kwargs=auth,
)
- response_api = operation.execute()
+ response_api = await operation.execute()
if response_api.finish_reason != "SUCCESS":
raise Exception(f"Stable Diffusion 3.5 Image generation failed: {response_api.finish_reason}.")
@@ -310,72 +341,75 @@ class StabilityStableImageSD_3_5Node:
image_data = base64.b64decode(response_api.image)
returned_image = bytesio_to_image_tensor(BytesIO(image_data))
- return (returned_image,)
+ return comfy_io.NodeOutput(returned_image)
-class StabilityUpscaleConservativeNode:
+class StabilityUpscaleConservativeNode(comfy_io.ComfyNode):
"""
Upscale image with minimal alterations to 4K resolution.
"""
- RETURN_TYPES = (IO.IMAGE,)
- DESCRIPTION = cleandoc(__doc__ or "") # Handle potential None value
- FUNCTION = "api_call"
- API_NODE = True
- CATEGORY = "api node/image/Stability AI"
+ @classmethod
+ def define_schema(cls):
+ return comfy_io.Schema(
+ node_id="StabilityUpscaleConservativeNode",
+ display_name="Stability AI Upscale Conservative",
+ category="api node/image/Stability AI",
+ description=cleandoc(cls.__doc__ or ""),
+ inputs=[
+ comfy_io.Image.Input("image"),
+ comfy_io.String.Input(
+ "prompt",
+ multiline=True,
+ default="",
+ tooltip="What you wish to see in the output image. A strong, descriptive prompt that clearly defines elements, colors, and subjects will lead to better results.",
+ ),
+ comfy_io.Float.Input(
+ "creativity",
+ default=0.35,
+ min=0.2,
+ max=0.5,
+ step=0.01,
+ tooltip="Controls the likelihood of creating additional details not heavily conditioned by the init image.",
+ ),
+ comfy_io.Int.Input(
+ "seed",
+ default=0,
+ min=0,
+ max=4294967294,
+ step=1,
+ display_mode=comfy_io.NumberDisplay.number,
+ control_after_generate=True,
+ tooltip="The random seed used for creating the noise.",
+ ),
+ comfy_io.String.Input(
+ "negative_prompt",
+ default="",
+ tooltip="Keywords of what you do not wish to see in the output image. This is an advanced feature.",
+ force_input=True,
+ optional=True,
+ ),
+ ],
+ outputs=[
+ comfy_io.Image.Output(),
+ ],
+ hidden=[
+ comfy_io.Hidden.auth_token_comfy_org,
+ comfy_io.Hidden.api_key_comfy_org,
+ comfy_io.Hidden.unique_id,
+ ],
+ is_api_node=True,
+ )
@classmethod
- def INPUT_TYPES(s):
- return {
- "required": {
- "image": (IO.IMAGE,),
- "prompt": (
- IO.STRING,
- {
- "multiline": True,
- "default": "",
- "tooltip": "What you wish to see in the output image. A strong, descriptive prompt that clearly defines elements, colors, and subjects will lead to better results."
- },
- ),
- "creativity": (
- IO.FLOAT,
- {
- "default": 0.35,
- "min": 0.2,
- "max": 0.5,
- "step": 0.01,
- "tooltip": "Controls the likelihood of creating additional details not heavily conditioned by the init image.",
- },
- ),
- "seed": (
- IO.INT,
- {
- "default": 0,
- "min": 0,
- "max": 4294967294,
- "control_after_generate": True,
- "tooltip": "The random seed used for creating the noise.",
- },
- ),
- },
- "optional": {
- "negative_prompt": (
- IO.STRING,
- {
- "default": "",
- "forceInput": True,
- "tooltip": "Keywords of what you do not wish to see in the output image. This is an advanced feature."
- },
- ),
- },
- "hidden": {
- "auth_token": "AUTH_TOKEN_COMFY_ORG",
- "comfy_api_key": "API_KEY_COMFY_ORG",
- },
- }
-
- def api_call(self, image: torch.Tensor, prompt: str, creativity: float, seed: int, negative_prompt: str=None,
- **kwargs):
+ async def execute(
+ cls,
+ image: torch.Tensor,
+ prompt: str,
+ creativity: float,
+ seed: int,
+ negative_prompt: str = "",
+ ) -> comfy_io.NodeOutput:
validate_string(prompt, strip_whitespace=False)
image_binary = tensor_to_bytesio(image, total_pixels=1024*1024).read()
@@ -386,6 +420,11 @@ class StabilityUpscaleConservativeNode:
"image": image_binary
}
+ auth = {
+ "auth_token": cls.hidden.auth_token_comfy_org,
+ "comfy_api_key": cls.hidden.api_key_comfy_org,
+ }
+
operation = SynchronousOperation(
endpoint=ApiEndpoint(
path="/proxy/stability/v2beta/stable-image/upscale/conservative",
@@ -401,9 +440,9 @@ class StabilityUpscaleConservativeNode:
),
files=files,
content_type="multipart/form-data",
- auth_kwargs=kwargs,
+ auth_kwargs=auth,
)
- response_api = operation.execute()
+ response_api = await operation.execute()
if response_api.finish_reason != "SUCCESS":
raise Exception(f"Stability Upscale Conservative generation failed: {response_api.finish_reason}.")
@@ -411,77 +450,81 @@ class StabilityUpscaleConservativeNode:
image_data = base64.b64decode(response_api.image)
returned_image = bytesio_to_image_tensor(BytesIO(image_data))
- return (returned_image,)
+ return comfy_io.NodeOutput(returned_image)
-class StabilityUpscaleCreativeNode:
+class StabilityUpscaleCreativeNode(comfy_io.ComfyNode):
"""
Upscale image with minimal alterations to 4K resolution.
"""
- RETURN_TYPES = (IO.IMAGE,)
- DESCRIPTION = cleandoc(__doc__ or "") # Handle potential None value
- FUNCTION = "api_call"
- API_NODE = True
- CATEGORY = "api node/image/Stability AI"
+ @classmethod
+ def define_schema(cls):
+ return comfy_io.Schema(
+ node_id="StabilityUpscaleCreativeNode",
+ display_name="Stability AI Upscale Creative",
+ category="api node/image/Stability AI",
+ description=cleandoc(cls.__doc__ or ""),
+ inputs=[
+ comfy_io.Image.Input("image"),
+ comfy_io.String.Input(
+ "prompt",
+ multiline=True,
+ default="",
+ tooltip="What you wish to see in the output image. A strong, descriptive prompt that clearly defines elements, colors, and subjects will lead to better results.",
+ ),
+ comfy_io.Float.Input(
+ "creativity",
+ default=0.3,
+ min=0.1,
+ max=0.5,
+ step=0.01,
+ tooltip="Controls the likelihood of creating additional details not heavily conditioned by the init image.",
+ ),
+ comfy_io.Combo.Input(
+ "style_preset",
+ options=get_stability_style_presets(),
+ tooltip="Optional desired style of generated image.",
+ ),
+ comfy_io.Int.Input(
+ "seed",
+ default=0,
+ min=0,
+ max=4294967294,
+ step=1,
+ display_mode=comfy_io.NumberDisplay.number,
+ control_after_generate=True,
+ tooltip="The random seed used for creating the noise.",
+ ),
+ comfy_io.String.Input(
+ "negative_prompt",
+ default="",
+ tooltip="Keywords of what you do not wish to see in the output image. This is an advanced feature.",
+ force_input=True,
+ optional=True,
+ ),
+ ],
+ outputs=[
+ comfy_io.Image.Output(),
+ ],
+ hidden=[
+ comfy_io.Hidden.auth_token_comfy_org,
+ comfy_io.Hidden.api_key_comfy_org,
+ comfy_io.Hidden.unique_id,
+ ],
+ is_api_node=True,
+ )
@classmethod
- def INPUT_TYPES(s):
- return {
- "required": {
- "image": (IO.IMAGE,),
- "prompt": (
- IO.STRING,
- {
- "multiline": True,
- "default": "",
- "tooltip": "What you wish to see in the output image. A strong, descriptive prompt that clearly defines elements, colors, and subjects will lead to better results."
- },
- ),
- "creativity": (
- IO.FLOAT,
- {
- "default": 0.3,
- "min": 0.1,
- "max": 0.5,
- "step": 0.01,
- "tooltip": "Controls the likelihood of creating additional details not heavily conditioned by the init image.",
- },
- ),
- "style_preset": (get_stability_style_presets(),
- {
- "tooltip": "Optional desired style of generated image.",
- },
- ),
- "seed": (
- IO.INT,
- {
- "default": 0,
- "min": 0,
- "max": 4294967294,
- "control_after_generate": True,
- "tooltip": "The random seed used for creating the noise.",
- },
- ),
- },
- "optional": {
- "negative_prompt": (
- IO.STRING,
- {
- "default": "",
- "forceInput": True,
- "tooltip": "Keywords of what you do not wish to see in the output image. This is an advanced feature."
- },
- ),
- },
- "hidden": {
- "auth_token": "AUTH_TOKEN_COMFY_ORG",
- "comfy_api_key": "API_KEY_COMFY_ORG",
- },
- }
-
- def api_call(self, image: torch.Tensor, prompt: str, creativity: float, style_preset: str, seed: int, negative_prompt: str=None,
- **kwargs):
+ async def execute(
+ cls,
+ image: torch.Tensor,
+ prompt: str,
+ creativity: float,
+ style_preset: str,
+ seed: int,
+ negative_prompt: str = "",
+ ) -> comfy_io.NodeOutput:
validate_string(prompt, strip_whitespace=False)
image_binary = tensor_to_bytesio(image, total_pixels=1024*1024).read()
@@ -494,6 +537,11 @@ class StabilityUpscaleCreativeNode:
"image": image_binary
}
+ auth = {
+ "auth_token": cls.hidden.auth_token_comfy_org,
+ "comfy_api_key": cls.hidden.api_key_comfy_org,
+ }
+
operation = SynchronousOperation(
endpoint=ApiEndpoint(
path="/proxy/stability/v2beta/stable-image/upscale/creative",
@@ -510,9 +558,9 @@ class StabilityUpscaleCreativeNode:
),
files=files,
content_type="multipart/form-data",
- auth_kwargs=kwargs,
+ auth_kwargs=auth,
)
- response_api = operation.execute()
+ response_api = await operation.execute()
operation = PollingOperation(
poll_endpoint=ApiEndpoint(
@@ -525,9 +573,10 @@ class StabilityUpscaleCreativeNode:
completed_statuses=[StabilityPollStatus.finished],
failed_statuses=[StabilityPollStatus.failed],
status_extractor=lambda x: get_async_dummy_status(x),
- auth_kwargs=kwargs,
+ auth_kwargs=auth,
+ node_id=cls.hidden.unique_id,
)
- response_poll: StabilityResultsGetResponse = operation.execute()
+ response_poll: StabilityResultsGetResponse = await operation.execute()
if response_poll.finish_reason != "SUCCESS":
raise Exception(f"Stability Upscale Creative generation failed: {response_poll.finish_reason}.")
@@ -535,42 +584,48 @@ class StabilityUpscaleCreativeNode:
image_data = base64.b64decode(response_poll.result)
returned_image = bytesio_to_image_tensor(BytesIO(image_data))
- return (returned_image,)
+ return comfy_io.NodeOutput(returned_image)
-class StabilityUpscaleFastNode:
+class StabilityUpscaleFastNode(comfy_io.ComfyNode):
"""
Quickly upscales an image via Stability API call to 4x its original size; intended for upscaling low-quality/compressed images.
"""
- RETURN_TYPES = (IO.IMAGE,)
- DESCRIPTION = cleandoc(__doc__ or "") # Handle potential None value
- FUNCTION = "api_call"
- API_NODE = True
- CATEGORY = "api node/image/Stability AI"
+ @classmethod
+ def define_schema(cls):
+ return comfy_io.Schema(
+ node_id="StabilityUpscaleFastNode",
+ display_name="Stability AI Upscale Fast",
+ category="api node/image/Stability AI",
+ description=cleandoc(cls.__doc__ or ""),
+ inputs=[
+ comfy_io.Image.Input("image"),
+ ],
+ outputs=[
+ comfy_io.Image.Output(),
+ ],
+ hidden=[
+ comfy_io.Hidden.auth_token_comfy_org,
+ comfy_io.Hidden.api_key_comfy_org,
+ comfy_io.Hidden.unique_id,
+ ],
+ is_api_node=True,
+ )
@classmethod
- def INPUT_TYPES(s):
- return {
- "required": {
- "image": (IO.IMAGE,),
- },
- "optional": {
- },
- "hidden": {
- "auth_token": "AUTH_TOKEN_COMFY_ORG",
- "comfy_api_key": "API_KEY_COMFY_ORG",
- },
- }
-
- def api_call(self, image: torch.Tensor,
- **kwargs):
+ async def execute(cls, image: torch.Tensor) -> comfy_io.NodeOutput:
image_binary = tensor_to_bytesio(image, total_pixels=4096*4096).read()
files = {
"image": image_binary
}
+ auth = {
+ "auth_token": cls.hidden.auth_token_comfy_org,
+ "comfy_api_key": cls.hidden.api_key_comfy_org,
+ }
+
operation = SynchronousOperation(
endpoint=ApiEndpoint(
path="/proxy/stability/v2beta/stable-image/upscale/fast",
@@ -581,9 +636,9 @@ class StabilityUpscaleFastNode:
request=EmptyRequest(),
files=files,
content_type="multipart/form-data",
- auth_kwargs=kwargs,
+ auth_kwargs=auth,
)
- response_api = operation.execute()
+ response_api = await operation.execute()
if response_api.finish_reason != "SUCCESS":
raise Exception(f"Stability Upscale Fast failed: {response_api.finish_reason}.")
@@ -591,24 +646,20 @@ class StabilityUpscaleFastNode:
image_data = base64.b64decode(response_api.image)
returned_image = bytesio_to_image_tensor(BytesIO(image_data))
- return (returned_image,)
+ return comfy_io.NodeOutput(returned_image)
-# A dictionary that contains all nodes you want to export with their names
-# NOTE: names should be globally unique
-NODE_CLASS_MAPPINGS = {
- "StabilityStableImageUltraNode": StabilityStableImageUltraNode,
- "StabilityStableImageSD_3_5Node": StabilityStableImageSD_3_5Node,
- "StabilityUpscaleConservativeNode": StabilityUpscaleConservativeNode,
- "StabilityUpscaleCreativeNode": StabilityUpscaleCreativeNode,
- "StabilityUpscaleFastNode": StabilityUpscaleFastNode,
-}
+class StabilityExtension(ComfyExtension):
+ @override
+ async def get_node_list(self) -> list[type[comfy_io.ComfyNode]]:
+ return [
+ StabilityStableImageUltraNode,
+ StabilityStableImageSD_3_5Node,
+ StabilityUpscaleConservativeNode,
+ StabilityUpscaleCreativeNode,
+ StabilityUpscaleFastNode,
+ ]
-# A dictionary that contains the friendly/humanly readable titles for the nodes
-NODE_DISPLAY_NAME_MAPPINGS = {
- "StabilityStableImageUltraNode": "Stability AI Stable Image Ultra",
- "StabilityStableImageSD_3_5Node": "Stability AI Stable Diffusion 3.5 Image",
- "StabilityUpscaleConservativeNode": "Stability AI Upscale Conservative",
- "StabilityUpscaleCreativeNode": "Stability AI Upscale Creative",
- "StabilityUpscaleFastNode": "Stability AI Upscale Fast",
-}
+
+async def comfy_entrypoint() -> StabilityExtension:
+ return StabilityExtension()
diff --git a/comfy_api_nodes/nodes_tripo.py b/comfy_api_nodes/nodes_tripo.py
index 65f3b21f5..d08cf9007 100644
--- a/comfy_api_nodes/nodes_tripo.py
+++ b/comfy_api_nodes/nodes_tripo.py
@@ -37,8 +37,8 @@ from comfy_api_nodes.apinode_utils import (
)
-def upload_image_to_tripo(image, **kwargs):
- urls = upload_images_to_comfyapi(image, max_images=1, auth_kwargs=kwargs)
+async def upload_image_to_tripo(image, **kwargs):
+ urls = await upload_images_to_comfyapi(image, max_images=1, auth_kwargs=kwargs)
return TripoFileReference(TripoUrlReference(url=urls[0], type="jpeg"))
def get_model_url_from_response(response: TripoTaskResponse) -> str:
@@ -49,7 +49,7 @@ def get_model_url_from_response(response: TripoTaskResponse) -> str:
raise RuntimeError(f"Failed to get model url from response: {response}")
-def poll_until_finished(
+async def poll_until_finished(
kwargs: dict[str, str],
response: TripoTaskResponse,
) -> tuple[str, str]:
@@ -57,7 +57,7 @@ def poll_until_finished(
if response.code != 0:
raise RuntimeError(f"Failed to generate mesh: {response.error}")
task_id = response.data.task_id
- response_poll = PollingOperation(
+ response_poll = await PollingOperation(
poll_endpoint=ApiEndpoint(
path=f"/proxy/tripo/v2/openapi/task/{task_id}",
method=HttpMethod.GET,
@@ -80,7 +80,7 @@ def poll_until_finished(
).execute()
if response_poll.data.status == TripoTaskStatus.SUCCESS:
url = get_model_url_from_response(response_poll)
- bytesio = download_url_to_bytesio(url)
+ bytesio = await download_url_to_bytesio(url)
# Save the downloaded model file
model_file = f"tripo_model_{task_id}.glb"
with open(os.path.join(get_output_directory(), model_file), "wb") as f:
@@ -88,6 +88,7 @@ def poll_until_finished(
return model_file, task_id
raise RuntimeError(f"Failed to generate mesh: {response_poll}")
+
class TripoTextToModelNode:
"""
Generates 3D models synchronously based on a text prompt using Tripo's API.
@@ -126,11 +127,11 @@ class TripoTextToModelNode:
API_NODE = True
OUTPUT_NODE = True
- def generate_mesh(self, prompt, negative_prompt=None, model_version=None, style=None, texture=None, pbr=None, image_seed=None, model_seed=None, texture_seed=None, texture_quality=None, face_limit=None, quad=None, **kwargs):
+ async def generate_mesh(self, prompt, negative_prompt=None, model_version=None, style=None, texture=None, pbr=None, image_seed=None, model_seed=None, texture_seed=None, texture_quality=None, face_limit=None, quad=None, **kwargs):
style_enum = None if style == "None" else style
if not prompt:
raise RuntimeError("Prompt is required")
- response = SynchronousOperation(
+ response = await SynchronousOperation(
endpoint=ApiEndpoint(
path="/proxy/tripo/v2/openapi/task",
method=HttpMethod.POST,
@@ -155,7 +156,8 @@ class TripoTextToModelNode:
),
auth_kwargs=kwargs,
).execute()
- return poll_until_finished(kwargs, response)
+ return await poll_until_finished(kwargs, response)
+
class TripoImageToModelNode:
"""
@@ -195,12 +197,12 @@ class TripoImageToModelNode:
API_NODE = True
OUTPUT_NODE = True
- def generate_mesh(self, image, model_version=None, style=None, texture=None, pbr=None, model_seed=None, orientation=None, texture_alignment=None, texture_seed=None, texture_quality=None, face_limit=None, quad=None, **kwargs):
+ async def generate_mesh(self, image, model_version=None, style=None, texture=None, pbr=None, model_seed=None, orientation=None, texture_alignment=None, texture_seed=None, texture_quality=None, face_limit=None, quad=None, **kwargs):
style_enum = None if style == "None" else style
if image is None:
raise RuntimeError("Image is required")
- tripo_file = upload_image_to_tripo(image, **kwargs)
- response = SynchronousOperation(
+ tripo_file = await upload_image_to_tripo(image, **kwargs)
+ response = await SynchronousOperation(
endpoint=ApiEndpoint(
path="/proxy/tripo/v2/openapi/task",
method=HttpMethod.POST,
@@ -225,7 +227,8 @@ class TripoImageToModelNode:
),
auth_kwargs=kwargs,
).execute()
- return poll_until_finished(kwargs, response)
+ return await poll_until_finished(kwargs, response)
+
class TripoMultiviewToModelNode:
"""
@@ -267,7 +270,7 @@ class TripoMultiviewToModelNode:
API_NODE = True
OUTPUT_NODE = True
- def generate_mesh(self, image, image_left=None, image_back=None, image_right=None, model_version=None, orientation=None, texture=None, pbr=None, model_seed=None, texture_seed=None, texture_quality=None, texture_alignment=None, face_limit=None, quad=None, **kwargs):
+ async def generate_mesh(self, image, image_left=None, image_back=None, image_right=None, model_version=None, orientation=None, texture=None, pbr=None, model_seed=None, texture_seed=None, texture_quality=None, texture_alignment=None, face_limit=None, quad=None, **kwargs):
if image is None:
raise RuntimeError("front image for multiview is required")
images = []
@@ -282,11 +285,11 @@ class TripoMultiviewToModelNode:
for image_name in ["image", "image_left", "image_back", "image_right"]:
image_ = image_dict[image_name]
if image_ is not None:
- tripo_file = upload_image_to_tripo(image_, **kwargs)
+ tripo_file = await upload_image_to_tripo(image_, **kwargs)
images.append(tripo_file)
else:
images.append(TripoFileEmptyReference())
- response = SynchronousOperation(
+ response = await SynchronousOperation(
endpoint=ApiEndpoint(
path="/proxy/tripo/v2/openapi/task",
method=HttpMethod.POST,
@@ -309,7 +312,8 @@ class TripoMultiviewToModelNode:
),
auth_kwargs=kwargs,
).execute()
- return poll_until_finished(kwargs, response)
+ return await poll_until_finished(kwargs, response)
+
class TripoTextureNode:
@classmethod
@@ -340,8 +344,8 @@ class TripoTextureNode:
OUTPUT_NODE = True
AVERAGE_DURATION = 80
- def generate_mesh(self, model_task_id, texture=None, pbr=None, texture_seed=None, texture_quality=None, texture_alignment=None, **kwargs):
- response = SynchronousOperation(
+ async def generate_mesh(self, model_task_id, texture=None, pbr=None, texture_seed=None, texture_quality=None, texture_alignment=None, **kwargs):
+ response = await SynchronousOperation(
endpoint=ApiEndpoint(
path="/proxy/tripo/v2/openapi/task",
method=HttpMethod.POST,
@@ -358,7 +362,7 @@ class TripoTextureNode:
),
auth_kwargs=kwargs,
).execute()
- return poll_until_finished(kwargs, response)
+ return await poll_until_finished(kwargs, response)
class TripoRefineNode:
@@ -387,8 +391,8 @@ class TripoRefineNode:
OUTPUT_NODE = True
AVERAGE_DURATION = 240
- def generate_mesh(self, model_task_id, **kwargs):
- response = SynchronousOperation(
+ async def generate_mesh(self, model_task_id, **kwargs):
+ response = await SynchronousOperation(
endpoint=ApiEndpoint(
path="/proxy/tripo/v2/openapi/task",
method=HttpMethod.POST,
@@ -400,7 +404,7 @@ class TripoRefineNode:
),
auth_kwargs=kwargs,
).execute()
- return poll_until_finished(kwargs, response)
+ return await poll_until_finished(kwargs, response)
class TripoRigNode:
@@ -425,8 +429,8 @@ class TripoRigNode:
OUTPUT_NODE = True
AVERAGE_DURATION = 180
- def generate_mesh(self, original_model_task_id, **kwargs):
- response = SynchronousOperation(
+ async def generate_mesh(self, original_model_task_id, **kwargs):
+ response = await SynchronousOperation(
endpoint=ApiEndpoint(
path="/proxy/tripo/v2/openapi/task",
method=HttpMethod.POST,
@@ -440,7 +444,8 @@ class TripoRigNode:
),
auth_kwargs=kwargs,
).execute()
- return poll_until_finished(kwargs, response)
+ return await poll_until_finished(kwargs, response)
+
class TripoRetargetNode:
@classmethod
@@ -475,8 +480,8 @@ class TripoRetargetNode:
OUTPUT_NODE = True
AVERAGE_DURATION = 30
- def generate_mesh(self, animation, original_model_task_id, **kwargs):
- response = SynchronousOperation(
+ async def generate_mesh(self, animation, original_model_task_id, **kwargs):
+ response = await SynchronousOperation(
endpoint=ApiEndpoint(
path="/proxy/tripo/v2/openapi/task",
method=HttpMethod.POST,
@@ -491,7 +496,8 @@ class TripoRetargetNode:
),
auth_kwargs=kwargs,
).execute()
- return poll_until_finished(kwargs, response)
+ return await poll_until_finished(kwargs, response)
+
class TripoConversionNode:
@classmethod
@@ -529,10 +535,10 @@ class TripoConversionNode:
OUTPUT_NODE = True
AVERAGE_DURATION = 30
- def generate_mesh(self, original_model_task_id, format, quad, face_limit, texture_size, texture_format, **kwargs):
+ async def generate_mesh(self, original_model_task_id, format, quad, face_limit, texture_size, texture_format, **kwargs):
if not original_model_task_id:
raise RuntimeError("original_model_task_id is required")
- response = SynchronousOperation(
+ response = await SynchronousOperation(
endpoint=ApiEndpoint(
path="/proxy/tripo/v2/openapi/task",
method=HttpMethod.POST,
@@ -549,7 +555,8 @@ class TripoConversionNode:
),
auth_kwargs=kwargs,
).execute()
- return poll_until_finished(kwargs, response)
+ return await poll_until_finished(kwargs, response)
+
NODE_CLASS_MAPPINGS = {
"TripoTextToModelNode": TripoTextToModelNode,
diff --git a/comfy_api_nodes/nodes_veo2.py b/comfy_api_nodes/nodes_veo2.py
index df846d5dd..251aecd42 100644
--- a/comfy_api_nodes/nodes_veo2.py
+++ b/comfy_api_nodes/nodes_veo2.py
@@ -1,17 +1,18 @@
-import io
import logging
import base64
-import requests
+import aiohttp
import torch
+from io import BytesIO
from typing import Optional
+from typing_extensions import override
-from comfy.comfy_types.node_typing import IO, ComfyNodeABC
+from comfy_api.latest import ComfyExtension, io as comfy_io
from comfy_api.input_impl.video_types import VideoFromFile
from comfy_api_nodes.apis import (
- Veo2GenVidRequest,
- Veo2GenVidResponse,
- Veo2GenVidPollRequest,
- Veo2GenVidPollResponse
+ VeoGenVidRequest,
+ VeoGenVidResponse,
+ VeoGenVidPollRequest,
+ VeoGenVidPollResponse,
)
from comfy_api_nodes.apis.client import (
ApiEndpoint,
@@ -22,7 +23,7 @@ from comfy_api_nodes.apis.client import (
from comfy_api_nodes.apinode_utils import (
downscale_image_tensor,
- tensor_to_base64_string
+ tensor_to_base64_string,
)
AVERAGE_DURATION_VIDEO_GEN = 32
@@ -35,7 +36,7 @@ def convert_image_to_base64(image: torch.Tensor):
return tensor_to_base64_string(scaled_image)
-def get_video_url_from_response(poll_response: Veo2GenVidPollResponse) -> Optional[str]:
+def get_video_url_from_response(poll_response: VeoGenVidPollResponse) -> Optional[str]:
if (
poll_response.response
and hasattr(poll_response.response, "videos")
@@ -50,7 +51,7 @@ def get_video_url_from_response(poll_response: Veo2GenVidPollResponse) -> Option
return None
-class VeoVideoGenerationNode(ComfyNodeABC):
+class VeoVideoGenerationNode(comfy_io.ComfyNode):
"""
Generates videos from text prompts using Google's Veo API.
@@ -59,93 +60,93 @@ class VeoVideoGenerationNode(ComfyNodeABC):
"""
@classmethod
- def INPUT_TYPES(s):
- return {
- "required": {
- "prompt": (
- IO.STRING,
- {
- "multiline": True,
- "default": "",
- "tooltip": "Text description of the video",
- },
+ def define_schema(cls):
+ return comfy_io.Schema(
+ node_id="VeoVideoGenerationNode",
+ display_name="Google Veo 2 Video Generation",
+ category="api node/video/Veo",
+ description="Generates videos from text prompts using Google's Veo 2 API",
+ inputs=[
+ comfy_io.String.Input(
+ "prompt",
+ multiline=True,
+ default="",
+ tooltip="Text description of the video",
),
- "aspect_ratio": (
- IO.COMBO,
- {
- "options": ["16:9", "9:16"],
- "default": "16:9",
- "tooltip": "Aspect ratio of the output video",
- },
+ comfy_io.Combo.Input(
+ "aspect_ratio",
+ options=["16:9", "9:16"],
+ default="16:9",
+ tooltip="Aspect ratio of the output video",
),
- },
- "optional": {
- "negative_prompt": (
- IO.STRING,
- {
- "multiline": True,
- "default": "",
- "tooltip": "Negative text prompt to guide what to avoid in the video",
- },
+ comfy_io.String.Input(
+ "negative_prompt",
+ multiline=True,
+ default="",
+ tooltip="Negative text prompt to guide what to avoid in the video",
+ optional=True,
),
- "duration_seconds": (
- IO.INT,
- {
- "default": 5,
- "min": 5,
- "max": 8,
- "step": 1,
- "display": "number",
- "tooltip": "Duration of the output video in seconds",
- },
+ comfy_io.Int.Input(
+ "duration_seconds",
+ default=5,
+ min=5,
+ max=8,
+ step=1,
+ display_mode=comfy_io.NumberDisplay.number,
+ tooltip="Duration of the output video in seconds",
+ optional=True,
),
- "enhance_prompt": (
- IO.BOOLEAN,
- {
- "default": True,
- "tooltip": "Whether to enhance the prompt with AI assistance",
- }
+ comfy_io.Boolean.Input(
+ "enhance_prompt",
+ default=True,
+ tooltip="Whether to enhance the prompt with AI assistance",
+ optional=True,
),
- "person_generation": (
- IO.COMBO,
- {
- "options": ["ALLOW", "BLOCK"],
- "default": "ALLOW",
- "tooltip": "Whether to allow generating people in the video",
- },
+ comfy_io.Combo.Input(
+ "person_generation",
+ options=["ALLOW", "BLOCK"],
+ default="ALLOW",
+ tooltip="Whether to allow generating people in the video",
+ optional=True,
),
- "seed": (
- IO.INT,
- {
- "default": 0,
- "min": 0,
- "max": 0xFFFFFFFF,
- "step": 1,
- "display": "number",
- "control_after_generate": True,
- "tooltip": "Seed for video generation (0 for random)",
- },
+ comfy_io.Int.Input(
+ "seed",
+ default=0,
+ min=0,
+ max=0xFFFFFFFF,
+ step=1,
+ display_mode=comfy_io.NumberDisplay.number,
+ control_after_generate=True,
+ tooltip="Seed for video generation (0 for random)",
+ optional=True,
),
- "image": (IO.IMAGE, {
- "default": None,
- "tooltip": "Optional reference image to guide video generation",
- }),
- },
- "hidden": {
- "auth_token": "AUTH_TOKEN_COMFY_ORG",
- "comfy_api_key": "API_KEY_COMFY_ORG",
- "unique_id": "UNIQUE_ID",
- },
- }
+ comfy_io.Image.Input(
+ "image",
+ tooltip="Optional reference image to guide video generation",
+ optional=True,
+ ),
+ comfy_io.Combo.Input(
+ "model",
+ options=["veo-2.0-generate-001"],
+ default="veo-2.0-generate-001",
+ tooltip="Veo 2 model to use for video generation",
+ optional=True,
+ ),
+ ],
+ outputs=[
+ comfy_io.Video.Output(),
+ ],
+ hidden=[
+ comfy_io.Hidden.auth_token_comfy_org,
+ comfy_io.Hidden.api_key_comfy_org,
+ comfy_io.Hidden.unique_id,
+ ],
+ is_api_node=True,
+ )
- RETURN_TYPES = (IO.VIDEO,)
- FUNCTION = "generate_video"
- CATEGORY = "api node/video/Veo"
- DESCRIPTION = "Generates videos from text prompts using Google's Veo API"
- API_NODE = True
-
- def generate_video(
- self,
+ @classmethod
+ async def execute(
+ cls,
prompt,
aspect_ratio="16:9",
negative_prompt="",
@@ -154,8 +155,8 @@ class VeoVideoGenerationNode(ComfyNodeABC):
person_generation="ALLOW",
seed=0,
image=None,
- unique_id: Optional[str] = None,
- **kwargs,
+ model="veo-2.0-generate-001",
+ generate_audio=False,
):
# Prepare the instances for the request
instances = []
@@ -188,23 +189,30 @@ class VeoVideoGenerationNode(ComfyNodeABC):
parameters["negativePrompt"] = negative_prompt
if seed > 0:
parameters["seed"] = seed
+ # Only add generateAudio for Veo 3 models
+ if "veo-3.0" in model:
+ parameters["generateAudio"] = generate_audio
+ auth = {
+ "auth_token": cls.hidden.auth_token_comfy_org,
+ "comfy_api_key": cls.hidden.api_key_comfy_org,
+ }
# Initial request to start video generation
initial_operation = SynchronousOperation(
endpoint=ApiEndpoint(
- path="/proxy/veo/generate",
+ path=f"/proxy/veo/{model}/generate",
method=HttpMethod.POST,
- request_model=Veo2GenVidRequest,
- response_model=Veo2GenVidResponse
+ request_model=VeoGenVidRequest,
+ response_model=VeoGenVidResponse
),
- request=Veo2GenVidRequest(
+ request=VeoGenVidRequest(
instances=instances,
parameters=parameters
),
- auth_kwargs=kwargs,
+ auth_kwargs=auth,
)
- initial_response = initial_operation.execute()
+ initial_response = await initial_operation.execute()
operation_name = initial_response.name
logging.info(f"Veo generation started with operation name: {operation_name}")
@@ -223,27 +231,27 @@ class VeoVideoGenerationNode(ComfyNodeABC):
# Define the polling operation
poll_operation = PollingOperation(
poll_endpoint=ApiEndpoint(
- path="/proxy/veo/poll",
+ path=f"/proxy/veo/{model}/poll",
method=HttpMethod.POST,
- request_model=Veo2GenVidPollRequest,
- response_model=Veo2GenVidPollResponse
+ request_model=VeoGenVidPollRequest,
+ response_model=VeoGenVidPollResponse
),
completed_statuses=["completed"],
failed_statuses=[], # No failed statuses, we'll handle errors after polling
status_extractor=status_extractor,
progress_extractor=progress_extractor,
- request=Veo2GenVidPollRequest(
+ request=VeoGenVidPollRequest(
operationName=operation_name
),
- auth_kwargs=kwargs,
+ auth_kwargs=auth,
poll_interval=5.0,
result_url_extractor=get_video_url_from_response,
- node_id=unique_id,
+ node_id=cls.hidden.unique_id,
estimated_duration=AVERAGE_DURATION_VIDEO_GEN,
)
# Execute the polling operation
- poll_response = poll_operation.execute()
+ poll_response = await poll_operation.execute()
# Now check for errors in the final response
# Check for error in poll response
@@ -268,7 +276,6 @@ class VeoVideoGenerationNode(ComfyNodeABC):
raise Exception(error_message)
# Extract video data
- video_data = None
if poll_response.response and hasattr(poll_response.response, 'videos') and poll_response.response.videos and len(poll_response.response.videos) > 0:
video = poll_response.response.videos[0]
@@ -278,9 +285,9 @@ class VeoVideoGenerationNode(ComfyNodeABC):
video_data = base64.b64decode(video.bytesBase64Encoded)
elif hasattr(video, 'gcsUri') and video.gcsUri:
# Download from URL
- video_url = video.gcsUri
- video_response = requests.get(video_url)
- video_data = video_response.content
+ async with aiohttp.ClientSession() as session:
+ async with session.get(video.gcsUri) as video_response:
+ video_data = await video_response.content.read()
else:
raise Exception("Video returned but no data or URL was provided")
else:
@@ -292,17 +299,123 @@ class VeoVideoGenerationNode(ComfyNodeABC):
logging.info("Video generation completed successfully")
# Convert video data to BytesIO object
- video_io = io.BytesIO(video_data)
+ video_io = BytesIO(video_data)
# Return VideoFromFile object
- return (VideoFromFile(video_io),)
+ return comfy_io.NodeOutput(VideoFromFile(video_io))
-# Register the node
-NODE_CLASS_MAPPINGS = {
- "VeoVideoGenerationNode": VeoVideoGenerationNode,
-}
+class Veo3VideoGenerationNode(VeoVideoGenerationNode):
+ """
+ Generates videos from text prompts using Google's Veo 3 API.
-NODE_DISPLAY_NAME_MAPPINGS = {
- "VeoVideoGenerationNode": "Google Veo2 Video Generation",
-}
+ Supported models:
+ - veo-3.0-generate-001
+ - veo-3.0-fast-generate-001
+
+ This node extends the base Veo node with Veo 3 specific features including
+ audio generation and fixed 8-second duration.
+ """
+
+ @classmethod
+ def define_schema(cls):
+ return comfy_io.Schema(
+ node_id="Veo3VideoGenerationNode",
+ display_name="Google Veo 3 Video Generation",
+ category="api node/video/Veo",
+ description="Generates videos from text prompts using Google's Veo 3 API",
+ inputs=[
+ comfy_io.String.Input(
+ "prompt",
+ multiline=True,
+ default="",
+ tooltip="Text description of the video",
+ ),
+ comfy_io.Combo.Input(
+ "aspect_ratio",
+ options=["16:9", "9:16"],
+ default="16:9",
+ tooltip="Aspect ratio of the output video",
+ ),
+ comfy_io.String.Input(
+ "negative_prompt",
+ multiline=True,
+ default="",
+ tooltip="Negative text prompt to guide what to avoid in the video",
+ optional=True,
+ ),
+ comfy_io.Int.Input(
+ "duration_seconds",
+ default=8,
+ min=8,
+ max=8,
+ step=1,
+ display_mode=comfy_io.NumberDisplay.number,
+ tooltip="Duration of the output video in seconds (Veo 3 only supports 8 seconds)",
+ optional=True,
+ ),
+ comfy_io.Boolean.Input(
+ "enhance_prompt",
+ default=True,
+ tooltip="Whether to enhance the prompt with AI assistance",
+ optional=True,
+ ),
+ comfy_io.Combo.Input(
+ "person_generation",
+ options=["ALLOW", "BLOCK"],
+ default="ALLOW",
+ tooltip="Whether to allow generating people in the video",
+ optional=True,
+ ),
+ comfy_io.Int.Input(
+ "seed",
+ default=0,
+ min=0,
+ max=0xFFFFFFFF,
+ step=1,
+ display_mode=comfy_io.NumberDisplay.number,
+ control_after_generate=True,
+ tooltip="Seed for video generation (0 for random)",
+ optional=True,
+ ),
+ comfy_io.Image.Input(
+ "image",
+ tooltip="Optional reference image to guide video generation",
+ optional=True,
+ ),
+ comfy_io.Combo.Input(
+ "model",
+ options=["veo-3.0-generate-001", "veo-3.0-fast-generate-001"],
+ default="veo-3.0-generate-001",
+ tooltip="Veo 3 model to use for video generation",
+ optional=True,
+ ),
+ comfy_io.Boolean.Input(
+ "generate_audio",
+ default=False,
+ tooltip="Generate audio for the video. Supported by all Veo 3 models.",
+ optional=True,
+ ),
+ ],
+ outputs=[
+ comfy_io.Video.Output(),
+ ],
+ hidden=[
+ comfy_io.Hidden.auth_token_comfy_org,
+ comfy_io.Hidden.api_key_comfy_org,
+ comfy_io.Hidden.unique_id,
+ ],
+ is_api_node=True,
+ )
+
+
+class VeoExtension(ComfyExtension):
+ @override
+ async def get_node_list(self) -> list[type[comfy_io.ComfyNode]]:
+ return [
+ VeoVideoGenerationNode,
+ Veo3VideoGenerationNode,
+ ]
+
+async def comfy_entrypoint() -> VeoExtension:
+ return VeoExtension()
diff --git a/comfy_api_nodes/nodes_vidu.py b/comfy_api_nodes/nodes_vidu.py
new file mode 100644
index 000000000..2f441948c
--- /dev/null
+++ b/comfy_api_nodes/nodes_vidu.py
@@ -0,0 +1,622 @@
+import logging
+from enum import Enum
+from typing import Any, Callable, Optional, Literal, TypeVar
+from typing_extensions import override
+
+import torch
+from pydantic import BaseModel, Field
+
+from comfy_api.latest import ComfyExtension, io as comfy_io
+from comfy_api_nodes.util.validation_utils import (
+ validate_aspect_ratio_closeness,
+ validate_image_dimensions,
+ validate_image_aspect_ratio_range,
+ get_number_of_images,
+)
+from comfy_api_nodes.apis.client import (
+ ApiEndpoint,
+ HttpMethod,
+ SynchronousOperation,
+ PollingOperation,
+ EmptyRequest,
+)
+from comfy_api_nodes.apinode_utils import download_url_to_video_output, upload_images_to_comfyapi
+
+
+VIDU_TEXT_TO_VIDEO = "/proxy/vidu/text2video"
+VIDU_IMAGE_TO_VIDEO = "/proxy/vidu/img2video"
+VIDU_REFERENCE_VIDEO = "/proxy/vidu/reference2video"
+VIDU_START_END_VIDEO = "/proxy/vidu/start-end2video"
+VIDU_GET_GENERATION_STATUS = "/proxy/vidu/tasks/%s/creations"
+
+R = TypeVar("R")
+
+class VideoModelName(str, Enum):
+ vidu_q1 = 'viduq1'
+
+
+class AspectRatio(str, Enum):
+ r_16_9 = "16:9"
+ r_9_16 = "9:16"
+ r_1_1 = "1:1"
+
+
+class Resolution(str, Enum):
+ r_1080p = "1080p"
+
+
+class MovementAmplitude(str, Enum):
+ auto = "auto"
+ small = "small"
+ medium = "medium"
+ large = "large"
+
+
+class TaskCreationRequest(BaseModel):
+ model: VideoModelName = VideoModelName.vidu_q1
+ prompt: Optional[str] = Field(None, max_length=1500)
+ duration: Optional[Literal[5]] = 5
+ seed: Optional[int] = Field(0, ge=0, le=2147483647)
+ aspect_ratio: Optional[AspectRatio] = AspectRatio.r_16_9
+ resolution: Optional[Resolution] = Resolution.r_1080p
+ movement_amplitude: Optional[MovementAmplitude] = MovementAmplitude.auto
+ images: Optional[list[str]] = Field(None, description="Base64 encoded string or image URL")
+
+
+class TaskStatus(str, Enum):
+ created = "created"
+ queueing = "queueing"
+ processing = "processing"
+ success = "success"
+ failed = "failed"
+
+
+class TaskCreationResponse(BaseModel):
+ task_id: str = Field(...)
+ state: TaskStatus = Field(...)
+ created_at: str = Field(...)
+ code: Optional[int] = Field(None, description="Error code")
+
+
+class TaskResult(BaseModel):
+ id: str = Field(..., description="Creation id")
+ url: str = Field(..., description="The URL of the generated results, valid for one hour")
+ cover_url: str = Field(..., description="The cover URL of the generated results, valid for one hour")
+
+
+class TaskStatusResponse(BaseModel):
+ state: TaskStatus = Field(...)
+ err_code: Optional[str] = Field(None)
+ creations: list[TaskResult] = Field(..., description="Generated results")
+
+
+async def poll_until_finished(
+ auth_kwargs: dict[str, str],
+ api_endpoint: ApiEndpoint[Any, R],
+ result_url_extractor: Optional[Callable[[R], str]] = None,
+ estimated_duration: Optional[int] = None,
+ node_id: Optional[str] = None,
+) -> R:
+ return await PollingOperation(
+ poll_endpoint=api_endpoint,
+ completed_statuses=[TaskStatus.success.value],
+ failed_statuses=[TaskStatus.failed.value],
+ status_extractor=lambda response: response.state.value,
+ auth_kwargs=auth_kwargs,
+ result_url_extractor=result_url_extractor,
+ estimated_duration=estimated_duration,
+ node_id=node_id,
+ poll_interval=16.0,
+ max_poll_attempts=256,
+ ).execute()
+
+
+def get_video_url_from_response(response) -> Optional[str]:
+ if response.creations:
+ return response.creations[0].url
+ return None
+
+
+def get_video_from_response(response) -> TaskResult:
+ if not response.creations:
+ error_msg = f"Vidu request does not contain results. State: {response.state}, Error Code: {response.err_code}"
+ logging.info(error_msg)
+ raise RuntimeError(error_msg)
+ logging.info("Vidu task %s succeeded. Video URL: %s", response.creations[0].id, response.creations[0].url)
+ return response.creations[0]
+
+
+async def execute_task(
+ vidu_endpoint: str,
+ auth_kwargs: Optional[dict[str, str]],
+ payload: TaskCreationRequest,
+ estimated_duration: int,
+ node_id: str,
+) -> R:
+ response = await SynchronousOperation(
+ endpoint=ApiEndpoint(
+ path=vidu_endpoint,
+ method=HttpMethod.POST,
+ request_model=TaskCreationRequest,
+ response_model=TaskCreationResponse,
+ ),
+ request=payload,
+ auth_kwargs=auth_kwargs,
+ ).execute()
+ if response.state == TaskStatus.failed:
+ error_msg = f"Vidu request failed. Code: {response.code}"
+ logging.error(error_msg)
+ raise RuntimeError(error_msg)
+ return await poll_until_finished(
+ auth_kwargs,
+ ApiEndpoint(
+ path=VIDU_GET_GENERATION_STATUS % response.task_id,
+ method=HttpMethod.GET,
+ request_model=EmptyRequest,
+ response_model=TaskStatusResponse,
+ ),
+ result_url_extractor=get_video_url_from_response,
+ estimated_duration=estimated_duration,
+ node_id=node_id,
+ )
+
+
+class ViduTextToVideoNode(comfy_io.ComfyNode):
+
+ @classmethod
+ def define_schema(cls):
+ return comfy_io.Schema(
+ node_id="ViduTextToVideoNode",
+ display_name="Vidu Text To Video Generation",
+ category="api node/video/Vidu",
+ description="Generate video from text prompt",
+ inputs=[
+ comfy_io.Combo.Input(
+ "model",
+ options=[model.value for model in VideoModelName],
+ default=VideoModelName.vidu_q1.value,
+ tooltip="Model name",
+ ),
+ comfy_io.String.Input(
+ "prompt",
+ multiline=True,
+ tooltip="A textual description for video generation",
+ ),
+ comfy_io.Int.Input(
+ "duration",
+ default=5,
+ min=5,
+ max=5,
+ step=1,
+ display_mode=comfy_io.NumberDisplay.number,
+ tooltip="Duration of the output video in seconds",
+ optional=True,
+ ),
+ comfy_io.Int.Input(
+ "seed",
+ default=0,
+ min=0,
+ max=2147483647,
+ step=1,
+ display_mode=comfy_io.NumberDisplay.number,
+ control_after_generate=True,
+ tooltip="Seed for video generation (0 for random)",
+ optional=True,
+ ),
+ comfy_io.Combo.Input(
+ "aspect_ratio",
+ options=[model.value for model in AspectRatio],
+ default=AspectRatio.r_16_9.value,
+ tooltip="The aspect ratio of the output video",
+ optional=True,
+ ),
+ comfy_io.Combo.Input(
+ "resolution",
+ options=[model.value for model in Resolution],
+ default=Resolution.r_1080p.value,
+ tooltip="Supported values may vary by model & duration",
+ optional=True,
+ ),
+ comfy_io.Combo.Input(
+ "movement_amplitude",
+ options=[model.value for model in MovementAmplitude],
+ default=MovementAmplitude.auto.value,
+ tooltip="The movement amplitude of objects in the frame",
+ optional=True,
+ ),
+ ],
+ outputs=[
+ comfy_io.Video.Output(),
+ ],
+ hidden=[
+ comfy_io.Hidden.auth_token_comfy_org,
+ comfy_io.Hidden.api_key_comfy_org,
+ comfy_io.Hidden.unique_id,
+ ],
+ is_api_node=True,
+ )
+
+ @classmethod
+ async def execute(
+ cls,
+ model: str,
+ prompt: str,
+ duration: int,
+ seed: int,
+ aspect_ratio: str,
+ resolution: str,
+ movement_amplitude: str,
+ ) -> comfy_io.NodeOutput:
+ if not prompt:
+ raise ValueError("The prompt field is required and cannot be empty.")
+ payload = TaskCreationRequest(
+ model_name=model,
+ prompt=prompt,
+ duration=duration,
+ seed=seed,
+ aspect_ratio=aspect_ratio,
+ resolution=resolution,
+ movement_amplitude=movement_amplitude,
+ )
+ auth = {
+ "auth_token": cls.hidden.auth_token_comfy_org,
+ "comfy_api_key": cls.hidden.api_key_comfy_org,
+ }
+ results = await execute_task(VIDU_TEXT_TO_VIDEO, auth, payload, 320, cls.hidden.unique_id)
+ return comfy_io.NodeOutput(await download_url_to_video_output(get_video_from_response(results).url))
+
+
+class ViduImageToVideoNode(comfy_io.ComfyNode):
+
+ @classmethod
+ def define_schema(cls):
+ return comfy_io.Schema(
+ node_id="ViduImageToVideoNode",
+ display_name="Vidu Image To Video Generation",
+ category="api node/video/Vidu",
+ description="Generate video from image and optional prompt",
+ inputs=[
+ comfy_io.Combo.Input(
+ "model",
+ options=[model.value for model in VideoModelName],
+ default=VideoModelName.vidu_q1.value,
+ tooltip="Model name",
+ ),
+ comfy_io.Image.Input(
+ "image",
+ tooltip="An image to be used as the start frame of the generated video",
+ ),
+ comfy_io.String.Input(
+ "prompt",
+ multiline=True,
+ default="",
+ tooltip="A textual description for video generation",
+ optional=True,
+ ),
+ comfy_io.Int.Input(
+ "duration",
+ default=5,
+ min=5,
+ max=5,
+ step=1,
+ display_mode=comfy_io.NumberDisplay.number,
+ tooltip="Duration of the output video in seconds",
+ optional=True,
+ ),
+ comfy_io.Int.Input(
+ "seed",
+ default=0,
+ min=0,
+ max=2147483647,
+ step=1,
+ display_mode=comfy_io.NumberDisplay.number,
+ control_after_generate=True,
+ tooltip="Seed for video generation (0 for random)",
+ optional=True,
+ ),
+ comfy_io.Combo.Input(
+ "resolution",
+ options=[model.value for model in Resolution],
+ default=Resolution.r_1080p.value,
+ tooltip="Supported values may vary by model & duration",
+ optional=True,
+ ),
+ comfy_io.Combo.Input(
+ "movement_amplitude",
+ options=[model.value for model in MovementAmplitude],
+ default=MovementAmplitude.auto.value,
+ tooltip="The movement amplitude of objects in the frame",
+ optional=True,
+ ),
+ ],
+ outputs=[
+ comfy_io.Video.Output(),
+ ],
+ hidden=[
+ comfy_io.Hidden.auth_token_comfy_org,
+ comfy_io.Hidden.api_key_comfy_org,
+ comfy_io.Hidden.unique_id,
+ ],
+ is_api_node=True,
+ )
+
+ @classmethod
+ async def execute(
+ cls,
+ model: str,
+ image: torch.Tensor,
+ prompt: str,
+ duration: int,
+ seed: int,
+ resolution: str,
+ movement_amplitude: str,
+ ) -> comfy_io.NodeOutput:
+ if get_number_of_images(image) > 1:
+ raise ValueError("Only one input image is allowed.")
+ validate_image_aspect_ratio_range(image, (1, 4), (4, 1))
+ payload = TaskCreationRequest(
+ model_name=model,
+ prompt=prompt,
+ duration=duration,
+ seed=seed,
+ resolution=resolution,
+ movement_amplitude=movement_amplitude,
+ )
+ auth = {
+ "auth_token": cls.hidden.auth_token_comfy_org,
+ "comfy_api_key": cls.hidden.api_key_comfy_org,
+ }
+ payload.images = await upload_images_to_comfyapi(
+ image,
+ max_images=1,
+ mime_type="image/png",
+ auth_kwargs=auth,
+ )
+ results = await execute_task(VIDU_IMAGE_TO_VIDEO, auth, payload, 120, cls.hidden.unique_id)
+ return comfy_io.NodeOutput(await download_url_to_video_output(get_video_from_response(results).url))
+
+
+class ViduReferenceVideoNode(comfy_io.ComfyNode):
+
+ @classmethod
+ def define_schema(cls):
+ return comfy_io.Schema(
+ node_id="ViduReferenceVideoNode",
+ display_name="Vidu Reference To Video Generation",
+ category="api node/video/Vidu",
+ description="Generate video from multiple images and prompt",
+ inputs=[
+ comfy_io.Combo.Input(
+ "model",
+ options=[model.value for model in VideoModelName],
+ default=VideoModelName.vidu_q1.value,
+ tooltip="Model name",
+ ),
+ comfy_io.Image.Input(
+ "images",
+ tooltip="Images to use as references to generate a video with consistent subjects (max 7 images).",
+ ),
+ comfy_io.String.Input(
+ "prompt",
+ multiline=True,
+ tooltip="A textual description for video generation",
+ ),
+ comfy_io.Int.Input(
+ "duration",
+ default=5,
+ min=5,
+ max=5,
+ step=1,
+ display_mode=comfy_io.NumberDisplay.number,
+ tooltip="Duration of the output video in seconds",
+ optional=True,
+ ),
+ comfy_io.Int.Input(
+ "seed",
+ default=0,
+ min=0,
+ max=2147483647,
+ step=1,
+ display_mode=comfy_io.NumberDisplay.number,
+ control_after_generate=True,
+ tooltip="Seed for video generation (0 for random)",
+ optional=True,
+ ),
+ comfy_io.Combo.Input(
+ "aspect_ratio",
+ options=[model.value for model in AspectRatio],
+ default=AspectRatio.r_16_9.value,
+ tooltip="The aspect ratio of the output video",
+ optional=True,
+ ),
+ comfy_io.Combo.Input(
+ "resolution",
+ options=[model.value for model in Resolution],
+ default=Resolution.r_1080p.value,
+ tooltip="Supported values may vary by model & duration",
+ optional=True,
+ ),
+ comfy_io.Combo.Input(
+ "movement_amplitude",
+ options=[model.value for model in MovementAmplitude],
+ default=MovementAmplitude.auto.value,
+ tooltip="The movement amplitude of objects in the frame",
+ optional=True,
+ ),
+ ],
+ outputs=[
+ comfy_io.Video.Output(),
+ ],
+ hidden=[
+ comfy_io.Hidden.auth_token_comfy_org,
+ comfy_io.Hidden.api_key_comfy_org,
+ comfy_io.Hidden.unique_id,
+ ],
+ is_api_node=True,
+ )
+
+ @classmethod
+ async def execute(
+ cls,
+ model: str,
+ images: torch.Tensor,
+ prompt: str,
+ duration: int,
+ seed: int,
+ aspect_ratio: str,
+ resolution: str,
+ movement_amplitude: str,
+ ) -> comfy_io.NodeOutput:
+ if not prompt:
+ raise ValueError("The prompt field is required and cannot be empty.")
+ a = get_number_of_images(images)
+ if a > 7:
+ raise ValueError("Too many images, maximum allowed is 7.")
+ for image in images:
+ validate_image_aspect_ratio_range(image, (1, 4), (4, 1))
+ validate_image_dimensions(image, min_width=128, min_height=128)
+ payload = TaskCreationRequest(
+ model_name=model,
+ prompt=prompt,
+ duration=duration,
+ seed=seed,
+ aspect_ratio=aspect_ratio,
+ resolution=resolution,
+ movement_amplitude=movement_amplitude,
+ )
+ auth = {
+ "auth_token": cls.hidden.auth_token_comfy_org,
+ "comfy_api_key": cls.hidden.api_key_comfy_org,
+ }
+ payload.images = await upload_images_to_comfyapi(
+ images,
+ max_images=7,
+ mime_type="image/png",
+ auth_kwargs=auth,
+ )
+ results = await execute_task(VIDU_REFERENCE_VIDEO, auth, payload, 120, cls.hidden.unique_id)
+ return comfy_io.NodeOutput(await download_url_to_video_output(get_video_from_response(results).url))
+
+
+class ViduStartEndToVideoNode(comfy_io.ComfyNode):
+
+ @classmethod
+ def define_schema(cls):
+ return comfy_io.Schema(
+ node_id="ViduStartEndToVideoNode",
+ display_name="Vidu Start End To Video Generation",
+ category="api node/video/Vidu",
+ description="Generate a video from start and end frames and a prompt",
+ inputs=[
+ comfy_io.Combo.Input(
+ "model",
+ options=[model.value for model in VideoModelName],
+ default=VideoModelName.vidu_q1.value,
+ tooltip="Model name",
+ ),
+ comfy_io.Image.Input(
+ "first_frame",
+ tooltip="Start frame",
+ ),
+ comfy_io.Image.Input(
+ "end_frame",
+ tooltip="End frame",
+ ),
+ comfy_io.String.Input(
+ "prompt",
+ multiline=True,
+ tooltip="A textual description for video generation",
+ optional=True,
+ ),
+ comfy_io.Int.Input(
+ "duration",
+ default=5,
+ min=5,
+ max=5,
+ step=1,
+ display_mode=comfy_io.NumberDisplay.number,
+ tooltip="Duration of the output video in seconds",
+ optional=True,
+ ),
+ comfy_io.Int.Input(
+ "seed",
+ default=0,
+ min=0,
+ max=2147483647,
+ step=1,
+ display_mode=comfy_io.NumberDisplay.number,
+ control_after_generate=True,
+ tooltip="Seed for video generation (0 for random)",
+ optional=True,
+ ),
+ comfy_io.Combo.Input(
+ "resolution",
+ options=[model.value for model in Resolution],
+ default=Resolution.r_1080p.value,
+ tooltip="Supported values may vary by model & duration",
+ optional=True,
+ ),
+ comfy_io.Combo.Input(
+ "movement_amplitude",
+ options=[model.value for model in MovementAmplitude],
+ default=MovementAmplitude.auto.value,
+ tooltip="The movement amplitude of objects in the frame",
+ optional=True,
+ ),
+ ],
+ outputs=[
+ comfy_io.Video.Output(),
+ ],
+ hidden=[
+ comfy_io.Hidden.auth_token_comfy_org,
+ comfy_io.Hidden.api_key_comfy_org,
+ comfy_io.Hidden.unique_id,
+ ],
+ is_api_node=True,
+ )
+
+ @classmethod
+ async def execute(
+ cls,
+ model: str,
+ first_frame: torch.Tensor,
+ end_frame: torch.Tensor,
+ prompt: str,
+ duration: int,
+ seed: int,
+ resolution: str,
+ movement_amplitude: str,
+ ) -> comfy_io.NodeOutput:
+ validate_aspect_ratio_closeness(first_frame, end_frame, min_rel=0.8, max_rel=1.25, strict=False)
+ payload = TaskCreationRequest(
+ model_name=model,
+ prompt=prompt,
+ duration=duration,
+ seed=seed,
+ resolution=resolution,
+ movement_amplitude=movement_amplitude,
+ )
+ auth = {
+ "auth_token": cls.hidden.auth_token_comfy_org,
+ "comfy_api_key": cls.hidden.api_key_comfy_org,
+ }
+ payload.images = [
+ (await upload_images_to_comfyapi(frame, max_images=1, mime_type="image/png", auth_kwargs=auth))[0]
+ for frame in (first_frame, end_frame)
+ ]
+ results = await execute_task(VIDU_START_END_VIDEO, auth, payload, 96, cls.hidden.unique_id)
+ return comfy_io.NodeOutput(await download_url_to_video_output(get_video_from_response(results).url))
+
+
+class ViduExtension(ComfyExtension):
+ @override
+ async def get_node_list(self) -> list[type[comfy_io.ComfyNode]]:
+ return [
+ ViduTextToVideoNode,
+ ViduImageToVideoNode,
+ ViduReferenceVideoNode,
+ ViduStartEndToVideoNode,
+ ]
+
+async def comfy_entrypoint() -> ViduExtension:
+ return ViduExtension()
diff --git a/comfy_api_nodes/util/validation_utils.py b/comfy_api_nodes/util/validation_utils.py
index 031b9fbd3..606b794bf 100644
--- a/comfy_api_nodes/util/validation_utils.py
+++ b/comfy_api_nodes/util/validation_utils.py
@@ -53,6 +53,53 @@ def validate_image_aspect_ratio(
)
+def validate_image_aspect_ratio_range(
+ image: torch.Tensor,
+ min_ratio: tuple[float, float], # e.g. (1, 4)
+ max_ratio: tuple[float, float], # e.g. (4, 1)
+ *,
+ strict: bool = True, # True -> (min, max); False -> [min, max]
+) -> float:
+ a1, b1 = min_ratio
+ a2, b2 = max_ratio
+ if a1 <= 0 or b1 <= 0 or a2 <= 0 or b2 <= 0:
+ raise ValueError("Ratios must be positive, like (1, 4) or (4, 1).")
+ lo, hi = (a1 / b1), (a2 / b2)
+ if lo > hi:
+ lo, hi = hi, lo
+ a1, b1, a2, b2 = a2, b2, a1, b1 # swap only for error text
+ w, h = get_image_dimensions(image)
+ if w <= 0 or h <= 0:
+ raise ValueError(f"Invalid image dimensions: {w}x{h}")
+ ar = w / h
+ ok = (lo < ar < hi) if strict else (lo <= ar <= hi)
+ if not ok:
+ op = "<" if strict else "≤"
+ raise ValueError(f"Image aspect ratio {ar:.6g} is outside allowed range: {a1}:{b1} {op} ratio {op} {a2}:{b2}")
+ return ar
+
+
+def validate_aspect_ratio_closeness(
+ start_img,
+ end_img,
+ min_rel: float,
+ max_rel: float,
+ *,
+ strict: bool = False, # True => exclusive, False => inclusive
+) -> None:
+ w1, h1 = get_image_dimensions(start_img)
+ w2, h2 = get_image_dimensions(end_img)
+ if min(w1, h1, w2, h2) <= 0:
+ raise ValueError("Invalid image dimensions")
+ ar1 = w1 / h1
+ ar2 = w2 / h2
+ # Normalize so it is symmetric (no need to check both ar1/ar2 and ar2/ar1)
+ closeness = max(ar1, ar2) / min(ar1, ar2)
+ limit = max(max_rel, 1.0 / min_rel) # for 0.8..1.25 this is 1.25
+ if (closeness >= limit) if strict else (closeness > limit):
+ raise ValueError(f"Aspect ratios must be close: start/end={ar1/ar2:.4f}, allowed range {min_rel}–{max_rel}.")
+
+
def validate_video_dimensions(
video: VideoInput,
min_width: Optional[int] = None,
@@ -98,3 +145,9 @@ def validate_video_duration(
raise ValueError(
f"Video duration must be at most {max_duration}s, got {duration}s"
)
+
+
+def get_number_of_images(images):
+ if isinstance(images, torch.Tensor):
+ return images.shape[0] if images.ndim >= 4 else 1
+ return len(images)
diff --git a/comfy_execution/graph.py b/comfy_execution/graph.py
index 60e2ab91e..f4b427265 100644
--- a/comfy_execution/graph.py
+++ b/comfy_execution/graph.py
@@ -4,9 +4,12 @@ from typing import Type, Literal
import nodes
import asyncio
import inspect
-from comfy_execution.graph_utils import is_link
+from comfy_execution.graph_utils import is_link, ExecutionBlocker
from comfy.comfy_types.node_typing import ComfyNodeABC, InputTypeDict, InputTypeOptions
+# NOTE: ExecutionBlocker code got moved to graph_utils.py to prevent torch being imported too soon during unit tests
+ExecutionBlocker = ExecutionBlocker
+
class DependencyCycleError(Exception):
pass
@@ -294,21 +297,3 @@ class ExecutionList(TopologicalSort):
del blocked_by[node_id]
to_remove = [node_id for node_id in blocked_by if len(blocked_by[node_id]) == 0]
return list(blocked_by.keys())
-
-class ExecutionBlocker:
- """
- Return this from a node and any users will be blocked with the given error message.
- If the message is None, execution will be blocked silently instead.
- Generally, you should avoid using this functionality unless absolutely necessary. Whenever it's
- possible, a lazy input will be more efficient and have a better user experience.
- This functionality is useful in two cases:
- 1. You want to conditionally prevent an output node from executing. (Particularly a built-in node
- like SaveImage. For your own output nodes, I would recommend just adding a BOOL input and using
- lazy evaluation to let it conditionally disable itself.)
- 2. You have a node with multiple possible outputs, some of which are invalid and should not be used.
- (I would recommend not making nodes like this in the future -- instead, make multiple nodes with
- different outputs. Unfortunately, there are several popular existing nodes using this pattern.)
- """
- def __init__(self, message):
- self.message = message
-
diff --git a/comfy_execution/graph_utils.py b/comfy_execution/graph_utils.py
index 8595e942d..496d2c634 100644
--- a/comfy_execution/graph_utils.py
+++ b/comfy_execution/graph_utils.py
@@ -137,3 +137,19 @@ def add_graph_prefix(graph, outputs, prefix):
return new_graph, tuple(new_outputs)
+class ExecutionBlocker:
+ """
+ Return this from a node and any users will be blocked with the given error message.
+ If the message is None, execution will be blocked silently instead.
+ Generally, you should avoid using this functionality unless absolutely necessary. Whenever it's
+ possible, a lazy input will be more efficient and have a better user experience.
+ This functionality is useful in two cases:
+ 1. You want to conditionally prevent an output node from executing. (Particularly a built-in node
+ like SaveImage. For your own output nodes, I would recommend just adding a BOOL input and using
+ lazy evaluation to let it conditionally disable itself.)
+ 2. You have a node with multiple possible outputs, some of which are invalid and should not be used.
+ (I would recommend not making nodes like this in the future -- instead, make multiple nodes with
+ different outputs. Unfortunately, there are several popular existing nodes using this pattern.)
+ """
+ def __init__(self, message):
+ self.message = message
diff --git a/comfy_execution/progress.py b/comfy_execution/progress.py
index 73dba3f75..e8f5ede1e 100644
--- a/comfy_execution/progress.py
+++ b/comfy_execution/progress.py
@@ -1,4 +1,6 @@
-from typing import TypedDict, Dict, Optional
+from __future__ import annotations
+
+from typing import TypedDict, Dict, Optional, Tuple
from typing_extensions import override
from PIL import Image
from enum import Enum
@@ -10,6 +12,7 @@ if TYPE_CHECKING:
from protocol import BinaryEventTypes
from comfy_api import feature_flags
+PreviewImageTuple = Tuple[str, Image.Image, Optional[int]]
class NodeState(Enum):
Pending = "pending"
@@ -52,7 +55,7 @@ class ProgressHandler(ABC):
max_value: float,
state: NodeProgressState,
prompt_id: str,
- image: Optional[Image.Image] = None,
+ image: PreviewImageTuple | None = None,
):
"""Called when a node's progress is updated"""
pass
@@ -103,7 +106,7 @@ class CLIProgressHandler(ProgressHandler):
max_value: float,
state: NodeProgressState,
prompt_id: str,
- image: Optional[Image.Image] = None,
+ image: PreviewImageTuple | None = None,
):
# Handle case where start_handler wasn't called
if node_id not in self.progress_bars:
@@ -196,7 +199,7 @@ class WebUIProgressHandler(ProgressHandler):
max_value: float,
state: NodeProgressState,
prompt_id: str,
- image: Optional[Image.Image] = None,
+ image: PreviewImageTuple | None = None,
):
# Send progress state of all nodes
if self.registry:
@@ -231,7 +234,6 @@ class WebUIProgressHandler(ProgressHandler):
if self.registry:
self._send_progress_state(prompt_id, self.registry.nodes)
-
class ProgressRegistry:
"""
Registry that maintains node progress state and notifies registered handlers.
@@ -285,7 +287,7 @@ class ProgressRegistry:
handler.start_handler(node_id, entry, self.prompt_id)
def update_progress(
- self, node_id: str, value: float, max_value: float, image: Optional[Image.Image]
+ self, node_id: str, value: float, max_value: float, image: PreviewImageTuple | None = None
) -> None:
"""Update progress for a node"""
entry = self.ensure_entry(node_id)
@@ -317,7 +319,7 @@ class ProgressRegistry:
handler.reset()
# Global registry instance
-global_progress_registry: ProgressRegistry = None
+global_progress_registry: ProgressRegistry | None = None
def reset_progress_state(prompt_id: str, dynprompt: "DynamicPrompt") -> None:
global global_progress_registry
diff --git a/comfy_extras/nodes_ace.py b/comfy_extras/nodes_ace.py
index cbfec15a2..1409233c9 100644
--- a/comfy_extras/nodes_ace.py
+++ b/comfy_extras/nodes_ace.py
@@ -1,49 +1,63 @@
import torch
+from typing_extensions import override
+
import comfy.model_management
import node_helpers
+from comfy_api.latest import ComfyExtension, io
-class TextEncodeAceStepAudio:
+
+class TextEncodeAceStepAudio(io.ComfyNode):
@classmethod
- def INPUT_TYPES(s):
- return {"required": {
- "clip": ("CLIP", ),
- "tags": ("STRING", {"multiline": True, "dynamicPrompts": True}),
- "lyrics": ("STRING", {"multiline": True, "dynamicPrompts": True}),
- "lyrics_strength": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 10.0, "step": 0.01}),
- }}
- RETURN_TYPES = ("CONDITIONING",)
- FUNCTION = "encode"
+ def define_schema(cls):
+ return io.Schema(
+ node_id="TextEncodeAceStepAudio",
+ category="conditioning",
+ inputs=[
+ io.Clip.Input("clip"),
+ io.String.Input("tags", multiline=True, dynamic_prompts=True),
+ io.String.Input("lyrics", multiline=True, dynamic_prompts=True),
+ io.Float.Input("lyrics_strength", default=1.0, min=0.0, max=10.0, step=0.01),
+ ],
+ outputs=[io.Conditioning.Output()],
+ )
- CATEGORY = "conditioning"
-
- def encode(self, clip, tags, lyrics, lyrics_strength):
+ @classmethod
+ def execute(cls, clip, tags, lyrics, lyrics_strength) -> io.NodeOutput:
tokens = clip.tokenize(tags, lyrics=lyrics)
conditioning = clip.encode_from_tokens_scheduled(tokens)
conditioning = node_helpers.conditioning_set_values(conditioning, {"lyrics_strength": lyrics_strength})
- return (conditioning, )
+ return io.NodeOutput(conditioning)
-class EmptyAceStepLatentAudio:
- def __init__(self):
- self.device = comfy.model_management.intermediate_device()
+class EmptyAceStepLatentAudio(io.ComfyNode):
+ @classmethod
+ def define_schema(cls):
+ return io.Schema(
+ node_id="EmptyAceStepLatentAudio",
+ category="latent/audio",
+ inputs=[
+ io.Float.Input("seconds", default=120.0, min=1.0, max=1000.0, step=0.1),
+ io.Int.Input(
+ "batch_size", default=1, min=1, max=4096, tooltip="The number of latent images in the batch."
+ ),
+ ],
+ outputs=[io.Latent.Output()],
+ )
@classmethod
- def INPUT_TYPES(s):
- return {"required": {"seconds": ("FLOAT", {"default": 120.0, "min": 1.0, "max": 1000.0, "step": 0.1}),
- "batch_size": ("INT", {"default": 1, "min": 1, "max": 4096, "tooltip": "The number of latent images in the batch."}),
- }}
- RETURN_TYPES = ("LATENT",)
- FUNCTION = "generate"
-
- CATEGORY = "latent/audio"
-
- def generate(self, seconds, batch_size):
+ def execute(cls, seconds, batch_size) -> io.NodeOutput:
length = int(seconds * 44100 / 512 / 8)
- latent = torch.zeros([batch_size, 8, 16, length], device=self.device)
- return ({"samples": latent, "type": "audio"}, )
+ latent = torch.zeros([batch_size, 8, 16, length], device=comfy.model_management.intermediate_device())
+ return io.NodeOutput({"samples": latent, "type": "audio"})
-NODE_CLASS_MAPPINGS = {
- "TextEncodeAceStepAudio": TextEncodeAceStepAudio,
- "EmptyAceStepLatentAudio": EmptyAceStepLatentAudio,
-}
+class AceExtension(ComfyExtension):
+ @override
+ async def get_node_list(self) -> list[type[io.ComfyNode]]:
+ return [
+ TextEncodeAceStepAudio,
+ EmptyAceStepLatentAudio,
+ ]
+
+async def comfy_entrypoint() -> AceExtension:
+ return AceExtension()
diff --git a/comfy_extras/nodes_advanced_samplers.py b/comfy_extras/nodes_advanced_samplers.py
index 5fbb096fb..5532ffe6a 100644
--- a/comfy_extras/nodes_advanced_samplers.py
+++ b/comfy_extras/nodes_advanced_samplers.py
@@ -1,8 +1,13 @@
+import numpy as np
+import torch
+from tqdm.auto import trange
+from typing_extensions import override
+
+import comfy.model_patcher
import comfy.samplers
import comfy.utils
-import torch
-import numpy as np
-from tqdm.auto import trange
+from comfy.k_diffusion.sampling import to_d
+from comfy_api.latest import ComfyExtension, io
@torch.no_grad()
@@ -33,30 +38,29 @@ def sample_lcm_upscale(model, x, sigmas, extra_args=None, callback=None, disable
return x
-class SamplerLCMUpscale:
- upscale_methods = ["bislerp", "nearest-exact", "bilinear", "area", "bicubic"]
+class SamplerLCMUpscale(io.ComfyNode):
+ UPSCALE_METHODS = ["bislerp", "nearest-exact", "bilinear", "area", "bicubic"]
@classmethod
- def INPUT_TYPES(s):
- return {"required":
- {"scale_ratio": ("FLOAT", {"default": 1.0, "min": 0.1, "max": 20.0, "step": 0.01}),
- "scale_steps": ("INT", {"default": -1, "min": -1, "max": 1000, "step": 1}),
- "upscale_method": (s.upscale_methods,),
- }
- }
- RETURN_TYPES = ("SAMPLER",)
- CATEGORY = "sampling/custom_sampling/samplers"
+ def define_schema(cls) -> io.Schema:
+ return io.Schema(
+ node_id="SamplerLCMUpscale",
+ category="sampling/custom_sampling/samplers",
+ inputs=[
+ io.Float.Input("scale_ratio", default=1.0, min=0.1, max=20.0, step=0.01),
+ io.Int.Input("scale_steps", default=-1, min=-1, max=1000, step=1),
+ io.Combo.Input("upscale_method", options=cls.UPSCALE_METHODS),
+ ],
+ outputs=[io.Sampler.Output()],
+ )
- FUNCTION = "get_sampler"
-
- def get_sampler(self, scale_ratio, scale_steps, upscale_method):
+ @classmethod
+ def execute(cls, scale_ratio, scale_steps, upscale_method) -> io.NodeOutput:
if scale_steps < 0:
scale_steps = None
sampler = comfy.samplers.KSAMPLER(sample_lcm_upscale, extra_options={"total_upscale": scale_ratio, "upscale_steps": scale_steps, "upscale_method": upscale_method})
- return (sampler, )
+ return io.NodeOutput(sampler)
-from comfy.k_diffusion.sampling import to_d
-import comfy.model_patcher
@torch.no_grad()
def sample_euler_pp(model, x, sigmas, extra_args=None, callback=None, disable=None):
@@ -82,30 +86,36 @@ def sample_euler_pp(model, x, sigmas, extra_args=None, callback=None, disable=No
return x
-class SamplerEulerCFGpp:
+class SamplerEulerCFGpp(io.ComfyNode):
@classmethod
- def INPUT_TYPES(s):
- return {"required":
- {"version": (["regular", "alternative"],),}
- }
- RETURN_TYPES = ("SAMPLER",)
- # CATEGORY = "sampling/custom_sampling/samplers"
- CATEGORY = "_for_testing"
+ def define_schema(cls) -> io.Schema:
+ return io.Schema(
+ node_id="SamplerEulerCFGpp",
+ display_name="SamplerEulerCFG++",
+ category="_for_testing", # "sampling/custom_sampling/samplers"
+ inputs=[
+ io.Combo.Input("version", options=["regular", "alternative"]),
+ ],
+ outputs=[io.Sampler.Output()],
+ is_experimental=True,
+ )
- FUNCTION = "get_sampler"
-
- def get_sampler(self, version):
+ @classmethod
+ def execute(cls, version) -> io.NodeOutput:
if version == "alternative":
sampler = comfy.samplers.KSAMPLER(sample_euler_pp)
else:
sampler = comfy.samplers.ksampler("euler_cfg_pp")
- return (sampler, )
+ return io.NodeOutput(sampler)
-NODE_CLASS_MAPPINGS = {
- "SamplerLCMUpscale": SamplerLCMUpscale,
- "SamplerEulerCFGpp": SamplerEulerCFGpp,
-}
-NODE_DISPLAY_NAME_MAPPINGS = {
- "SamplerEulerCFGpp": "SamplerEulerCFG++",
-}
+class AdvancedSamplersExtension(ComfyExtension):
+ @override
+ async def get_node_list(self) -> list[type[io.ComfyNode]]:
+ return [
+ SamplerLCMUpscale,
+ SamplerEulerCFGpp,
+ ]
+
+async def comfy_entrypoint() -> AdvancedSamplersExtension:
+ return AdvancedSamplersExtension()
diff --git a/comfy_extras/nodes_apg.py b/comfy_extras/nodes_apg.py
index 25b21b1b8..f27ae7da8 100644
--- a/comfy_extras/nodes_apg.py
+++ b/comfy_extras/nodes_apg.py
@@ -1,4 +1,8 @@
import torch
+from typing_extensions import override
+
+from comfy_api.latest import ComfyExtension, io
+
def project(v0, v1):
v1 = torch.nn.functional.normalize(v1, dim=[-1, -2, -3])
@@ -6,22 +10,45 @@ def project(v0, v1):
v0_orthogonal = v0 - v0_parallel
return v0_parallel, v0_orthogonal
-class APG:
+class APG(io.ComfyNode):
@classmethod
- def INPUT_TYPES(s):
- return {
- "required": {
- "model": ("MODEL",),
- "eta": ("FLOAT", {"default": 1.0, "min": -10.0, "max": 10.0, "step": 0.01, "tooltip": "Controls the scale of the parallel guidance vector. Default CFG behavior at a setting of 1."}),
- "norm_threshold": ("FLOAT", {"default": 5.0, "min": 0.0, "max": 50.0, "step": 0.1, "tooltip": "Normalize guidance vector to this value, normalization disable at a setting of 0."}),
- "momentum": ("FLOAT", {"default": 0.0, "min": -5.0, "max": 1.0, "step": 0.01, "tooltip":"Controls a running average of guidance during diffusion, disabled at a setting of 0."}),
- }
- }
- RETURN_TYPES = ("MODEL",)
- FUNCTION = "patch"
- CATEGORY = "sampling/custom_sampling"
+ def define_schema(cls) -> io.Schema:
+ return io.Schema(
+ node_id="APG",
+ display_name="Adaptive Projected Guidance",
+ category="sampling/custom_sampling",
+ inputs=[
+ io.Model.Input("model"),
+ io.Float.Input(
+ "eta",
+ default=1.0,
+ min=-10.0,
+ max=10.0,
+ step=0.01,
+ tooltip="Controls the scale of the parallel guidance vector. Default CFG behavior at a setting of 1.",
+ ),
+ io.Float.Input(
+ "norm_threshold",
+ default=5.0,
+ min=0.0,
+ max=50.0,
+ step=0.1,
+ tooltip="Normalize guidance vector to this value, normalization disable at a setting of 0.",
+ ),
+ io.Float.Input(
+ "momentum",
+ default=0.0,
+ min=-5.0,
+ max=1.0,
+ step=0.01,
+ tooltip="Controls a running average of guidance during diffusion, disabled at a setting of 0.",
+ ),
+ ],
+ outputs=[io.Model.Output()],
+ )
- def patch(self, model, eta, norm_threshold, momentum):
+ @classmethod
+ def execute(cls, model, eta, norm_threshold, momentum) -> io.NodeOutput:
running_avg = 0
prev_sigma = None
@@ -65,12 +92,15 @@ class APG:
m = model.clone()
m.set_model_sampler_pre_cfg_function(pre_cfg_function)
- return (m,)
+ return io.NodeOutput(m)
-NODE_CLASS_MAPPINGS = {
- "APG": APG,
-}
-NODE_DISPLAY_NAME_MAPPINGS = {
- "APG": "Adaptive Projected Guidance",
-}
+class ApgExtension(ComfyExtension):
+ @override
+ async def get_node_list(self) -> list[type[io.ComfyNode]]:
+ return [
+ APG,
+ ]
+
+async def comfy_entrypoint() -> ApgExtension:
+ return ApgExtension()
diff --git a/comfy_extras/nodes_attention_multiply.py b/comfy_extras/nodes_attention_multiply.py
index 4747eb395..c0e494c2a 100644
--- a/comfy_extras/nodes_attention_multiply.py
+++ b/comfy_extras/nodes_attention_multiply.py
@@ -1,3 +1,7 @@
+from typing_extensions import override
+
+from comfy_api.latest import ComfyExtension, io
+
def attention_multiply(attn, model, q, k, v, out):
m = model.clone()
@@ -16,57 +20,71 @@ def attention_multiply(attn, model, q, k, v, out):
return m
-class UNetSelfAttentionMultiply:
+class UNetSelfAttentionMultiply(io.ComfyNode):
@classmethod
- def INPUT_TYPES(s):
- return {"required": { "model": ("MODEL",),
- "q": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 10.0, "step": 0.01}),
- "k": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 10.0, "step": 0.01}),
- "v": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 10.0, "step": 0.01}),
- "out": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 10.0, "step": 0.01}),
- }}
- RETURN_TYPES = ("MODEL",)
- FUNCTION = "patch"
+ def define_schema(cls) -> io.Schema:
+ return io.Schema(
+ node_id="UNetSelfAttentionMultiply",
+ category="_for_testing/attention_experiments",
+ inputs=[
+ io.Model.Input("model"),
+ io.Float.Input("q", default=1.0, min=0.0, max=10.0, step=0.01),
+ io.Float.Input("k", default=1.0, min=0.0, max=10.0, step=0.01),
+ io.Float.Input("v", default=1.0, min=0.0, max=10.0, step=0.01),
+ io.Float.Input("out", default=1.0, min=0.0, max=10.0, step=0.01),
+ ],
+ outputs=[io.Model.Output()],
+ is_experimental=True,
+ )
- CATEGORY = "_for_testing/attention_experiments"
-
- def patch(self, model, q, k, v, out):
+ @classmethod
+ def execute(cls, model, q, k, v, out) -> io.NodeOutput:
m = attention_multiply("attn1", model, q, k, v, out)
- return (m, )
+ return io.NodeOutput(m)
-class UNetCrossAttentionMultiply:
+
+class UNetCrossAttentionMultiply(io.ComfyNode):
@classmethod
- def INPUT_TYPES(s):
- return {"required": { "model": ("MODEL",),
- "q": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 10.0, "step": 0.01}),
- "k": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 10.0, "step": 0.01}),
- "v": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 10.0, "step": 0.01}),
- "out": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 10.0, "step": 0.01}),
- }}
- RETURN_TYPES = ("MODEL",)
- FUNCTION = "patch"
+ def define_schema(cls) -> io.Schema:
+ return io.Schema(
+ node_id="UNetCrossAttentionMultiply",
+ category="_for_testing/attention_experiments",
+ inputs=[
+ io.Model.Input("model"),
+ io.Float.Input("q", default=1.0, min=0.0, max=10.0, step=0.01),
+ io.Float.Input("k", default=1.0, min=0.0, max=10.0, step=0.01),
+ io.Float.Input("v", default=1.0, min=0.0, max=10.0, step=0.01),
+ io.Float.Input("out", default=1.0, min=0.0, max=10.0, step=0.01),
+ ],
+ outputs=[io.Model.Output()],
+ is_experimental=True,
+ )
- CATEGORY = "_for_testing/attention_experiments"
-
- def patch(self, model, q, k, v, out):
+ @classmethod
+ def execute(cls, model, q, k, v, out) -> io.NodeOutput:
m = attention_multiply("attn2", model, q, k, v, out)
- return (m, )
+ return io.NodeOutput(m)
-class CLIPAttentionMultiply:
+
+class CLIPAttentionMultiply(io.ComfyNode):
@classmethod
- def INPUT_TYPES(s):
- return {"required": { "clip": ("CLIP",),
- "q": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 10.0, "step": 0.01}),
- "k": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 10.0, "step": 0.01}),
- "v": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 10.0, "step": 0.01}),
- "out": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 10.0, "step": 0.01}),
- }}
- RETURN_TYPES = ("CLIP",)
- FUNCTION = "patch"
+ def define_schema(cls) -> io.Schema:
+ return io.Schema(
+ node_id="CLIPAttentionMultiply",
+ category="_for_testing/attention_experiments",
+ inputs=[
+ io.Clip.Input("clip"),
+ io.Float.Input("q", default=1.0, min=0.0, max=10.0, step=0.01),
+ io.Float.Input("k", default=1.0, min=0.0, max=10.0, step=0.01),
+ io.Float.Input("v", default=1.0, min=0.0, max=10.0, step=0.01),
+ io.Float.Input("out", default=1.0, min=0.0, max=10.0, step=0.01),
+ ],
+ outputs=[io.Clip.Output()],
+ is_experimental=True,
+ )
- CATEGORY = "_for_testing/attention_experiments"
-
- def patch(self, clip, q, k, v, out):
+ @classmethod
+ def execute(cls, clip, q, k, v, out) -> io.NodeOutput:
m = clip.clone()
sd = m.patcher.model_state_dict()
@@ -79,23 +97,28 @@ class CLIPAttentionMultiply:
m.add_patches({key: (None,)}, 0.0, v)
if key.endswith("self_attn.out_proj.weight") or key.endswith("self_attn.out_proj.bias"):
m.add_patches({key: (None,)}, 0.0, out)
- return (m, )
+ return io.NodeOutput(m)
-class UNetTemporalAttentionMultiply:
+
+class UNetTemporalAttentionMultiply(io.ComfyNode):
@classmethod
- def INPUT_TYPES(s):
- return {"required": { "model": ("MODEL",),
- "self_structural": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 10.0, "step": 0.01}),
- "self_temporal": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 10.0, "step": 0.01}),
- "cross_structural": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 10.0, "step": 0.01}),
- "cross_temporal": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 10.0, "step": 0.01}),
- }}
- RETURN_TYPES = ("MODEL",)
- FUNCTION = "patch"
+ def define_schema(cls) -> io.Schema:
+ return io.Schema(
+ node_id="UNetTemporalAttentionMultiply",
+ category="_for_testing/attention_experiments",
+ inputs=[
+ io.Model.Input("model"),
+ io.Float.Input("self_structural", default=1.0, min=0.0, max=10.0, step=0.01),
+ io.Float.Input("self_temporal", default=1.0, min=0.0, max=10.0, step=0.01),
+ io.Float.Input("cross_structural", default=1.0, min=0.0, max=10.0, step=0.01),
+ io.Float.Input("cross_temporal", default=1.0, min=0.0, max=10.0, step=0.01),
+ ],
+ outputs=[io.Model.Output()],
+ is_experimental=True,
+ )
- CATEGORY = "_for_testing/attention_experiments"
-
- def patch(self, model, self_structural, self_temporal, cross_structural, cross_temporal):
+ @classmethod
+ def execute(cls, model, self_structural, self_temporal, cross_structural, cross_temporal) -> io.NodeOutput:
m = model.clone()
sd = model.model_state_dict()
@@ -110,11 +133,18 @@ class UNetTemporalAttentionMultiply:
m.add_patches({k: (None,)}, 0.0, cross_temporal)
else:
m.add_patches({k: (None,)}, 0.0, cross_structural)
- return (m, )
+ return io.NodeOutput(m)
-NODE_CLASS_MAPPINGS = {
- "UNetSelfAttentionMultiply": UNetSelfAttentionMultiply,
- "UNetCrossAttentionMultiply": UNetCrossAttentionMultiply,
- "CLIPAttentionMultiply": CLIPAttentionMultiply,
- "UNetTemporalAttentionMultiply": UNetTemporalAttentionMultiply,
-}
+
+class AttentionMultiplyExtension(ComfyExtension):
+ @override
+ async def get_node_list(self) -> list[type[io.ComfyNode]]:
+ return [
+ UNetSelfAttentionMultiply,
+ UNetCrossAttentionMultiply,
+ CLIPAttentionMultiply,
+ UNetTemporalAttentionMultiply,
+ ]
+
+async def comfy_entrypoint() -> AttentionMultiplyExtension:
+ return AttentionMultiplyExtension()
diff --git a/comfy_extras/nodes_audio.py b/comfy_extras/nodes_audio.py
index a90b31779..3b23f65d8 100644
--- a/comfy_extras/nodes_audio.py
+++ b/comfy_extras/nodes_audio.py
@@ -346,6 +346,24 @@ class LoadAudio:
return "Invalid audio file: {}".format(audio)
return True
+class RecordAudio:
+ @classmethod
+ def INPUT_TYPES(s):
+ return {"required": {"audio": ("AUDIO_RECORD", {})}}
+
+ CATEGORY = "audio"
+
+ RETURN_TYPES = ("AUDIO", )
+ FUNCTION = "load"
+
+ def load(self, audio):
+ audio_path = folder_paths.get_annotated_filepath(audio)
+
+ waveform, sample_rate = torchaudio.load(audio_path)
+ audio = {"waveform": waveform.unsqueeze(0), "sample_rate": sample_rate}
+ return (audio, )
+
+
NODE_CLASS_MAPPINGS = {
"EmptyLatentAudio": EmptyLatentAudio,
"VAEEncodeAudio": VAEEncodeAudio,
@@ -356,6 +374,7 @@ NODE_CLASS_MAPPINGS = {
"LoadAudio": LoadAudio,
"PreviewAudio": PreviewAudio,
"ConditioningStableAudio": ConditioningStableAudio,
+ "RecordAudio": RecordAudio,
}
NODE_DISPLAY_NAME_MAPPINGS = {
@@ -367,4 +386,5 @@ NODE_DISPLAY_NAME_MAPPINGS = {
"SaveAudio": "Save Audio (FLAC)",
"SaveAudioMP3": "Save Audio (MP3)",
"SaveAudioOpus": "Save Audio (Opus)",
+ "RecordAudio": "Record Audio",
}
diff --git a/comfy_extras/nodes_audio_encoder.py b/comfy_extras/nodes_audio_encoder.py
new file mode 100644
index 000000000..39a140fef
--- /dev/null
+++ b/comfy_extras/nodes_audio_encoder.py
@@ -0,0 +1,44 @@
+import folder_paths
+import comfy.audio_encoders.audio_encoders
+import comfy.utils
+
+
+class AudioEncoderLoader:
+ @classmethod
+ def INPUT_TYPES(s):
+ return {"required": { "audio_encoder_name": (folder_paths.get_filename_list("audio_encoders"), ),
+ }}
+ RETURN_TYPES = ("AUDIO_ENCODER",)
+ FUNCTION = "load_model"
+
+ CATEGORY = "loaders"
+
+ def load_model(self, audio_encoder_name):
+ audio_encoder_name = folder_paths.get_full_path_or_raise("audio_encoders", audio_encoder_name)
+ sd = comfy.utils.load_torch_file(audio_encoder_name, safe_load=True)
+ audio_encoder = comfy.audio_encoders.audio_encoders.load_audio_encoder_from_sd(sd)
+ if audio_encoder is None:
+ raise RuntimeError("ERROR: audio encoder file is invalid and does not contain a valid model.")
+ return (audio_encoder,)
+
+
+class AudioEncoderEncode:
+ @classmethod
+ def INPUT_TYPES(s):
+ return {"required": { "audio_encoder": ("AUDIO_ENCODER",),
+ "audio": ("AUDIO",),
+ }}
+ RETURN_TYPES = ("AUDIO_ENCODER_OUTPUT",)
+ FUNCTION = "encode"
+
+ CATEGORY = "conditioning"
+
+ def encode(self, audio_encoder, audio):
+ output = audio_encoder.encode_audio(audio["waveform"], audio["sample_rate"])
+ return (output,)
+
+
+NODE_CLASS_MAPPINGS = {
+ "AudioEncoderLoader": AudioEncoderLoader,
+ "AudioEncoderEncode": AudioEncoderEncode,
+}
diff --git a/comfy_extras/nodes_context_windows.py b/comfy_extras/nodes_context_windows.py
new file mode 100644
index 000000000..1c3d9e697
--- /dev/null
+++ b/comfy_extras/nodes_context_windows.py
@@ -0,0 +1,89 @@
+from __future__ import annotations
+from comfy_api.latest import ComfyExtension, io
+import comfy.context_windows
+import nodes
+
+
+class ContextWindowsManualNode(io.ComfyNode):
+ @classmethod
+ def define_schema(cls) -> io.Schema:
+ return io.Schema(
+ node_id="ContextWindowsManual",
+ display_name="Context Windows (Manual)",
+ category="context",
+ description="Manually set context windows.",
+ inputs=[
+ io.Model.Input("model", tooltip="The model to apply context windows to during sampling."),
+ io.Int.Input("context_length", min=1, default=16, tooltip="The length of the context window."),
+ io.Int.Input("context_overlap", min=0, default=4, tooltip="The overlap of the context window."),
+ io.Combo.Input("context_schedule", options=[
+ comfy.context_windows.ContextSchedules.STATIC_STANDARD,
+ comfy.context_windows.ContextSchedules.UNIFORM_STANDARD,
+ comfy.context_windows.ContextSchedules.UNIFORM_LOOPED,
+ comfy.context_windows.ContextSchedules.BATCHED,
+ ], tooltip="The stride of the context window."),
+ io.Int.Input("context_stride", min=1, default=1, tooltip="The stride of the context window; only applicable to uniform schedules."),
+ io.Boolean.Input("closed_loop", default=False, tooltip="Whether to close the context window loop; only applicable to looped schedules."),
+ io.Combo.Input("fuse_method", options=comfy.context_windows.ContextFuseMethods.LIST_STATIC, default=comfy.context_windows.ContextFuseMethods.PYRAMID, tooltip="The method to use to fuse the context windows."),
+ io.Int.Input("dim", min=0, max=5, default=0, tooltip="The dimension to apply the context windows to."),
+ ],
+ outputs=[
+ io.Model.Output(tooltip="The model with context windows applied during sampling."),
+ ],
+ is_experimental=True,
+ )
+
+ @classmethod
+ def execute(cls, model: io.Model.Type, context_length: int, context_overlap: int, context_schedule: str, context_stride: int, closed_loop: bool, fuse_method: str, dim: int) -> io.Model:
+ model = model.clone()
+ model.model_options["context_handler"] = comfy.context_windows.IndexListContextHandler(
+ context_schedule=comfy.context_windows.get_matching_context_schedule(context_schedule),
+ fuse_method=comfy.context_windows.get_matching_fuse_method(fuse_method),
+ context_length=context_length,
+ context_overlap=context_overlap,
+ context_stride=context_stride,
+ closed_loop=closed_loop,
+ dim=dim)
+ # make memory usage calculation only take into account the context window latents
+ comfy.context_windows.create_prepare_sampling_wrapper(model)
+ return io.NodeOutput(model)
+
+class WanContextWindowsManualNode(ContextWindowsManualNode):
+ @classmethod
+ def define_schema(cls) -> io.Schema:
+ schema = super().define_schema()
+ schema.node_id = "WanContextWindowsManual"
+ schema.display_name = "WAN Context Windows (Manual)"
+ schema.description = "Manually set context windows for WAN-like models (dim=2)."
+ schema.inputs = [
+ io.Model.Input("model", tooltip="The model to apply context windows to during sampling."),
+ io.Int.Input("context_length", min=1, max=nodes.MAX_RESOLUTION, step=4, default=81, tooltip="The length of the context window."),
+ io.Int.Input("context_overlap", min=0, default=30, tooltip="The overlap of the context window."),
+ io.Combo.Input("context_schedule", options=[
+ comfy.context_windows.ContextSchedules.STATIC_STANDARD,
+ comfy.context_windows.ContextSchedules.UNIFORM_STANDARD,
+ comfy.context_windows.ContextSchedules.UNIFORM_LOOPED,
+ comfy.context_windows.ContextSchedules.BATCHED,
+ ], tooltip="The stride of the context window."),
+ io.Int.Input("context_stride", min=1, default=1, tooltip="The stride of the context window; only applicable to uniform schedules."),
+ io.Boolean.Input("closed_loop", default=False, tooltip="Whether to close the context window loop; only applicable to looped schedules."),
+ io.Combo.Input("fuse_method", options=comfy.context_windows.ContextFuseMethods.LIST_STATIC, default=comfy.context_windows.ContextFuseMethods.PYRAMID, tooltip="The method to use to fuse the context windows."),
+ ]
+ return schema
+
+ @classmethod
+ def execute(cls, model: io.Model.Type, context_length: int, context_overlap: int, context_schedule: str, context_stride: int, closed_loop: bool, fuse_method: str) -> io.Model:
+ context_length = max(((context_length - 1) // 4) + 1, 1) # at least length 1
+ context_overlap = max(((context_overlap - 1) // 4) + 1, 0) # at least overlap 0
+ return super().execute(model, context_length, context_overlap, context_schedule, context_stride, closed_loop, fuse_method, dim=2)
+
+
+class ContextWindowsExtension(ComfyExtension):
+ async def get_node_list(self) -> list[type[io.ComfyNode]]:
+ return [
+ ContextWindowsManualNode,
+ WanContextWindowsManualNode,
+ ]
+
+def comfy_entrypoint():
+ return ContextWindowsExtension()
diff --git a/comfy_extras/nodes_easycache.py b/comfy_extras/nodes_easycache.py
new file mode 100644
index 000000000..9d2988f5f
--- /dev/null
+++ b/comfy_extras/nodes_easycache.py
@@ -0,0 +1,493 @@
+from __future__ import annotations
+from typing import TYPE_CHECKING, Union
+from comfy_api.latest import io, ComfyExtension
+import comfy.patcher_extension
+import logging
+import torch
+import comfy.model_patcher
+if TYPE_CHECKING:
+ from uuid import UUID
+
+
+def easycache_forward_wrapper(executor, *args, **kwargs):
+ # get values from args
+ x: torch.Tensor = args[0]
+ transformer_options: dict[str] = args[-1]
+ if not isinstance(transformer_options, dict):
+ transformer_options = kwargs.get("transformer_options")
+ if not transformer_options:
+ transformer_options = args[-2]
+ easycache: EasyCacheHolder = transformer_options["easycache"]
+ sigmas = transformer_options["sigmas"]
+ uuids = transformer_options["uuids"]
+ if sigmas is not None and easycache.is_past_end_timestep(sigmas):
+ return executor(*args, **kwargs)
+ # prepare next x_prev
+ has_first_cond_uuid = easycache.has_first_cond_uuid(uuids)
+ next_x_prev = x
+ input_change = None
+ do_easycache = easycache.should_do_easycache(sigmas)
+ if do_easycache:
+ easycache.check_metadata(x)
+ # if first cond marked this step for skipping, skip it and use appropriate cached values
+ if easycache.skip_current_step:
+ if easycache.verbose:
+ logging.info(f"EasyCache [verbose] - was marked to skip this step by {easycache.first_cond_uuid}. Present uuids: {uuids}")
+ return easycache.apply_cache_diff(x, uuids)
+ if easycache.initial_step:
+ easycache.first_cond_uuid = uuids[0]
+ has_first_cond_uuid = easycache.has_first_cond_uuid(uuids)
+ easycache.initial_step = False
+ if has_first_cond_uuid:
+ if easycache.has_x_prev_subsampled():
+ input_change = (easycache.subsample(x, uuids, clone=False) - easycache.x_prev_subsampled).flatten().abs().mean()
+ if easycache.has_output_prev_norm() and easycache.has_relative_transformation_rate():
+ approx_output_change_rate = (easycache.relative_transformation_rate * input_change) / easycache.output_prev_norm
+ easycache.cumulative_change_rate += approx_output_change_rate
+ if easycache.cumulative_change_rate < easycache.reuse_threshold:
+ if easycache.verbose:
+ logging.info(f"EasyCache [verbose] - skipping step; cumulative_change_rate: {easycache.cumulative_change_rate}, reuse_threshold: {easycache.reuse_threshold}")
+ # other conds should also skip this step, and instead use their cached values
+ easycache.skip_current_step = True
+ return easycache.apply_cache_diff(x, uuids)
+ else:
+ if easycache.verbose:
+ logging.info(f"EasyCache [verbose] - NOT skipping step; cumulative_change_rate: {easycache.cumulative_change_rate}, reuse_threshold: {easycache.reuse_threshold}")
+ easycache.cumulative_change_rate = 0.0
+
+ output: torch.Tensor = executor(*args, **kwargs)
+ if has_first_cond_uuid and easycache.has_output_prev_norm():
+ output_change = (easycache.subsample(output, uuids, clone=False) - easycache.output_prev_subsampled).flatten().abs().mean()
+ if easycache.verbose:
+ output_change_rate = output_change / easycache.output_prev_norm
+ easycache.output_change_rates.append(output_change_rate.item())
+ if easycache.has_relative_transformation_rate():
+ approx_output_change_rate = (easycache.relative_transformation_rate * input_change) / easycache.output_prev_norm
+ easycache.approx_output_change_rates.append(approx_output_change_rate.item())
+ if easycache.verbose:
+ logging.info(f"EasyCache [verbose] - approx_output_change_rate: {approx_output_change_rate}")
+ if input_change is not None:
+ easycache.relative_transformation_rate = output_change / input_change
+ if easycache.verbose:
+ logging.info(f"EasyCache [verbose] - output_change_rate: {output_change_rate}")
+ # TODO: allow cache_diff to be offloaded
+ easycache.update_cache_diff(output, next_x_prev, uuids)
+ if has_first_cond_uuid:
+ easycache.x_prev_subsampled = easycache.subsample(next_x_prev, uuids)
+ easycache.output_prev_subsampled = easycache.subsample(output, uuids)
+ easycache.output_prev_norm = output.flatten().abs().mean()
+ if easycache.verbose:
+ logging.info(f"EasyCache [verbose] - x_prev_subsampled: {easycache.x_prev_subsampled.shape}")
+ return output
+
+def lazycache_predict_noise_wrapper(executor, *args, **kwargs):
+ # get values from args
+ x: torch.Tensor = args[0]
+ timestep: float = args[1]
+ model_options: dict[str] = args[2]
+ easycache: LazyCacheHolder = model_options["transformer_options"]["easycache"]
+ if easycache.is_past_end_timestep(timestep):
+ return executor(*args, **kwargs)
+ # prepare next x_prev
+ next_x_prev = x
+ input_change = None
+ do_easycache = easycache.should_do_easycache(timestep)
+ if do_easycache:
+ easycache.check_metadata(x)
+ if easycache.has_x_prev_subsampled():
+ if easycache.has_x_prev_subsampled():
+ input_change = (easycache.subsample(x, clone=False) - easycache.x_prev_subsampled).flatten().abs().mean()
+ if easycache.has_output_prev_norm() and easycache.has_relative_transformation_rate():
+ approx_output_change_rate = (easycache.relative_transformation_rate * input_change) / easycache.output_prev_norm
+ easycache.cumulative_change_rate += approx_output_change_rate
+ if easycache.cumulative_change_rate < easycache.reuse_threshold:
+ if easycache.verbose:
+ logging.info(f"LazyCache [verbose] - skipping step; cumulative_change_rate: {easycache.cumulative_change_rate}, reuse_threshold: {easycache.reuse_threshold}")
+ # other conds should also skip this step, and instead use their cached values
+ easycache.skip_current_step = True
+ return easycache.apply_cache_diff(x)
+ else:
+ if easycache.verbose:
+ logging.info(f"LazyCache [verbose] - NOT skipping step; cumulative_change_rate: {easycache.cumulative_change_rate}, reuse_threshold: {easycache.reuse_threshold}")
+ easycache.cumulative_change_rate = 0.0
+ output: torch.Tensor = executor(*args, **kwargs)
+ if easycache.has_output_prev_norm():
+ output_change = (easycache.subsample(output, clone=False) - easycache.output_prev_subsampled).flatten().abs().mean()
+ if easycache.verbose:
+ output_change_rate = output_change / easycache.output_prev_norm
+ easycache.output_change_rates.append(output_change_rate.item())
+ if easycache.has_relative_transformation_rate():
+ approx_output_change_rate = (easycache.relative_transformation_rate * input_change) / easycache.output_prev_norm
+ easycache.approx_output_change_rates.append(approx_output_change_rate.item())
+ if easycache.verbose:
+ logging.info(f"LazyCache [verbose] - approx_output_change_rate: {approx_output_change_rate}")
+ if input_change is not None:
+ easycache.relative_transformation_rate = output_change / input_change
+ if easycache.verbose:
+ logging.info(f"LazyCache [verbose] - output_change_rate: {output_change_rate}")
+ # TODO: allow cache_diff to be offloaded
+ easycache.update_cache_diff(output, next_x_prev)
+ easycache.x_prev_subsampled = easycache.subsample(next_x_prev)
+ easycache.output_prev_subsampled = easycache.subsample(output)
+ easycache.output_prev_norm = output.flatten().abs().mean()
+ if easycache.verbose:
+ logging.info(f"LazyCache [verbose] - x_prev_subsampled: {easycache.x_prev_subsampled.shape}")
+ return output
+
+def easycache_calc_cond_batch_wrapper(executor, *args, **kwargs):
+ model_options = args[-1]
+ easycache: EasyCacheHolder = model_options["transformer_options"]["easycache"]
+ easycache.skip_current_step = False
+ # TODO: check if first_cond_uuid is active at this timestep; otherwise, EasyCache needs to be partially reset
+ return executor(*args, **kwargs)
+
+def easycache_sample_wrapper(executor, *args, **kwargs):
+ """
+ This OUTER_SAMPLE wrapper makes sure easycache is prepped for current run, and all memory usage is cleared at the end.
+ """
+ try:
+ guider = executor.class_obj
+ orig_model_options = guider.model_options
+ guider.model_options = comfy.model_patcher.create_model_options_clone(orig_model_options)
+ # clone and prepare timesteps
+ guider.model_options["transformer_options"]["easycache"] = guider.model_options["transformer_options"]["easycache"].clone().prepare_timesteps(guider.model_patcher.model.model_sampling)
+ easycache: Union[EasyCacheHolder, LazyCacheHolder] = guider.model_options['transformer_options']['easycache']
+ logging.info(f"{easycache.name} enabled - threshold: {easycache.reuse_threshold}, start_percent: {easycache.start_percent}, end_percent: {easycache.end_percent}")
+ return executor(*args, **kwargs)
+ finally:
+ easycache = guider.model_options['transformer_options']['easycache']
+ output_change_rates = easycache.output_change_rates
+ approx_output_change_rates = easycache.approx_output_change_rates
+ if easycache.verbose:
+ logging.info(f"{easycache.name} [verbose] - output_change_rates {len(output_change_rates)}: {output_change_rates}")
+ logging.info(f"{easycache.name} [verbose] - approx_output_change_rates {len(approx_output_change_rates)}: {approx_output_change_rates}")
+ total_steps = len(args[3])-1
+ logging.info(f"{easycache.name} - skipped {easycache.total_steps_skipped}/{total_steps} steps ({total_steps/(total_steps-easycache.total_steps_skipped):.2f}x speedup).")
+ easycache.reset()
+ guider.model_options = orig_model_options
+
+
+class EasyCacheHolder:
+ def __init__(self, reuse_threshold: float, start_percent: float, end_percent: float, subsample_factor: int, offload_cache_diff: bool, verbose: bool=False):
+ self.name = "EasyCache"
+ self.reuse_threshold = reuse_threshold
+ self.start_percent = start_percent
+ self.end_percent = end_percent
+ self.subsample_factor = subsample_factor
+ self.offload_cache_diff = offload_cache_diff
+ self.verbose = verbose
+ # timestep values
+ self.start_t = 0.0
+ self.end_t = 0.0
+ # control values
+ self.relative_transformation_rate: float = None
+ self.cumulative_change_rate = 0.0
+ self.initial_step = True
+ self.skip_current_step = False
+ # cache values
+ self.first_cond_uuid = None
+ self.x_prev_subsampled: torch.Tensor = None
+ self.output_prev_subsampled: torch.Tensor = None
+ self.output_prev_norm: torch.Tensor = None
+ self.uuid_cache_diffs: dict[UUID, torch.Tensor] = {}
+ self.output_change_rates = []
+ self.approx_output_change_rates = []
+ self.total_steps_skipped = 0
+ # how to deal with mismatched dims
+ self.allow_mismatch = True
+ self.cut_from_start = True
+ self.state_metadata = None
+
+ def is_past_end_timestep(self, timestep: float) -> bool:
+ return not (timestep[0] > self.end_t).item()
+
+ def should_do_easycache(self, timestep: float) -> bool:
+ return (timestep[0] <= self.start_t).item()
+
+ def has_x_prev_subsampled(self) -> bool:
+ return self.x_prev_subsampled is not None
+
+ def has_output_prev_subsampled(self) -> bool:
+ return self.output_prev_subsampled is not None
+
+ def has_output_prev_norm(self) -> bool:
+ return self.output_prev_norm is not None
+
+ def has_relative_transformation_rate(self) -> bool:
+ return self.relative_transformation_rate is not None
+
+ def prepare_timesteps(self, model_sampling):
+ self.start_t = model_sampling.percent_to_sigma(self.start_percent)
+ self.end_t = model_sampling.percent_to_sigma(self.end_percent)
+ return self
+
+ def subsample(self, x: torch.Tensor, uuids: list[UUID], clone: bool = True) -> torch.Tensor:
+ batch_offset = x.shape[0] // len(uuids)
+ uuid_idx = uuids.index(self.first_cond_uuid)
+ if self.subsample_factor > 1:
+ to_return = x[uuid_idx*batch_offset:(uuid_idx+1)*batch_offset, ..., ::self.subsample_factor, ::self.subsample_factor]
+ if clone:
+ return to_return.clone()
+ return to_return
+ to_return = x[uuid_idx*batch_offset:(uuid_idx+1)*batch_offset, ...]
+ if clone:
+ return to_return.clone()
+ return to_return
+
+ def apply_cache_diff(self, x: torch.Tensor, uuids: list[UUID]):
+ if self.first_cond_uuid in uuids:
+ self.total_steps_skipped += 1
+ batch_offset = x.shape[0] // len(uuids)
+ for i, uuid in enumerate(uuids):
+ # if cached dims don't match x dims, cut off excess and hope for the best (cosmos world2video)
+ if x.shape[1:] != self.uuid_cache_diffs[uuid].shape[1:]:
+ if not self.allow_mismatch:
+ raise ValueError(f"Cached dims {self.uuid_cache_diffs[uuid].shape} don't match x dims {x.shape} - this is no good")
+ slicing = []
+ skip_this_dim = True
+ for dim_u, dim_x in zip(self.uuid_cache_diffs[uuid].shape, x.shape):
+ if skip_this_dim:
+ skip_this_dim = False
+ continue
+ if dim_u != dim_x:
+ if self.cut_from_start:
+ slicing.append(slice(dim_x-dim_u, None))
+ else:
+ slicing.append(slice(None, dim_u))
+ else:
+ slicing.append(slice(None))
+ slicing = [slice(i*batch_offset,(i+1)*batch_offset)] + slicing
+ x = x[slicing]
+ x += self.uuid_cache_diffs[uuid].to(x.device)
+ return x
+
+ def update_cache_diff(self, output: torch.Tensor, x: torch.Tensor, uuids: list[UUID]):
+ # if output dims don't match x dims, cut off excess and hope for the best (cosmos world2video)
+ if output.shape[1:] != x.shape[1:]:
+ if not self.allow_mismatch:
+ raise ValueError(f"Output dims {output.shape} don't match x dims {x.shape} - this is no good")
+ slicing = []
+ skip_dim = True
+ for dim_o, dim_x in zip(output.shape, x.shape):
+ if not skip_dim and dim_o != dim_x:
+ if self.cut_from_start:
+ slicing.append(slice(dim_x-dim_o, None))
+ else:
+ slicing.append(slice(None, dim_o))
+ else:
+ slicing.append(slice(None))
+ skip_dim = False
+ x = x[slicing]
+ diff = output - x
+ batch_offset = diff.shape[0] // len(uuids)
+ for i, uuid in enumerate(uuids):
+ self.uuid_cache_diffs[uuid] = diff[i*batch_offset:(i+1)*batch_offset, ...]
+
+ def has_first_cond_uuid(self, uuids: list[UUID]) -> bool:
+ return self.first_cond_uuid in uuids
+
+ def check_metadata(self, x: torch.Tensor) -> bool:
+ metadata = (x.device, x.dtype, x.shape[1:])
+ if self.state_metadata is None:
+ self.state_metadata = metadata
+ return True
+ if metadata == self.state_metadata:
+ return True
+ logging.warn(f"{self.name} - Tensor shape, dtype or device changed, resetting state")
+ self.reset()
+ return False
+
+ def reset(self):
+ self.relative_transformation_rate = 0.0
+ self.cumulative_change_rate = 0.0
+ self.initial_step = True
+ self.skip_current_step = False
+ self.output_change_rates = []
+ self.first_cond_uuid = None
+ del self.x_prev_subsampled
+ self.x_prev_subsampled = None
+ del self.output_prev_subsampled
+ self.output_prev_subsampled = None
+ del self.output_prev_norm
+ self.output_prev_norm = None
+ del self.uuid_cache_diffs
+ self.uuid_cache_diffs = {}
+ self.total_steps_skipped = 0
+ self.state_metadata = None
+ return self
+
+ def clone(self):
+ return EasyCacheHolder(self.reuse_threshold, self.start_percent, self.end_percent, self.subsample_factor, self.offload_cache_diff, self.verbose)
+
+
+class EasyCacheNode(io.ComfyNode):
+ @classmethod
+ def define_schema(cls) -> io.Schema:
+ return io.Schema(
+ node_id="EasyCache",
+ display_name="EasyCache",
+ description="Native EasyCache implementation.",
+ category="advanced/debug/model",
+ is_experimental=True,
+ inputs=[
+ io.Model.Input("model", tooltip="The model to add EasyCache to."),
+ io.Float.Input("reuse_threshold", min=0.0, default=0.2, max=3.0, step=0.01, tooltip="The threshold for reusing cached steps."),
+ io.Float.Input("start_percent", min=0.0, default=0.15, max=1.0, step=0.01, tooltip="The relative sampling step to begin use of EasyCache."),
+ io.Float.Input("end_percent", min=0.0, default=0.95, max=1.0, step=0.01, tooltip="The relative sampling step to end use of EasyCache."),
+ io.Boolean.Input("verbose", default=False, tooltip="Whether to log verbose information."),
+ ],
+ outputs=[
+ io.Model.Output(tooltip="The model with EasyCache."),
+ ],
+ )
+
+ @classmethod
+ def execute(cls, model: io.Model.Type, reuse_threshold: float, start_percent: float, end_percent: float, verbose: bool) -> io.NodeOutput:
+ model = model.clone()
+ model.model_options["transformer_options"]["easycache"] = EasyCacheHolder(reuse_threshold, start_percent, end_percent, subsample_factor=8, offload_cache_diff=False, verbose=verbose)
+ model.add_wrapper_with_key(comfy.patcher_extension.WrappersMP.OUTER_SAMPLE, "easycache", easycache_sample_wrapper)
+ model.add_wrapper_with_key(comfy.patcher_extension.WrappersMP.CALC_COND_BATCH, "easycache", easycache_calc_cond_batch_wrapper)
+ model.add_wrapper_with_key(comfy.patcher_extension.WrappersMP.DIFFUSION_MODEL, "easycache", easycache_forward_wrapper)
+ return io.NodeOutput(model)
+
+
+class LazyCacheHolder:
+ def __init__(self, reuse_threshold: float, start_percent: float, end_percent: float, subsample_factor: int, offload_cache_diff: bool, verbose: bool=False):
+ self.name = "LazyCache"
+ self.reuse_threshold = reuse_threshold
+ self.start_percent = start_percent
+ self.end_percent = end_percent
+ self.subsample_factor = subsample_factor
+ self.offload_cache_diff = offload_cache_diff
+ self.verbose = verbose
+ # timestep values
+ self.start_t = 0.0
+ self.end_t = 0.0
+ # control values
+ self.relative_transformation_rate: float = None
+ self.cumulative_change_rate = 0.0
+ self.initial_step = True
+ # cache values
+ self.x_prev_subsampled: torch.Tensor = None
+ self.output_prev_subsampled: torch.Tensor = None
+ self.output_prev_norm: torch.Tensor = None
+ self.cache_diff: torch.Tensor = None
+ self.output_change_rates = []
+ self.approx_output_change_rates = []
+ self.total_steps_skipped = 0
+ self.state_metadata = None
+
+ def has_cache_diff(self) -> bool:
+ return self.cache_diff is not None
+
+ def is_past_end_timestep(self, timestep: float) -> bool:
+ return not (timestep[0] > self.end_t).item()
+
+ def should_do_easycache(self, timestep: float) -> bool:
+ return (timestep[0] <= self.start_t).item()
+
+ def has_x_prev_subsampled(self) -> bool:
+ return self.x_prev_subsampled is not None
+
+ def has_output_prev_subsampled(self) -> bool:
+ return self.output_prev_subsampled is not None
+
+ def has_output_prev_norm(self) -> bool:
+ return self.output_prev_norm is not None
+
+ def has_relative_transformation_rate(self) -> bool:
+ return self.relative_transformation_rate is not None
+
+ def prepare_timesteps(self, model_sampling):
+ self.start_t = model_sampling.percent_to_sigma(self.start_percent)
+ self.end_t = model_sampling.percent_to_sigma(self.end_percent)
+ return self
+
+ def subsample(self, x: torch.Tensor, clone: bool = True) -> torch.Tensor:
+ if self.subsample_factor > 1:
+ to_return = x[..., ::self.subsample_factor, ::self.subsample_factor]
+ if clone:
+ return to_return.clone()
+ return to_return
+ if clone:
+ return x.clone()
+ return x
+
+ def apply_cache_diff(self, x: torch.Tensor):
+ self.total_steps_skipped += 1
+ return x + self.cache_diff.to(x.device)
+
+ def update_cache_diff(self, output: torch.Tensor, x: torch.Tensor):
+ self.cache_diff = output - x
+
+ def check_metadata(self, x: torch.Tensor) -> bool:
+ metadata = (x.device, x.dtype, x.shape)
+ if self.state_metadata is None:
+ self.state_metadata = metadata
+ return True
+ if metadata == self.state_metadata:
+ return True
+ logging.warn(f"{self.name} - Tensor shape, dtype or device changed, resetting state")
+ self.reset()
+ return False
+
+ def reset(self):
+ self.relative_transformation_rate = 0.0
+ self.cumulative_change_rate = 0.0
+ self.initial_step = True
+ self.output_change_rates = []
+ self.approx_output_change_rates = []
+ del self.cache_diff
+ self.cache_diff = None
+ del self.x_prev_subsampled
+ self.x_prev_subsampled = None
+ del self.output_prev_subsampled
+ self.output_prev_subsampled = None
+ del self.output_prev_norm
+ self.output_prev_norm = None
+ self.total_steps_skipped = 0
+ self.state_metadata = None
+ return self
+
+ def clone(self):
+ return LazyCacheHolder(self.reuse_threshold, self.start_percent, self.end_percent, self.subsample_factor, self.offload_cache_diff, self.verbose)
+
+class LazyCacheNode(io.ComfyNode):
+ @classmethod
+ def define_schema(cls) -> io.Schema:
+ return io.Schema(
+ node_id="LazyCache",
+ display_name="LazyCache",
+ description="A homebrew version of EasyCache - even 'easier' version of EasyCache to implement. Overall works worse than EasyCache, but better in some rare cases AND universal compatibility with everything in ComfyUI.",
+ category="advanced/debug/model",
+ is_experimental=True,
+ inputs=[
+ io.Model.Input("model", tooltip="The model to add LazyCache to."),
+ io.Float.Input("reuse_threshold", min=0.0, default=0.2, max=3.0, step=0.01, tooltip="The threshold for reusing cached steps."),
+ io.Float.Input("start_percent", min=0.0, default=0.15, max=1.0, step=0.01, tooltip="The relative sampling step to begin use of LazyCache."),
+ io.Float.Input("end_percent", min=0.0, default=0.95, max=1.0, step=0.01, tooltip="The relative sampling step to end use of LazyCache."),
+ io.Boolean.Input("verbose", default=False, tooltip="Whether to log verbose information."),
+ ],
+ outputs=[
+ io.Model.Output(tooltip="The model with LazyCache."),
+ ],
+ )
+
+ @classmethod
+ def execute(cls, model: io.Model.Type, reuse_threshold: float, start_percent: float, end_percent: float, verbose: bool) -> io.NodeOutput:
+ model = model.clone()
+ model.model_options["transformer_options"]["easycache"] = LazyCacheHolder(reuse_threshold, start_percent, end_percent, subsample_factor=8, offload_cache_diff=False, verbose=verbose)
+ model.add_wrapper_with_key(comfy.patcher_extension.WrappersMP.OUTER_SAMPLE, "lazycache", easycache_sample_wrapper)
+ model.add_wrapper_with_key(comfy.patcher_extension.WrappersMP.PREDICT_NOISE, "lazycache", lazycache_predict_noise_wrapper)
+ return io.NodeOutput(model)
+
+
+class EasyCacheExtension(ComfyExtension):
+ async def get_node_list(self) -> list[type[io.ComfyNode]]:
+ return [
+ EasyCacheNode,
+ LazyCacheNode,
+ ]
+
+def comfy_entrypoint():
+ return EasyCacheExtension()
diff --git a/comfy_extras/nodes_flux.py b/comfy_extras/nodes_flux.py
index 8a8a17698..c8db75bb3 100644
--- a/comfy_extras/nodes_flux.py
+++ b/comfy_extras/nodes_flux.py
@@ -100,9 +100,28 @@ class FluxKontextImageScale:
return (image, )
+class FluxKontextMultiReferenceLatentMethod:
+ @classmethod
+ def INPUT_TYPES(s):
+ return {"required": {
+ "conditioning": ("CONDITIONING", ),
+ "reference_latents_method": (("offset", "index"), ),
+ }}
+
+ RETURN_TYPES = ("CONDITIONING",)
+ FUNCTION = "append"
+ EXPERIMENTAL = True
+
+ CATEGORY = "advanced/conditioning/flux"
+
+ def append(self, conditioning, reference_latents_method):
+ c = node_helpers.conditioning_set_values(conditioning, {"reference_latents_method": reference_latents_method})
+ return (c, )
+
NODE_CLASS_MAPPINGS = {
"CLIPTextEncodeFlux": CLIPTextEncodeFlux,
"FluxGuidance": FluxGuidance,
"FluxDisableGuidance": FluxDisableGuidance,
"FluxKontextImageScale": FluxKontextImageScale,
+ "FluxKontextMultiReferenceLatentMethod": FluxKontextMultiReferenceLatentMethod,
}
diff --git a/comfy_extras/nodes_latent.py b/comfy_extras/nodes_latent.py
index f33ed1bee..0f90cf60c 100644
--- a/comfy_extras/nodes_latent.py
+++ b/comfy_extras/nodes_latent.py
@@ -1,6 +1,7 @@
import comfy.utils
import comfy_extras.nodes_post_processing
import torch
+import nodes
def reshape_latent_to(target_shape, latent, repeat_batch=True):
@@ -105,6 +106,73 @@ class LatentInterpolate:
samples_out["samples"] = st * (m1 * ratio + m2 * (1.0 - ratio))
return (samples_out,)
+class LatentConcat:
+ @classmethod
+ def INPUT_TYPES(s):
+ return {"required": { "samples1": ("LATENT",), "samples2": ("LATENT",), "dim": (["x", "-x", "y", "-y", "t", "-t"], )}}
+
+ RETURN_TYPES = ("LATENT",)
+ FUNCTION = "op"
+
+ CATEGORY = "latent/advanced"
+
+ def op(self, samples1, samples2, dim):
+ samples_out = samples1.copy()
+
+ s1 = samples1["samples"]
+ s2 = samples2["samples"]
+ s2 = comfy.utils.repeat_to_batch_size(s2, s1.shape[0])
+
+ if "-" in dim:
+ c = (s2, s1)
+ else:
+ c = (s1, s2)
+
+ if "x" in dim:
+ dim = -1
+ elif "y" in dim:
+ dim = -2
+ elif "t" in dim:
+ dim = -3
+
+ samples_out["samples"] = torch.cat(c, dim=dim)
+ return (samples_out,)
+
+class LatentCut:
+ @classmethod
+ def INPUT_TYPES(s):
+ return {"required": {"samples": ("LATENT",),
+ "dim": (["x", "y", "t"], ),
+ "index": ("INT", {"default": 0, "min": -nodes.MAX_RESOLUTION, "max": nodes.MAX_RESOLUTION, "step": 1}),
+ "amount": ("INT", {"default": 1, "min": 1, "max": nodes.MAX_RESOLUTION, "step": 1})}}
+
+ RETURN_TYPES = ("LATENT",)
+ FUNCTION = "op"
+
+ CATEGORY = "latent/advanced"
+
+ def op(self, samples, dim, index, amount):
+ samples_out = samples.copy()
+
+ s1 = samples["samples"]
+
+ if "x" in dim:
+ dim = s1.ndim - 1
+ elif "y" in dim:
+ dim = s1.ndim - 2
+ elif "t" in dim:
+ dim = s1.ndim - 3
+
+ if index >= 0:
+ index = min(index, s1.shape[dim] - 1)
+ amount = min(s1.shape[dim] - index, amount)
+ else:
+ index = max(index, -s1.shape[dim])
+ amount = min(-index, amount)
+
+ samples_out["samples"] = torch.narrow(s1, dim, index, amount)
+ return (samples_out,)
+
class LatentBatch:
@classmethod
def INPUT_TYPES(s):
@@ -279,6 +347,8 @@ NODE_CLASS_MAPPINGS = {
"LatentSubtract": LatentSubtract,
"LatentMultiply": LatentMultiply,
"LatentInterpolate": LatentInterpolate,
+ "LatentConcat": LatentConcat,
+ "LatentCut": LatentCut,
"LatentBatch": LatentBatch,
"LatentBatchSeedBehavior": LatentBatchSeedBehavior,
"LatentApplyOperation": LatentApplyOperation,
diff --git a/comfy_extras/nodes_lt.py b/comfy_extras/nodes_lt.py
index b5058667a..f82337a67 100644
--- a/comfy_extras/nodes_lt.py
+++ b/comfy_extras/nodes_lt.py
@@ -166,7 +166,7 @@ class LTXVAddGuide:
negative = self.add_keyframe_index(negative, frame_idx, guiding_latent, scale_factors)
mask = torch.full(
- (noise_mask.shape[0], 1, guiding_latent.shape[2], 1, 1),
+ (noise_mask.shape[0], 1, guiding_latent.shape[2], noise_mask.shape[3], noise_mask.shape[4]),
1.0 - strength,
dtype=noise_mask.dtype,
device=noise_mask.device,
diff --git a/comfy_extras/nodes_model_merging_model_specific.py b/comfy_extras/nodes_model_merging_model_specific.py
index 2c93cd84f..55eb3ccfe 100644
--- a/comfy_extras/nodes_model_merging_model_specific.py
+++ b/comfy_extras/nodes_model_merging_model_specific.py
@@ -314,6 +314,29 @@ class ModelMergeCosmosPredict2_14B(comfy_extras.nodes_model_merging.ModelMergeBl
return {"required": arg_dict}
+class ModelMergeQwenImage(comfy_extras.nodes_model_merging.ModelMergeBlocks):
+ CATEGORY = "advanced/model_merging/model_specific"
+
+ @classmethod
+ def INPUT_TYPES(s):
+ arg_dict = { "model1": ("MODEL",),
+ "model2": ("MODEL",)}
+
+ argument = ("FLOAT", {"default": 1.0, "min": 0.0, "max": 1.0, "step": 0.01})
+
+ arg_dict["pos_embeds."] = argument
+ arg_dict["img_in."] = argument
+ arg_dict["txt_norm."] = argument
+ arg_dict["txt_in."] = argument
+ arg_dict["time_text_embed."] = argument
+
+ for i in range(60):
+ arg_dict["transformer_blocks.{}.".format(i)] = argument
+
+ arg_dict["proj_out."] = argument
+
+ return {"required": arg_dict}
+
NODE_CLASS_MAPPINGS = {
"ModelMergeSD1": ModelMergeSD1,
"ModelMergeSD2": ModelMergeSD1, #SD1 and SD2 have the same blocks
@@ -329,4 +352,5 @@ NODE_CLASS_MAPPINGS = {
"ModelMergeWAN2_1": ModelMergeWAN2_1,
"ModelMergeCosmosPredict2_2B": ModelMergeCosmosPredict2_2B,
"ModelMergeCosmosPredict2_14B": ModelMergeCosmosPredict2_14B,
+ "ModelMergeQwenImage": ModelMergeQwenImage,
}
diff --git a/comfy_extras/nodes_model_patch.py b/comfy_extras/nodes_model_patch.py
new file mode 100644
index 000000000..65e766b52
--- /dev/null
+++ b/comfy_extras/nodes_model_patch.py
@@ -0,0 +1,163 @@
+import torch
+import folder_paths
+import comfy.utils
+import comfy.ops
+import comfy.model_management
+import comfy.ldm.common_dit
+import comfy.latent_formats
+
+
+class BlockWiseControlBlock(torch.nn.Module):
+ # [linear, gelu, linear]
+ def __init__(self, dim: int = 3072, device=None, dtype=None, operations=None):
+ super().__init__()
+ self.x_rms = operations.RMSNorm(dim, eps=1e-6)
+ self.y_rms = operations.RMSNorm(dim, eps=1e-6)
+ self.input_proj = operations.Linear(dim, dim)
+ self.act = torch.nn.GELU()
+ self.output_proj = operations.Linear(dim, dim)
+
+ def forward(self, x, y):
+ x, y = self.x_rms(x), self.y_rms(y)
+ x = self.input_proj(x + y)
+ x = self.act(x)
+ x = self.output_proj(x)
+ return x
+
+
+class QwenImageBlockWiseControlNet(torch.nn.Module):
+ def __init__(
+ self,
+ num_layers: int = 60,
+ in_dim: int = 64,
+ additional_in_dim: int = 0,
+ dim: int = 3072,
+ device=None, dtype=None, operations=None
+ ):
+ super().__init__()
+ self.additional_in_dim = additional_in_dim
+ self.img_in = operations.Linear(in_dim + additional_in_dim, dim, device=device, dtype=dtype)
+ self.controlnet_blocks = torch.nn.ModuleList(
+ [
+ BlockWiseControlBlock(dim, device=device, dtype=dtype, operations=operations)
+ for _ in range(num_layers)
+ ]
+ )
+
+ def process_input_latent_image(self, latent_image):
+ latent_image[:, :16] = comfy.latent_formats.Wan21().process_in(latent_image[:, :16])
+ patch_size = 2
+ hidden_states = comfy.ldm.common_dit.pad_to_patch_size(latent_image, (1, patch_size, patch_size))
+ orig_shape = hidden_states.shape
+ hidden_states = hidden_states.view(orig_shape[0], orig_shape[1], orig_shape[-2] // 2, 2, orig_shape[-1] // 2, 2)
+ hidden_states = hidden_states.permute(0, 2, 4, 1, 3, 5)
+ hidden_states = hidden_states.reshape(orig_shape[0], (orig_shape[-2] // 2) * (orig_shape[-1] // 2), orig_shape[1] * 4)
+ return self.img_in(hidden_states)
+
+ def control_block(self, img, controlnet_conditioning, block_id):
+ return self.controlnet_blocks[block_id](img, controlnet_conditioning)
+
+
+class ModelPatchLoader:
+ @classmethod
+ def INPUT_TYPES(s):
+ return {"required": { "name": (folder_paths.get_filename_list("model_patches"), ),
+ }}
+ RETURN_TYPES = ("MODEL_PATCH",)
+ FUNCTION = "load_model_patch"
+ EXPERIMENTAL = True
+
+ CATEGORY = "advanced/loaders"
+
+ def load_model_patch(self, name):
+ model_patch_path = folder_paths.get_full_path_or_raise("model_patches", name)
+ sd = comfy.utils.load_torch_file(model_patch_path, safe_load=True)
+ dtype = comfy.utils.weight_dtype(sd)
+ # TODO: this node will work with more types of model patches
+ additional_in_dim = sd["img_in.weight"].shape[1] - 64
+ model = QwenImageBlockWiseControlNet(additional_in_dim=additional_in_dim, device=comfy.model_management.unet_offload_device(), dtype=dtype, operations=comfy.ops.manual_cast)
+ model.load_state_dict(sd)
+ model = comfy.model_patcher.ModelPatcher(model, load_device=comfy.model_management.get_torch_device(), offload_device=comfy.model_management.unet_offload_device())
+ return (model,)
+
+
+class DiffSynthCnetPatch:
+ def __init__(self, model_patch, vae, image, strength, mask=None):
+ self.model_patch = model_patch
+ self.vae = vae
+ self.image = image
+ self.strength = strength
+ self.mask = mask
+ self.encoded_image = model_patch.model.process_input_latent_image(self.encode_latent_cond(image))
+ self.encoded_image_size = (image.shape[1], image.shape[2])
+
+ def encode_latent_cond(self, image):
+ latent_image = self.vae.encode(image)
+ if self.model_patch.model.additional_in_dim > 0:
+ if self.mask is None:
+ mask_ = torch.ones_like(latent_image)[:, :self.model_patch.model.additional_in_dim // 4]
+ else:
+ mask_ = comfy.utils.common_upscale(self.mask.mean(dim=1, keepdim=True), latent_image.shape[-1], latent_image.shape[-2], "bilinear", "none")
+
+ return torch.cat([latent_image, mask_], dim=1)
+ else:
+ return latent_image
+
+ def __call__(self, kwargs):
+ x = kwargs.get("x")
+ img = kwargs.get("img")
+ block_index = kwargs.get("block_index")
+ spacial_compression = self.vae.spacial_compression_encode()
+ if self.encoded_image is None or self.encoded_image_size != (x.shape[-2] * spacial_compression, x.shape[-1] * spacial_compression):
+ image_scaled = comfy.utils.common_upscale(self.image.movedim(-1, 1), x.shape[-1] * spacial_compression, x.shape[-2] * spacial_compression, "area", "center")
+ loaded_models = comfy.model_management.loaded_models(only_currently_used=True)
+ self.encoded_image = self.model_patch.model.process_input_latent_image(self.encode_latent_cond(image_scaled.movedim(1, -1)))
+ self.encoded_image_size = (image_scaled.shape[-2], image_scaled.shape[-1])
+ comfy.model_management.load_models_gpu(loaded_models)
+
+ img[:, :self.encoded_image.shape[1]] += (self.model_patch.model.control_block(img[:, :self.encoded_image.shape[1]], self.encoded_image.to(img.dtype), block_index) * self.strength)
+ kwargs['img'] = img
+ return kwargs
+
+ def to(self, device_or_dtype):
+ if isinstance(device_or_dtype, torch.device):
+ self.encoded_image = self.encoded_image.to(device_or_dtype)
+ return self
+
+ def models(self):
+ return [self.model_patch]
+
+class QwenImageDiffsynthControlnet:
+ @classmethod
+ def INPUT_TYPES(s):
+ return {"required": { "model": ("MODEL",),
+ "model_patch": ("MODEL_PATCH",),
+ "vae": ("VAE",),
+ "image": ("IMAGE",),
+ "strength": ("FLOAT", {"default": 1.0, "min": -10.0, "max": 10.0, "step": 0.01}),
+ },
+ "optional": {"mask": ("MASK",)}}
+ RETURN_TYPES = ("MODEL",)
+ FUNCTION = "diffsynth_controlnet"
+ EXPERIMENTAL = True
+
+ CATEGORY = "advanced/loaders/qwen"
+
+ def diffsynth_controlnet(self, model, model_patch, vae, image, strength, mask=None):
+ model_patched = model.clone()
+ image = image[:, :, :, :3]
+ if mask is not None:
+ if mask.ndim == 3:
+ mask = mask.unsqueeze(1)
+ if mask.ndim == 4:
+ mask = mask.unsqueeze(2)
+ mask = 1.0 - mask
+
+ model_patched.set_model_double_block_patch(DiffSynthCnetPatch(model_patch, vae, image, strength, mask))
+ return (model_patched,)
+
+
+NODE_CLASS_MAPPINGS = {
+ "ModelPatchLoader": ModelPatchLoader,
+ "QwenImageDiffsynthControlnet": QwenImageDiffsynthControlnet,
+}
diff --git a/comfy_extras/nodes_primitive.py b/comfy_extras/nodes_primitive.py
index 1f93f87a7..5a1aeba80 100644
--- a/comfy_extras/nodes_primitive.py
+++ b/comfy_extras/nodes_primitive.py
@@ -1,98 +1,109 @@
-# Primitive nodes that are evaluated at backend.
-from __future__ import annotations
-
import sys
+from typing_extensions import override
-from comfy.comfy_types.node_typing import ComfyNodeABC, InputTypeDict, IO
+from comfy_api.latest import ComfyExtension, io
-class String(ComfyNodeABC):
+class String(io.ComfyNode):
@classmethod
- def INPUT_TYPES(cls) -> InputTypeDict:
- return {
- "required": {"value": (IO.STRING, {})},
- }
+ def define_schema(cls):
+ return io.Schema(
+ node_id="PrimitiveString",
+ display_name="String",
+ category="utils/primitive",
+ inputs=[
+ io.String.Input("value"),
+ ],
+ outputs=[io.String.Output()],
+ )
- RETURN_TYPES = (IO.STRING,)
- FUNCTION = "execute"
- CATEGORY = "utils/primitive"
-
- def execute(self, value: str) -> tuple[str]:
- return (value,)
-
-
-class StringMultiline(ComfyNodeABC):
@classmethod
- def INPUT_TYPES(cls) -> InputTypeDict:
- return {
- "required": {"value": (IO.STRING, {"multiline": True,},)},
- }
-
- RETURN_TYPES = (IO.STRING,)
- FUNCTION = "execute"
- CATEGORY = "utils/primitive"
-
- def execute(self, value: str) -> tuple[str]:
- return (value,)
+ def execute(cls, value: str) -> io.NodeOutput:
+ return io.NodeOutput(value)
-class Int(ComfyNodeABC):
+class StringMultiline(io.ComfyNode):
@classmethod
- def INPUT_TYPES(cls) -> InputTypeDict:
- return {
- "required": {"value": (IO.INT, {"min": -sys.maxsize, "max": sys.maxsize, "control_after_generate": True})},
- }
+ def define_schema(cls):
+ return io.Schema(
+ node_id="PrimitiveStringMultiline",
+ display_name="String (Multiline)",
+ category="utils/primitive",
+ inputs=[
+ io.String.Input("value", multiline=True),
+ ],
+ outputs=[io.String.Output()],
+ )
- RETURN_TYPES = (IO.INT,)
- FUNCTION = "execute"
- CATEGORY = "utils/primitive"
-
- def execute(self, value: int) -> tuple[int]:
- return (value,)
-
-
-class Float(ComfyNodeABC):
@classmethod
- def INPUT_TYPES(cls) -> InputTypeDict:
- return {
- "required": {"value": (IO.FLOAT, {"min": -sys.maxsize, "max": sys.maxsize})},
- }
-
- RETURN_TYPES = (IO.FLOAT,)
- FUNCTION = "execute"
- CATEGORY = "utils/primitive"
-
- def execute(self, value: float) -> tuple[float]:
- return (value,)
+ def execute(cls, value: str) -> io.NodeOutput:
+ return io.NodeOutput(value)
-class Boolean(ComfyNodeABC):
+class Int(io.ComfyNode):
@classmethod
- def INPUT_TYPES(cls) -> InputTypeDict:
- return {
- "required": {"value": (IO.BOOLEAN, {})},
- }
+ def define_schema(cls):
+ return io.Schema(
+ node_id="PrimitiveInt",
+ display_name="Int",
+ category="utils/primitive",
+ inputs=[
+ io.Int.Input("value", min=-sys.maxsize, max=sys.maxsize, control_after_generate=True),
+ ],
+ outputs=[io.Int.Output()],
+ )
- RETURN_TYPES = (IO.BOOLEAN,)
- FUNCTION = "execute"
- CATEGORY = "utils/primitive"
-
- def execute(self, value: bool) -> tuple[bool]:
- return (value,)
+ @classmethod
+ def execute(cls, value: int) -> io.NodeOutput:
+ return io.NodeOutput(value)
-NODE_CLASS_MAPPINGS = {
- "PrimitiveString": String,
- "PrimitiveStringMultiline": StringMultiline,
- "PrimitiveInt": Int,
- "PrimitiveFloat": Float,
- "PrimitiveBoolean": Boolean,
-}
+class Float(io.ComfyNode):
+ @classmethod
+ def define_schema(cls):
+ return io.Schema(
+ node_id="PrimitiveFloat",
+ display_name="Float",
+ category="utils/primitive",
+ inputs=[
+ io.Float.Input("value", min=-sys.maxsize, max=sys.maxsize),
+ ],
+ outputs=[io.Float.Output()],
+ )
-NODE_DISPLAY_NAME_MAPPINGS = {
- "PrimitiveString": "String",
- "PrimitiveStringMultiline": "String (Multiline)",
- "PrimitiveInt": "Int",
- "PrimitiveFloat": "Float",
- "PrimitiveBoolean": "Boolean",
-}
+ @classmethod
+ def execute(cls, value: float) -> io.NodeOutput:
+ return io.NodeOutput(value)
+
+
+class Boolean(io.ComfyNode):
+ @classmethod
+ def define_schema(cls):
+ return io.Schema(
+ node_id="PrimitiveBoolean",
+ display_name="Boolean",
+ category="utils/primitive",
+ inputs=[
+ io.Boolean.Input("value"),
+ ],
+ outputs=[io.Boolean.Output()],
+ )
+
+ @classmethod
+ def execute(cls, value: bool) -> io.NodeOutput:
+ return io.NodeOutput(value)
+
+
+class PrimitivesExtension(ComfyExtension):
+ @override
+ async def get_node_list(self) -> list[type[io.ComfyNode]]:
+ return [
+ String,
+ StringMultiline,
+ Int,
+ Float,
+ Boolean,
+ ]
+
+async def comfy_entrypoint() -> PrimitivesExtension:
+ return PrimitivesExtension()
diff --git a/comfy_extras/nodes_qwen.py b/comfy_extras/nodes_qwen.py
new file mode 100644
index 000000000..fff89556f
--- /dev/null
+++ b/comfy_extras/nodes_qwen.py
@@ -0,0 +1,48 @@
+import node_helpers
+import comfy.utils
+import math
+
+
+class TextEncodeQwenImageEdit:
+ @classmethod
+ def INPUT_TYPES(s):
+ return {"required": {
+ "clip": ("CLIP", ),
+ "prompt": ("STRING", {"multiline": True, "dynamicPrompts": True}),
+ },
+ "optional": {"vae": ("VAE", ),
+ "image": ("IMAGE", ),}}
+
+ RETURN_TYPES = ("CONDITIONING",)
+ FUNCTION = "encode"
+
+ CATEGORY = "advanced/conditioning"
+
+ def encode(self, clip, prompt, vae=None, image=None):
+ ref_latent = None
+ if image is None:
+ images = []
+ else:
+ samples = image.movedim(-1, 1)
+ total = int(1024 * 1024)
+
+ scale_by = math.sqrt(total / (samples.shape[3] * samples.shape[2]))
+ width = round(samples.shape[3] * scale_by)
+ height = round(samples.shape[2] * scale_by)
+
+ s = comfy.utils.common_upscale(samples, width, height, "area", "disabled")
+ image = s.movedim(1, -1)
+ images = [image[:, :, :, :3]]
+ if vae is not None:
+ ref_latent = vae.encode(image[:, :, :, :3])
+
+ tokens = clip.tokenize(prompt, images=images)
+ conditioning = clip.encode_from_tokens_scheduled(tokens)
+ if ref_latent is not None:
+ conditioning = node_helpers.conditioning_set_values(conditioning, {"reference_latents": [ref_latent]}, append=True)
+ return (conditioning, )
+
+
+NODE_CLASS_MAPPINGS = {
+ "TextEncodeQwenImageEdit": TextEncodeQwenImageEdit,
+}
diff --git a/comfy_extras/nodes_stable_cascade.py b/comfy_extras/nodes_stable_cascade.py
index 003403215..04c0b366a 100644
--- a/comfy_extras/nodes_stable_cascade.py
+++ b/comfy_extras/nodes_stable_cascade.py
@@ -17,55 +17,61 @@
"""
import torch
-import nodes
+from typing_extensions import override
+
import comfy.utils
+import nodes
+from comfy_api.latest import ComfyExtension, io
-class StableCascade_EmptyLatentImage:
- def __init__(self, device="cpu"):
- self.device = device
+class StableCascade_EmptyLatentImage(io.ComfyNode):
+ @classmethod
+ def define_schema(cls):
+ return io.Schema(
+ node_id="StableCascade_EmptyLatentImage",
+ category="latent/stable_cascade",
+ inputs=[
+ io.Int.Input("width", default=1024, min=256, max=nodes.MAX_RESOLUTION, step=8),
+ io.Int.Input("height", default=1024, min=256, max=nodes.MAX_RESOLUTION, step=8),
+ io.Int.Input("compression", default=42, min=4, max=128, step=1),
+ io.Int.Input("batch_size", default=1, min=1, max=4096),
+ ],
+ outputs=[
+ io.Latent.Output(display_name="stage_c"),
+ io.Latent.Output(display_name="stage_b"),
+ ],
+ )
@classmethod
- def INPUT_TYPES(s):
- return {"required": {
- "width": ("INT", {"default": 1024, "min": 256, "max": nodes.MAX_RESOLUTION, "step": 8}),
- "height": ("INT", {"default": 1024, "min": 256, "max": nodes.MAX_RESOLUTION, "step": 8}),
- "compression": ("INT", {"default": 42, "min": 4, "max": 128, "step": 1}),
- "batch_size": ("INT", {"default": 1, "min": 1, "max": 4096})
- }}
- RETURN_TYPES = ("LATENT", "LATENT")
- RETURN_NAMES = ("stage_c", "stage_b")
- FUNCTION = "generate"
-
- CATEGORY = "latent/stable_cascade"
-
- def generate(self, width, height, compression, batch_size=1):
+ def execute(cls, width, height, compression, batch_size=1):
c_latent = torch.zeros([batch_size, 16, height // compression, width // compression])
b_latent = torch.zeros([batch_size, 4, height // 4, width // 4])
- return ({
+ return io.NodeOutput({
"samples": c_latent,
}, {
"samples": b_latent,
})
-class StableCascade_StageC_VAEEncode:
- def __init__(self, device="cpu"):
- self.device = device
+
+class StableCascade_StageC_VAEEncode(io.ComfyNode):
+ @classmethod
+ def define_schema(cls):
+ return io.Schema(
+ node_id="StableCascade_StageC_VAEEncode",
+ category="latent/stable_cascade",
+ inputs=[
+ io.Image.Input("image"),
+ io.Vae.Input("vae"),
+ io.Int.Input("compression", default=42, min=4, max=128, step=1),
+ ],
+ outputs=[
+ io.Latent.Output(display_name="stage_c"),
+ io.Latent.Output(display_name="stage_b"),
+ ],
+ )
@classmethod
- def INPUT_TYPES(s):
- return {"required": {
- "image": ("IMAGE",),
- "vae": ("VAE", ),
- "compression": ("INT", {"default": 42, "min": 4, "max": 128, "step": 1}),
- }}
- RETURN_TYPES = ("LATENT", "LATENT")
- RETURN_NAMES = ("stage_c", "stage_b")
- FUNCTION = "generate"
-
- CATEGORY = "latent/stable_cascade"
-
- def generate(self, image, vae, compression):
+ def execute(cls, image, vae, compression):
width = image.shape[-2]
height = image.shape[-3]
out_width = (width // compression) * vae.downscale_ratio
@@ -75,51 +81,59 @@ class StableCascade_StageC_VAEEncode:
c_latent = vae.encode(s[:,:,:,:3])
b_latent = torch.zeros([c_latent.shape[0], 4, (height // 8) * 2, (width // 8) * 2])
- return ({
+ return io.NodeOutput({
"samples": c_latent,
}, {
"samples": b_latent,
})
-class StableCascade_StageB_Conditioning:
+
+class StableCascade_StageB_Conditioning(io.ComfyNode):
@classmethod
- def INPUT_TYPES(s):
- return {"required": { "conditioning": ("CONDITIONING",),
- "stage_c": ("LATENT",),
- }}
- RETURN_TYPES = ("CONDITIONING",)
+ def define_schema(cls):
+ return io.Schema(
+ node_id="StableCascade_StageB_Conditioning",
+ category="conditioning/stable_cascade",
+ inputs=[
+ io.Conditioning.Input("conditioning"),
+ io.Latent.Input("stage_c"),
+ ],
+ outputs=[
+ io.Conditioning.Output(),
+ ],
+ )
- FUNCTION = "set_prior"
-
- CATEGORY = "conditioning/stable_cascade"
-
- def set_prior(self, conditioning, stage_c):
+ @classmethod
+ def execute(cls, conditioning, stage_c):
c = []
for t in conditioning:
d = t[1].copy()
- d['stable_cascade_prior'] = stage_c['samples']
+ d["stable_cascade_prior"] = stage_c["samples"]
n = [t[0], d]
c.append(n)
- return (c, )
+ return io.NodeOutput(c)
-class StableCascade_SuperResolutionControlnet:
- def __init__(self, device="cpu"):
- self.device = device
+
+class StableCascade_SuperResolutionControlnet(io.ComfyNode):
+ @classmethod
+ def define_schema(cls):
+ return io.Schema(
+ node_id="StableCascade_SuperResolutionControlnet",
+ category="_for_testing/stable_cascade",
+ is_experimental=True,
+ inputs=[
+ io.Image.Input("image"),
+ io.Vae.Input("vae"),
+ ],
+ outputs=[
+ io.Image.Output(display_name="controlnet_input"),
+ io.Latent.Output(display_name="stage_c"),
+ io.Latent.Output(display_name="stage_b"),
+ ],
+ )
@classmethod
- def INPUT_TYPES(s):
- return {"required": {
- "image": ("IMAGE",),
- "vae": ("VAE", ),
- }}
- RETURN_TYPES = ("IMAGE", "LATENT", "LATENT")
- RETURN_NAMES = ("controlnet_input", "stage_c", "stage_b")
- FUNCTION = "generate"
-
- EXPERIMENTAL = True
- CATEGORY = "_for_testing/stable_cascade"
-
- def generate(self, image, vae):
+ def execute(cls, image, vae):
width = image.shape[-2]
height = image.shape[-3]
batch_size = image.shape[0]
@@ -127,15 +141,22 @@ class StableCascade_SuperResolutionControlnet:
c_latent = torch.zeros([batch_size, 16, height // 16, width // 16])
b_latent = torch.zeros([batch_size, 4, height // 2, width // 2])
- return (controlnet_input, {
+ return io.NodeOutput(controlnet_input, {
"samples": c_latent,
}, {
"samples": b_latent,
})
-NODE_CLASS_MAPPINGS = {
- "StableCascade_EmptyLatentImage": StableCascade_EmptyLatentImage,
- "StableCascade_StageB_Conditioning": StableCascade_StageB_Conditioning,
- "StableCascade_StageC_VAEEncode": StableCascade_StageC_VAEEncode,
- "StableCascade_SuperResolutionControlnet": StableCascade_SuperResolutionControlnet,
-}
+
+class StableCascadeExtension(ComfyExtension):
+ @override
+ async def get_node_list(self) -> list[type[io.ComfyNode]]:
+ return [
+ StableCascade_EmptyLatentImage,
+ StableCascade_StageB_Conditioning,
+ StableCascade_StageC_VAEEncode,
+ StableCascade_SuperResolutionControlnet,
+ ]
+
+async def comfy_entrypoint() -> StableCascadeExtension:
+ return StableCascadeExtension()
diff --git a/comfy_extras/nodes_string.py b/comfy_extras/nodes_string.py
index b1a8ceef0..571d89f62 100644
--- a/comfy_extras/nodes_string.py
+++ b/comfy_extras/nodes_string.py
@@ -1,77 +1,91 @@
import re
+from typing_extensions import override
-from comfy.comfy_types.node_typing import IO
+from comfy_api.latest import ComfyExtension, io
-class StringConcatenate():
+
+class StringConcatenate(io.ComfyNode):
@classmethod
- def INPUT_TYPES(s):
- return {
- "required": {
- "string_a": (IO.STRING, {"multiline": True}),
- "string_b": (IO.STRING, {"multiline": True}),
- "delimiter": (IO.STRING, {"multiline": False, "default": ""})
- }
- }
+ def define_schema(cls):
+ return io.Schema(
+ node_id="StringConcatenate",
+ display_name="Concatenate",
+ category="utils/string",
+ inputs=[
+ io.String.Input("string_a", multiline=True),
+ io.String.Input("string_b", multiline=True),
+ io.String.Input("delimiter", multiline=False, default=""),
+ ],
+ outputs=[
+ io.String.Output(),
+ ]
+ )
- RETURN_TYPES = (IO.STRING,)
- FUNCTION = "execute"
- CATEGORY = "utils/string"
-
- def execute(self, string_a, string_b, delimiter, **kwargs):
- return delimiter.join((string_a, string_b)),
-
-class StringSubstring():
@classmethod
- def INPUT_TYPES(s):
- return {
- "required": {
- "string": (IO.STRING, {"multiline": True}),
- "start": (IO.INT, {}),
- "end": (IO.INT, {}),
- }
- }
+ def execute(cls, string_a, string_b, delimiter):
+ return io.NodeOutput(delimiter.join((string_a, string_b)))
- RETURN_TYPES = (IO.STRING,)
- FUNCTION = "execute"
- CATEGORY = "utils/string"
- def execute(self, string, start, end, **kwargs):
- return string[start:end],
-
-class StringLength():
+class StringSubstring(io.ComfyNode):
@classmethod
- def INPUT_TYPES(s):
- return {
- "required": {
- "string": (IO.STRING, {"multiline": True})
- }
- }
+ def define_schema(cls):
+ return io.Schema(
+ node_id="StringSubstring",
+ display_name="Substring",
+ category="utils/string",
+ inputs=[
+ io.String.Input("string", multiline=True),
+ io.Int.Input("start"),
+ io.Int.Input("end"),
+ ],
+ outputs=[
+ io.String.Output(),
+ ]
+ )
- RETURN_TYPES = (IO.INT,)
- RETURN_NAMES = ("length",)
- FUNCTION = "execute"
- CATEGORY = "utils/string"
-
- def execute(self, string, **kwargs):
- length = len(string)
-
- return length,
-
-class CaseConverter():
@classmethod
- def INPUT_TYPES(s):
- return {
- "required": {
- "string": (IO.STRING, {"multiline": True}),
- "mode": (IO.COMBO, {"options": ["UPPERCASE", "lowercase", "Capitalize", "Title Case"]})
- }
- }
+ def execute(cls, string, start, end):
+ return io.NodeOutput(string[start:end])
- RETURN_TYPES = (IO.STRING,)
- FUNCTION = "execute"
- CATEGORY = "utils/string"
- def execute(self, string, mode, **kwargs):
+class StringLength(io.ComfyNode):
+ @classmethod
+ def define_schema(cls):
+ return io.Schema(
+ node_id="StringLength",
+ display_name="Length",
+ category="utils/string",
+ inputs=[
+ io.String.Input("string", multiline=True),
+ ],
+ outputs=[
+ io.Int.Output(display_name="length"),
+ ]
+ )
+
+ @classmethod
+ def execute(cls, string):
+ return io.NodeOutput(len(string))
+
+
+class CaseConverter(io.ComfyNode):
+ @classmethod
+ def define_schema(cls):
+ return io.Schema(
+ node_id="CaseConverter",
+ display_name="Case Converter",
+ category="utils/string",
+ inputs=[
+ io.String.Input("string", multiline=True),
+ io.Combo.Input("mode", options=["UPPERCASE", "lowercase", "Capitalize", "Title Case"]),
+ ],
+ outputs=[
+ io.String.Output(),
+ ]
+ )
+
+ @classmethod
+ def execute(cls, string, mode):
if mode == "UPPERCASE":
result = string.upper()
elif mode == "lowercase":
@@ -83,24 +97,27 @@ class CaseConverter():
else:
result = string
- return result,
+ return io.NodeOutput(result)
-class StringTrim():
+class StringTrim(io.ComfyNode):
@classmethod
- def INPUT_TYPES(s):
- return {
- "required": {
- "string": (IO.STRING, {"multiline": True}),
- "mode": (IO.COMBO, {"options": ["Both", "Left", "Right"]})
- }
- }
+ def define_schema(cls):
+ return io.Schema(
+ node_id="StringTrim",
+ display_name="Trim",
+ category="utils/string",
+ inputs=[
+ io.String.Input("string", multiline=True),
+ io.Combo.Input("mode", options=["Both", "Left", "Right"]),
+ ],
+ outputs=[
+ io.String.Output(),
+ ]
+ )
- RETURN_TYPES = (IO.STRING,)
- FUNCTION = "execute"
- CATEGORY = "utils/string"
-
- def execute(self, string, mode, **kwargs):
+ @classmethod
+ def execute(cls, string, mode):
if mode == "Both":
result = string.strip()
elif mode == "Left":
@@ -110,70 +127,78 @@ class StringTrim():
else:
result = string
- return result,
+ return io.NodeOutput(result)
-class StringReplace():
+
+class StringReplace(io.ComfyNode):
@classmethod
- def INPUT_TYPES(s):
- return {
- "required": {
- "string": (IO.STRING, {"multiline": True}),
- "find": (IO.STRING, {"multiline": True}),
- "replace": (IO.STRING, {"multiline": True})
- }
- }
+ def define_schema(cls):
+ return io.Schema(
+ node_id="StringReplace",
+ display_name="Replace",
+ category="utils/string",
+ inputs=[
+ io.String.Input("string", multiline=True),
+ io.String.Input("find", multiline=True),
+ io.String.Input("replace", multiline=True),
+ ],
+ outputs=[
+ io.String.Output(),
+ ]
+ )
- RETURN_TYPES = (IO.STRING,)
- FUNCTION = "execute"
- CATEGORY = "utils/string"
-
- def execute(self, string, find, replace, **kwargs):
- result = string.replace(find, replace)
- return result,
-
-
-class StringContains():
@classmethod
- def INPUT_TYPES(s):
- return {
- "required": {
- "string": (IO.STRING, {"multiline": True}),
- "substring": (IO.STRING, {"multiline": True}),
- "case_sensitive": (IO.BOOLEAN, {"default": True})
- }
- }
+ def execute(cls, string, find, replace):
+ return io.NodeOutput(string.replace(find, replace))
- RETURN_TYPES = (IO.BOOLEAN,)
- RETURN_NAMES = ("contains",)
- FUNCTION = "execute"
- CATEGORY = "utils/string"
- def execute(self, string, substring, case_sensitive, **kwargs):
+class StringContains(io.ComfyNode):
+ @classmethod
+ def define_schema(cls):
+ return io.Schema(
+ node_id="StringContains",
+ display_name="Contains",
+ category="utils/string",
+ inputs=[
+ io.String.Input("string", multiline=True),
+ io.String.Input("substring", multiline=True),
+ io.Boolean.Input("case_sensitive", default=True),
+ ],
+ outputs=[
+ io.Boolean.Output(display_name="contains"),
+ ]
+ )
+
+ @classmethod
+ def execute(cls, string, substring, case_sensitive):
if case_sensitive:
contains = substring in string
else:
contains = substring.lower() in string.lower()
- return contains,
+ return io.NodeOutput(contains)
-class StringCompare():
+class StringCompare(io.ComfyNode):
@classmethod
- def INPUT_TYPES(s):
- return {
- "required": {
- "string_a": (IO.STRING, {"multiline": True}),
- "string_b": (IO.STRING, {"multiline": True}),
- "mode": (IO.COMBO, {"options": ["Starts With", "Ends With", "Equal"]}),
- "case_sensitive": (IO.BOOLEAN, {"default": True})
- }
- }
+ def define_schema(cls):
+ return io.Schema(
+ node_id="StringCompare",
+ display_name="Compare",
+ category="utils/string",
+ inputs=[
+ io.String.Input("string_a", multiline=True),
+ io.String.Input("string_b", multiline=True),
+ io.Combo.Input("mode", options=["Starts With", "Ends With", "Equal"]),
+ io.Boolean.Input("case_sensitive", default=True),
+ ],
+ outputs=[
+ io.Boolean.Output(),
+ ]
+ )
- RETURN_TYPES = (IO.BOOLEAN,)
- FUNCTION = "execute"
- CATEGORY = "utils/string"
-
- def execute(self, string_a, string_b, mode, case_sensitive, **kwargs):
+ @classmethod
+ def execute(cls, string_a, string_b, mode, case_sensitive):
if case_sensitive:
a = string_a
b = string_b
@@ -182,31 +207,34 @@ class StringCompare():
b = string_b.lower()
if mode == "Equal":
- return a == b,
+ return io.NodeOutput(a == b)
elif mode == "Starts With":
- return a.startswith(b),
+ return io.NodeOutput(a.startswith(b))
elif mode == "Ends With":
- return a.endswith(b),
+ return io.NodeOutput(a.endswith(b))
-class RegexMatch():
+
+class RegexMatch(io.ComfyNode):
@classmethod
- def INPUT_TYPES(s):
- return {
- "required": {
- "string": (IO.STRING, {"multiline": True}),
- "regex_pattern": (IO.STRING, {"multiline": True}),
- "case_insensitive": (IO.BOOLEAN, {"default": True}),
- "multiline": (IO.BOOLEAN, {"default": False}),
- "dotall": (IO.BOOLEAN, {"default": False})
- }
- }
+ def define_schema(cls):
+ return io.Schema(
+ node_id="RegexMatch",
+ display_name="Regex Match",
+ category="utils/string",
+ inputs=[
+ io.String.Input("string", multiline=True),
+ io.String.Input("regex_pattern", multiline=True),
+ io.Boolean.Input("case_insensitive", default=True),
+ io.Boolean.Input("multiline", default=False),
+ io.Boolean.Input("dotall", default=False),
+ ],
+ outputs=[
+ io.Boolean.Output(display_name="matches"),
+ ]
+ )
- RETURN_TYPES = (IO.BOOLEAN,)
- RETURN_NAMES = ("matches",)
- FUNCTION = "execute"
- CATEGORY = "utils/string"
-
- def execute(self, string, regex_pattern, case_insensitive, multiline, dotall, **kwargs):
+ @classmethod
+ def execute(cls, string, regex_pattern, case_insensitive, multiline, dotall):
flags = 0
if case_insensitive:
@@ -223,29 +251,32 @@ class RegexMatch():
except re.error:
result = False
- return result,
+ return io.NodeOutput(result)
-class RegexExtract():
+class RegexExtract(io.ComfyNode):
@classmethod
- def INPUT_TYPES(s):
- return {
- "required": {
- "string": (IO.STRING, {"multiline": True}),
- "regex_pattern": (IO.STRING, {"multiline": True}),
- "mode": (IO.COMBO, {"options": ["First Match", "All Matches", "First Group", "All Groups"]}),
- "case_insensitive": (IO.BOOLEAN, {"default": True}),
- "multiline": (IO.BOOLEAN, {"default": False}),
- "dotall": (IO.BOOLEAN, {"default": False}),
- "group_index": (IO.INT, {"default": 1, "min": 0, "max": 100})
- }
- }
+ def define_schema(cls):
+ return io.Schema(
+ node_id="RegexExtract",
+ display_name="Regex Extract",
+ category="utils/string",
+ inputs=[
+ io.String.Input("string", multiline=True),
+ io.String.Input("regex_pattern", multiline=True),
+ io.Combo.Input("mode", options=["First Match", "All Matches", "First Group", "All Groups"]),
+ io.Boolean.Input("case_insensitive", default=True),
+ io.Boolean.Input("multiline", default=False),
+ io.Boolean.Input("dotall", default=False),
+ io.Int.Input("group_index", default=1, min=0, max=100),
+ ],
+ outputs=[
+ io.String.Output(),
+ ]
+ )
- RETURN_TYPES = (IO.STRING,)
- FUNCTION = "execute"
- CATEGORY = "utils/string"
-
- def execute(self, string, regex_pattern, mode, case_insensitive, multiline, dotall, group_index, **kwargs):
+ @classmethod
+ def execute(cls, string, regex_pattern, mode, case_insensitive, multiline, dotall, group_index):
join_delimiter = "\n"
flags = 0
@@ -294,32 +325,33 @@ class RegexExtract():
except re.error:
result = ""
- return result,
+ return io.NodeOutput(result)
-class RegexReplace():
- DESCRIPTION = "Find and replace text using regex patterns."
+class RegexReplace(io.ComfyNode):
@classmethod
- def INPUT_TYPES(s):
- return {
- "required": {
- "string": (IO.STRING, {"multiline": True}),
- "regex_pattern": (IO.STRING, {"multiline": True}),
- "replace": (IO.STRING, {"multiline": True}),
- },
- "optional": {
- "case_insensitive": (IO.BOOLEAN, {"default": True}),
- "multiline": (IO.BOOLEAN, {"default": False}),
- "dotall": (IO.BOOLEAN, {"default": False, "tooltip": "When enabled, the dot (.) character will match any character including newline characters. When disabled, dots won't match newlines."}),
- "count": (IO.INT, {"default": 0, "min": 0, "max": 100, "tooltip": "Maximum number of replacements to make. Set to 0 to replace all occurrences (default). Set to 1 to replace only the first match, 2 for the first two matches, etc."}),
- }
- }
+ def define_schema(cls):
+ return io.Schema(
+ node_id="RegexReplace",
+ display_name="Regex Replace",
+ category="utils/string",
+ description="Find and replace text using regex patterns.",
+ inputs=[
+ io.String.Input("string", multiline=True),
+ io.String.Input("regex_pattern", multiline=True),
+ io.String.Input("replace", multiline=True),
+ io.Boolean.Input("case_insensitive", default=True, optional=True),
+ io.Boolean.Input("multiline", default=False, optional=True),
+ io.Boolean.Input("dotall", default=False, optional=True, tooltip="When enabled, the dot (.) character will match any character including newline characters. When disabled, dots won't match newlines."),
+ io.Int.Input("count", default=0, min=0, max=100, optional=True, tooltip="Maximum number of replacements to make. Set to 0 to replace all occurrences (default). Set to 1 to replace only the first match, 2 for the first two matches, etc."),
+ ],
+ outputs=[
+ io.String.Output(),
+ ]
+ )
- RETURN_TYPES = (IO.STRING,)
- FUNCTION = "execute"
- CATEGORY = "utils/string"
-
- def execute(self, string, regex_pattern, replace, case_insensitive=True, multiline=False, dotall=False, count=0, **kwargs):
+ @classmethod
+ def execute(cls, string, regex_pattern, replace, case_insensitive=True, multiline=False, dotall=False, count=0):
flags = 0
if case_insensitive:
@@ -329,32 +361,25 @@ class RegexReplace():
if dotall:
flags |= re.DOTALL
result = re.sub(regex_pattern, replace, string, count=count, flags=flags)
- return result,
+ return io.NodeOutput(result)
-NODE_CLASS_MAPPINGS = {
- "StringConcatenate": StringConcatenate,
- "StringSubstring": StringSubstring,
- "StringLength": StringLength,
- "CaseConverter": CaseConverter,
- "StringTrim": StringTrim,
- "StringReplace": StringReplace,
- "StringContains": StringContains,
- "StringCompare": StringCompare,
- "RegexMatch": RegexMatch,
- "RegexExtract": RegexExtract,
- "RegexReplace": RegexReplace,
-}
-NODE_DISPLAY_NAME_MAPPINGS = {
- "StringConcatenate": "Concatenate",
- "StringSubstring": "Substring",
- "StringLength": "Length",
- "CaseConverter": "Case Converter",
- "StringTrim": "Trim",
- "StringReplace": "Replace",
- "StringContains": "Contains",
- "StringCompare": "Compare",
- "RegexMatch": "Regex Match",
- "RegexExtract": "Regex Extract",
- "RegexReplace": "Regex Replace",
-}
+class StringExtension(ComfyExtension):
+ @override
+ async def get_node_list(self) -> list[type[io.ComfyNode]]:
+ return [
+ StringConcatenate,
+ StringSubstring,
+ StringLength,
+ CaseConverter,
+ StringTrim,
+ StringReplace,
+ StringContains,
+ StringCompare,
+ RegexMatch,
+ RegexExtract,
+ RegexReplace,
+ ]
+
+async def comfy_entrypoint() -> StringExtension:
+ return StringExtension()
diff --git a/comfy_extras/nodes_video.py b/comfy_extras/nodes_video.py
index 61f7171b2..69fabb12e 100644
--- a/comfy_extras/nodes_video.py
+++ b/comfy_extras/nodes_video.py
@@ -5,54 +5,49 @@ import av
import torch
import folder_paths
import json
-from typing import Optional, Literal
+from typing import Optional
+from typing_extensions import override
from fractions import Fraction
-from comfy.comfy_types import IO, FileLocator, ComfyNodeABC
-from comfy_api.input import ImageInput, AudioInput, VideoInput
-from comfy_api.util import VideoContainer, VideoCodec, VideoComponents
-from comfy_api.input_impl import VideoFromFile, VideoFromComponents
+from comfy_api.input import AudioInput, ImageInput, VideoInput
+from comfy_api.input_impl import VideoFromComponents, VideoFromFile
+from comfy_api.util import VideoCodec, VideoComponents, VideoContainer
+from comfy_api.latest import ComfyExtension, io, ui
from comfy.cli_args import args
-class SaveWEBM:
- def __init__(self):
- self.output_dir = folder_paths.get_output_directory()
- self.type = "output"
- self.prefix_append = ""
+class SaveWEBM(io.ComfyNode):
+ @classmethod
+ def define_schema(cls):
+ return io.Schema(
+ node_id="SaveWEBM",
+ category="image/video",
+ is_experimental=True,
+ inputs=[
+ io.Image.Input("images"),
+ io.String.Input("filename_prefix", default="ComfyUI"),
+ io.Combo.Input("codec", options=["vp9", "av1"]),
+ io.Float.Input("fps", default=24.0, min=0.01, max=1000.0, step=0.01),
+ io.Float.Input("crf", default=32.0, min=0, max=63.0, step=1, tooltip="Higher crf means lower quality with a smaller file size, lower crf means higher quality higher filesize."),
+ ],
+ outputs=[],
+ hidden=[io.Hidden.prompt, io.Hidden.extra_pnginfo],
+ is_output_node=True,
+ )
@classmethod
- def INPUT_TYPES(s):
- return {"required":
- {"images": ("IMAGE", ),
- "filename_prefix": ("STRING", {"default": "ComfyUI"}),
- "codec": (["vp9", "av1"],),
- "fps": ("FLOAT", {"default": 24.0, "min": 0.01, "max": 1000.0, "step": 0.01}),
- "crf": ("FLOAT", {"default": 32.0, "min": 0, "max": 63.0, "step": 1, "tooltip": "Higher crf means lower quality with a smaller file size, lower crf means higher quality higher filesize."}),
- },
- "hidden": {"prompt": "PROMPT", "extra_pnginfo": "EXTRA_PNGINFO"},
- }
-
- RETURN_TYPES = ()
- FUNCTION = "save_images"
-
- OUTPUT_NODE = True
-
- CATEGORY = "image/video"
-
- EXPERIMENTAL = True
-
- def save_images(self, images, codec, fps, filename_prefix, crf, prompt=None, extra_pnginfo=None):
- filename_prefix += self.prefix_append
- full_output_folder, filename, counter, subfolder, filename_prefix = folder_paths.get_save_image_path(filename_prefix, self.output_dir, images[0].shape[1], images[0].shape[0])
+ def execute(cls, images, codec, fps, filename_prefix, crf) -> io.NodeOutput:
+ full_output_folder, filename, counter, subfolder, filename_prefix = folder_paths.get_save_image_path(
+ filename_prefix, folder_paths.get_output_directory(), images[0].shape[1], images[0].shape[0]
+ )
file = f"{filename}_{counter:05}_.webm"
container = av.open(os.path.join(full_output_folder, file), mode="w")
- if prompt is not None:
- container.metadata["prompt"] = json.dumps(prompt)
+ if cls.hidden.prompt is not None:
+ container.metadata["prompt"] = json.dumps(cls.hidden.prompt)
- if extra_pnginfo is not None:
- for x in extra_pnginfo:
- container.metadata[x] = json.dumps(extra_pnginfo[x])
+ if cls.hidden.extra_pnginfo is not None:
+ for x in cls.hidden.extra_pnginfo:
+ container.metadata[x] = json.dumps(cls.hidden.extra_pnginfo[x])
codec_map = {"vp9": "libvpx-vp9", "av1": "libsvtav1"}
stream = container.add_stream(codec_map[codec], rate=Fraction(round(fps * 1000), 1000))
@@ -71,60 +66,43 @@ class SaveWEBM:
container.mux(stream.encode())
container.close()
- results: list[FileLocator] = [{
- "filename": file,
- "subfolder": subfolder,
- "type": self.type
- }]
+ return io.NodeOutput(ui=ui.PreviewVideo([ui.SavedResult(file, subfolder, io.FolderType.output)]))
- return {"ui": {"images": results, "animated": (True,)}} # TODO: frontend side
-
-class SaveVideo(ComfyNodeABC):
- def __init__(self):
- self.output_dir = folder_paths.get_output_directory()
- self.type: Literal["output"] = "output"
- self.prefix_append = ""
+class SaveVideo(io.ComfyNode):
+ @classmethod
+ def define_schema(cls):
+ return io.Schema(
+ node_id="SaveVideo",
+ display_name="Save Video",
+ category="image/video",
+ description="Saves the input images to your ComfyUI output directory.",
+ inputs=[
+ io.Video.Input("video", tooltip="The video to save."),
+ io.String.Input("filename_prefix", default="video/ComfyUI", tooltip="The prefix for the file to save. This may include formatting information such as %date:yyyy-MM-dd% or %Empty Latent Image.width% to include values from nodes."),
+ io.Combo.Input("format", options=VideoContainer.as_input(), default="auto", tooltip="The format to save the video as."),
+ io.Combo.Input("codec", options=VideoCodec.as_input(), default="auto", tooltip="The codec to use for the video."),
+ ],
+ outputs=[],
+ hidden=[io.Hidden.prompt, io.Hidden.extra_pnginfo],
+ is_output_node=True,
+ )
@classmethod
- def INPUT_TYPES(cls):
- return {
- "required": {
- "video": (IO.VIDEO, {"tooltip": "The video to save."}),
- "filename_prefix": ("STRING", {"default": "video/ComfyUI", "tooltip": "The prefix for the file to save. This may include formatting information such as %date:yyyy-MM-dd% or %Empty Latent Image.width% to include values from nodes."}),
- "format": (VideoContainer.as_input(), {"default": "auto", "tooltip": "The format to save the video as."}),
- "codec": (VideoCodec.as_input(), {"default": "auto", "tooltip": "The codec to use for the video."}),
- },
- "hidden": {
- "prompt": "PROMPT",
- "extra_pnginfo": "EXTRA_PNGINFO"
- },
- }
-
- RETURN_TYPES = ()
- FUNCTION = "save_video"
-
- OUTPUT_NODE = True
-
- CATEGORY = "image/video"
- DESCRIPTION = "Saves the input images to your ComfyUI output directory."
-
- def save_video(self, video: VideoInput, filename_prefix, format, codec, prompt=None, extra_pnginfo=None):
- filename_prefix += self.prefix_append
+ def execute(cls, video: VideoInput, filename_prefix, format, codec) -> io.NodeOutput:
width, height = video.get_dimensions()
full_output_folder, filename, counter, subfolder, filename_prefix = folder_paths.get_save_image_path(
filename_prefix,
- self.output_dir,
+ folder_paths.get_output_directory(),
width,
height
)
- results: list[FileLocator] = list()
saved_metadata = None
if not args.disable_metadata:
metadata = {}
- if extra_pnginfo is not None:
- metadata.update(extra_pnginfo)
- if prompt is not None:
- metadata["prompt"] = prompt
+ if cls.hidden.extra_pnginfo is not None:
+ metadata.update(cls.hidden.extra_pnginfo)
+ if cls.hidden.prompt is not None:
+ metadata["prompt"] = cls.hidden.prompt
if len(metadata) > 0:
saved_metadata = metadata
file = f"{filename}_{counter:05}_.{VideoContainer.get_extension(format)}"
@@ -135,83 +113,82 @@ class SaveVideo(ComfyNodeABC):
metadata=saved_metadata
)
- results.append({
- "filename": file,
- "subfolder": subfolder,
- "type": self.type
- })
- counter += 1
+ return io.NodeOutput(ui=ui.PreviewVideo([ui.SavedResult(file, subfolder, io.FolderType.output)]))
- return { "ui": { "images": results, "animated": (True,) } }
-class CreateVideo(ComfyNodeABC):
+class CreateVideo(io.ComfyNode):
@classmethod
- def INPUT_TYPES(cls):
- return {
- "required": {
- "images": (IO.IMAGE, {"tooltip": "The images to create a video from."}),
- "fps": ("FLOAT", {"default": 30.0, "min": 1.0, "max": 120.0, "step": 1.0}),
- },
- "optional": {
- "audio": (IO.AUDIO, {"tooltip": "The audio to add to the video."}),
- }
- }
+ def define_schema(cls):
+ return io.Schema(
+ node_id="CreateVideo",
+ display_name="Create Video",
+ category="image/video",
+ description="Create a video from images.",
+ inputs=[
+ io.Image.Input("images", tooltip="The images to create a video from."),
+ io.Float.Input("fps", default=30.0, min=1.0, max=120.0, step=1.0),
+ io.Audio.Input("audio", optional=True, tooltip="The audio to add to the video."),
+ ],
+ outputs=[
+ io.Video.Output(),
+ ],
+ )
- RETURN_TYPES = (IO.VIDEO,)
- FUNCTION = "create_video"
-
- CATEGORY = "image/video"
- DESCRIPTION = "Create a video from images."
-
- def create_video(self, images: ImageInput, fps: float, audio: Optional[AudioInput] = None):
- return (VideoFromComponents(
- VideoComponents(
- images=images,
- audio=audio,
- frame_rate=Fraction(fps),
- )
- ),)
-
-class GetVideoComponents(ComfyNodeABC):
@classmethod
- def INPUT_TYPES(cls):
- return {
- "required": {
- "video": (IO.VIDEO, {"tooltip": "The video to extract components from."}),
- }
- }
- RETURN_TYPES = (IO.IMAGE, IO.AUDIO, IO.FLOAT)
- RETURN_NAMES = ("images", "audio", "fps")
- FUNCTION = "get_components"
+ def execute(cls, images: ImageInput, fps: float, audio: Optional[AudioInput] = None) -> io.NodeOutput:
+ return io.NodeOutput(
+ VideoFromComponents(VideoComponents(images=images, audio=audio, frame_rate=Fraction(fps)))
+ )
- CATEGORY = "image/video"
- DESCRIPTION = "Extracts all components from a video: frames, audio, and framerate."
+class GetVideoComponents(io.ComfyNode):
+ @classmethod
+ def define_schema(cls):
+ return io.Schema(
+ node_id="GetVideoComponents",
+ display_name="Get Video Components",
+ category="image/video",
+ description="Extracts all components from a video: frames, audio, and framerate.",
+ inputs=[
+ io.Video.Input("video", tooltip="The video to extract components from."),
+ ],
+ outputs=[
+ io.Image.Output(display_name="images"),
+ io.Audio.Output(display_name="audio"),
+ io.Float.Output(display_name="fps"),
+ ],
+ )
- def get_components(self, video: VideoInput):
+ @classmethod
+ def execute(cls, video: VideoInput) -> io.NodeOutput:
components = video.get_components()
- return (components.images, components.audio, float(components.frame_rate))
+ return io.NodeOutput(components.images, components.audio, float(components.frame_rate))
-class LoadVideo(ComfyNodeABC):
+class LoadVideo(io.ComfyNode):
@classmethod
- def INPUT_TYPES(cls):
+ def define_schema(cls):
input_dir = folder_paths.get_input_directory()
files = [f for f in os.listdir(input_dir) if os.path.isfile(os.path.join(input_dir, f))]
files = folder_paths.filter_files_content_types(files, ["video"])
- return {"required":
- {"file": (sorted(files), {"video_upload": True})},
- }
-
- CATEGORY = "image/video"
-
- RETURN_TYPES = (IO.VIDEO,)
- FUNCTION = "load_video"
- def load_video(self, file):
- video_path = folder_paths.get_annotated_filepath(file)
- return (VideoFromFile(video_path),)
+ return io.Schema(
+ node_id="LoadVideo",
+ display_name="Load Video",
+ category="image/video",
+ inputs=[
+ io.Combo.Input("file", options=sorted(files), upload=io.UploadType.video),
+ ],
+ outputs=[
+ io.Video.Output(),
+ ],
+ )
@classmethod
- def IS_CHANGED(cls, file):
+ def execute(cls, file) -> io.NodeOutput:
+ video_path = folder_paths.get_annotated_filepath(file)
+ return io.NodeOutput(VideoFromFile(video_path))
+
+ @classmethod
+ def fingerprint_inputs(s, file):
video_path = folder_paths.get_annotated_filepath(file)
mod_time = os.path.getmtime(video_path)
# Instead of hashing the file, we can just use the modification time to avoid
@@ -219,23 +196,23 @@ class LoadVideo(ComfyNodeABC):
return mod_time
@classmethod
- def VALIDATE_INPUTS(cls, file):
+ def validate_inputs(s, file):
if not folder_paths.exists_annotated_filepath(file):
return "Invalid video file: {}".format(file)
return True
-NODE_CLASS_MAPPINGS = {
- "SaveWEBM": SaveWEBM,
- "SaveVideo": SaveVideo,
- "CreateVideo": CreateVideo,
- "GetVideoComponents": GetVideoComponents,
- "LoadVideo": LoadVideo,
-}
-NODE_DISPLAY_NAME_MAPPINGS = {
- "SaveVideo": "Save Video",
- "CreateVideo": "Create Video",
- "GetVideoComponents": "Get Video Components",
- "LoadVideo": "Load Video",
-}
+class VideoExtension(ComfyExtension):
+ @override
+ async def get_node_list(self) -> list[type[io.ComfyNode]]:
+ return [
+ SaveWEBM,
+ SaveVideo,
+ CreateVideo,
+ GetVideoComponents,
+ LoadVideo,
+ ]
+
+async def comfy_entrypoint() -> VideoExtension:
+ return VideoExtension()
diff --git a/comfy_extras/nodes_wan.py b/comfy_extras/nodes_wan.py
index 0b92c68ac..4f73369f5 100644
--- a/comfy_extras/nodes_wan.py
+++ b/comfy_extras/nodes_wan.py
@@ -9,29 +9,35 @@ import comfy.clip_vision
import json
import numpy as np
from typing import Tuple
+from typing_extensions import override
+from comfy_api.latest import ComfyExtension, io
-class WanImageToVideo:
+class WanImageToVideo(io.ComfyNode):
@classmethod
- def INPUT_TYPES(s):
- return {"required": {"positive": ("CONDITIONING", ),
- "negative": ("CONDITIONING", ),
- "vae": ("VAE", ),
- "width": ("INT", {"default": 832, "min": 16, "max": nodes.MAX_RESOLUTION, "step": 16}),
- "height": ("INT", {"default": 480, "min": 16, "max": nodes.MAX_RESOLUTION, "step": 16}),
- "length": ("INT", {"default": 81, "min": 1, "max": nodes.MAX_RESOLUTION, "step": 4}),
- "batch_size": ("INT", {"default": 1, "min": 1, "max": 4096}),
- },
- "optional": {"clip_vision_output": ("CLIP_VISION_OUTPUT", ),
- "start_image": ("IMAGE", ),
- }}
+ def define_schema(cls):
+ return io.Schema(
+ node_id="WanImageToVideo",
+ category="conditioning/video_models",
+ inputs=[
+ io.Conditioning.Input("positive"),
+ io.Conditioning.Input("negative"),
+ io.Vae.Input("vae"),
+ io.Int.Input("width", default=832, min=16, max=nodes.MAX_RESOLUTION, step=16),
+ io.Int.Input("height", default=480, min=16, max=nodes.MAX_RESOLUTION, step=16),
+ io.Int.Input("length", default=81, min=1, max=nodes.MAX_RESOLUTION, step=4),
+ io.Int.Input("batch_size", default=1, min=1, max=4096),
+ io.ClipVisionOutput.Input("clip_vision_output", optional=True),
+ io.Image.Input("start_image", optional=True),
+ ],
+ outputs=[
+ io.Conditioning.Output(display_name="positive"),
+ io.Conditioning.Output(display_name="negative"),
+ io.Latent.Output(display_name="latent"),
+ ],
+ )
- RETURN_TYPES = ("CONDITIONING", "CONDITIONING", "LATENT")
- RETURN_NAMES = ("positive", "negative", "latent")
- FUNCTION = "encode"
-
- CATEGORY = "conditioning/video_models"
-
- def encode(self, positive, negative, vae, width, height, length, batch_size, start_image=None, clip_vision_output=None):
+ @classmethod
+ def execute(cls, positive, negative, vae, width, height, length, batch_size, start_image=None, clip_vision_output=None) -> io.NodeOutput:
latent = torch.zeros([batch_size, 16, ((length - 1) // 4) + 1, height // 8, width // 8], device=comfy.model_management.intermediate_device())
if start_image is not None:
start_image = comfy.utils.common_upscale(start_image[:length].movedim(-1, 1), width, height, "bilinear", "center").movedim(1, -1)
@@ -51,32 +57,36 @@ class WanImageToVideo:
out_latent = {}
out_latent["samples"] = latent
- return (positive, negative, out_latent)
+ return io.NodeOutput(positive, negative, out_latent)
-class WanFunControlToVideo:
+class WanFunControlToVideo(io.ComfyNode):
@classmethod
- def INPUT_TYPES(s):
- return {"required": {"positive": ("CONDITIONING", ),
- "negative": ("CONDITIONING", ),
- "vae": ("VAE", ),
- "width": ("INT", {"default": 832, "min": 16, "max": nodes.MAX_RESOLUTION, "step": 16}),
- "height": ("INT", {"default": 480, "min": 16, "max": nodes.MAX_RESOLUTION, "step": 16}),
- "length": ("INT", {"default": 81, "min": 1, "max": nodes.MAX_RESOLUTION, "step": 4}),
- "batch_size": ("INT", {"default": 1, "min": 1, "max": 4096}),
- },
- "optional": {"clip_vision_output": ("CLIP_VISION_OUTPUT", ),
- "start_image": ("IMAGE", ),
- "control_video": ("IMAGE", ),
- }}
+ def define_schema(cls):
+ return io.Schema(
+ node_id="WanFunControlToVideo",
+ category="conditioning/video_models",
+ inputs=[
+ io.Conditioning.Input("positive"),
+ io.Conditioning.Input("negative"),
+ io.Vae.Input("vae"),
+ io.Int.Input("width", default=832, min=16, max=nodes.MAX_RESOLUTION, step=16),
+ io.Int.Input("height", default=480, min=16, max=nodes.MAX_RESOLUTION, step=16),
+ io.Int.Input("length", default=81, min=1, max=nodes.MAX_RESOLUTION, step=4),
+ io.Int.Input("batch_size", default=1, min=1, max=4096),
+ io.ClipVisionOutput.Input("clip_vision_output", optional=True),
+ io.Image.Input("start_image", optional=True),
+ io.Image.Input("control_video", optional=True),
+ ],
+ outputs=[
+ io.Conditioning.Output(display_name="positive"),
+ io.Conditioning.Output(display_name="negative"),
+ io.Latent.Output(display_name="latent"),
+ ],
+ )
- RETURN_TYPES = ("CONDITIONING", "CONDITIONING", "LATENT")
- RETURN_NAMES = ("positive", "negative", "latent")
- FUNCTION = "encode"
-
- CATEGORY = "conditioning/video_models"
-
- def encode(self, positive, negative, vae, width, height, length, batch_size, start_image=None, clip_vision_output=None, control_video=None):
+ @classmethod
+ def execute(cls, positive, negative, vae, width, height, length, batch_size, start_image=None, clip_vision_output=None, control_video=None) -> io.NodeOutput:
latent = torch.zeros([batch_size, 16, ((length - 1) // 4) + 1, height // 8, width // 8], device=comfy.model_management.intermediate_device())
concat_latent = torch.zeros([batch_size, 16, ((length - 1) // 4) + 1, height // 8, width // 8], device=comfy.model_management.intermediate_device())
concat_latent = comfy.latent_formats.Wan21().process_out(concat_latent)
@@ -101,33 +111,103 @@ class WanFunControlToVideo:
out_latent = {}
out_latent["samples"] = latent
- return (positive, negative, out_latent)
+ return io.NodeOutput(positive, negative, out_latent)
-class WanFirstLastFrameToVideo:
+class Wan22FunControlToVideo(io.ComfyNode):
@classmethod
- def INPUT_TYPES(s):
- return {"required": {"positive": ("CONDITIONING", ),
- "negative": ("CONDITIONING", ),
- "vae": ("VAE", ),
- "width": ("INT", {"default": 832, "min": 16, "max": nodes.MAX_RESOLUTION, "step": 16}),
- "height": ("INT", {"default": 480, "min": 16, "max": nodes.MAX_RESOLUTION, "step": 16}),
- "length": ("INT", {"default": 81, "min": 1, "max": nodes.MAX_RESOLUTION, "step": 4}),
- "batch_size": ("INT", {"default": 1, "min": 1, "max": 4096}),
- },
- "optional": {"clip_vision_start_image": ("CLIP_VISION_OUTPUT", ),
- "clip_vision_end_image": ("CLIP_VISION_OUTPUT", ),
- "start_image": ("IMAGE", ),
- "end_image": ("IMAGE", ),
- }}
+ def define_schema(cls):
+ return io.Schema(
+ node_id="Wan22FunControlToVideo",
+ category="conditioning/video_models",
+ inputs=[
+ io.Conditioning.Input("positive"),
+ io.Conditioning.Input("negative"),
+ io.Vae.Input("vae"),
+ io.Int.Input("width", default=832, min=16, max=nodes.MAX_RESOLUTION, step=16),
+ io.Int.Input("height", default=480, min=16, max=nodes.MAX_RESOLUTION, step=16),
+ io.Int.Input("length", default=81, min=1, max=nodes.MAX_RESOLUTION, step=4),
+ io.Int.Input("batch_size", default=1, min=1, max=4096),
+ io.Image.Input("ref_image", optional=True),
+ io.Image.Input("control_video", optional=True),
+ ],
+ outputs=[
+ io.Conditioning.Output(display_name="positive"),
+ io.Conditioning.Output(display_name="negative"),
+ io.Latent.Output(display_name="latent"),
+ ],
+ )
- RETURN_TYPES = ("CONDITIONING", "CONDITIONING", "LATENT")
- RETURN_NAMES = ("positive", "negative", "latent")
- FUNCTION = "encode"
+ @classmethod
+ def execute(cls, positive, negative, vae, width, height, length, batch_size, ref_image=None, start_image=None, control_video=None) -> io.NodeOutput:
+ spacial_scale = vae.spacial_compression_encode()
+ latent_channels = vae.latent_channels
+ latent = torch.zeros([batch_size, latent_channels, ((length - 1) // 4) + 1, height // spacial_scale, width // spacial_scale], device=comfy.model_management.intermediate_device())
+ concat_latent = torch.zeros([batch_size, latent_channels, ((length - 1) // 4) + 1, height // spacial_scale, width // spacial_scale], device=comfy.model_management.intermediate_device())
+ if latent_channels == 48:
+ concat_latent = comfy.latent_formats.Wan22().process_out(concat_latent)
+ else:
+ concat_latent = comfy.latent_formats.Wan21().process_out(concat_latent)
+ concat_latent = concat_latent.repeat(1, 2, 1, 1, 1)
+ mask = torch.ones((1, 1, latent.shape[2] * 4, latent.shape[-2], latent.shape[-1]))
- CATEGORY = "conditioning/video_models"
+ if start_image is not None:
+ start_image = comfy.utils.common_upscale(start_image[:length].movedim(-1, 1), width, height, "bilinear", "center").movedim(1, -1)
+ concat_latent_image = vae.encode(start_image[:, :, :, :3])
+ concat_latent[:,latent_channels:,:concat_latent_image.shape[2]] = concat_latent_image[:,:,:concat_latent.shape[2]]
+ mask[:, :, :start_image.shape[0] + 3] = 0.0
- def encode(self, positive, negative, vae, width, height, length, batch_size, start_image=None, end_image=None, clip_vision_start_image=None, clip_vision_end_image=None):
- latent = torch.zeros([batch_size, 16, ((length - 1) // 4) + 1, height // 8, width // 8], device=comfy.model_management.intermediate_device())
+ ref_latent = None
+ if ref_image is not None:
+ ref_image = comfy.utils.common_upscale(ref_image[:1].movedim(-1, 1), width, height, "bilinear", "center").movedim(1, -1)
+ ref_latent = vae.encode(ref_image[:, :, :, :3])
+
+ if control_video is not None:
+ control_video = comfy.utils.common_upscale(control_video[:length].movedim(-1, 1), width, height, "bilinear", "center").movedim(1, -1)
+ concat_latent_image = vae.encode(control_video[:, :, :, :3])
+ concat_latent[:,:latent_channels,:concat_latent_image.shape[2]] = concat_latent_image[:,:,:concat_latent.shape[2]]
+
+ mask = mask.view(1, mask.shape[2] // 4, 4, mask.shape[3], mask.shape[4]).transpose(1, 2)
+ positive = node_helpers.conditioning_set_values(positive, {"concat_latent_image": concat_latent, "concat_mask": mask, "concat_mask_index": latent_channels})
+ negative = node_helpers.conditioning_set_values(negative, {"concat_latent_image": concat_latent, "concat_mask": mask, "concat_mask_index": latent_channels})
+
+ if ref_latent is not None:
+ positive = node_helpers.conditioning_set_values(positive, {"reference_latents": [ref_latent]}, append=True)
+ negative = node_helpers.conditioning_set_values(negative, {"reference_latents": [ref_latent]}, append=True)
+
+ out_latent = {}
+ out_latent["samples"] = latent
+ return io.NodeOutput(positive, negative, out_latent)
+
+class WanFirstLastFrameToVideo(io.ComfyNode):
+ @classmethod
+ def define_schema(cls):
+ return io.Schema(
+ node_id="WanFirstLastFrameToVideo",
+ category="conditioning/video_models",
+ inputs=[
+ io.Conditioning.Input("positive"),
+ io.Conditioning.Input("negative"),
+ io.Vae.Input("vae"),
+ io.Int.Input("width", default=832, min=16, max=nodes.MAX_RESOLUTION, step=16),
+ io.Int.Input("height", default=480, min=16, max=nodes.MAX_RESOLUTION, step=16),
+ io.Int.Input("length", default=81, min=1, max=nodes.MAX_RESOLUTION, step=4),
+ io.Int.Input("batch_size", default=1, min=1, max=4096),
+ io.ClipVisionOutput.Input("clip_vision_start_image", optional=True),
+ io.ClipVisionOutput.Input("clip_vision_end_image", optional=True),
+ io.Image.Input("start_image", optional=True),
+ io.Image.Input("end_image", optional=True),
+ ],
+ outputs=[
+ io.Conditioning.Output(display_name="positive"),
+ io.Conditioning.Output(display_name="negative"),
+ io.Latent.Output(display_name="latent"),
+ ],
+ )
+
+ @classmethod
+ def execute(cls, positive, negative, vae, width, height, length, batch_size, start_image=None, end_image=None, clip_vision_start_image=None, clip_vision_end_image=None) -> io.NodeOutput:
+ spacial_scale = vae.spacial_compression_encode()
+ latent = torch.zeros([batch_size, vae.latent_channels, ((length - 1) // 4) + 1, height // spacial_scale, width // spacial_scale], device=comfy.model_management.intermediate_device())
if start_image is not None:
start_image = comfy.utils.common_upscale(start_image[:length].movedim(-1, 1), width, height, "bilinear", "center").movedim(1, -1)
if end_image is not None:
@@ -149,6 +229,7 @@ class WanFirstLastFrameToVideo:
positive = node_helpers.conditioning_set_values(positive, {"concat_latent_image": concat_latent_image, "concat_mask": mask})
negative = node_helpers.conditioning_set_values(negative, {"concat_latent_image": concat_latent_image, "concat_mask": mask})
+ clip_vision_output = None
if clip_vision_start_image is not None:
clip_vision_output = clip_vision_start_image
@@ -166,62 +247,70 @@ class WanFirstLastFrameToVideo:
out_latent = {}
out_latent["samples"] = latent
- return (positive, negative, out_latent)
+ return io.NodeOutput(positive, negative, out_latent)
-class WanFunInpaintToVideo:
+class WanFunInpaintToVideo(io.ComfyNode):
@classmethod
- def INPUT_TYPES(s):
- return {"required": {"positive": ("CONDITIONING", ),
- "negative": ("CONDITIONING", ),
- "vae": ("VAE", ),
- "width": ("INT", {"default": 832, "min": 16, "max": nodes.MAX_RESOLUTION, "step": 16}),
- "height": ("INT", {"default": 480, "min": 16, "max": nodes.MAX_RESOLUTION, "step": 16}),
- "length": ("INT", {"default": 81, "min": 1, "max": nodes.MAX_RESOLUTION, "step": 4}),
- "batch_size": ("INT", {"default": 1, "min": 1, "max": 4096}),
- },
- "optional": {"clip_vision_output": ("CLIP_VISION_OUTPUT", ),
- "start_image": ("IMAGE", ),
- "end_image": ("IMAGE", ),
- }}
+ def define_schema(cls):
+ return io.Schema(
+ node_id="WanFunInpaintToVideo",
+ category="conditioning/video_models",
+ inputs=[
+ io.Conditioning.Input("positive"),
+ io.Conditioning.Input("negative"),
+ io.Vae.Input("vae"),
+ io.Int.Input("width", default=832, min=16, max=nodes.MAX_RESOLUTION, step=16),
+ io.Int.Input("height", default=480, min=16, max=nodes.MAX_RESOLUTION, step=16),
+ io.Int.Input("length", default=81, min=1, max=nodes.MAX_RESOLUTION, step=4),
+ io.Int.Input("batch_size", default=1, min=1, max=4096),
+ io.ClipVisionOutput.Input("clip_vision_output", optional=True),
+ io.Image.Input("start_image", optional=True),
+ io.Image.Input("end_image", optional=True),
+ ],
+ outputs=[
+ io.Conditioning.Output(display_name="positive"),
+ io.Conditioning.Output(display_name="negative"),
+ io.Latent.Output(display_name="latent"),
+ ],
+ )
- RETURN_TYPES = ("CONDITIONING", "CONDITIONING", "LATENT")
- RETURN_NAMES = ("positive", "negative", "latent")
- FUNCTION = "encode"
-
- CATEGORY = "conditioning/video_models"
-
- def encode(self, positive, negative, vae, width, height, length, batch_size, start_image=None, end_image=None, clip_vision_output=None):
+ @classmethod
+ def execute(cls, positive, negative, vae, width, height, length, batch_size, start_image=None, end_image=None, clip_vision_output=None) -> io.NodeOutput:
flfv = WanFirstLastFrameToVideo()
- return flfv.encode(positive, negative, vae, width, height, length, batch_size, start_image=start_image, end_image=end_image, clip_vision_start_image=clip_vision_output)
+ return flfv.execute(positive, negative, vae, width, height, length, batch_size, start_image=start_image, end_image=end_image, clip_vision_start_image=clip_vision_output)
-class WanVaceToVideo:
+class WanVaceToVideo(io.ComfyNode):
@classmethod
- def INPUT_TYPES(s):
- return {"required": {"positive": ("CONDITIONING", ),
- "negative": ("CONDITIONING", ),
- "vae": ("VAE", ),
- "width": ("INT", {"default": 832, "min": 16, "max": nodes.MAX_RESOLUTION, "step": 16}),
- "height": ("INT", {"default": 480, "min": 16, "max": nodes.MAX_RESOLUTION, "step": 16}),
- "length": ("INT", {"default": 81, "min": 1, "max": nodes.MAX_RESOLUTION, "step": 4}),
- "batch_size": ("INT", {"default": 1, "min": 1, "max": 4096}),
- "strength": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 1000.0, "step": 0.01}),
- },
- "optional": {"control_video": ("IMAGE", ),
- "control_masks": ("MASK", ),
- "reference_image": ("IMAGE", ),
- }}
+ def define_schema(cls):
+ return io.Schema(
+ node_id="WanVaceToVideo",
+ category="conditioning/video_models",
+ is_experimental=True,
+ inputs=[
+ io.Conditioning.Input("positive"),
+ io.Conditioning.Input("negative"),
+ io.Vae.Input("vae"),
+ io.Int.Input("width", default=832, min=16, max=nodes.MAX_RESOLUTION, step=16),
+ io.Int.Input("height", default=480, min=16, max=nodes.MAX_RESOLUTION, step=16),
+ io.Int.Input("length", default=81, min=1, max=nodes.MAX_RESOLUTION, step=4),
+ io.Int.Input("batch_size", default=1, min=1, max=4096),
+ io.Float.Input("strength", default=1.0, min=0.0, max=1000.0, step=0.01),
+ io.Image.Input("control_video", optional=True),
+ io.Mask.Input("control_masks", optional=True),
+ io.Image.Input("reference_image", optional=True),
+ ],
+ outputs=[
+ io.Conditioning.Output(display_name="positive"),
+ io.Conditioning.Output(display_name="negative"),
+ io.Latent.Output(display_name="latent"),
+ io.Int.Output(display_name="trim_latent"),
+ ],
+ )
- RETURN_TYPES = ("CONDITIONING", "CONDITIONING", "LATENT", "INT")
- RETURN_NAMES = ("positive", "negative", "latent", "trim_latent")
- FUNCTION = "encode"
-
- CATEGORY = "conditioning/video_models"
-
- EXPERIMENTAL = True
-
- def encode(self, positive, negative, vae, width, height, length, batch_size, strength, control_video=None, control_masks=None, reference_image=None):
+ @classmethod
+ def execute(cls, positive, negative, vae, width, height, length, batch_size, strength, control_video=None, control_masks=None, reference_image=None) -> io.NodeOutput:
latent_length = ((length - 1) // 4) + 1
if control_video is not None:
control_video = comfy.utils.common_upscale(control_video[:length].movedim(-1, 1), width, height, "bilinear", "center").movedim(1, -1)
@@ -278,52 +367,59 @@ class WanVaceToVideo:
latent = torch.zeros([batch_size, 16, latent_length, height // 8, width // 8], device=comfy.model_management.intermediate_device())
out_latent = {}
out_latent["samples"] = latent
- return (positive, negative, out_latent, trim_latent)
+ return io.NodeOutput(positive, negative, out_latent, trim_latent)
-class TrimVideoLatent:
+class TrimVideoLatent(io.ComfyNode):
@classmethod
- def INPUT_TYPES(s):
- return {"required": { "samples": ("LATENT",),
- "trim_amount": ("INT", {"default": 0, "min": 0, "max": 99999}),
- }}
+ def define_schema(cls):
+ return io.Schema(
+ node_id="TrimVideoLatent",
+ category="latent/video",
+ is_experimental=True,
+ inputs=[
+ io.Latent.Input("samples"),
+ io.Int.Input("trim_amount", default=0, min=0, max=99999),
+ ],
+ outputs=[
+ io.Latent.Output(),
+ ],
+ )
- RETURN_TYPES = ("LATENT",)
- FUNCTION = "op"
-
- CATEGORY = "latent/video"
-
- EXPERIMENTAL = True
-
- def op(self, samples, trim_amount):
+ @classmethod
+ def execute(cls, samples, trim_amount) -> io.NodeOutput:
samples_out = samples.copy()
s1 = samples["samples"]
samples_out["samples"] = s1[:, :, trim_amount:]
- return (samples_out,)
+ return io.NodeOutput(samples_out)
-class WanCameraImageToVideo:
+class WanCameraImageToVideo(io.ComfyNode):
@classmethod
- def INPUT_TYPES(s):
- return {"required": {"positive": ("CONDITIONING", ),
- "negative": ("CONDITIONING", ),
- "vae": ("VAE", ),
- "width": ("INT", {"default": 832, "min": 16, "max": nodes.MAX_RESOLUTION, "step": 16}),
- "height": ("INT", {"default": 480, "min": 16, "max": nodes.MAX_RESOLUTION, "step": 16}),
- "length": ("INT", {"default": 81, "min": 1, "max": nodes.MAX_RESOLUTION, "step": 4}),
- "batch_size": ("INT", {"default": 1, "min": 1, "max": 4096}),
- },
- "optional": {"clip_vision_output": ("CLIP_VISION_OUTPUT", ),
- "start_image": ("IMAGE", ),
- "camera_conditions": ("WAN_CAMERA_EMBEDDING", ),
- }}
+ def define_schema(cls):
+ return io.Schema(
+ node_id="WanCameraImageToVideo",
+ category="conditioning/video_models",
+ inputs=[
+ io.Conditioning.Input("positive"),
+ io.Conditioning.Input("negative"),
+ io.Vae.Input("vae"),
+ io.Int.Input("width", default=832, min=16, max=nodes.MAX_RESOLUTION, step=16),
+ io.Int.Input("height", default=480, min=16, max=nodes.MAX_RESOLUTION, step=16),
+ io.Int.Input("length", default=81, min=1, max=nodes.MAX_RESOLUTION, step=4),
+ io.Int.Input("batch_size", default=1, min=1, max=4096),
+ io.ClipVisionOutput.Input("clip_vision_output", optional=True),
+ io.Image.Input("start_image", optional=True),
+ io.WanCameraEmbedding.Input("camera_conditions", optional=True),
+ ],
+ outputs=[
+ io.Conditioning.Output(display_name="positive"),
+ io.Conditioning.Output(display_name="negative"),
+ io.Latent.Output(display_name="latent"),
+ ],
+ )
- RETURN_TYPES = ("CONDITIONING", "CONDITIONING", "LATENT")
- RETURN_NAMES = ("positive", "negative", "latent")
- FUNCTION = "encode"
-
- CATEGORY = "conditioning/video_models"
-
- def encode(self, positive, negative, vae, width, height, length, batch_size, start_image=None, clip_vision_output=None, camera_conditions=None):
+ @classmethod
+ def execute(cls, positive, negative, vae, width, height, length, batch_size, start_image=None, clip_vision_output=None, camera_conditions=None) -> io.NodeOutput:
latent = torch.zeros([batch_size, 16, ((length - 1) // 4) + 1, height // 8, width // 8], device=comfy.model_management.intermediate_device())
concat_latent = torch.zeros([batch_size, 16, ((length - 1) // 4) + 1, height // 8, width // 8], device=comfy.model_management.intermediate_device())
concat_latent = comfy.latent_formats.Wan21().process_out(concat_latent)
@@ -332,9 +428,12 @@ class WanCameraImageToVideo:
start_image = comfy.utils.common_upscale(start_image[:length].movedim(-1, 1), width, height, "bilinear", "center").movedim(1, -1)
concat_latent_image = vae.encode(start_image[:, :, :, :3])
concat_latent[:,:,:concat_latent_image.shape[2]] = concat_latent_image[:,:,:concat_latent.shape[2]]
+ mask = torch.ones((1, 1, latent.shape[2] * 4, latent.shape[-2], latent.shape[-1]))
+ mask[:, :, :start_image.shape[0] + 3] = 0.0
+ mask = mask.view(1, mask.shape[2] // 4, 4, mask.shape[3], mask.shape[4]).transpose(1, 2)
- positive = node_helpers.conditioning_set_values(positive, {"concat_latent_image": concat_latent})
- negative = node_helpers.conditioning_set_values(negative, {"concat_latent_image": concat_latent})
+ positive = node_helpers.conditioning_set_values(positive, {"concat_latent_image": concat_latent, "concat_mask": mask})
+ negative = node_helpers.conditioning_set_values(negative, {"concat_latent_image": concat_latent, "concat_mask": mask})
if camera_conditions is not None:
positive = node_helpers.conditioning_set_values(positive, {'camera_conditions': camera_conditions})
@@ -346,29 +445,34 @@ class WanCameraImageToVideo:
out_latent = {}
out_latent["samples"] = latent
- return (positive, negative, out_latent)
+ return io.NodeOutput(positive, negative, out_latent)
-class WanPhantomSubjectToVideo:
+class WanPhantomSubjectToVideo(io.ComfyNode):
@classmethod
- def INPUT_TYPES(s):
- return {"required": {"positive": ("CONDITIONING", ),
- "negative": ("CONDITIONING", ),
- "vae": ("VAE", ),
- "width": ("INT", {"default": 832, "min": 16, "max": nodes.MAX_RESOLUTION, "step": 16}),
- "height": ("INT", {"default": 480, "min": 16, "max": nodes.MAX_RESOLUTION, "step": 16}),
- "length": ("INT", {"default": 81, "min": 1, "max": nodes.MAX_RESOLUTION, "step": 4}),
- "batch_size": ("INT", {"default": 1, "min": 1, "max": 4096}),
- },
- "optional": {"images": ("IMAGE", ),
- }}
+ def define_schema(cls):
+ return io.Schema(
+ node_id="WanPhantomSubjectToVideo",
+ category="conditioning/video_models",
+ inputs=[
+ io.Conditioning.Input("positive"),
+ io.Conditioning.Input("negative"),
+ io.Vae.Input("vae"),
+ io.Int.Input("width", default=832, min=16, max=nodes.MAX_RESOLUTION, step=16),
+ io.Int.Input("height", default=480, min=16, max=nodes.MAX_RESOLUTION, step=16),
+ io.Int.Input("length", default=81, min=1, max=nodes.MAX_RESOLUTION, step=4),
+ io.Int.Input("batch_size", default=1, min=1, max=4096),
+ io.Image.Input("images", optional=True),
+ ],
+ outputs=[
+ io.Conditioning.Output(display_name="positive"),
+ io.Conditioning.Output(display_name="negative_text"),
+ io.Conditioning.Output(display_name="negative_img_text"),
+ io.Latent.Output(display_name="latent"),
+ ],
+ )
- RETURN_TYPES = ("CONDITIONING", "CONDITIONING", "CONDITIONING", "LATENT")
- RETURN_NAMES = ("positive", "negative_text", "negative_img_text", "latent")
- FUNCTION = "encode"
-
- CATEGORY = "conditioning/video_models"
-
- def encode(self, positive, negative, vae, width, height, length, batch_size, images):
+ @classmethod
+ def execute(cls, positive, negative, vae, width, height, length, batch_size, images) -> io.NodeOutput:
latent = torch.zeros([batch_size, 16, ((length - 1) // 4) + 1, height // 8, width // 8], device=comfy.model_management.intermediate_device())
cond2 = negative
if images is not None:
@@ -384,7 +488,7 @@ class WanPhantomSubjectToVideo:
out_latent = {}
out_latent["samples"] = latent
- return (positive, cond2, negative, out_latent)
+ return io.NodeOutput(positive, cond2, negative, out_latent)
def parse_json_tracks(tracks):
"""Parse JSON track data into a standardized format"""
@@ -597,39 +701,41 @@ def patch_motion(
return out_mask_full, out_feature_full
-class WanTrackToVideo:
+class WanTrackToVideo(io.ComfyNode):
@classmethod
- def INPUT_TYPES(s):
- return {"required": {
- "positive": ("CONDITIONING", ),
- "negative": ("CONDITIONING", ),
- "vae": ("VAE", ),
- "tracks": ("STRING", {"multiline": True, "default": "[]"}),
- "width": ("INT", {"default": 832, "min": 16, "max": nodes.MAX_RESOLUTION, "step": 16}),
- "height": ("INT", {"default": 480, "min": 16, "max": nodes.MAX_RESOLUTION, "step": 16}),
- "length": ("INT", {"default": 81, "min": 1, "max": nodes.MAX_RESOLUTION, "step": 4}),
- "batch_size": ("INT", {"default": 1, "min": 1, "max": 4096}),
- "temperature": ("FLOAT", {"default": 220.0, "min": 1.0, "max": 1000.0, "step": 0.1}),
- "topk": ("INT", {"default": 2, "min": 1, "max": 10}),
- "start_image": ("IMAGE", ),
- },
- "optional": {
- "clip_vision_output": ("CLIP_VISION_OUTPUT", ),
- }}
+ def define_schema(cls):
+ return io.Schema(
+ node_id="WanTrackToVideo",
+ category="conditioning/video_models",
+ inputs=[
+ io.Conditioning.Input("positive"),
+ io.Conditioning.Input("negative"),
+ io.Vae.Input("vae"),
+ io.String.Input("tracks", multiline=True, default="[]"),
+ io.Int.Input("width", default=832, min=16, max=nodes.MAX_RESOLUTION, step=16),
+ io.Int.Input("height", default=480, min=16, max=nodes.MAX_RESOLUTION, step=16),
+ io.Int.Input("length", default=81, min=1, max=nodes.MAX_RESOLUTION, step=4),
+ io.Int.Input("batch_size", default=1, min=1, max=4096),
+ io.Float.Input("temperature", default=220.0, min=1.0, max=1000.0, step=0.1),
+ io.Int.Input("topk", default=2, min=1, max=10),
+ io.Image.Input("start_image"),
+ io.ClipVisionOutput.Input("clip_vision_output", optional=True),
+ ],
+ outputs=[
+ io.Conditioning.Output(display_name="positive"),
+ io.Conditioning.Output(display_name="negative"),
+ io.Latent.Output(display_name="latent"),
+ ],
+ )
- RETURN_TYPES = ("CONDITIONING", "CONDITIONING", "LATENT")
- RETURN_NAMES = ("positive", "negative", "latent")
- FUNCTION = "encode"
-
- CATEGORY = "conditioning/video_models"
-
- def encode(self, positive, negative, vae, tracks, width, height, length, batch_size,
- temperature, topk, start_image=None, clip_vision_output=None):
+ @classmethod
+ def execute(cls, positive, negative, vae, tracks, width, height, length, batch_size,
+ temperature, topk, start_image=None, clip_vision_output=None) -> io.NodeOutput:
tracks_data = parse_json_tracks(tracks)
if not tracks_data:
- return WanImageToVideo().encode(positive, negative, vae, width, height, length, batch_size, start_image=start_image, clip_vision_output=clip_vision_output)
+ return WanImageToVideo().execute(positive, negative, vae, width, height, length, batch_size, start_image=start_image, clip_vision_output=clip_vision_output)
latent = torch.zeros([batch_size, 16, ((length - 1) // 4) + 1, height // 8, width // 8],
device=comfy.model_management.intermediate_device())
@@ -683,34 +789,259 @@ class WanTrackToVideo:
out_latent = {}
out_latent["samples"] = latent
- return (positive, negative, out_latent)
+ return io.NodeOutput(positive, negative, out_latent)
-class Wan22ImageToVideoLatent:
+def linear_interpolation(features, input_fps, output_fps, output_len=None):
+ """
+ features: shape=[1, T, 512]
+ input_fps: fps for audio, f_a
+ output_fps: fps for video, f_m
+ output_len: video length
+ """
+ features = features.transpose(1, 2) # [1, 512, T]
+ seq_len = features.shape[2] / float(input_fps) # T/f_a
+ if output_len is None:
+ output_len = int(seq_len * output_fps) # f_m*T/f_a
+ output_features = torch.nn.functional.interpolate(
+ features, size=output_len, align_corners=True,
+ mode='linear') # [1, 512, output_len]
+ return output_features.transpose(1, 2) # [1, output_len, 512]
+
+
+def get_sample_indices(original_fps,
+ total_frames,
+ target_fps,
+ num_sample,
+ fixed_start=None):
+ required_duration = num_sample / target_fps
+ required_origin_frames = int(np.ceil(required_duration * original_fps))
+ if required_duration > total_frames / original_fps:
+ raise ValueError("required_duration must be less than video length")
+
+ if not fixed_start is None and fixed_start >= 0:
+ start_frame = fixed_start
+ else:
+ max_start = total_frames - required_origin_frames
+ if max_start < 0:
+ raise ValueError("video length is too short")
+ start_frame = np.random.randint(0, max_start + 1)
+ start_time = start_frame / original_fps
+
+ end_time = start_time + required_duration
+ time_points = np.linspace(start_time, end_time, num_sample, endpoint=False)
+
+ frame_indices = np.round(np.array(time_points) * original_fps).astype(int)
+ frame_indices = np.clip(frame_indices, 0, total_frames - 1)
+ return frame_indices
+
+
+def get_audio_embed_bucket_fps(audio_embed, fps=16, batch_frames=81, m=0, video_rate=30):
+ num_layers, audio_frame_num, audio_dim = audio_embed.shape
+
+ if num_layers > 1:
+ return_all_layers = True
+ else:
+ return_all_layers = False
+
+ scale = video_rate / fps
+
+ min_batch_num = int(audio_frame_num / (batch_frames * scale)) + 1
+
+ bucket_num = min_batch_num * batch_frames
+ padd_audio_num = math.ceil(min_batch_num * batch_frames / fps * video_rate) - audio_frame_num
+ batch_idx = get_sample_indices(
+ original_fps=video_rate,
+ total_frames=audio_frame_num + padd_audio_num,
+ target_fps=fps,
+ num_sample=bucket_num,
+ fixed_start=0)
+ batch_audio_eb = []
+ audio_sample_stride = int(video_rate / fps)
+ for bi in batch_idx:
+ if bi < audio_frame_num:
+
+ chosen_idx = list(
+ range(bi - m * audio_sample_stride, bi + (m + 1) * audio_sample_stride, audio_sample_stride))
+ chosen_idx = [0 if c < 0 else c for c in chosen_idx]
+ chosen_idx = [
+ audio_frame_num - 1 if c >= audio_frame_num else c
+ for c in chosen_idx
+ ]
+
+ if return_all_layers:
+ frame_audio_embed = audio_embed[:, chosen_idx].flatten(
+ start_dim=-2, end_dim=-1)
+ else:
+ frame_audio_embed = audio_embed[0][chosen_idx].flatten()
+ else:
+ frame_audio_embed = torch.zeros([audio_dim * (2 * m + 1)], device=audio_embed.device) if not return_all_layers \
+ else torch.zeros([num_layers, audio_dim * (2 * m + 1)], device=audio_embed.device)
+ batch_audio_eb.append(frame_audio_embed)
+ batch_audio_eb = torch.cat([c.unsqueeze(0) for c in batch_audio_eb], dim=0)
+
+ return batch_audio_eb, min_batch_num
+
+
+def wan_sound_to_video(positive, negative, vae, width, height, length, batch_size, frame_offset=0, ref_image=None, audio_encoder_output=None, control_video=None, ref_motion=None, ref_motion_latent=None):
+ latent_t = ((length - 1) // 4) + 1
+ if audio_encoder_output is not None:
+ feat = torch.cat(audio_encoder_output["encoded_audio_all_layers"])
+ video_rate = 30
+ fps = 16
+ feat = linear_interpolation(feat, input_fps=50, output_fps=video_rate)
+ batch_frames = latent_t * 4
+ audio_embed_bucket, num_repeat = get_audio_embed_bucket_fps(feat, fps=fps, batch_frames=batch_frames, m=0, video_rate=video_rate)
+ audio_embed_bucket = audio_embed_bucket.unsqueeze(0)
+ if len(audio_embed_bucket.shape) == 3:
+ audio_embed_bucket = audio_embed_bucket.permute(0, 2, 1)
+ elif len(audio_embed_bucket.shape) == 4:
+ audio_embed_bucket = audio_embed_bucket.permute(0, 2, 3, 1)
+
+ audio_embed_bucket = audio_embed_bucket[:, :, :, frame_offset:frame_offset + batch_frames]
+ if audio_embed_bucket.shape[3] > 0:
+ positive = node_helpers.conditioning_set_values(positive, {"audio_embed": audio_embed_bucket})
+ negative = node_helpers.conditioning_set_values(negative, {"audio_embed": audio_embed_bucket * 0.0})
+ frame_offset += batch_frames
+
+ if ref_image is not None:
+ ref_image = comfy.utils.common_upscale(ref_image[:1].movedim(-1, 1), width, height, "bilinear", "center").movedim(1, -1)
+ ref_latent = vae.encode(ref_image[:, :, :, :3])
+ positive = node_helpers.conditioning_set_values(positive, {"reference_latents": [ref_latent]}, append=True)
+ negative = node_helpers.conditioning_set_values(negative, {"reference_latents": [ref_latent]}, append=True)
+
+ if ref_motion is not None:
+ if ref_motion.shape[0] > 73:
+ ref_motion = ref_motion[-73:]
+
+ ref_motion = comfy.utils.common_upscale(ref_motion.movedim(-1, 1), width, height, "bilinear", "center").movedim(1, -1)
+
+ if ref_motion.shape[0] < 73:
+ r = torch.ones([73, height, width, 3]) * 0.5
+ r[-ref_motion.shape[0]:] = ref_motion
+ ref_motion = r
+
+ ref_motion_latent = vae.encode(ref_motion[:, :, :, :3])
+
+ if ref_motion_latent is not None:
+ ref_motion_latent = ref_motion_latent[:, :, -19:]
+ positive = node_helpers.conditioning_set_values(positive, {"reference_motion": ref_motion_latent})
+ negative = node_helpers.conditioning_set_values(negative, {"reference_motion": ref_motion_latent})
+
+ latent = torch.zeros([batch_size, 16, latent_t, height // 8, width // 8], device=comfy.model_management.intermediate_device())
+
+ control_video_out = comfy.latent_formats.Wan21().process_out(torch.zeros_like(latent))
+ if control_video is not None:
+ control_video = comfy.utils.common_upscale(control_video[:length].movedim(-1, 1), width, height, "bilinear", "center").movedim(1, -1)
+ control_video = vae.encode(control_video[:, :, :, :3])
+ control_video_out[:, :, :control_video.shape[2]] = control_video
+
+ # TODO: check if zero is better than none if none provided
+ positive = node_helpers.conditioning_set_values(positive, {"control_video": control_video_out})
+ negative = node_helpers.conditioning_set_values(negative, {"control_video": control_video_out})
+
+ out_latent = {}
+ out_latent["samples"] = latent
+ return positive, negative, out_latent, frame_offset
+
+
+class WanSoundImageToVideo(io.ComfyNode):
@classmethod
- def INPUT_TYPES(s):
- return {"required": {"vae": ("VAE", ),
- "width": ("INT", {"default": 1280, "min": 32, "max": nodes.MAX_RESOLUTION, "step": 32}),
- "height": ("INT", {"default": 704, "min": 32, "max": nodes.MAX_RESOLUTION, "step": 32}),
- "length": ("INT", {"default": 49, "min": 1, "max": nodes.MAX_RESOLUTION, "step": 4}),
- "batch_size": ("INT", {"default": 1, "min": 1, "max": 4096}),
- },
- "optional": {"start_image": ("IMAGE", ),
- }}
+ def define_schema(cls):
+ return io.Schema(
+ node_id="WanSoundImageToVideo",
+ category="conditioning/video_models",
+ inputs=[
+ io.Conditioning.Input("positive"),
+ io.Conditioning.Input("negative"),
+ io.Vae.Input("vae"),
+ io.Int.Input("width", default=832, min=16, max=nodes.MAX_RESOLUTION, step=16),
+ io.Int.Input("height", default=480, min=16, max=nodes.MAX_RESOLUTION, step=16),
+ io.Int.Input("length", default=77, min=1, max=nodes.MAX_RESOLUTION, step=4),
+ io.Int.Input("batch_size", default=1, min=1, max=4096),
+ io.AudioEncoderOutput.Input("audio_encoder_output", optional=True),
+ io.Image.Input("ref_image", optional=True),
+ io.Image.Input("control_video", optional=True),
+ io.Image.Input("ref_motion", optional=True),
+ ],
+ outputs=[
+ io.Conditioning.Output(display_name="positive"),
+ io.Conditioning.Output(display_name="negative"),
+ io.Latent.Output(display_name="latent"),
+ ],
+ is_experimental=True,
+ )
+
+ @classmethod
+ def execute(cls, positive, negative, vae, width, height, length, batch_size, ref_image=None, audio_encoder_output=None, control_video=None, ref_motion=None) -> io.NodeOutput:
+ positive, negative, out_latent, frame_offset = wan_sound_to_video(positive, negative, vae, width, height, length, batch_size, ref_image=ref_image, audio_encoder_output=audio_encoder_output,
+ control_video=control_video, ref_motion=ref_motion)
+ return io.NodeOutput(positive, negative, out_latent)
- RETURN_TYPES = ("LATENT",)
- FUNCTION = "encode"
+class WanSoundImageToVideoExtend(io.ComfyNode):
+ @classmethod
+ def define_schema(cls):
+ return io.Schema(
+ node_id="WanSoundImageToVideoExtend",
+ category="conditioning/video_models",
+ inputs=[
+ io.Conditioning.Input("positive"),
+ io.Conditioning.Input("negative"),
+ io.Vae.Input("vae"),
+ io.Int.Input("length", default=77, min=1, max=nodes.MAX_RESOLUTION, step=4),
+ io.Latent.Input("video_latent"),
+ io.AudioEncoderOutput.Input("audio_encoder_output", optional=True),
+ io.Image.Input("ref_image", optional=True),
+ io.Image.Input("control_video", optional=True),
+ ],
+ outputs=[
+ io.Conditioning.Output(display_name="positive"),
+ io.Conditioning.Output(display_name="negative"),
+ io.Latent.Output(display_name="latent"),
+ ],
+ is_experimental=True,
+ )
- CATEGORY = "conditioning/inpaint"
+ @classmethod
+ def execute(cls, positive, negative, vae, length, video_latent, ref_image=None, audio_encoder_output=None, control_video=None) -> io.NodeOutput:
+ video_latent = video_latent["samples"]
+ width = video_latent.shape[-1] * 8
+ height = video_latent.shape[-2] * 8
+ batch_size = video_latent.shape[0]
+ frame_offset = video_latent.shape[-3] * 4
+ positive, negative, out_latent, frame_offset = wan_sound_to_video(positive, negative, vae, width, height, length, batch_size, frame_offset=frame_offset, ref_image=ref_image, audio_encoder_output=audio_encoder_output,
+ control_video=control_video, ref_motion=None, ref_motion_latent=video_latent)
+ return io.NodeOutput(positive, negative, out_latent)
- def encode(self, vae, width, height, length, batch_size, start_image=None):
+
+class Wan22ImageToVideoLatent(io.ComfyNode):
+ @classmethod
+ def define_schema(cls):
+ return io.Schema(
+ node_id="Wan22ImageToVideoLatent",
+ category="conditioning/inpaint",
+ inputs=[
+ io.Vae.Input("vae"),
+ io.Int.Input("width", default=1280, min=32, max=nodes.MAX_RESOLUTION, step=32),
+ io.Int.Input("height", default=704, min=32, max=nodes.MAX_RESOLUTION, step=32),
+ io.Int.Input("length", default=49, min=1, max=nodes.MAX_RESOLUTION, step=4),
+ io.Int.Input("batch_size", default=1, min=1, max=4096),
+ io.Image.Input("start_image", optional=True),
+ ],
+ outputs=[
+ io.Latent.Output(),
+ ],
+ )
+
+ @classmethod
+ def execute(cls, vae, width, height, length, batch_size, start_image=None) -> io.NodeOutput:
latent = torch.zeros([1, 48, ((length - 1) // 4) + 1, height // 16, width // 16], device=comfy.model_management.intermediate_device())
if start_image is None:
out_latent = {}
out_latent["samples"] = latent
- return (out_latent,)
+ return io.NodeOutput(out_latent)
mask = torch.ones([latent.shape[0], 1, ((length - 1) // 4) + 1, latent.shape[-2], latent.shape[-1]], device=comfy.model_management.intermediate_device())
@@ -725,18 +1056,27 @@ class Wan22ImageToVideoLatent:
latent = latent_format.process_out(latent) * mask + latent * (1.0 - mask)
out_latent["samples"] = latent.repeat((batch_size, ) + (1,) * (latent.ndim - 1))
out_latent["noise_mask"] = mask.repeat((batch_size, ) + (1,) * (mask.ndim - 1))
- return (out_latent,)
+ return io.NodeOutput(out_latent)
-NODE_CLASS_MAPPINGS = {
- "WanTrackToVideo": WanTrackToVideo,
- "WanImageToVideo": WanImageToVideo,
- "WanFunControlToVideo": WanFunControlToVideo,
- "WanFunInpaintToVideo": WanFunInpaintToVideo,
- "WanFirstLastFrameToVideo": WanFirstLastFrameToVideo,
- "WanVaceToVideo": WanVaceToVideo,
- "TrimVideoLatent": TrimVideoLatent,
- "WanCameraImageToVideo": WanCameraImageToVideo,
- "WanPhantomSubjectToVideo": WanPhantomSubjectToVideo,
- "Wan22ImageToVideoLatent": Wan22ImageToVideoLatent,
-}
+class WanExtension(ComfyExtension):
+ @override
+ async def get_node_list(self) -> list[type[io.ComfyNode]]:
+ return [
+ WanTrackToVideo,
+ WanImageToVideo,
+ WanFunControlToVideo,
+ Wan22FunControlToVideo,
+ WanFunInpaintToVideo,
+ WanFirstLastFrameToVideo,
+ WanVaceToVideo,
+ TrimVideoLatent,
+ WanCameraImageToVideo,
+ WanPhantomSubjectToVideo,
+ WanSoundImageToVideo,
+ WanSoundImageToVideoExtend,
+ Wan22ImageToVideoLatent,
+ ]
+
+async def comfy_entrypoint() -> WanExtension:
+ return WanExtension()
diff --git a/comfyui_version.py b/comfyui_version.py
index 315710dd2..e8e039373 100644
--- a/comfyui_version.py
+++ b/comfyui_version.py
@@ -1,3 +1,3 @@
# This file is automatically generated by the build process when version is
# updated in pyproject.toml.
-__version__ = "0.3.46"
+__version__ = "0.3.56"
diff --git a/execution.py b/execution.py
index 8a9663a7d..1dc35738b 100644
--- a/execution.py
+++ b/execution.py
@@ -7,7 +7,7 @@ import threading
import time
import traceback
from enum import Enum
-from typing import List, Literal, NamedTuple, Optional
+from typing import List, Literal, NamedTuple, Optional, Union
import asyncio
import torch
@@ -32,6 +32,8 @@ from comfy_execution.graph_utils import GraphBuilder, is_link
from comfy_execution.validation import validate_node_input
from comfy_execution.progress import get_progress_state, reset_progress_state, add_progress_handler, WebUIProgressHandler
from comfy_execution.utils import CurrentNodeContext
+from comfy_api.internal import _ComfyNodeInternal, _NodeOutputInternal, first_real_override, is_class, make_locked_method_func
+from comfy_api.latest import io
class ExecutionResult(Enum):
@@ -56,7 +58,15 @@ class IsChangedCache:
node = self.dynprompt.get_node(node_id)
class_type = node["class_type"]
class_def = nodes.NODE_CLASS_MAPPINGS[class_type]
- if not hasattr(class_def, "IS_CHANGED"):
+ has_is_changed = False
+ is_changed_name = None
+ if issubclass(class_def, _ComfyNodeInternal) and first_real_override(class_def, "fingerprint_inputs") is not None:
+ has_is_changed = True
+ is_changed_name = "fingerprint_inputs"
+ elif hasattr(class_def, "IS_CHANGED"):
+ has_is_changed = True
+ is_changed_name = "IS_CHANGED"
+ if not has_is_changed:
self.is_changed[node_id] = False
return self.is_changed[node_id]
@@ -65,9 +75,9 @@ class IsChangedCache:
return self.is_changed[node_id]
# Intentionally do not use cached outputs here. We only want constants in IS_CHANGED
- input_data_all, _ = get_input_data(node["inputs"], class_def, node_id, None)
+ input_data_all, _, hidden_inputs = get_input_data(node["inputs"], class_def, node_id, None)
try:
- is_changed = await _async_map_node_over_list(self.prompt_id, node_id, class_def, input_data_all, "IS_CHANGED")
+ is_changed = await _async_map_node_over_list(self.prompt_id, node_id, class_def, input_data_all, is_changed_name)
is_changed = await resolve_map_node_over_list_results(is_changed)
node["is_changed"] = [None if isinstance(x, ExecutionBlocker) else x for x in is_changed]
except Exception as e:
@@ -126,9 +136,14 @@ class CacheSet:
SENSITIVE_EXTRA_DATA_KEYS = ("auth_token_comfy_org", "api_key_comfy_org")
def get_input_data(inputs, class_def, unique_id, outputs=None, dynprompt=None, extra_data={}):
- valid_inputs = class_def.INPUT_TYPES()
+ is_v3 = issubclass(class_def, _ComfyNodeInternal)
+ if is_v3:
+ valid_inputs, schema = class_def.INPUT_TYPES(include_hidden=False, return_schema=True)
+ else:
+ valid_inputs = class_def.INPUT_TYPES()
input_data_all = {}
missing_keys = {}
+ hidden_inputs_v3 = {}
for x in inputs:
input_data = inputs[x]
_, input_category, input_info = get_input_info(class_def, x, valid_inputs)
@@ -153,22 +168,37 @@ def get_input_data(inputs, class_def, unique_id, outputs=None, dynprompt=None, e
elif input_category is not None:
input_data_all[x] = [input_data]
- if "hidden" in valid_inputs:
- h = valid_inputs["hidden"]
- for x in h:
- if h[x] == "PROMPT":
- input_data_all[x] = [dynprompt.get_original_prompt() if dynprompt is not None else {}]
- if h[x] == "DYNPROMPT":
- input_data_all[x] = [dynprompt]
- if h[x] == "EXTRA_PNGINFO":
- input_data_all[x] = [extra_data.get('extra_pnginfo', None)]
- if h[x] == "UNIQUE_ID":
- input_data_all[x] = [unique_id]
- if h[x] == "AUTH_TOKEN_COMFY_ORG":
- input_data_all[x] = [extra_data.get("auth_token_comfy_org", None)]
- if h[x] == "API_KEY_COMFY_ORG":
- input_data_all[x] = [extra_data.get("api_key_comfy_org", None)]
- return input_data_all, missing_keys
+ if is_v3:
+ if schema.hidden:
+ if io.Hidden.prompt in schema.hidden:
+ hidden_inputs_v3[io.Hidden.prompt] = dynprompt.get_original_prompt() if dynprompt is not None else {}
+ if io.Hidden.dynprompt in schema.hidden:
+ hidden_inputs_v3[io.Hidden.dynprompt] = dynprompt
+ if io.Hidden.extra_pnginfo in schema.hidden:
+ hidden_inputs_v3[io.Hidden.extra_pnginfo] = extra_data.get('extra_pnginfo', None)
+ if io.Hidden.unique_id in schema.hidden:
+ hidden_inputs_v3[io.Hidden.unique_id] = unique_id
+ if io.Hidden.auth_token_comfy_org in schema.hidden:
+ hidden_inputs_v3[io.Hidden.auth_token_comfy_org] = extra_data.get("auth_token_comfy_org", None)
+ if io.Hidden.api_key_comfy_org in schema.hidden:
+ hidden_inputs_v3[io.Hidden.api_key_comfy_org] = extra_data.get("api_key_comfy_org", None)
+ else:
+ if "hidden" in valid_inputs:
+ h = valid_inputs["hidden"]
+ for x in h:
+ if h[x] == "PROMPT":
+ input_data_all[x] = [dynprompt.get_original_prompt() if dynprompt is not None else {}]
+ if h[x] == "DYNPROMPT":
+ input_data_all[x] = [dynprompt]
+ if h[x] == "EXTRA_PNGINFO":
+ input_data_all[x] = [extra_data.get('extra_pnginfo', None)]
+ if h[x] == "UNIQUE_ID":
+ input_data_all[x] = [unique_id]
+ if h[x] == "AUTH_TOKEN_COMFY_ORG":
+ input_data_all[x] = [extra_data.get("auth_token_comfy_org", None)]
+ if h[x] == "API_KEY_COMFY_ORG":
+ input_data_all[x] = [extra_data.get("api_key_comfy_org", None)]
+ return input_data_all, missing_keys, hidden_inputs_v3
map_node_over_list = None #Don't hook this please
@@ -184,7 +214,7 @@ async def resolve_map_node_over_list_results(results):
raise exc
return [x.result() if isinstance(x, asyncio.Task) else x for x in results]
-async def _async_map_node_over_list(prompt_id, unique_id, obj, input_data_all, func, allow_interrupt=False, execution_block_cb=None, pre_execute_cb=None):
+async def _async_map_node_over_list(prompt_id, unique_id, obj, input_data_all, func, allow_interrupt=False, execution_block_cb=None, pre_execute_cb=None, hidden_inputs=None):
# check if node wants the lists
input_is_list = getattr(obj, "INPUT_IS_LIST", False)
@@ -214,7 +244,22 @@ async def _async_map_node_over_list(prompt_id, unique_id, obj, input_data_all, f
if execution_block is None:
if pre_execute_cb is not None and index is not None:
pre_execute_cb(index)
- f = getattr(obj, func)
+ # V3
+ if isinstance(obj, _ComfyNodeInternal) or (is_class(obj) and issubclass(obj, _ComfyNodeInternal)):
+ # if is just a class, then assign no resources or state, just create clone
+ if is_class(obj):
+ type_obj = obj
+ obj.VALIDATE_CLASS()
+ class_clone = obj.PREPARE_CLASS_CLONE(hidden_inputs)
+ # otherwise, use class instance to populate/reuse some fields
+ else:
+ type_obj = type(obj)
+ type_obj.VALIDATE_CLASS()
+ class_clone = type_obj.PREPARE_CLASS_CLONE(hidden_inputs)
+ f = make_locked_method_func(type_obj, func, class_clone)
+ # V1
+ else:
+ f = getattr(obj, func)
if inspect.iscoroutinefunction(f):
async def async_wrapper(f, prompt_id, unique_id, list_index, args):
with CurrentNodeContext(prompt_id, unique_id, list_index):
@@ -266,8 +311,8 @@ def merge_result_data(results, obj):
output.append([o[i] for o in results])
return output
-async def get_output_data(prompt_id, unique_id, obj, input_data_all, execution_block_cb=None, pre_execute_cb=None):
- return_values = await _async_map_node_over_list(prompt_id, unique_id, obj, input_data_all, obj.FUNCTION, allow_interrupt=True, execution_block_cb=execution_block_cb, pre_execute_cb=pre_execute_cb)
+async def get_output_data(prompt_id, unique_id, obj, input_data_all, execution_block_cb=None, pre_execute_cb=None, hidden_inputs=None):
+ return_values = await _async_map_node_over_list(prompt_id, unique_id, obj, input_data_all, obj.FUNCTION, allow_interrupt=True, execution_block_cb=execution_block_cb, pre_execute_cb=pre_execute_cb, hidden_inputs=hidden_inputs)
has_pending_task = any(isinstance(r, asyncio.Task) and not r.done() for r in return_values)
if has_pending_task:
return return_values, {}, False, has_pending_task
@@ -298,6 +343,26 @@ def get_output_from_returns(return_values, obj):
result = tuple([result] * len(obj.RETURN_TYPES))
results.append(result)
subgraph_results.append((None, result))
+ elif isinstance(r, _NodeOutputInternal):
+ # V3
+ if r.ui is not None:
+ if isinstance(r.ui, dict):
+ uis.append(r.ui)
+ else:
+ uis.append(r.ui.as_dict())
+ if r.expand is not None:
+ has_subgraph = True
+ new_graph = r.expand
+ result = r.result
+ if r.block_execution is not None:
+ result = tuple([ExecutionBlocker(r.block_execution)] * len(obj.RETURN_TYPES))
+ subgraph_results.append((new_graph, result))
+ elif r.result is not None:
+ result = r.result
+ if r.block_execution is not None:
+ result = tuple([ExecutionBlocker(r.block_execution)] * len(obj.RETURN_TYPES))
+ results.append(result)
+ subgraph_results.append((None, result))
else:
if isinstance(r, ExecutionBlocker):
r = tuple([r] * len(obj.RETURN_TYPES))
@@ -381,7 +446,7 @@ async def execute(server, dynprompt, caches, current_item, extra_data, executed,
has_subgraph = False
else:
get_progress_state().start_progress(unique_id)
- input_data_all, missing_keys = get_input_data(inputs, class_def, unique_id, caches.outputs, dynprompt, extra_data)
+ input_data_all, missing_keys, hidden_inputs = get_input_data(inputs, class_def, unique_id, caches.outputs, dynprompt, extra_data)
if server.client_id is not None:
server.last_node_id = display_node_id
server.send_sync("executing", { "node": unique_id, "display_node": display_node_id, "prompt_id": prompt_id }, server.client_id)
@@ -391,8 +456,12 @@ async def execute(server, dynprompt, caches, current_item, extra_data, executed,
obj = class_def()
caches.objects.set(unique_id, obj)
- if hasattr(obj, "check_lazy_status"):
- required_inputs = await _async_map_node_over_list(prompt_id, unique_id, obj, input_data_all, "check_lazy_status", allow_interrupt=True)
+ if issubclass(class_def, _ComfyNodeInternal):
+ lazy_status_present = first_real_override(class_def, "check_lazy_status") is not None
+ else:
+ lazy_status_present = getattr(obj, "check_lazy_status", None) is not None
+ if lazy_status_present:
+ required_inputs = await _async_map_node_over_list(prompt_id, unique_id, obj, input_data_all, "check_lazy_status", allow_interrupt=True, hidden_inputs=hidden_inputs)
required_inputs = await resolve_map_node_over_list_results(required_inputs)
required_inputs = set(sum([r for r in required_inputs if isinstance(r,list)], []))
required_inputs = [x for x in required_inputs if isinstance(x,str) and (
@@ -424,7 +493,7 @@ async def execute(server, dynprompt, caches, current_item, extra_data, executed,
def pre_execute_cb(call_index):
# TODO - How to handle this with async functions without contextvars (which requires Python 3.12)?
GraphBuilder.set_default_prefix(unique_id, call_index, 0)
- output_data, output_ui, has_subgraph, has_pending_tasks = await get_output_data(prompt_id, unique_id, obj, input_data_all, execution_block_cb=execution_block_cb, pre_execute_cb=pre_execute_cb)
+ output_data, output_ui, has_subgraph, has_pending_tasks = await get_output_data(prompt_id, unique_id, obj, input_data_all, execution_block_cb=execution_block_cb, pre_execute_cb=pre_execute_cb, hidden_inputs=hidden_inputs)
if has_pending_tasks:
pending_async_nodes[unique_id] = output_data
unblock = execution_list.add_external_block(unique_id)
@@ -577,8 +646,6 @@ class PromptExecutor:
self.add_message("execution_error", mes, broadcast=False)
def execute(self, prompt, prompt_id, extra_data={}, execute_outputs=[]):
- asyncio_loop = asyncio.new_event_loop()
- asyncio.set_event_loop(asyncio_loop)
asyncio.run(self.execute_async(prompt, prompt_id, extra_data, execute_outputs))
async def execute_async(self, prompt, prompt_id, extra_data={}, execute_outputs=[]):
@@ -672,8 +739,14 @@ async def validate_inputs(prompt_id, prompt, item, validated):
validate_function_inputs = []
validate_has_kwargs = False
- if hasattr(obj_class, "VALIDATE_INPUTS"):
- argspec = inspect.getfullargspec(obj_class.VALIDATE_INPUTS)
+ if issubclass(obj_class, _ComfyNodeInternal):
+ validate_function_name = "validate_inputs"
+ validate_function = first_real_override(obj_class, validate_function_name)
+ else:
+ validate_function_name = "VALIDATE_INPUTS"
+ validate_function = getattr(obj_class, validate_function_name, None)
+ if validate_function is not None:
+ argspec = inspect.getfullargspec(validate_function)
validate_function_inputs = argspec.args
validate_has_kwargs = argspec.varkw is not None
received_types = {}
@@ -848,7 +921,7 @@ async def validate_inputs(prompt_id, prompt, item, validated):
continue
if len(validate_function_inputs) > 0 or validate_has_kwargs:
- input_data_all, _ = get_input_data(inputs, obj_class, unique_id)
+ input_data_all, _, hidden_inputs = get_input_data(inputs, obj_class, unique_id)
input_filtered = {}
for x in input_data_all:
if x in validate_function_inputs or validate_has_kwargs:
@@ -856,8 +929,7 @@ async def validate_inputs(prompt_id, prompt, item, validated):
if 'input_types' in validate_function_inputs:
input_filtered['input_types'] = [received_types]
- #ret = obj_class.VALIDATE_INPUTS(**input_filtered)
- ret = await _async_map_node_over_list(prompt_id, unique_id, obj_class, input_filtered, "VALIDATE_INPUTS")
+ ret = await _async_map_node_over_list(prompt_id, unique_id, obj_class, input_filtered, validate_function_name, hidden_inputs=hidden_inputs)
ret = await resolve_map_node_over_list_results(ret)
for x in input_filtered:
for i, r in enumerate(ret):
@@ -891,7 +963,7 @@ def full_type_name(klass):
return klass.__qualname__
return module + '.' + klass.__qualname__
-async def validate_prompt(prompt_id, prompt):
+async def validate_prompt(prompt_id, prompt, partial_execution_list: Union[list[str], None]):
outputs = set()
for x in prompt:
if 'class_type' not in prompt[x]:
@@ -915,7 +987,8 @@ async def validate_prompt(prompt_id, prompt):
return (False, error, [], {})
if hasattr(class_, 'OUTPUT_NODE') and class_.OUTPUT_NODE is True:
- outputs.add(x)
+ if partial_execution_list is None or x in partial_execution_list:
+ outputs.add(x)
if len(outputs) == 0:
error = {
diff --git a/folder_paths.py b/folder_paths.py
index 9ec952940..f110d832b 100644
--- a/folder_paths.py
+++ b/folder_paths.py
@@ -46,6 +46,10 @@ folder_names_and_paths["photomaker"] = ([os.path.join(models_dir, "photomaker")]
folder_names_and_paths["classifiers"] = ([os.path.join(models_dir, "classifiers")], {""})
+folder_names_and_paths["model_patches"] = ([os.path.join(models_dir, "model_patches")], supported_pt_extensions)
+
+folder_names_and_paths["audio_encoders"] = ([os.path.join(models_dir, "audio_encoders")], supported_pt_extensions)
+
output_directory = os.path.join(base_path, "output")
temp_directory = os.path.join(base_path, "temp")
input_directory = os.path.join(base_path, "input")
diff --git a/main.py b/main.py
index e8ca8152a..c33f0e17b 100644
--- a/main.py
+++ b/main.py
@@ -112,7 +112,7 @@ import gc
if os.name == "nt":
- logging.getLogger("xformers").addFilter(lambda record: 'A matching Triton is not available' not in record.getMessage())
+ os.environ['MIMALLOC_PURGE_DELAY'] = '0'
if __name__ == "__main__":
if args.default_device is not None:
@@ -313,10 +313,10 @@ def start_comfyui(asyncio_loop=None):
prompt_server = server.PromptServer(asyncio_loop)
hook_breaker_ac10a0.save_functions()
- nodes.init_extra_nodes(
+ asyncio_loop.run_until_complete(nodes.init_extra_nodes(
init_custom_nodes=(not args.disable_all_custom_nodes) or len(args.whitelist_custom_nodes) > 0,
init_api_nodes=not args.disable_api_nodes
- )
+ ))
hook_breaker_ac10a0.restore_functions()
cuda_malloc_warning()
diff --git a/models/audio_encoders/put_audio_encoder_models_here b/models/audio_encoders/put_audio_encoder_models_here
new file mode 100644
index 000000000..e69de29bb
diff --git a/models/model_patches/put_model_patches_here b/models/model_patches/put_model_patches_here
new file mode 100644
index 000000000..e69de29bb
diff --git a/nodes.py b/nodes.py
index 70875eb65..cbc74291d 100644
--- a/nodes.py
+++ b/nodes.py
@@ -1,10 +1,12 @@
from __future__ import annotations
import torch
+
import os
import sys
import json
import hashlib
+import inspect
import traceback
import math
import time
@@ -26,6 +28,9 @@ import comfy.sd
import comfy.utils
import comfy.controlnet
from comfy.comfy_types import IO, ComfyNodeABC, InputTypeDict, FileLocator
+from comfy_api.internal import register_versions, ComfyAPIWithVersion
+from comfy_api.version_list import supported_versions
+from comfy_api.latest import io, ComfyExtension
import comfy.clip_vision
@@ -920,7 +925,7 @@ class CLIPLoader:
@classmethod
def INPUT_TYPES(s):
return {"required": { "clip_name": (folder_paths.get_filename_list("text_encoders"), ),
- "type": (["stable_diffusion", "stable_cascade", "sd3", "stable_audio", "mochi", "ltxv", "pixart", "cosmos", "lumina2", "wan", "hidream", "chroma", "ace", "omnigen2"], ),
+ "type": (["stable_diffusion", "stable_cascade", "sd3", "stable_audio", "mochi", "ltxv", "pixart", "cosmos", "lumina2", "wan", "hidream", "chroma", "ace", "omnigen2", "qwen_image"], ),
},
"optional": {
"device": (["default", "cpu"], {"advanced": True}),
@@ -1235,12 +1240,12 @@ class RepeatLatentBatch:
s = samples.copy()
s_in = samples["samples"]
- s["samples"] = s_in.repeat((amount, 1,1,1))
+ s["samples"] = s_in.repeat((amount,) + ((1,) * (s_in.ndim - 1)))
if "noise_mask" in samples and samples["noise_mask"].shape[0] > 1:
masks = samples["noise_mask"]
if masks.shape[0] < s_in.shape[0]:
- masks = masks.repeat(math.ceil(s_in.shape[0] / masks.shape[0]), 1, 1, 1)[:s_in.shape[0]]
- s["noise_mask"] = samples["noise_mask"].repeat((amount, 1,1,1))
+ masks = masks.repeat((math.ceil(s_in.shape[0] / masks.shape[0]),) + ((1,) * (masks.ndim - 1)))[:s_in.shape[0]]
+ s["noise_mask"] = samples["noise_mask"].repeat((amount,) + ((1,) * (samples["noise_mask"].ndim - 1)))
if "batch_index" in s:
offset = max(s["batch_index"]) - min(s["batch_index"]) + 1
s["batch_index"] = s["batch_index"] + [x + (i * offset) for i in range(1, amount) for x in s["batch_index"]]
@@ -2112,7 +2117,7 @@ def get_module_name(module_path: str) -> str:
return base_path
-def load_custom_node(module_path: str, ignore=set(), module_parent="custom_nodes") -> bool:
+async def load_custom_node(module_path: str, ignore=set(), module_parent="custom_nodes") -> bool:
module_name = get_module_name(module_path)
if os.path.isfile(module_path):
sp = os.path.splitext(module_path)
@@ -2160,6 +2165,7 @@ def load_custom_node(module_path: str, ignore=set(), module_parent="custom_nodes
if os.path.isdir(web_dir):
EXTENSION_WEB_DIRS[module_name] = web_dir
+ # V1 node definition
if hasattr(module, "NODE_CLASS_MAPPINGS") and getattr(module, "NODE_CLASS_MAPPINGS") is not None:
for name, node_cls in module.NODE_CLASS_MAPPINGS.items():
if name not in ignore:
@@ -2168,15 +2174,45 @@ def load_custom_node(module_path: str, ignore=set(), module_parent="custom_nodes
if hasattr(module, "NODE_DISPLAY_NAME_MAPPINGS") and getattr(module, "NODE_DISPLAY_NAME_MAPPINGS") is not None:
NODE_DISPLAY_NAME_MAPPINGS.update(module.NODE_DISPLAY_NAME_MAPPINGS)
return True
+ # V3 Extension Definition
+ elif hasattr(module, "comfy_entrypoint"):
+ entrypoint = getattr(module, "comfy_entrypoint")
+ if not callable(entrypoint):
+ logging.warning(f"comfy_entrypoint in {module_path} is not callable, skipping.")
+ return False
+ try:
+ if inspect.iscoroutinefunction(entrypoint):
+ extension = await entrypoint()
+ else:
+ extension = entrypoint()
+ if not isinstance(extension, ComfyExtension):
+ logging.warning(f"comfy_entrypoint in {module_path} did not return a ComfyExtension, skipping.")
+ return False
+ node_list = await extension.get_node_list()
+ if not isinstance(node_list, list):
+ logging.warning(f"comfy_entrypoint in {module_path} did not return a list of nodes, skipping.")
+ return False
+ for node_cls in node_list:
+ node_cls: io.ComfyNode
+ schema = node_cls.GET_SCHEMA()
+ if schema.node_id not in ignore:
+ NODE_CLASS_MAPPINGS[schema.node_id] = node_cls
+ node_cls.RELATIVE_PYTHON_MODULE = "{}.{}".format(module_parent, get_module_name(module_path))
+ if schema.display_name is not None:
+ NODE_DISPLAY_NAME_MAPPINGS[schema.node_id] = schema.display_name
+ return True
+ except Exception as e:
+ logging.warning(f"Error while calling comfy_entrypoint in {module_path}: {e}")
+ return False
else:
- logging.warning(f"Skip {module_path} module for custom nodes due to the lack of NODE_CLASS_MAPPINGS.")
+ logging.warning(f"Skip {module_path} module for custom nodes due to the lack of NODE_CLASS_MAPPINGS or NODES_LIST (need one).")
return False
except Exception as e:
logging.warning(traceback.format_exc())
logging.warning(f"Cannot import {module_path} module for custom nodes: {e}")
return False
-def init_external_custom_nodes():
+async def init_external_custom_nodes():
"""
Initializes the external custom nodes.
@@ -2202,7 +2238,7 @@ def init_external_custom_nodes():
logging.info(f"Skipping {possible_module} due to disable_all_custom_nodes and whitelist_custom_nodes")
continue
time_before = time.perf_counter()
- success = load_custom_node(module_path, base_node_names, module_parent="custom_nodes")
+ success = await load_custom_node(module_path, base_node_names, module_parent="custom_nodes")
node_import_times.append((time.perf_counter() - time_before, module_path, success))
if len(node_import_times) > 0:
@@ -2215,7 +2251,7 @@ def init_external_custom_nodes():
logging.info("{:6.1f} seconds{}: {}".format(n[0], import_message, n[1]))
logging.info("")
-def init_builtin_extra_nodes():
+async def init_builtin_extra_nodes():
"""
Initializes the built-in extra nodes in ComfyUI.
@@ -2294,18 +2330,23 @@ def init_builtin_extra_nodes():
"nodes_string.py",
"nodes_camera_trajectory.py",
"nodes_edit_model.py",
- "nodes_tcfg.py"
+ "nodes_tcfg.py",
+ "nodes_context_windows.py",
+ "nodes_qwen.py",
+ "nodes_model_patch.py",
+ "nodes_easycache.py",
+ "nodes_audio_encoder.py",
]
import_failed = []
for node_file in extras_files:
- if not load_custom_node(os.path.join(extras_dir, node_file), module_parent="comfy_extras"):
+ if not await load_custom_node(os.path.join(extras_dir, node_file), module_parent="comfy_extras"):
import_failed.append(node_file)
return import_failed
-def init_builtin_api_nodes():
+async def init_builtin_api_nodes():
api_nodes_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), "comfy_api_nodes")
api_nodes_files = [
"nodes_ideogram.py",
@@ -2324,28 +2365,38 @@ def init_builtin_api_nodes():
"nodes_moonvalley.py",
"nodes_rodin.py",
"nodes_gemini.py",
+ "nodes_vidu.py",
]
- if not load_custom_node(os.path.join(api_nodes_dir, "canary.py"), module_parent="comfy_api_nodes"):
+ if not await load_custom_node(os.path.join(api_nodes_dir, "canary.py"), module_parent="comfy_api_nodes"):
return api_nodes_files
import_failed = []
for node_file in api_nodes_files:
- if not load_custom_node(os.path.join(api_nodes_dir, node_file), module_parent="comfy_api_nodes"):
+ if not await load_custom_node(os.path.join(api_nodes_dir, node_file), module_parent="comfy_api_nodes"):
import_failed.append(node_file)
return import_failed
+async def init_public_apis():
+ register_versions([
+ ComfyAPIWithVersion(
+ version=getattr(v, "VERSION"),
+ api_class=v
+ ) for v in supported_versions
+ ])
-def init_extra_nodes(init_custom_nodes=True, init_api_nodes=True):
- import_failed = init_builtin_extra_nodes()
+async def init_extra_nodes(init_custom_nodes=True, init_api_nodes=True):
+ await init_public_apis()
+
+ import_failed = await init_builtin_extra_nodes()
import_failed_api = []
if init_api_nodes:
- import_failed_api = init_builtin_api_nodes()
+ import_failed_api = await init_builtin_api_nodes()
if init_custom_nodes:
- init_external_custom_nodes()
+ await init_external_custom_nodes()
else:
logging.info("Skipping loading of custom nodes")
diff --git a/pyproject.toml b/pyproject.toml
index 59c4c70fb..cfd5d45ef 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,6 +1,6 @@
[project]
name = "ComfyUI"
-version = "0.3.46"
+version = "0.3.56"
readme = "README.md"
license = { file = "LICENSE" }
requires-python = ">=3.9"
@@ -21,4 +21,4 @@ lint.select = [
# See all rules here: https://docs.astral.sh/ruff/rules/#pyflakes-f
"F",
]
-exclude = ["*.ipynb"]
+exclude = ["*.ipynb", "**/generated/*.pyi"]
diff --git a/requirements.txt b/requirements.txt
index 78d952a7d..ff70bc63a 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,6 +1,6 @@
-comfyui-frontend-package==1.23.4
-comfyui-workflow-templates==0.1.41
-comfyui-embedded-docs==0.2.4
+comfyui-frontend-package==1.25.11
+comfyui-workflow-templates==0.1.70
+comfyui-embedded-docs==0.2.6
torch
torchsde
torchvision
@@ -20,11 +20,11 @@ tqdm
psutil
alembic
SQLAlchemy
+av>=14.2.0
#non essential dependencies:
kornia>=0.7.1
spandrel
soundfile
-av>=14.2.0
pydantic~=2.0
pydantic-settings~=2.0
\ No newline at end of file
diff --git a/server.py b/server.py
index f4de0079b..8f9c88ebf 100644
--- a/server.py
+++ b/server.py
@@ -30,6 +30,7 @@ from comfy_api import feature_flags
import node_helpers
from comfyui_version import __version__
from app.frontend_management import FrontendManager
+from comfy_api.internal import _ComfyNodeInternal
from app.user_manager import UserManager
from app.model_manager import ModelFileManager
@@ -234,7 +235,7 @@ class PromptServer():
sid,
)
- logging.info(
+ logging.debug(
f"Feature flags negotiated for client {sid}: {client_flags}"
)
first_message = False
@@ -591,6 +592,8 @@ class PromptServer():
def node_info(node_class):
obj_class = nodes.NODE_CLASS_MAPPINGS[node_class]
+ if issubclass(obj_class, _ComfyNodeInternal):
+ return obj_class.GET_NODE_INFO_V1()
info = {}
info['input'] = obj_class.INPUT_TYPES()
info['input_order'] = {key: list(value.keys()) for (key, value) in obj_class.INPUT_TYPES().items()}
@@ -681,7 +684,12 @@ class PromptServer():
if "prompt" in json_data:
prompt = json_data["prompt"]
prompt_id = str(json_data.get("prompt_id", uuid.uuid4()))
- valid = await execution.validate_prompt(prompt_id, prompt)
+
+ partial_execution_targets = None
+ if "partial_execution_targets" in json_data:
+ partial_execution_targets = json_data["partial_execution_targets"]
+
+ valid = await execution.validate_prompt(prompt_id, prompt, partial_execution_targets)
extra_data = {}
if "extra_data" in json_data:
extra_data = json_data["extra_data"]
diff --git a/tests/inference/test_async_nodes.py b/tests/inference/test_async_nodes.py
index b243bbca9..f029953dd 100644
--- a/tests/inference/test_async_nodes.py
+++ b/tests/inference/test_async_nodes.py
@@ -7,7 +7,7 @@ import subprocess
from pytest import fixture
from comfy_execution.graph_utils import GraphBuilder
-from tests.inference.test_execution import ComfyClient
+from tests.inference.test_execution import ComfyClient, run_warmup
@pytest.mark.execution
@@ -24,6 +24,7 @@ class TestAsyncNodes:
'--listen', args_pytest["listen"],
'--port', str(args_pytest["port"]),
'--extra-model-paths-config', 'tests/inference/extra_model_paths.yaml',
+ '--cpu',
]
use_lru, lru_size = request.param
if use_lru:
@@ -82,6 +83,9 @@ class TestAsyncNodes:
def test_multiple_async_parallel_execution(self, client: ComfyClient, builder: GraphBuilder):
"""Test that multiple async nodes execute in parallel."""
+ # Warmup execution to ensure server is fully initialized
+ run_warmup(client)
+
g = builder
image = g.node("StubImage", content="BLACK", height=512, width=512, batch_size=1)
@@ -148,6 +152,9 @@ class TestAsyncNodes:
def test_async_lazy_evaluation(self, client: ComfyClient, builder: GraphBuilder):
"""Test async nodes with lazy evaluation."""
+ # Warmup execution to ensure server is fully initialized
+ run_warmup(client, prefix="warmup_lazy")
+
g = builder
input1 = g.node("StubImage", content="BLACK", height=512, width=512, batch_size=1)
input2 = g.node("StubImage", content="WHITE", height=512, width=512, batch_size=1)
@@ -305,6 +312,9 @@ class TestAsyncNodes:
def test_async_caching_behavior(self, client: ComfyClient, builder: GraphBuilder):
"""Test that async nodes are properly cached."""
+ # Warmup execution to ensure server is fully initialized
+ run_warmup(client, prefix="warmup_cache")
+
g = builder
image = g.node("StubImage", content="BLACK", height=512, width=512, batch_size=1)
sleep_node = g.node("TestSleep", value=image.out(0), seconds=0.2)
@@ -324,6 +334,9 @@ class TestAsyncNodes:
def test_async_with_dynamic_prompts(self, client: ComfyClient, builder: GraphBuilder):
"""Test async nodes within dynamically generated prompts."""
+ # Warmup execution to ensure server is fully initialized
+ run_warmup(client, prefix="warmup_dynamic")
+
g = builder
image1 = g.node("StubImage", content="BLACK", height=512, width=512, batch_size=1)
image2 = g.node("StubImage", content="WHITE", height=512, width=512, batch_size=1)
diff --git a/tests/inference/test_execution.py b/tests/inference/test_execution.py
index 9d3d685cc..e7b29302e 100644
--- a/tests/inference/test_execution.py
+++ b/tests/inference/test_execution.py
@@ -15,10 +15,18 @@ import urllib.parse
import urllib.error
from comfy_execution.graph_utils import GraphBuilder, Node
+def run_warmup(client, prefix="warmup"):
+ """Run a simple workflow to warm up the server."""
+ warmup_g = GraphBuilder(prefix=prefix)
+ warmup_image = warmup_g.node("StubImage", content="BLACK", height=32, width=32, batch_size=1)
+ warmup_g.node("PreviewImage", images=warmup_image.out(0))
+ client.run(warmup_g)
+
class RunResult:
def __init__(self, prompt_id: str):
self.outputs: Dict[str,Dict] = {}
self.runs: Dict[str,bool] = {}
+ self.cached: Dict[str,bool] = {}
self.prompt_id: str = prompt_id
def get_output(self, node: Node):
@@ -27,6 +35,13 @@ class RunResult:
def did_run(self, node: Node):
return self.runs.get(node.id, False)
+ def was_cached(self, node: Node):
+ return self.cached.get(node.id, False)
+
+ def was_executed(self, node: Node):
+ """Returns True if node was either run or cached"""
+ return self.did_run(node) or self.was_cached(node)
+
def get_images(self, node: Node):
output = self.get_output(node)
if output is None:
@@ -51,8 +66,10 @@ class ComfyClient:
ws.connect("ws://{}/ws?clientId={}".format(self.server_address, self.client_id))
self.ws = ws
- def queue_prompt(self, prompt):
+ def queue_prompt(self, prompt, partial_execution_targets=None):
p = {"prompt": prompt, "client_id": self.client_id}
+ if partial_execution_targets is not None:
+ p["partial_execution_targets"] = partial_execution_targets
data = json.dumps(p).encode('utf-8')
req = urllib.request.Request("http://{}/prompt".format(self.server_address), data=data)
return json.loads(urllib.request.urlopen(req).read())
@@ -70,13 +87,13 @@ class ComfyClient:
def set_test_name(self, name):
self.test_name = name
- def run(self, graph):
+ def run(self, graph, partial_execution_targets=None):
prompt = graph.finalize()
for node in graph.nodes.values():
if node.class_type == 'SaveImage':
node.inputs['filename_prefix'] = self.test_name
- prompt_id = self.queue_prompt(prompt)['prompt_id']
+ prompt_id = self.queue_prompt(prompt, partial_execution_targets)['prompt_id']
result = RunResult(prompt_id)
while True:
out = self.ws.recv()
@@ -92,7 +109,10 @@ class ComfyClient:
elif message['type'] == 'execution_error':
raise Exception(message['data'])
elif message['type'] == 'execution_cached':
- pass # Probably want to store this off for testing
+ if message['data']['prompt_id'] == prompt_id:
+ cached_nodes = message['data'].get('nodes', [])
+ for node_id in cached_nodes:
+ result.cached[node_id] = True
history = self.get_history(prompt_id)[prompt_id]
for node_id in history['outputs']:
@@ -130,6 +150,7 @@ class TestExecution:
'--listen', args_pytest["listen"],
'--port', str(args_pytest["port"]),
'--extra-model-paths-config', 'tests/inference/extra_model_paths.yaml',
+ '--cpu',
]
use_lru, lru_size = request.param
if use_lru:
@@ -498,12 +519,15 @@ class TestExecution:
assert not result.did_run(test_node), "The execution should have been cached"
def test_parallel_sleep_nodes(self, client: ComfyClient, builder: GraphBuilder):
+ # Warmup execution to ensure server is fully initialized
+ run_warmup(client)
+
g = builder
image = g.node("StubImage", content="BLACK", height=512, width=512, batch_size=1)
# Create sleep nodes for each duration
- sleep_node1 = g.node("TestSleep", value=image.out(0), seconds=2.8)
- sleep_node2 = g.node("TestSleep", value=image.out(0), seconds=2.9)
+ sleep_node1 = g.node("TestSleep", value=image.out(0), seconds=2.9)
+ sleep_node2 = g.node("TestSleep", value=image.out(0), seconds=3.1)
sleep_node3 = g.node("TestSleep", value=image.out(0), seconds=3.0)
# Add outputs to verify the execution
@@ -515,10 +539,9 @@ class TestExecution:
result = client.run(g)
elapsed_time = time.time() - start_time
- # The test should take around 0.4 seconds (the longest sleep duration)
- # plus some overhead, but definitely less than the sum of all sleeps (0.9s)
- # We'll allow for up to 0.8s total to account for overhead
- assert elapsed_time < 4.0, f"Parallel execution took {elapsed_time}s, expected less than 0.8s"
+ # The test should take around 3.0 seconds (the longest sleep duration)
+ # plus some overhead, but definitely less than the sum of all sleeps (9.0s)
+ assert elapsed_time < 8.9, f"Parallel execution took {elapsed_time}s, expected less than 8.9s"
# Verify that all nodes executed
assert result.did_run(sleep_node1), "Sleep node 1 should have run"
@@ -526,6 +549,9 @@ class TestExecution:
assert result.did_run(sleep_node3), "Sleep node 3 should have run"
def test_parallel_sleep_expansion(self, client: ComfyClient, builder: GraphBuilder):
+ # Warmup execution to ensure server is fully initialized
+ run_warmup(client)
+
g = builder
# Create input images with different values
image1 = g.node("StubImage", content="BLACK", height=512, width=512, batch_size=1)
@@ -537,9 +563,9 @@ class TestExecution:
image1=image1.out(0),
image2=image2.out(0),
image3=image3.out(0),
- sleep1=0.4,
- sleep2=0.5,
- sleep3=0.6)
+ sleep1=4.8,
+ sleep2=4.9,
+ sleep3=5.0)
output = g.node("SaveImage", images=parallel_sleep.out(0))
start_time = time.time()
@@ -548,7 +574,7 @@ class TestExecution:
# Similar to the previous test, expect parallel execution of the sleep nodes
# which should complete in less than the sum of all sleeps
- assert elapsed_time < 0.8, f"Expansion execution took {elapsed_time}s, expected less than 0.8s"
+ assert elapsed_time < 10.0, f"Expansion execution took {elapsed_time}s, expected less than 5.5s"
# Verify the parallel sleep node executed
assert result.did_run(parallel_sleep), "ParallelSleep node should have run"
@@ -585,3 +611,151 @@ class TestExecution:
assert len(images) == 2, "Should have 2 images"
assert numpy.array(images[0]).min() == 0 and numpy.array(images[0]).max() == 0, "First image should be black"
assert numpy.array(images[1]).min() == 0 and numpy.array(images[1]).max() == 0, "Second image should also be black"
+
+ # Output nodes included in the partial execution list are executed
+ def test_partial_execution_included_outputs(self, client: ComfyClient, builder: GraphBuilder):
+ g = builder
+ input1 = g.node("StubImage", content="BLACK", height=512, width=512, batch_size=1)
+ input2 = g.node("StubImage", content="WHITE", height=512, width=512, batch_size=1)
+
+ # Create two separate output nodes
+ output1 = g.node("SaveImage", images=input1.out(0))
+ output2 = g.node("SaveImage", images=input2.out(0))
+
+ # Run with partial execution targeting only output1
+ result = client.run(g, partial_execution_targets=[output1.id])
+
+ assert result.was_executed(input1), "Input1 should have been executed (run or cached)"
+ assert result.was_executed(output1), "Output1 should have been executed (run or cached)"
+ assert not result.did_run(input2), "Input2 should not have run"
+ assert not result.did_run(output2), "Output2 should not have run"
+
+ # Verify only output1 produced results
+ assert len(result.get_images(output1)) == 1, "Output1 should have produced an image"
+ assert len(result.get_images(output2)) == 0, "Output2 should not have produced an image"
+
+ # Output nodes NOT included in the partial execution list are NOT executed
+ def test_partial_execution_excluded_outputs(self, client: ComfyClient, builder: GraphBuilder):
+ g = builder
+ input1 = g.node("StubImage", content="BLACK", height=512, width=512, batch_size=1)
+ input2 = g.node("StubImage", content="WHITE", height=512, width=512, batch_size=1)
+ input3 = g.node("StubImage", content="NOISE", height=512, width=512, batch_size=1)
+
+ # Create three output nodes
+ output1 = g.node("SaveImage", images=input1.out(0))
+ output2 = g.node("SaveImage", images=input2.out(0))
+ output3 = g.node("SaveImage", images=input3.out(0))
+
+ # Run with partial execution targeting only output1 and output3
+ result = client.run(g, partial_execution_targets=[output1.id, output3.id])
+
+ assert result.was_executed(input1), "Input1 should have been executed"
+ assert result.was_executed(input3), "Input3 should have been executed"
+ assert result.was_executed(output1), "Output1 should have been executed"
+ assert result.was_executed(output3), "Output3 should have been executed"
+ assert not result.did_run(input2), "Input2 should not have run"
+ assert not result.did_run(output2), "Output2 should not have run"
+
+ # Output nodes NOT in list ARE executed if necessary for nodes that are in the list
+ def test_partial_execution_dependencies(self, client: ComfyClient, builder: GraphBuilder):
+ g = builder
+ input1 = g.node("StubImage", content="BLACK", height=512, width=512, batch_size=1)
+
+ # Create a processing chain with an OUTPUT_NODE that has socket outputs
+ output_with_socket = g.node("TestOutputNodeWithSocketOutput", image=input1.out(0), value=2.0)
+
+ # Create another node that depends on the output_with_socket
+ dependent_node = g.node("TestLazyMixImages",
+ image1=output_with_socket.out(0),
+ image2=input1.out(0),
+ mask=g.node("StubMask", value=0.5, height=512, width=512, batch_size=1).out(0))
+
+ # Create the final output
+ final_output = g.node("SaveImage", images=dependent_node.out(0))
+
+ # Run with partial execution targeting only the final output
+ result = client.run(g, partial_execution_targets=[final_output.id])
+
+ # All nodes should have been executed because they're dependencies
+ assert result.was_executed(input1), "Input1 should have been executed"
+ assert result.was_executed(output_with_socket), "Output with socket should have been executed (dependency)"
+ assert result.was_executed(dependent_node), "Dependent node should have been executed"
+ assert result.was_executed(final_output), "Final output should have been executed"
+
+ # Lazy execution works with partial execution
+ def test_partial_execution_with_lazy_nodes(self, client: ComfyClient, builder: GraphBuilder):
+ g = builder
+ input1 = g.node("StubImage", content="BLACK", height=512, width=512, batch_size=1)
+ input2 = g.node("StubImage", content="WHITE", height=512, width=512, batch_size=1)
+ input3 = g.node("StubImage", content="NOISE", height=512, width=512, batch_size=1)
+
+ # Create masks that will trigger different lazy execution paths
+ mask1 = g.node("StubMask", value=0.0, height=512, width=512, batch_size=1) # Will only need image1
+ mask2 = g.node("StubMask", value=0.5, height=512, width=512, batch_size=1) # Will need both images
+
+ # Create two lazy mix nodes
+ lazy_mix1 = g.node("TestLazyMixImages", image1=input1.out(0), image2=input2.out(0), mask=mask1.out(0))
+ lazy_mix2 = g.node("TestLazyMixImages", image1=input2.out(0), image2=input3.out(0), mask=mask2.out(0))
+
+ output1 = g.node("SaveImage", images=lazy_mix1.out(0))
+ output2 = g.node("SaveImage", images=lazy_mix2.out(0))
+
+ # Run with partial execution targeting only output1
+ result = client.run(g, partial_execution_targets=[output1.id])
+
+ # For output1 path - only input1 should run due to lazy evaluation (mask=0.0)
+ assert result.was_executed(input1), "Input1 should have been executed"
+ assert not result.did_run(input2), "Input2 should not have run (lazy evaluation)"
+ assert result.was_executed(mask1), "Mask1 should have been executed"
+ assert result.was_executed(lazy_mix1), "Lazy mix1 should have been executed"
+ assert result.was_executed(output1), "Output1 should have been executed"
+
+ # Nothing from output2 path should run
+ assert not result.did_run(input3), "Input3 should not have run"
+ assert not result.did_run(mask2), "Mask2 should not have run"
+ assert not result.did_run(lazy_mix2), "Lazy mix2 should not have run"
+ assert not result.did_run(output2), "Output2 should not have run"
+
+ # Multiple OUTPUT_NODEs with dependencies
+ def test_partial_execution_multiple_output_nodes(self, client: ComfyClient, builder: GraphBuilder):
+ g = builder
+ input1 = g.node("StubImage", content="BLACK", height=512, width=512, batch_size=1)
+ input2 = g.node("StubImage", content="WHITE", height=512, width=512, batch_size=1)
+
+ # Create a chain of OUTPUT_NODEs
+ output_node1 = g.node("TestOutputNodeWithSocketOutput", image=input1.out(0), value=1.5)
+ output_node2 = g.node("TestOutputNodeWithSocketOutput", image=output_node1.out(0), value=2.0)
+
+ # Create regular output nodes
+ save1 = g.node("SaveImage", images=output_node1.out(0))
+ save2 = g.node("SaveImage", images=output_node2.out(0))
+ save3 = g.node("SaveImage", images=input2.out(0))
+
+ # Run targeting only save2
+ result = client.run(g, partial_execution_targets=[save2.id])
+
+ # Should run: input1, output_node1, output_node2, save2
+ assert result.was_executed(input1), "Input1 should have been executed"
+ assert result.was_executed(output_node1), "Output node 1 should have been executed (dependency)"
+ assert result.was_executed(output_node2), "Output node 2 should have been executed (dependency)"
+ assert result.was_executed(save2), "Save2 should have been executed"
+
+ # Should NOT run: input2, save1, save3
+ assert not result.did_run(input2), "Input2 should not have run"
+ assert not result.did_run(save1), "Save1 should not have run"
+ assert not result.did_run(save3), "Save3 should not have run"
+
+ # Empty partial execution list (should execute nothing)
+ def test_partial_execution_empty_list(self, client: ComfyClient, builder: GraphBuilder):
+ g = builder
+ input1 = g.node("StubImage", content="BLACK", height=512, width=512, batch_size=1)
+ _output1 = g.node("SaveImage", images=input1.out(0))
+
+ # Run with empty partial execution list
+ try:
+ _result = client.run(g, partial_execution_targets=[])
+ # Should get an error because no outputs are selected
+ assert False, "Should have raised an error for empty partial execution list"
+ except urllib.error.HTTPError:
+ pass # Expected behavior
+
diff --git a/tests/inference/testing_nodes/testing-pack/__init__.py b/tests/inference/testing_nodes/testing-pack/__init__.py
index 20f9533c7..3d5ac8a94 100644
--- a/tests/inference/testing_nodes/testing-pack/__init__.py
+++ b/tests/inference/testing_nodes/testing-pack/__init__.py
@@ -4,6 +4,7 @@ from .util import UTILITY_NODE_CLASS_MAPPINGS, UTILITY_NODE_DISPLAY_NAME_MAPPING
from .conditions import CONDITION_NODE_CLASS_MAPPINGS, CONDITION_NODE_DISPLAY_NAME_MAPPINGS
from .stubs import TEST_STUB_NODE_CLASS_MAPPINGS, TEST_STUB_NODE_DISPLAY_NAME_MAPPINGS
from .async_test_nodes import ASYNC_TEST_NODE_CLASS_MAPPINGS, ASYNC_TEST_NODE_DISPLAY_NAME_MAPPINGS
+from .api_test_nodes import API_TEST_NODE_CLASS_MAPPINGS, API_TEST_NODE_DISPLAY_NAME_MAPPINGS
# NODE_CLASS_MAPPINGS = GENERAL_NODE_CLASS_MAPPINGS.update(COMPONENT_NODE_CLASS_MAPPINGS)
# NODE_DISPLAY_NAME_MAPPINGS = GENERAL_NODE_DISPLAY_NAME_MAPPINGS.update(COMPONENT_NODE_DISPLAY_NAME_MAPPINGS)
@@ -15,6 +16,7 @@ NODE_CLASS_MAPPINGS.update(UTILITY_NODE_CLASS_MAPPINGS)
NODE_CLASS_MAPPINGS.update(CONDITION_NODE_CLASS_MAPPINGS)
NODE_CLASS_MAPPINGS.update(TEST_STUB_NODE_CLASS_MAPPINGS)
NODE_CLASS_MAPPINGS.update(ASYNC_TEST_NODE_CLASS_MAPPINGS)
+NODE_CLASS_MAPPINGS.update(API_TEST_NODE_CLASS_MAPPINGS)
NODE_DISPLAY_NAME_MAPPINGS = {}
NODE_DISPLAY_NAME_MAPPINGS.update(TEST_NODE_DISPLAY_NAME_MAPPINGS)
@@ -23,4 +25,4 @@ NODE_DISPLAY_NAME_MAPPINGS.update(UTILITY_NODE_DISPLAY_NAME_MAPPINGS)
NODE_DISPLAY_NAME_MAPPINGS.update(CONDITION_NODE_DISPLAY_NAME_MAPPINGS)
NODE_DISPLAY_NAME_MAPPINGS.update(TEST_STUB_NODE_DISPLAY_NAME_MAPPINGS)
NODE_DISPLAY_NAME_MAPPINGS.update(ASYNC_TEST_NODE_DISPLAY_NAME_MAPPINGS)
-
+NODE_DISPLAY_NAME_MAPPINGS.update(API_TEST_NODE_DISPLAY_NAME_MAPPINGS)
diff --git a/tests/inference/testing_nodes/testing-pack/api_test_nodes.py b/tests/inference/testing_nodes/testing-pack/api_test_nodes.py
new file mode 100644
index 000000000..b2eaae05e
--- /dev/null
+++ b/tests/inference/testing_nodes/testing-pack/api_test_nodes.py
@@ -0,0 +1,78 @@
+import asyncio
+import time
+from comfy.comfy_types import IO, ComfyNodeABC, InputTypeDict
+from comfy_api.v0_0_2 import ComfyAPI, ComfyAPISync
+
+api = ComfyAPI()
+api_sync = ComfyAPISync()
+
+
+class TestAsyncProgressUpdate(ComfyNodeABC):
+ """Test node with async VALIDATE_INPUTS."""
+
+ @classmethod
+ def INPUT_TYPES(cls) -> InputTypeDict:
+ return {
+ "required": {
+ "value": (IO.ANY, {}),
+ "sleep_seconds": (IO.FLOAT, {"default": 1.0}),
+ },
+ }
+
+ RETURN_TYPES = (IO.ANY,)
+ FUNCTION = "execute"
+ CATEGORY = "_for_testing/async"
+
+ async def execute(self, value, sleep_seconds):
+ start = time.time()
+ expiration = start + sleep_seconds
+ now = start
+ while now < expiration:
+ now = time.time()
+ await api.execution.set_progress(
+ value=(now - start) / sleep_seconds,
+ max_value=1.0,
+ )
+ await asyncio.sleep(0.01)
+ return (value,)
+
+
+class TestSyncProgressUpdate(ComfyNodeABC):
+ """Test node with async VALIDATE_INPUTS."""
+
+ @classmethod
+ def INPUT_TYPES(cls) -> InputTypeDict:
+ return {
+ "required": {
+ "value": (IO.ANY, {}),
+ "sleep_seconds": (IO.FLOAT, {"default": 1.0}),
+ },
+ }
+
+ RETURN_TYPES = (IO.ANY,)
+ FUNCTION = "execute"
+ CATEGORY = "_for_testing/async"
+
+ def execute(self, value, sleep_seconds):
+ start = time.time()
+ expiration = start + sleep_seconds
+ now = start
+ while now < expiration:
+ now = time.time()
+ api_sync.execution.set_progress(
+ value=(now - start) / sleep_seconds,
+ max_value=1.0,
+ )
+ time.sleep(0.01)
+ return (value,)
+
+
+API_TEST_NODE_CLASS_MAPPINGS = {
+ "TestAsyncProgressUpdate": TestAsyncProgressUpdate,
+ "TestSyncProgressUpdate": TestSyncProgressUpdate,
+}
+
+API_TEST_NODE_DISPLAY_NAME_MAPPINGS = {
+ "TestAsyncProgressUpdate": "Async Progress Update Test Node",
+ "TestSyncProgressUpdate": "Sync Progress Update Test Node",
+}
diff --git a/tests/inference/testing_nodes/testing-pack/specific_tests.py b/tests/inference/testing_nodes/testing-pack/specific_tests.py
index 657d49f2f..4f8f01ae4 100644
--- a/tests/inference/testing_nodes/testing-pack/specific_tests.py
+++ b/tests/inference/testing_nodes/testing-pack/specific_tests.py
@@ -463,6 +463,25 @@ class TestParallelSleep(ComfyNodeABC):
"expand": g.finalize(),
}
+class TestOutputNodeWithSocketOutput:
+ @classmethod
+ def INPUT_TYPES(cls):
+ return {
+ "required": {
+ "image": ("IMAGE",),
+ "value": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 10.0}),
+ },
+ }
+ RETURN_TYPES = ("IMAGE",)
+ FUNCTION = "process"
+ CATEGORY = "_for_testing"
+ OUTPUT_NODE = True
+
+ def process(self, image, value):
+ # Apply value scaling and return both as output and socket
+ result = image * value
+ return (result,)
+
TEST_NODE_CLASS_MAPPINGS = {
"TestLazyMixImages": TestLazyMixImages,
"TestVariadicAverage": TestVariadicAverage,
@@ -478,6 +497,7 @@ TEST_NODE_CLASS_MAPPINGS = {
"TestSamplingInExpansion": TestSamplingInExpansion,
"TestSleep": TestSleep,
"TestParallelSleep": TestParallelSleep,
+ "TestOutputNodeWithSocketOutput": TestOutputNodeWithSocketOutput,
}
TEST_NODE_DISPLAY_NAME_MAPPINGS = {
@@ -495,4 +515,5 @@ TEST_NODE_DISPLAY_NAME_MAPPINGS = {
"TestSamplingInExpansion": "Sampling In Expansion",
"TestSleep": "Test Sleep",
"TestParallelSleep": "Test Parallel Sleep",
+ "TestOutputNodeWithSocketOutput": "Test Output Node With Socket Output",
}