diff --git a/.gitattributes b/.gitattributes
index 4391de678..5b3c15bb4 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -1,2 +1,3 @@
/web/assets/** linguist-generated
/web/** linguist-vendored
+comfy_api_nodes/apis/__init__.py linguist-generated
diff --git a/.github/ISSUE_TEMPLATE/bug-report.yml b/.github/ISSUE_TEMPLATE/bug-report.yml
index 69ce998eb..3cf2717b7 100644
--- a/.github/ISSUE_TEMPLATE/bug-report.yml
+++ b/.github/ISSUE_TEMPLATE/bug-report.yml
@@ -22,7 +22,7 @@ body:
description: Please confirm you have tried to reproduce the issue with all custom nodes disabled.
options:
- label: I have tried disabling custom nodes and the issue persists (see [how to disable custom nodes](https://docs.comfy.org/troubleshooting/custom-node-issues#step-1%3A-test-with-all-custom-nodes-disabled) if you need help)
- required: true
+ required: false
- type: textarea
attributes:
label: Expected Behavior
diff --git a/.github/ISSUE_TEMPLATE/user-support.yml b/.github/ISSUE_TEMPLATE/user-support.yml
index 50657d493..281661f92 100644
--- a/.github/ISSUE_TEMPLATE/user-support.yml
+++ b/.github/ISSUE_TEMPLATE/user-support.yml
@@ -18,7 +18,7 @@ body:
description: Please confirm you have tried to reproduce the issue with all custom nodes disabled.
options:
- label: I have tried disabling custom nodes and the issue persists (see [how to disable custom nodes](https://docs.comfy.org/troubleshooting/custom-node-issues#step-1%3A-test-with-all-custom-nodes-disabled) if you need help)
- required: true
+ required: false
- type: textarea
attributes:
label: Your question
diff --git a/.github/workflows/stable-release.yml b/.github/workflows/stable-release.yml
index 61105abe4..2bc8e5905 100644
--- a/.github/workflows/stable-release.yml
+++ b/.github/workflows/stable-release.yml
@@ -12,17 +12,17 @@ on:
description: 'CUDA version'
required: true
type: string
- default: "128"
+ default: "129"
python_minor:
description: 'Python minor version'
required: true
type: string
- default: "12"
+ default: "13"
python_patch:
description: 'Python patch version'
required: true
type: string
- default: "10"
+ default: "6"
jobs:
@@ -66,8 +66,13 @@ jobs:
curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py
./python.exe get-pip.py
./python.exe -s -m pip install ../cu${{ inputs.cu }}_python_deps/*
- sed -i '1i../ComfyUI' ./python3${{ inputs.python_minor }}._pth
- cd ..
+ sed -i '1i../ComfyUI' ./python3${{ inputs.python_minor }}._pth
+
+ rm ./Lib/site-packages/torch/lib/dnnl.lib #I don't think this is actually used and I need the space
+ rm ./Lib/site-packages/torch/lib/libprotoc.lib
+ rm ./Lib/site-packages/torch/lib/libprotobuf.lib
+
+ cd ..
git clone --depth 1 https://github.com/comfyanonymous/taesd
cp taesd/*.safetensors ./ComfyUI_copy/models/vae_approx/
@@ -85,7 +90,7 @@ jobs:
cd ..
- "C:\Program Files\7-Zip\7z.exe" a -t7z -m0=lzma2 -mx=9 -mfb=128 -md=512m -ms=on -mf=BCJ2 ComfyUI_windows_portable.7z ComfyUI_windows_portable
+ "C:\Program Files\7-Zip\7z.exe" a -t7z -m0=lzma2 -mx=9 -mfb=128 -md=768m -ms=on -mf=BCJ2 ComfyUI_windows_portable.7z ComfyUI_windows_portable
mv ComfyUI_windows_portable.7z ComfyUI/ComfyUI_windows_portable_nvidia.7z
cd ComfyUI_windows_portable
diff --git a/.github/workflows/windows_release_dependencies.yml b/.github/workflows/windows_release_dependencies.yml
index dfdb96d50..7761cc1ed 100644
--- a/.github/workflows/windows_release_dependencies.yml
+++ b/.github/workflows/windows_release_dependencies.yml
@@ -17,19 +17,19 @@ on:
description: 'cuda version'
required: true
type: string
- default: "128"
+ default: "129"
python_minor:
description: 'python minor version'
required: true
type: string
- default: "12"
+ default: "13"
python_patch:
description: 'python patch version'
required: true
type: string
- default: "10"
+ default: "6"
# push:
# branches:
# - master
diff --git a/.github/workflows/windows_release_package.yml b/.github/workflows/windows_release_package.yml
index 3926a65f3..46375698e 100644
--- a/.github/workflows/windows_release_package.yml
+++ b/.github/workflows/windows_release_package.yml
@@ -7,19 +7,19 @@ on:
description: 'cuda version'
required: true
type: string
- default: "128"
+ default: "129"
python_minor:
description: 'python minor version'
required: true
type: string
- default: "12"
+ default: "13"
python_patch:
description: 'python patch version'
required: true
type: string
- default: "10"
+ default: "6"
# push:
# branches:
# - master
@@ -64,6 +64,10 @@ jobs:
./python.exe get-pip.py
./python.exe -s -m pip install ../cu${{ inputs.cu }}_python_deps/*
sed -i '1i../ComfyUI' ./python3${{ inputs.python_minor }}._pth
+
+ rm ./Lib/site-packages/torch/lib/dnnl.lib #I don't think this is actually used and I need the space
+ rm ./Lib/site-packages/torch/lib/libprotoc.lib
+ rm ./Lib/site-packages/torch/lib/libprotobuf.lib
cd ..
git clone --depth 1 https://github.com/comfyanonymous/taesd
@@ -82,7 +86,7 @@ jobs:
cd ..
- "C:\Program Files\7-Zip\7z.exe" a -t7z -m0=lzma2 -mx=9 -mfb=128 -md=512m -ms=on -mf=BCJ2 ComfyUI_windows_portable.7z ComfyUI_windows_portable
+ "C:\Program Files\7-Zip\7z.exe" a -t7z -m0=lzma2 -mx=9 -mfb=128 -md=768m -ms=on -mf=BCJ2 ComfyUI_windows_portable.7z ComfyUI_windows_portable
mv ComfyUI_windows_portable.7z ComfyUI/new_ComfyUI_windows_portable_nvidia_cu${{ inputs.cu }}_or_cpu.7z
cd ComfyUI_windows_portable
diff --git a/CODEOWNERS b/CODEOWNERS
index c4acbf06e..c8acd66d5 100644
--- a/CODEOWNERS
+++ b/CODEOWNERS
@@ -5,20 +5,21 @@
# Inlined the team members for now.
# Maintainers
-*.md @yoland68 @robinjhuang @webfiltered @pythongosssss @ltdrdata @Kosinkadink @christian-byrne
-/tests/ @yoland68 @robinjhuang @webfiltered @pythongosssss @ltdrdata @Kosinkadink @christian-byrne
-/tests-unit/ @yoland68 @robinjhuang @webfiltered @pythongosssss @ltdrdata @Kosinkadink @christian-byrne
-/notebooks/ @yoland68 @robinjhuang @webfiltered @pythongosssss @ltdrdata @Kosinkadink @christian-byrne
-/script_examples/ @yoland68 @robinjhuang @webfiltered @pythongosssss @ltdrdata @Kosinkadink @christian-byrne
-/.github/ @yoland68 @robinjhuang @webfiltered @pythongosssss @ltdrdata @Kosinkadink @christian-byrne
-/requirements.txt @yoland68 @robinjhuang @webfiltered @pythongosssss @ltdrdata @Kosinkadink @christian-byrne
-/pyproject.toml @yoland68 @robinjhuang @webfiltered @pythongosssss @ltdrdata @Kosinkadink @christian-byrne
+*.md @yoland68 @robinjhuang @webfiltered @pythongosssss @ltdrdata @Kosinkadink @christian-byrne @guill
+/tests/ @yoland68 @robinjhuang @webfiltered @pythongosssss @ltdrdata @Kosinkadink @christian-byrne @guill
+/tests-unit/ @yoland68 @robinjhuang @webfiltered @pythongosssss @ltdrdata @Kosinkadink @christian-byrne @guill
+/notebooks/ @yoland68 @robinjhuang @webfiltered @pythongosssss @ltdrdata @Kosinkadink @christian-byrne @guill
+/script_examples/ @yoland68 @robinjhuang @webfiltered @pythongosssss @ltdrdata @Kosinkadink @christian-byrne @guill
+/.github/ @yoland68 @robinjhuang @webfiltered @pythongosssss @ltdrdata @Kosinkadink @christian-byrne @guill
+/requirements.txt @yoland68 @robinjhuang @webfiltered @pythongosssss @ltdrdata @Kosinkadink @christian-byrne @guill
+/pyproject.toml @yoland68 @robinjhuang @webfiltered @pythongosssss @ltdrdata @Kosinkadink @christian-byrne @guill
# Python web server
-/api_server/ @yoland68 @robinjhuang @webfiltered @pythongosssss @ltdrdata @christian-byrne
-/app/ @yoland68 @robinjhuang @webfiltered @pythongosssss @ltdrdata @christian-byrne
-/utils/ @yoland68 @robinjhuang @webfiltered @pythongosssss @ltdrdata @christian-byrne
+/api_server/ @yoland68 @robinjhuang @webfiltered @pythongosssss @ltdrdata @christian-byrne @guill
+/app/ @yoland68 @robinjhuang @webfiltered @pythongosssss @ltdrdata @christian-byrne @guill
+/utils/ @yoland68 @robinjhuang @webfiltered @pythongosssss @ltdrdata @christian-byrne @guill
# Node developers
-/comfy_extras/ @yoland68 @robinjhuang @pythongosssss @ltdrdata @Kosinkadink @webfiltered @christian-byrne
-/comfy/comfy_types/ @yoland68 @robinjhuang @pythongosssss @ltdrdata @Kosinkadink @webfiltered @christian-byrne
+/comfy_extras/ @yoland68 @robinjhuang @pythongosssss @ltdrdata @Kosinkadink @webfiltered @christian-byrne @guill
+/comfy/comfy_types/ @yoland68 @robinjhuang @pythongosssss @ltdrdata @Kosinkadink @webfiltered @christian-byrne @guill
+/comfy_api_nodes/ @yoland68 @robinjhuang @pythongosssss @ltdrdata @Kosinkadink @webfiltered @christian-byrne @guill
diff --git a/README.md b/README.md
index 2abd8e600..79a8a8c79 100644
--- a/README.md
+++ b/README.md
@@ -39,7 +39,7 @@ ComfyUI lets you design and execute advanced stable diffusion pipelines using a
## Get Started
#### [Desktop Application](https://www.comfy.org/download)
-- The easiest way to get started.
+- The easiest way to get started.
- Available on Windows & macOS.
#### [Windows Portable Package](#installing)
@@ -66,10 +66,12 @@ See what ComfyUI can do with the [example workflows](https://comfyanonymous.gith
- [Lumina Image 2.0](https://comfyanonymous.github.io/ComfyUI_examples/lumina2/)
- [HiDream](https://comfyanonymous.github.io/ComfyUI_examples/hidream/)
- [Cosmos Predict2](https://comfyanonymous.github.io/ComfyUI_examples/cosmos_predict2/)
+ - [Qwen Image](https://comfyanonymous.github.io/ComfyUI_examples/qwen_image/)
- Image Editing Models
- [Omnigen 2](https://comfyanonymous.github.io/ComfyUI_examples/omnigen/)
- [Flux Kontext](https://comfyanonymous.github.io/ComfyUI_examples/flux/#flux-kontext-image-editing-model)
- [HiDream E1.1](https://comfyanonymous.github.io/ComfyUI_examples/hidream/#hidream-e11)
+ - [Qwen Image Edit](https://comfyanonymous.github.io/ComfyUI_examples/qwen_image/#edit-model)
- Video Models
- [Stable Video Diffusion](https://comfyanonymous.github.io/ComfyUI_examples/video/)
- [Mochi](https://comfyanonymous.github.io/ComfyUI_examples/mochi/)
@@ -202,7 +204,7 @@ Put your VAE in: models/vae
### AMD GPUs (Linux only)
AMD users can install rocm and pytorch with pip if you don't have it already installed, this is the command to install the stable version:
-```pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/rocm6.3```
+```pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/rocm6.4```
This is the command to install the nightly with ROCm 6.4 which might have some performance improvements:
@@ -210,33 +212,25 @@ This is the command to install the nightly with ROCm 6.4 which might have some p
### Intel GPUs (Windows and Linux)
-(Option 1) Intel Arc GPU users can install native PyTorch with torch.xpu support using pip (currently available in PyTorch nightly builds). More information can be found [here](https://pytorch.org/docs/main/notes/get_start_xpu.html)
-
-1. To install PyTorch nightly, use the following command:
+(Option 1) Intel Arc GPU users can install native PyTorch with torch.xpu support using pip. More information can be found [here](https://pytorch.org/docs/main/notes/get_start_xpu.html)
+
+1. To install PyTorch xpu, use the following command:
+
+```pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/xpu```
+
+This is the command to install the Pytorch xpu nightly which might have some performance improvements:
```pip install --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/xpu```
-2. Launch ComfyUI by running `python main.py`
-
-
(Option 2) Alternatively, Intel GPUs supported by Intel Extension for PyTorch (IPEX) can leverage IPEX for improved performance.
-1. For Intel® Arc™ A-Series Graphics utilizing IPEX, create a conda environment and use the commands below:
-
-```
-conda install libuv
-pip install torch==2.3.1.post0+cxx11.abi torchvision==0.18.1.post0+cxx11.abi torchaudio==2.3.1.post0+cxx11.abi intel-extension-for-pytorch==2.3.110.post0+xpu --extra-index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/us/ --extra-index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/cn/
-```
-
-For other supported Intel GPUs with IPEX, visit [Installation](https://intel.github.io/intel-extension-for-pytorch/index.html#installation?platform=gpu) for more information.
-
-Additional discussion and help can be found [here](https://github.com/comfyanonymous/ComfyUI/discussions/476).
+1. visit [Installation](https://intel.github.io/intel-extension-for-pytorch/index.html#installation?platform=gpu) for more information.
### NVIDIA
Nvidia users should install stable pytorch using this command:
-```pip install torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cu128```
+```pip install torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cu129```
This is the command to install pytorch nightly instead which might have performance improvements.
@@ -351,7 +345,7 @@ Generate a self-signed certificate (not appropriate for shared/production use) a
Use `--tls-keyfile key.pem --tls-certfile cert.pem` to enable TLS/SSL, the app will now be accessible with `https://...` instead of `http://...`.
-> Note: Windows users can use [alexisrolland/docker-openssl](https://github.com/alexisrolland/docker-openssl) or one of the [3rd party binary distributions](https://wiki.openssl.org/index.php/Binaries) to run the command example above.
+> Note: Windows users can use [alexisrolland/docker-openssl](https://github.com/alexisrolland/docker-openssl) or one of the [3rd party binary distributions](https://wiki.openssl.org/index.php/Binaries) to run the command example above.
If you use a container, note that the volume mount `-v` can be a relative path so `... -v ".\:/openssl-certs" ...` would create the key & cert files in the current directory of your command prompt or powershell terminal.
## Support and dev channel
diff --git a/app/user_manager.py b/app/user_manager.py
index 0ec3e46ea..a2d376c0c 100644
--- a/app/user_manager.py
+++ b/app/user_manager.py
@@ -363,10 +363,17 @@ class UserManager():
if not overwrite and os.path.exists(path):
return web.Response(status=409, text="File already exists")
- body = await request.read()
+ try:
+ body = await request.read()
- with open(path, "wb") as f:
- f.write(body)
+ with open(path, "wb") as f:
+ f.write(body)
+ except OSError as e:
+ logging.warning(f"Error saving file '{path}': {e}")
+ return web.Response(
+ status=400,
+ reason="Invalid filename. Please avoid special characters like :\\/*?\"<>|"
+ )
user_path = self.get_request_user_filepath(request, None)
if full_info:
diff --git a/comfy/cli_args.py b/comfy/cli_args.py
index 0d760d524..de3e85c08 100644
--- a/comfy/cli_args.py
+++ b/comfy/cli_args.py
@@ -132,6 +132,8 @@ parser.add_argument("--reserve-vram", type=float, default=None, help="Set the am
parser.add_argument("--async-offload", action="store_true", help="Use async weight offloading.")
+parser.add_argument("--force-non-blocking", action="store_true", help="Force ComfyUI to use non-blocking operations for all applicable tensors. This may improve performance on some non-Nvidia systems but can cause issues with some workflows.")
+
parser.add_argument("--default-hashing-function", type=str, choices=['md5', 'sha1', 'sha256', 'sha512'], default='sha256', help="Allows you to choose the hash function to use for duplicate filename / contents comparison. Default is sha256.")
parser.add_argument("--disable-smart-memory", action="store_true", help="Force ComfyUI to agressively offload to regular ram instead of keeping models in vram when it can.")
diff --git a/comfy/clip_model.py b/comfy/clip_model.py
index c8294d483..7e47d8a55 100644
--- a/comfy/clip_model.py
+++ b/comfy/clip_model.py
@@ -97,7 +97,7 @@ class CLIPTextModel_(torch.nn.Module):
self.encoder = CLIPEncoder(num_layers, embed_dim, heads, intermediate_size, intermediate_activation, dtype, device, operations)
self.final_layer_norm = operations.LayerNorm(embed_dim, dtype=dtype, device=device)
- def forward(self, input_tokens=None, attention_mask=None, embeds=None, num_tokens=None, intermediate_output=None, final_layer_norm_intermediate=True, dtype=torch.float32):
+ def forward(self, input_tokens=None, attention_mask=None, embeds=None, num_tokens=None, intermediate_output=None, final_layer_norm_intermediate=True, dtype=torch.float32, embeds_info=[]):
if embeds is not None:
x = embeds + comfy.ops.cast_to(self.embeddings.position_embedding.weight, dtype=dtype, device=embeds.device)
else:
diff --git a/comfy/context_windows.py b/comfy/context_windows.py
new file mode 100644
index 000000000..041f380f9
--- /dev/null
+++ b/comfy/context_windows.py
@@ -0,0 +1,540 @@
+from __future__ import annotations
+from typing import TYPE_CHECKING, Callable
+import torch
+import numpy as np
+import collections
+from dataclasses import dataclass
+from abc import ABC, abstractmethod
+import logging
+import comfy.model_management
+import comfy.patcher_extension
+if TYPE_CHECKING:
+ from comfy.model_base import BaseModel
+ from comfy.model_patcher import ModelPatcher
+ from comfy.controlnet import ControlBase
+
+
+class ContextWindowABC(ABC):
+ def __init__(self):
+ ...
+
+ @abstractmethod
+ def get_tensor(self, full: torch.Tensor) -> torch.Tensor:
+ """
+ Get torch.Tensor applicable to current window.
+ """
+ raise NotImplementedError("Not implemented.")
+
+ @abstractmethod
+ def add_window(self, full: torch.Tensor, to_add: torch.Tensor) -> torch.Tensor:
+ """
+ Apply torch.Tensor of window to the full tensor, in place. Returns reference to updated full tensor, not a copy.
+ """
+ raise NotImplementedError("Not implemented.")
+
+class ContextHandlerABC(ABC):
+ def __init__(self):
+ ...
+
+ @abstractmethod
+ def should_use_context(self, model: BaseModel, conds: list[list[dict]], x_in: torch.Tensor, timestep: torch.Tensor, model_options: dict[str]) -> bool:
+ raise NotImplementedError("Not implemented.")
+
+ @abstractmethod
+ def get_resized_cond(self, cond_in: list[dict], x_in: torch.Tensor, window: ContextWindowABC, device=None) -> list:
+ raise NotImplementedError("Not implemented.")
+
+ @abstractmethod
+ def execute(self, calc_cond_batch: Callable, model: BaseModel, conds: list[list[dict]], x_in: torch.Tensor, timestep: torch.Tensor, model_options: dict[str]):
+ raise NotImplementedError("Not implemented.")
+
+
+
+class IndexListContextWindow(ContextWindowABC):
+ def __init__(self, index_list: list[int], dim: int=0):
+ self.index_list = index_list
+ self.context_length = len(index_list)
+ self.dim = dim
+
+ def get_tensor(self, full: torch.Tensor, device=None, dim=None) -> torch.Tensor:
+ if dim is None:
+ dim = self.dim
+ if dim == 0 and full.shape[dim] == 1:
+ return full
+ idx = [slice(None)] * dim + [self.index_list]
+ return full[idx].to(device)
+
+ def add_window(self, full: torch.Tensor, to_add: torch.Tensor, dim=None) -> torch.Tensor:
+ if dim is None:
+ dim = self.dim
+ idx = [slice(None)] * dim + [self.index_list]
+ full[idx] += to_add
+ return full
+
+
+class IndexListCallbacks:
+ EVALUATE_CONTEXT_WINDOWS = "evaluate_context_windows"
+ COMBINE_CONTEXT_WINDOW_RESULTS = "combine_context_window_results"
+ EXECUTE_START = "execute_start"
+ EXECUTE_CLEANUP = "execute_cleanup"
+
+ def init_callbacks(self):
+ return {}
+
+
+@dataclass
+class ContextSchedule:
+ name: str
+ func: Callable
+
+@dataclass
+class ContextFuseMethod:
+ name: str
+ func: Callable
+
+ContextResults = collections.namedtuple("ContextResults", ['window_idx', 'sub_conds_out', 'sub_conds', 'window'])
+class IndexListContextHandler(ContextHandlerABC):
+ def __init__(self, context_schedule: ContextSchedule, fuse_method: ContextFuseMethod, context_length: int=1, context_overlap: int=0, context_stride: int=1, closed_loop=False, dim=0):
+ self.context_schedule = context_schedule
+ self.fuse_method = fuse_method
+ self.context_length = context_length
+ self.context_overlap = context_overlap
+ self.context_stride = context_stride
+ self.closed_loop = closed_loop
+ self.dim = dim
+ self._step = 0
+
+ self.callbacks = {}
+
+ def should_use_context(self, model: BaseModel, conds: list[list[dict]], x_in: torch.Tensor, timestep: torch.Tensor, model_options: dict[str]) -> bool:
+ # for now, assume first dim is batch - should have stored on BaseModel in actual implementation
+ if x_in.size(self.dim) > self.context_length:
+ logging.info(f"Using context windows {self.context_length} for {x_in.size(self.dim)} frames.")
+ return True
+ return False
+
+ def prepare_control_objects(self, control: ControlBase, device=None) -> ControlBase:
+ if control.previous_controlnet is not None:
+ self.prepare_control_objects(control.previous_controlnet, device)
+ return control
+
+ def get_resized_cond(self, cond_in: list[dict], x_in: torch.Tensor, window: IndexListContextWindow, device=None) -> list:
+ if cond_in is None:
+ return None
+ # reuse or resize cond items to match context requirements
+ resized_cond = []
+ # cond object is a list containing a dict - outer list is irrelevant, so just loop through it
+ for actual_cond in cond_in:
+ resized_actual_cond = actual_cond.copy()
+ # now we are in the inner dict - "pooled_output" is a tensor, "control" is a ControlBase object, "model_conds" is dictionary
+ for key in actual_cond:
+ try:
+ cond_item = actual_cond[key]
+ if isinstance(cond_item, torch.Tensor):
+ # check that tensor is the expected length - x.size(0)
+ if self.dim < cond_item.ndim and cond_item.size(self.dim) == x_in.size(self.dim):
+ # if so, it's subsetting time - tell controls the expected indeces so they can handle them
+ actual_cond_item = window.get_tensor(cond_item)
+ resized_actual_cond[key] = actual_cond_item.to(device)
+ else:
+ resized_actual_cond[key] = cond_item.to(device)
+ # look for control
+ elif key == "control":
+ resized_actual_cond[key] = self.prepare_control_objects(cond_item, device)
+ elif isinstance(cond_item, dict):
+ new_cond_item = cond_item.copy()
+ # when in dictionary, look for tensors and CONDCrossAttn [comfy/conds.py] (has cond attr that is a tensor)
+ for cond_key, cond_value in new_cond_item.items():
+ if isinstance(cond_value, torch.Tensor):
+ if cond_value.ndim < self.dim and cond_value.size(0) == x_in.size(self.dim):
+ new_cond_item[cond_key] = window.get_tensor(cond_value, device)
+ # if has cond that is a Tensor, check if needs to be subset
+ elif hasattr(cond_value, "cond") and isinstance(cond_value.cond, torch.Tensor):
+ if cond_value.cond.ndim < self.dim and cond_value.cond.size(0) == x_in.size(self.dim):
+ new_cond_item[cond_key] = cond_value._copy_with(window.get_tensor(cond_value.cond, device))
+ elif cond_key == "num_video_frames": # for SVD
+ new_cond_item[cond_key] = cond_value._copy_with(cond_value.cond)
+ new_cond_item[cond_key].cond = window.context_length
+ resized_actual_cond[key] = new_cond_item
+ else:
+ resized_actual_cond[key] = cond_item
+ finally:
+ del cond_item # just in case to prevent VRAM issues
+ resized_cond.append(resized_actual_cond)
+ return resized_cond
+
+ def set_step(self, timestep: torch.Tensor, model_options: dict[str]):
+ mask = torch.isclose(model_options["transformer_options"]["sample_sigmas"], timestep, rtol=0.0001)
+ matches = torch.nonzero(mask)
+ if torch.numel(matches) == 0:
+ raise Exception("No sample_sigmas matched current timestep; something went wrong.")
+ self._step = int(matches[0].item())
+
+ def get_context_windows(self, model: BaseModel, x_in: torch.Tensor, model_options: dict[str]) -> list[IndexListContextWindow]:
+ full_length = x_in.size(self.dim) # TODO: choose dim based on model
+ context_windows = self.context_schedule.func(full_length, self, model_options)
+ context_windows = [IndexListContextWindow(window, dim=self.dim) for window in context_windows]
+ return context_windows
+
+ def execute(self, calc_cond_batch: Callable, model: BaseModel, conds: list[list[dict]], x_in: torch.Tensor, timestep: torch.Tensor, model_options: dict[str]):
+ self.set_step(timestep, model_options)
+ context_windows = self.get_context_windows(model, x_in, model_options)
+ enumerated_context_windows = list(enumerate(context_windows))
+
+ conds_final = [torch.zeros_like(x_in) for _ in conds]
+ if self.fuse_method.name == ContextFuseMethods.RELATIVE:
+ counts_final = [torch.ones(get_shape_for_dim(x_in, self.dim), device=x_in.device) for _ in conds]
+ else:
+ counts_final = [torch.zeros(get_shape_for_dim(x_in, self.dim), device=x_in.device) for _ in conds]
+ biases_final = [([0.0] * x_in.shape[self.dim]) for _ in conds]
+
+ for callback in comfy.patcher_extension.get_all_callbacks(IndexListCallbacks.EXECUTE_START, self.callbacks):
+ callback(self, model, x_in, conds, timestep, model_options)
+
+ for enum_window in enumerated_context_windows:
+ results = self.evaluate_context_windows(calc_cond_batch, model, x_in, conds, timestep, [enum_window], model_options)
+ for result in results:
+ self.combine_context_window_results(x_in, result.sub_conds_out, result.sub_conds, result.window, result.window_idx, len(enumerated_context_windows), timestep,
+ conds_final, counts_final, biases_final)
+ try:
+ # finalize conds
+ if self.fuse_method.name == ContextFuseMethods.RELATIVE:
+ # relative is already normalized, so return as is
+ del counts_final
+ return conds_final
+ else:
+ # normalize conds via division by context usage counts
+ for i in range(len(conds_final)):
+ conds_final[i] /= counts_final[i]
+ del counts_final
+ return conds_final
+ finally:
+ for callback in comfy.patcher_extension.get_all_callbacks(IndexListCallbacks.EXECUTE_CLEANUP, self.callbacks):
+ callback(self, model, x_in, conds, timestep, model_options)
+
+ def evaluate_context_windows(self, calc_cond_batch: Callable, model: BaseModel, x_in: torch.Tensor, conds, timestep: torch.Tensor, enumerated_context_windows: list[tuple[int, IndexListContextWindow]],
+ model_options, device=None, first_device=None):
+ results: list[ContextResults] = []
+ for window_idx, window in enumerated_context_windows:
+ # allow processing to end between context window executions for faster Cancel
+ comfy.model_management.throw_exception_if_processing_interrupted()
+
+ for callback in comfy.patcher_extension.get_all_callbacks(IndexListCallbacks.EVALUATE_CONTEXT_WINDOWS, self.callbacks):
+ callback(self, model, x_in, conds, timestep, model_options, window_idx, window, model_options, device, first_device)
+
+ # update exposed params
+ model_options["transformer_options"]["context_window"] = window
+ # get subsections of x, timestep, conds
+ sub_x = window.get_tensor(x_in, device)
+ sub_timestep = window.get_tensor(timestep, device, dim=0)
+ sub_conds = [self.get_resized_cond(cond, x_in, window, device) for cond in conds]
+
+ sub_conds_out = calc_cond_batch(model, sub_conds, sub_x, sub_timestep, model_options)
+ if device is not None:
+ for i in range(len(sub_conds_out)):
+ sub_conds_out[i] = sub_conds_out[i].to(x_in.device)
+ results.append(ContextResults(window_idx, sub_conds_out, sub_conds, window))
+ return results
+
+
+ def combine_context_window_results(self, x_in: torch.Tensor, sub_conds_out, sub_conds, window: IndexListContextWindow, window_idx: int, total_windows: int, timestep: torch.Tensor,
+ conds_final: list[torch.Tensor], counts_final: list[torch.Tensor], biases_final: list[torch.Tensor]):
+ if self.fuse_method.name == ContextFuseMethods.RELATIVE:
+ for pos, idx in enumerate(window.index_list):
+ # bias is the influence of a specific index in relation to the whole context window
+ bias = 1 - abs(idx - (window.index_list[0] + window.index_list[-1]) / 2) / ((window.index_list[-1] - window.index_list[0] + 1e-2) / 2)
+ bias = max(1e-2, bias)
+ # take weighted average relative to total bias of current idx
+ for i in range(len(sub_conds_out)):
+ bias_total = biases_final[i][idx]
+ prev_weight = (bias_total / (bias_total + bias))
+ new_weight = (bias / (bias_total + bias))
+ # account for dims of tensors
+ idx_window = [slice(None)] * self.dim + [idx]
+ pos_window = [slice(None)] * self.dim + [pos]
+ # apply new values
+ conds_final[i][idx_window] = conds_final[i][idx_window] * prev_weight + sub_conds_out[i][pos_window] * new_weight
+ biases_final[i][idx] = bias_total + bias
+ else:
+ # add conds and counts based on weights of fuse method
+ weights = get_context_weights(window.context_length, x_in.shape[self.dim], window.index_list, self, sigma=timestep)
+ weights_tensor = match_weights_to_dim(weights, x_in, self.dim, device=x_in.device)
+ for i in range(len(sub_conds_out)):
+ window.add_window(conds_final[i], sub_conds_out[i] * weights_tensor)
+ window.add_window(counts_final[i], weights_tensor)
+
+ for callback in comfy.patcher_extension.get_all_callbacks(IndexListCallbacks.COMBINE_CONTEXT_WINDOW_RESULTS, self.callbacks):
+ callback(self, x_in, sub_conds_out, sub_conds, window, window_idx, total_windows, timestep, conds_final, counts_final, biases_final)
+
+
+def _prepare_sampling_wrapper(executor, model, noise_shape: torch.Tensor, *args, **kwargs):
+ # limit noise_shape length to context_length for more accurate vram use estimation
+ model_options = kwargs.get("model_options", None)
+ if model_options is None:
+ raise Exception("model_options not found in prepare_sampling_wrapper; this should never happen, something went wrong.")
+ handler: IndexListContextHandler = model_options.get("context_handler", None)
+ if handler is not None:
+ noise_shape = list(noise_shape)
+ noise_shape[handler.dim] = min(noise_shape[handler.dim], handler.context_length)
+ return executor(model, noise_shape, *args, **kwargs)
+
+
+def create_prepare_sampling_wrapper(model: ModelPatcher):
+ model.add_wrapper_with_key(
+ comfy.patcher_extension.WrappersMP.PREPARE_SAMPLING,
+ "ContextWindows_prepare_sampling",
+ _prepare_sampling_wrapper
+ )
+
+
+def match_weights_to_dim(weights: list[float], x_in: torch.Tensor, dim: int, device=None) -> torch.Tensor:
+ total_dims = len(x_in.shape)
+ weights_tensor = torch.Tensor(weights).to(device=device)
+ for _ in range(dim):
+ weights_tensor = weights_tensor.unsqueeze(0)
+ for _ in range(total_dims - dim - 1):
+ weights_tensor = weights_tensor.unsqueeze(-1)
+ return weights_tensor
+
+def get_shape_for_dim(x_in: torch.Tensor, dim: int) -> list[int]:
+ total_dims = len(x_in.shape)
+ shape = []
+ for _ in range(dim):
+ shape.append(1)
+ shape.append(x_in.shape[dim])
+ for _ in range(total_dims - dim - 1):
+ shape.append(1)
+ return shape
+
+class ContextSchedules:
+ UNIFORM_LOOPED = "looped_uniform"
+ UNIFORM_STANDARD = "standard_uniform"
+ STATIC_STANDARD = "standard_static"
+ BATCHED = "batched"
+
+
+# from https://github.com/neggles/animatediff-cli/blob/main/src/animatediff/pipelines/context.py
+def create_windows_uniform_looped(num_frames: int, handler: IndexListContextHandler, model_options: dict[str]):
+ windows = []
+ if num_frames < handler.context_length:
+ windows.append(list(range(num_frames)))
+ return windows
+
+ context_stride = min(handler.context_stride, int(np.ceil(np.log2(num_frames / handler.context_length))) + 1)
+ # obtain uniform windows as normal, looping and all
+ for context_step in 1 << np.arange(context_stride):
+ pad = int(round(num_frames * ordered_halving(handler._step)))
+ for j in range(
+ int(ordered_halving(handler._step) * context_step) + pad,
+ num_frames + pad + (0 if handler.closed_loop else -handler.context_overlap),
+ (handler.context_length * context_step - handler.context_overlap),
+ ):
+ windows.append([e % num_frames for e in range(j, j + handler.context_length * context_step, context_step)])
+
+ return windows
+
+def create_windows_uniform_standard(num_frames: int, handler: IndexListContextHandler, model_options: dict[str]):
+ # unlike looped, uniform_straight does NOT allow windows that loop back to the beginning;
+ # instead, they get shifted to the corresponding end of the frames.
+ # in the case that a window (shifted or not) is identical to the previous one, it gets skipped.
+ windows = []
+ if num_frames <= handler.context_length:
+ windows.append(list(range(num_frames)))
+ return windows
+
+ context_stride = min(handler.context_stride, int(np.ceil(np.log2(num_frames / handler.context_length))) + 1)
+ # first, obtain uniform windows as normal, looping and all
+ for context_step in 1 << np.arange(context_stride):
+ pad = int(round(num_frames * ordered_halving(handler._step)))
+ for j in range(
+ int(ordered_halving(handler._step) * context_step) + pad,
+ num_frames + pad + (-handler.context_overlap),
+ (handler.context_length * context_step - handler.context_overlap),
+ ):
+ windows.append([e % num_frames for e in range(j, j + handler.context_length * context_step, context_step)])
+
+ # now that windows are created, shift any windows that loop, and delete duplicate windows
+ delete_idxs = []
+ win_i = 0
+ while win_i < len(windows):
+ # if window is rolls over itself, need to shift it
+ is_roll, roll_idx = does_window_roll_over(windows[win_i], num_frames)
+ if is_roll:
+ roll_val = windows[win_i][roll_idx] # roll_val might not be 0 for windows of higher strides
+ shift_window_to_end(windows[win_i], num_frames=num_frames)
+ # check if next window (cyclical) is missing roll_val
+ if roll_val not in windows[(win_i+1) % len(windows)]:
+ # need to insert new window here - just insert window starting at roll_val
+ windows.insert(win_i+1, list(range(roll_val, roll_val + handler.context_length)))
+ # delete window if it's not unique
+ for pre_i in range(0, win_i):
+ if windows[win_i] == windows[pre_i]:
+ delete_idxs.append(win_i)
+ break
+ win_i += 1
+
+ # reverse delete_idxs so that they will be deleted in an order that doesn't break idx correlation
+ delete_idxs.reverse()
+ for i in delete_idxs:
+ windows.pop(i)
+
+ return windows
+
+
+def create_windows_static_standard(num_frames: int, handler: IndexListContextHandler, model_options: dict[str]):
+ windows = []
+ if num_frames <= handler.context_length:
+ windows.append(list(range(num_frames)))
+ return windows
+ # always return the same set of windows
+ delta = handler.context_length - handler.context_overlap
+ for start_idx in range(0, num_frames, delta):
+ # if past the end of frames, move start_idx back to allow same context_length
+ ending = start_idx + handler.context_length
+ if ending >= num_frames:
+ final_delta = ending - num_frames
+ final_start_idx = start_idx - final_delta
+ windows.append(list(range(final_start_idx, final_start_idx + handler.context_length)))
+ break
+ windows.append(list(range(start_idx, start_idx + handler.context_length)))
+ return windows
+
+
+def create_windows_batched(num_frames: int, handler: IndexListContextHandler, model_options: dict[str]):
+ windows = []
+ if num_frames <= handler.context_length:
+ windows.append(list(range(num_frames)))
+ return windows
+ # always return the same set of windows;
+ # no overlap, just cut up based on context_length;
+ # last window size will be different if num_frames % opts.context_length != 0
+ for start_idx in range(0, num_frames, handler.context_length):
+ windows.append(list(range(start_idx, min(start_idx + handler.context_length, num_frames))))
+ return windows
+
+
+def create_windows_default(num_frames: int, handler: IndexListContextHandler):
+ return [list(range(num_frames))]
+
+
+CONTEXT_MAPPING = {
+ ContextSchedules.UNIFORM_LOOPED: create_windows_uniform_looped,
+ ContextSchedules.UNIFORM_STANDARD: create_windows_uniform_standard,
+ ContextSchedules.STATIC_STANDARD: create_windows_static_standard,
+ ContextSchedules.BATCHED: create_windows_batched,
+}
+
+
+def get_matching_context_schedule(context_schedule: str) -> ContextSchedule:
+ func = CONTEXT_MAPPING.get(context_schedule, None)
+ if func is None:
+ raise ValueError(f"Unknown context_schedule '{context_schedule}'.")
+ return ContextSchedule(context_schedule, func)
+
+
+def get_context_weights(length: int, full_length: int, idxs: list[int], handler: IndexListContextHandler, sigma: torch.Tensor=None):
+ return handler.fuse_method.func(length, sigma=sigma, handler=handler, full_length=full_length, idxs=idxs)
+
+
+def create_weights_flat(length: int, **kwargs) -> list[float]:
+ # weight is the same for all
+ return [1.0] * length
+
+def create_weights_pyramid(length: int, **kwargs) -> list[float]:
+ # weight is based on the distance away from the edge of the context window;
+ # based on weighted average concept in FreeNoise paper
+ if length % 2 == 0:
+ max_weight = length // 2
+ weight_sequence = list(range(1, max_weight + 1, 1)) + list(range(max_weight, 0, -1))
+ else:
+ max_weight = (length + 1) // 2
+ weight_sequence = list(range(1, max_weight, 1)) + [max_weight] + list(range(max_weight - 1, 0, -1))
+ return weight_sequence
+
+def create_weights_overlap_linear(length: int, full_length: int, idxs: list[int], handler: IndexListContextHandler, **kwargs):
+ # based on code in Kijai's WanVideoWrapper: https://github.com/kijai/ComfyUI-WanVideoWrapper/blob/dbb2523b37e4ccdf45127e5ae33e31362f755c8e/nodes.py#L1302
+ # only expected overlap is given different weights
+ weights_torch = torch.ones((length))
+ # blend left-side on all except first window
+ if min(idxs) > 0:
+ ramp_up = torch.linspace(1e-37, 1, handler.context_overlap)
+ weights_torch[:handler.context_overlap] = ramp_up
+ # blend right-side on all except last window
+ if max(idxs) < full_length-1:
+ ramp_down = torch.linspace(1, 1e-37, handler.context_overlap)
+ weights_torch[-handler.context_overlap:] = ramp_down
+ return weights_torch
+
+class ContextFuseMethods:
+ FLAT = "flat"
+ PYRAMID = "pyramid"
+ RELATIVE = "relative"
+ OVERLAP_LINEAR = "overlap-linear"
+
+ LIST = [PYRAMID, FLAT, OVERLAP_LINEAR]
+ LIST_STATIC = [PYRAMID, RELATIVE, FLAT, OVERLAP_LINEAR]
+
+
+FUSE_MAPPING = {
+ ContextFuseMethods.FLAT: create_weights_flat,
+ ContextFuseMethods.PYRAMID: create_weights_pyramid,
+ ContextFuseMethods.RELATIVE: create_weights_pyramid,
+ ContextFuseMethods.OVERLAP_LINEAR: create_weights_overlap_linear,
+}
+
+def get_matching_fuse_method(fuse_method: str) -> ContextFuseMethod:
+ func = FUSE_MAPPING.get(fuse_method, None)
+ if func is None:
+ raise ValueError(f"Unknown fuse_method '{fuse_method}'.")
+ return ContextFuseMethod(fuse_method, func)
+
+# Returns fraction that has denominator that is a power of 2
+def ordered_halving(val):
+ # get binary value, padded with 0s for 64 bits
+ bin_str = f"{val:064b}"
+ # flip binary value, padding included
+ bin_flip = bin_str[::-1]
+ # convert binary to int
+ as_int = int(bin_flip, 2)
+ # divide by 1 << 64, equivalent to 2**64, or 18446744073709551616,
+ # or b10000000000000000000000000000000000000000000000000000000000000000 (1 with 64 zero's)
+ return as_int / (1 << 64)
+
+
+def get_missing_indexes(windows: list[list[int]], num_frames: int) -> list[int]:
+ all_indexes = list(range(num_frames))
+ for w in windows:
+ for val in w:
+ try:
+ all_indexes.remove(val)
+ except ValueError:
+ pass
+ return all_indexes
+
+
+def does_window_roll_over(window: list[int], num_frames: int) -> tuple[bool, int]:
+ prev_val = -1
+ for i, val in enumerate(window):
+ val = val % num_frames
+ if val < prev_val:
+ return True, i
+ prev_val = val
+ return False, -1
+
+
+def shift_window_to_start(window: list[int], num_frames: int):
+ start_val = window[0]
+ for i in range(len(window)):
+ # 1) subtract each element by start_val to move vals relative to the start of all frames
+ # 2) add num_frames and take modulus to get adjusted vals
+ window[i] = ((window[i] - start_val) + num_frames) % num_frames
+
+
+def shift_window_to_end(window: list[int], num_frames: int):
+ # 1) shift window to start
+ shift_window_to_start(window, num_frames)
+ end_val = window[-1]
+ end_delta = num_frames - end_val - 1
+ for i in range(len(window)):
+ # 2) add end_delta to each val to slide windows to end
+ window[i] = window[i] + end_delta
diff --git a/comfy/ldm/flux/model.py b/comfy/ldm/flux/model.py
index c09594862..f84475bd9 100644
--- a/comfy/ldm/flux/model.py
+++ b/comfy/ldm/flux/model.py
@@ -229,19 +229,27 @@ class Flux(nn.Module):
if ref_latents is not None:
h = 0
w = 0
+ index = 0
+ index_ref_method = kwargs.get("ref_latents_method", "offset") == "index"
for ref in ref_latents:
- h_offset = 0
- w_offset = 0
- if ref.shape[-2] + h > ref.shape[-1] + w:
- w_offset = w
+ if index_ref_method:
+ index += 1
+ h_offset = 0
+ w_offset = 0
else:
- h_offset = h
+ index = 1
+ h_offset = 0
+ w_offset = 0
+ if ref.shape[-2] + h > ref.shape[-1] + w:
+ w_offset = w
+ else:
+ h_offset = h
+ h = max(h, ref.shape[-2] + h_offset)
+ w = max(w, ref.shape[-1] + w_offset)
- kontext, kontext_ids = self.process_img(ref, index=1, h_offset=h_offset, w_offset=w_offset)
+ kontext, kontext_ids = self.process_img(ref, index=index, h_offset=h_offset, w_offset=w_offset)
img = torch.cat([img, kontext], dim=1)
img_ids = torch.cat([img_ids, kontext_ids], dim=1)
- h = max(h, ref.shape[-2] + h_offset)
- w = max(w, ref.shape[-1] + w_offset)
txt_ids = torch.zeros((bs, context.shape[1], 3), device=x.device, dtype=x.dtype)
out = self.forward_orig(img, img_ids, context, txt_ids, timestep, y, guidance, control, transformer_options, attn_mask=kwargs.get("attention_mask", None))
diff --git a/comfy/ldm/hunyuan3d/vae.py b/comfy/ldm/hunyuan3d/vae.py
index 5eb2c6548..6e8cbf1d9 100644
--- a/comfy/ldm/hunyuan3d/vae.py
+++ b/comfy/ldm/hunyuan3d/vae.py
@@ -178,7 +178,7 @@ class FourierEmbedder(nn.Module):
class CrossAttentionProcessor:
def __call__(self, attn, q, k, v):
- out = F.scaled_dot_product_attention(q, k, v)
+ out = comfy.ops.scaled_dot_product_attention(q, k, v)
return out
diff --git a/comfy/ldm/modules/attention.py b/comfy/ldm/modules/attention.py
index 35d2270ee..043df28df 100644
--- a/comfy/ldm/modules/attention.py
+++ b/comfy/ldm/modules/attention.py
@@ -448,7 +448,7 @@ def attention_pytorch(q, k, v, heads, mask=None, attn_precision=None, skip_resha
mask = mask.unsqueeze(1)
if SDP_BATCH_LIMIT >= b:
- out = torch.nn.functional.scaled_dot_product_attention(q, k, v, attn_mask=mask, dropout_p=0.0, is_causal=False)
+ out = comfy.ops.scaled_dot_product_attention(q, k, v, attn_mask=mask, dropout_p=0.0, is_causal=False)
if not skip_output_reshape:
out = (
out.transpose(1, 2).reshape(b, -1, heads * dim_head)
@@ -461,7 +461,7 @@ def attention_pytorch(q, k, v, heads, mask=None, attn_precision=None, skip_resha
if mask.shape[0] > 1:
m = mask[i : i + SDP_BATCH_LIMIT]
- out[i : i + SDP_BATCH_LIMIT] = torch.nn.functional.scaled_dot_product_attention(
+ out[i : i + SDP_BATCH_LIMIT] = comfy.ops.scaled_dot_product_attention(
q[i : i + SDP_BATCH_LIMIT],
k[i : i + SDP_BATCH_LIMIT],
v[i : i + SDP_BATCH_LIMIT],
diff --git a/comfy/ldm/modules/diffusionmodules/model.py b/comfy/ldm/modules/diffusionmodules/model.py
index 5c0373b74..1fd12b35a 100644
--- a/comfy/ldm/modules/diffusionmodules/model.py
+++ b/comfy/ldm/modules/diffusionmodules/model.py
@@ -285,7 +285,7 @@ def pytorch_attention(q, k, v):
)
try:
- out = torch.nn.functional.scaled_dot_product_attention(q, k, v, attn_mask=None, dropout_p=0.0, is_causal=False)
+ out = comfy.ops.scaled_dot_product_attention(q, k, v, attn_mask=None, dropout_p=0.0, is_causal=False)
out = out.transpose(2, 3).reshape(orig_shape)
except model_management.OOM_EXCEPTION:
logging.warning("scaled_dot_product_attention OOMed: switched to slice attention")
diff --git a/comfy/ldm/qwen_image/model.py b/comfy/ldm/qwen_image/model.py
new file mode 100644
index 000000000..2503583cb
--- /dev/null
+++ b/comfy/ldm/qwen_image/model.py
@@ -0,0 +1,451 @@
+# https://github.com/QwenLM/Qwen-Image (Apache 2.0)
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from typing import Optional, Tuple
+from einops import repeat
+
+from comfy.ldm.lightricks.model import TimestepEmbedding, Timesteps
+from comfy.ldm.modules.attention import optimized_attention_masked
+from comfy.ldm.flux.layers import EmbedND
+import comfy.ldm.common_dit
+
+class GELU(nn.Module):
+ def __init__(self, dim_in: int, dim_out: int, approximate: str = "none", bias: bool = True, dtype=None, device=None, operations=None):
+ super().__init__()
+ self.proj = operations.Linear(dim_in, dim_out, bias=bias, dtype=dtype, device=device)
+ self.approximate = approximate
+
+ def forward(self, hidden_states):
+ hidden_states = self.proj(hidden_states)
+ hidden_states = F.gelu(hidden_states, approximate=self.approximate)
+ return hidden_states
+
+
+class FeedForward(nn.Module):
+ def __init__(
+ self,
+ dim: int,
+ dim_out: Optional[int] = None,
+ mult: int = 4,
+ dropout: float = 0.0,
+ inner_dim=None,
+ bias: bool = True,
+ dtype=None, device=None, operations=None
+ ):
+ super().__init__()
+ if inner_dim is None:
+ inner_dim = int(dim * mult)
+ dim_out = dim_out if dim_out is not None else dim
+
+ self.net = nn.ModuleList([])
+ self.net.append(GELU(dim, inner_dim, approximate="tanh", bias=bias, dtype=dtype, device=device, operations=operations))
+ self.net.append(nn.Dropout(dropout))
+ self.net.append(operations.Linear(inner_dim, dim_out, bias=bias, dtype=dtype, device=device))
+
+ def forward(self, hidden_states: torch.Tensor, *args, **kwargs) -> torch.Tensor:
+ for module in self.net:
+ hidden_states = module(hidden_states)
+ return hidden_states
+
+
+def apply_rotary_emb(x, freqs_cis):
+ if x.shape[1] == 0:
+ return x
+
+ t_ = x.reshape(*x.shape[:-1], -1, 1, 2)
+ t_out = freqs_cis[..., 0] * t_[..., 0] + freqs_cis[..., 1] * t_[..., 1]
+ return t_out.reshape(*x.shape)
+
+
+class QwenTimestepProjEmbeddings(nn.Module):
+ def __init__(self, embedding_dim, pooled_projection_dim, dtype=None, device=None, operations=None):
+ super().__init__()
+ self.time_proj = Timesteps(num_channels=256, flip_sin_to_cos=True, downscale_freq_shift=0, scale=1000)
+ self.timestep_embedder = TimestepEmbedding(
+ in_channels=256,
+ time_embed_dim=embedding_dim,
+ dtype=dtype,
+ device=device,
+ operations=operations
+ )
+
+ def forward(self, timestep, hidden_states):
+ timesteps_proj = self.time_proj(timestep)
+ timesteps_emb = self.timestep_embedder(timesteps_proj.to(dtype=hidden_states.dtype))
+ return timesteps_emb
+
+
+class Attention(nn.Module):
+ def __init__(
+ self,
+ query_dim: int,
+ dim_head: int = 64,
+ heads: int = 8,
+ dropout: float = 0.0,
+ bias: bool = False,
+ eps: float = 1e-5,
+ out_bias: bool = True,
+ out_dim: int = None,
+ out_context_dim: int = None,
+ dtype=None,
+ device=None,
+ operations=None
+ ):
+ super().__init__()
+ self.inner_dim = out_dim if out_dim is not None else dim_head * heads
+ self.inner_kv_dim = self.inner_dim
+ self.heads = heads
+ self.dim_head = dim_head
+ self.out_dim = out_dim if out_dim is not None else query_dim
+ self.out_context_dim = out_context_dim if out_context_dim is not None else query_dim
+ self.dropout = dropout
+
+ # Q/K normalization
+ self.norm_q = operations.RMSNorm(dim_head, eps=eps, elementwise_affine=True, dtype=dtype, device=device)
+ self.norm_k = operations.RMSNorm(dim_head, eps=eps, elementwise_affine=True, dtype=dtype, device=device)
+ self.norm_added_q = operations.RMSNorm(dim_head, eps=eps, dtype=dtype, device=device)
+ self.norm_added_k = operations.RMSNorm(dim_head, eps=eps, dtype=dtype, device=device)
+
+ # Image stream projections
+ self.to_q = operations.Linear(query_dim, self.inner_dim, bias=bias, dtype=dtype, device=device)
+ self.to_k = operations.Linear(query_dim, self.inner_kv_dim, bias=bias, dtype=dtype, device=device)
+ self.to_v = operations.Linear(query_dim, self.inner_kv_dim, bias=bias, dtype=dtype, device=device)
+
+ # Text stream projections
+ self.add_q_proj = operations.Linear(query_dim, self.inner_dim, bias=bias, dtype=dtype, device=device)
+ self.add_k_proj = operations.Linear(query_dim, self.inner_kv_dim, bias=bias, dtype=dtype, device=device)
+ self.add_v_proj = operations.Linear(query_dim, self.inner_kv_dim, bias=bias, dtype=dtype, device=device)
+
+ # Output projections
+ self.to_out = nn.ModuleList([
+ operations.Linear(self.inner_dim, self.out_dim, bias=out_bias, dtype=dtype, device=device),
+ nn.Dropout(dropout)
+ ])
+ self.to_add_out = operations.Linear(self.inner_dim, self.out_context_dim, bias=out_bias, dtype=dtype, device=device)
+
+ def forward(
+ self,
+ hidden_states: torch.FloatTensor, # Image stream
+ encoder_hidden_states: torch.FloatTensor = None, # Text stream
+ encoder_hidden_states_mask: torch.FloatTensor = None,
+ attention_mask: Optional[torch.FloatTensor] = None,
+ image_rotary_emb: Optional[torch.Tensor] = None,
+ ) -> Tuple[torch.Tensor, torch.Tensor]:
+ seq_txt = encoder_hidden_states.shape[1]
+
+ img_query = self.to_q(hidden_states).unflatten(-1, (self.heads, -1))
+ img_key = self.to_k(hidden_states).unflatten(-1, (self.heads, -1))
+ img_value = self.to_v(hidden_states).unflatten(-1, (self.heads, -1))
+
+ txt_query = self.add_q_proj(encoder_hidden_states).unflatten(-1, (self.heads, -1))
+ txt_key = self.add_k_proj(encoder_hidden_states).unflatten(-1, (self.heads, -1))
+ txt_value = self.add_v_proj(encoder_hidden_states).unflatten(-1, (self.heads, -1))
+
+ img_query = self.norm_q(img_query)
+ img_key = self.norm_k(img_key)
+ txt_query = self.norm_added_q(txt_query)
+ txt_key = self.norm_added_k(txt_key)
+
+ joint_query = torch.cat([txt_query, img_query], dim=1)
+ joint_key = torch.cat([txt_key, img_key], dim=1)
+ joint_value = torch.cat([txt_value, img_value], dim=1)
+
+ joint_query = apply_rotary_emb(joint_query, image_rotary_emb)
+ joint_key = apply_rotary_emb(joint_key, image_rotary_emb)
+
+ joint_query = joint_query.flatten(start_dim=2)
+ joint_key = joint_key.flatten(start_dim=2)
+ joint_value = joint_value.flatten(start_dim=2)
+
+ joint_hidden_states = optimized_attention_masked(joint_query, joint_key, joint_value, self.heads, attention_mask)
+
+ txt_attn_output = joint_hidden_states[:, :seq_txt, :]
+ img_attn_output = joint_hidden_states[:, seq_txt:, :]
+
+ img_attn_output = self.to_out[0](img_attn_output)
+ img_attn_output = self.to_out[1](img_attn_output)
+ txt_attn_output = self.to_add_out(txt_attn_output)
+
+ return img_attn_output, txt_attn_output
+
+
+class QwenImageTransformerBlock(nn.Module):
+ def __init__(
+ self,
+ dim: int,
+ num_attention_heads: int,
+ attention_head_dim: int,
+ eps: float = 1e-6,
+ dtype=None,
+ device=None,
+ operations=None
+ ):
+ super().__init__()
+ self.dim = dim
+ self.num_attention_heads = num_attention_heads
+ self.attention_head_dim = attention_head_dim
+
+ self.img_mod = nn.Sequential(
+ nn.SiLU(),
+ operations.Linear(dim, 6 * dim, bias=True, dtype=dtype, device=device),
+ )
+ self.img_norm1 = operations.LayerNorm(dim, elementwise_affine=False, eps=eps, dtype=dtype, device=device)
+ self.img_norm2 = operations.LayerNorm(dim, elementwise_affine=False, eps=eps, dtype=dtype, device=device)
+ self.img_mlp = FeedForward(dim=dim, dim_out=dim, dtype=dtype, device=device, operations=operations)
+
+ self.txt_mod = nn.Sequential(
+ nn.SiLU(),
+ operations.Linear(dim, 6 * dim, bias=True, dtype=dtype, device=device),
+ )
+ self.txt_norm1 = operations.LayerNorm(dim, elementwise_affine=False, eps=eps, dtype=dtype, device=device)
+ self.txt_norm2 = operations.LayerNorm(dim, elementwise_affine=False, eps=eps, dtype=dtype, device=device)
+ self.txt_mlp = FeedForward(dim=dim, dim_out=dim, dtype=dtype, device=device, operations=operations)
+
+ self.attn = Attention(
+ query_dim=dim,
+ dim_head=attention_head_dim,
+ heads=num_attention_heads,
+ out_dim=dim,
+ bias=True,
+ eps=eps,
+ dtype=dtype,
+ device=device,
+ operations=operations,
+ )
+
+ def _modulate(self, x, mod_params):
+ shift, scale, gate = mod_params.chunk(3, dim=-1)
+ return x * (1 + scale.unsqueeze(1)) + shift.unsqueeze(1), gate.unsqueeze(1)
+
+ def forward(
+ self,
+ hidden_states: torch.Tensor,
+ encoder_hidden_states: torch.Tensor,
+ encoder_hidden_states_mask: torch.Tensor,
+ temb: torch.Tensor,
+ image_rotary_emb: Optional[Tuple[torch.Tensor, torch.Tensor]] = None,
+ ) -> Tuple[torch.Tensor, torch.Tensor]:
+ img_mod_params = self.img_mod(temb)
+ txt_mod_params = self.txt_mod(temb)
+ img_mod1, img_mod2 = img_mod_params.chunk(2, dim=-1)
+ txt_mod1, txt_mod2 = txt_mod_params.chunk(2, dim=-1)
+
+ img_normed = self.img_norm1(hidden_states)
+ img_modulated, img_gate1 = self._modulate(img_normed, img_mod1)
+ txt_normed = self.txt_norm1(encoder_hidden_states)
+ txt_modulated, txt_gate1 = self._modulate(txt_normed, txt_mod1)
+
+ img_attn_output, txt_attn_output = self.attn(
+ hidden_states=img_modulated,
+ encoder_hidden_states=txt_modulated,
+ encoder_hidden_states_mask=encoder_hidden_states_mask,
+ image_rotary_emb=image_rotary_emb,
+ )
+
+ hidden_states = hidden_states + img_gate1 * img_attn_output
+ encoder_hidden_states = encoder_hidden_states + txt_gate1 * txt_attn_output
+
+ img_normed2 = self.img_norm2(hidden_states)
+ img_modulated2, img_gate2 = self._modulate(img_normed2, img_mod2)
+ hidden_states = hidden_states + img_gate2 * self.img_mlp(img_modulated2)
+
+ txt_normed2 = self.txt_norm2(encoder_hidden_states)
+ txt_modulated2, txt_gate2 = self._modulate(txt_normed2, txt_mod2)
+ encoder_hidden_states = encoder_hidden_states + txt_gate2 * self.txt_mlp(txt_modulated2)
+
+ return encoder_hidden_states, hidden_states
+
+
+class LastLayer(nn.Module):
+ def __init__(
+ self,
+ embedding_dim: int,
+ conditioning_embedding_dim: int,
+ elementwise_affine=False,
+ eps=1e-6,
+ bias=True,
+ dtype=None, device=None, operations=None
+ ):
+ super().__init__()
+ self.silu = nn.SiLU()
+ self.linear = operations.Linear(conditioning_embedding_dim, embedding_dim * 2, bias=bias, dtype=dtype, device=device)
+ self.norm = operations.LayerNorm(embedding_dim, eps, elementwise_affine=False, bias=bias, dtype=dtype, device=device)
+
+ def forward(self, x: torch.Tensor, conditioning_embedding: torch.Tensor) -> torch.Tensor:
+ emb = self.linear(self.silu(conditioning_embedding))
+ scale, shift = torch.chunk(emb, 2, dim=1)
+ x = self.norm(x) * (1 + scale)[:, None, :] + shift[:, None, :]
+ return x
+
+
+class QwenImageTransformer2DModel(nn.Module):
+ def __init__(
+ self,
+ patch_size: int = 2,
+ in_channels: int = 64,
+ out_channels: Optional[int] = 16,
+ num_layers: int = 60,
+ attention_head_dim: int = 128,
+ num_attention_heads: int = 24,
+ joint_attention_dim: int = 3584,
+ pooled_projection_dim: int = 768,
+ guidance_embeds: bool = False,
+ axes_dims_rope: Tuple[int, int, int] = (16, 56, 56),
+ image_model=None,
+ dtype=None,
+ device=None,
+ operations=None,
+ ):
+ super().__init__()
+ self.dtype = dtype
+ self.patch_size = patch_size
+ self.out_channels = out_channels or in_channels
+ self.inner_dim = num_attention_heads * attention_head_dim
+
+ self.pe_embedder = EmbedND(dim=attention_head_dim, theta=10000, axes_dim=list(axes_dims_rope))
+
+ self.time_text_embed = QwenTimestepProjEmbeddings(
+ embedding_dim=self.inner_dim,
+ pooled_projection_dim=pooled_projection_dim,
+ dtype=dtype,
+ device=device,
+ operations=operations
+ )
+
+ self.txt_norm = operations.RMSNorm(joint_attention_dim, eps=1e-6, dtype=dtype, device=device)
+ self.img_in = operations.Linear(in_channels, self.inner_dim, dtype=dtype, device=device)
+ self.txt_in = operations.Linear(joint_attention_dim, self.inner_dim, dtype=dtype, device=device)
+
+ self.transformer_blocks = nn.ModuleList([
+ QwenImageTransformerBlock(
+ dim=self.inner_dim,
+ num_attention_heads=num_attention_heads,
+ attention_head_dim=attention_head_dim,
+ dtype=dtype,
+ device=device,
+ operations=operations
+ )
+ for _ in range(num_layers)
+ ])
+
+ self.norm_out = LastLayer(self.inner_dim, self.inner_dim, dtype=dtype, device=device, operations=operations)
+ self.proj_out = operations.Linear(self.inner_dim, patch_size * patch_size * self.out_channels, bias=True, dtype=dtype, device=device)
+ self.gradient_checkpointing = False
+
+ def process_img(self, x, index=0, h_offset=0, w_offset=0):
+ bs, c, t, h, w = x.shape
+ patch_size = self.patch_size
+ hidden_states = comfy.ldm.common_dit.pad_to_patch_size(x, (1, self.patch_size, self.patch_size))
+ orig_shape = hidden_states.shape
+ hidden_states = hidden_states.view(orig_shape[0], orig_shape[1], orig_shape[-2] // 2, 2, orig_shape[-1] // 2, 2)
+ hidden_states = hidden_states.permute(0, 2, 4, 1, 3, 5)
+ hidden_states = hidden_states.reshape(orig_shape[0], (orig_shape[-2] // 2) * (orig_shape[-1] // 2), orig_shape[1] * 4)
+ h_len = ((h + (patch_size // 2)) // patch_size)
+ w_len = ((w + (patch_size // 2)) // patch_size)
+
+ h_offset = ((h_offset + (patch_size // 2)) // patch_size)
+ w_offset = ((w_offset + (patch_size // 2)) // patch_size)
+
+ img_ids = torch.zeros((h_len, w_len, 3), device=x.device)
+ img_ids[:, :, 0] = img_ids[:, :, 1] + index
+ img_ids[:, :, 1] = img_ids[:, :, 1] + torch.linspace(h_offset, h_len - 1 + h_offset, steps=h_len, device=x.device, dtype=x.dtype).unsqueeze(1) - (h_len // 2)
+ img_ids[:, :, 2] = img_ids[:, :, 2] + torch.linspace(w_offset, w_len - 1 + w_offset, steps=w_len, device=x.device, dtype=x.dtype).unsqueeze(0) - (w_len // 2)
+ return hidden_states, repeat(img_ids, "h w c -> b (h w) c", b=bs), orig_shape
+
+ def forward(
+ self,
+ x,
+ timesteps,
+ context,
+ attention_mask=None,
+ guidance: torch.Tensor = None,
+ ref_latents=None,
+ transformer_options={},
+ **kwargs
+ ):
+ timestep = timesteps
+ encoder_hidden_states = context
+ encoder_hidden_states_mask = attention_mask
+
+ hidden_states, img_ids, orig_shape = self.process_img(x)
+ num_embeds = hidden_states.shape[1]
+
+ if ref_latents is not None:
+ h = 0
+ w = 0
+ index = 0
+ index_ref_method = kwargs.get("ref_latents_method", "index") == "index"
+ for ref in ref_latents:
+ if index_ref_method:
+ index += 1
+ h_offset = 0
+ w_offset = 0
+ else:
+ index = 1
+ h_offset = 0
+ w_offset = 0
+ if ref.shape[-2] + h > ref.shape[-1] + w:
+ w_offset = w
+ else:
+ h_offset = h
+ h = max(h, ref.shape[-2] + h_offset)
+ w = max(w, ref.shape[-1] + w_offset)
+
+ kontext, kontext_ids, _ = self.process_img(ref, index=index, h_offset=h_offset, w_offset=w_offset)
+ hidden_states = torch.cat([hidden_states, kontext], dim=1)
+ img_ids = torch.cat([img_ids, kontext_ids], dim=1)
+
+ txt_start = round(max(((x.shape[-1] + (self.patch_size // 2)) // self.patch_size) // 2, ((x.shape[-2] + (self.patch_size // 2)) // self.patch_size) // 2))
+ txt_ids = torch.arange(txt_start, txt_start + context.shape[1], device=x.device).reshape(1, -1, 1).repeat(x.shape[0], 1, 3)
+ ids = torch.cat((txt_ids, img_ids), dim=1)
+ image_rotary_emb = self.pe_embedder(ids).squeeze(1).unsqueeze(2).to(x.dtype)
+ del ids, txt_ids, img_ids
+
+ hidden_states = self.img_in(hidden_states)
+ encoder_hidden_states = self.txt_norm(encoder_hidden_states)
+ encoder_hidden_states = self.txt_in(encoder_hidden_states)
+
+ if guidance is not None:
+ guidance = guidance * 1000
+
+ temb = (
+ self.time_text_embed(timestep, hidden_states)
+ if guidance is None
+ else self.time_text_embed(timestep, guidance, hidden_states)
+ )
+
+ patches_replace = transformer_options.get("patches_replace", {})
+ patches = transformer_options.get("patches", {})
+ blocks_replace = patches_replace.get("dit", {})
+
+ for i, block in enumerate(self.transformer_blocks):
+ if ("double_block", i) in blocks_replace:
+ def block_wrap(args):
+ out = {}
+ out["txt"], out["img"] = block(hidden_states=args["img"], encoder_hidden_states=args["txt"], encoder_hidden_states_mask=encoder_hidden_states_mask, temb=args["vec"], image_rotary_emb=args["pe"])
+ return out
+ out = blocks_replace[("double_block", i)]({"img": hidden_states, "txt": encoder_hidden_states, "vec": temb, "pe": image_rotary_emb}, {"original_block": block_wrap})
+ hidden_states = out["img"]
+ encoder_hidden_states = out["txt"]
+ else:
+ encoder_hidden_states, hidden_states = block(
+ hidden_states=hidden_states,
+ encoder_hidden_states=encoder_hidden_states,
+ encoder_hidden_states_mask=encoder_hidden_states_mask,
+ temb=temb,
+ image_rotary_emb=image_rotary_emb,
+ )
+
+ if "double_block" in patches:
+ for p in patches["double_block"]:
+ out = p({"img": hidden_states, "txt": encoder_hidden_states, "x": x, "block_index": i})
+ hidden_states = out["img"]
+ encoder_hidden_states = out["txt"]
+
+ hidden_states = self.norm_out(hidden_states, temb)
+ hidden_states = self.proj_out(hidden_states)
+
+ hidden_states = hidden_states[:, :num_embeds].view(orig_shape[0], orig_shape[-2] // 2, orig_shape[-1] // 2, orig_shape[1], 2, 2)
+ hidden_states = hidden_states.permute(0, 3, 1, 4, 2, 5)
+ return hidden_states.reshape(orig_shape)[:, :, :, :x.shape[-2], :x.shape[-1]]
diff --git a/comfy/ldm/wan/model.py b/comfy/ldm/wan/model.py
index 86d0795e9..9d3741be3 100644
--- a/comfy/ldm/wan/model.py
+++ b/comfy/ldm/wan/model.py
@@ -391,6 +391,7 @@ class WanModel(torch.nn.Module):
cross_attn_norm=True,
eps=1e-6,
flf_pos_embed_token_number=None,
+ in_dim_ref_conv=None,
image_model=None,
device=None,
dtype=None,
@@ -484,6 +485,11 @@ class WanModel(torch.nn.Module):
else:
self.img_emb = None
+ if in_dim_ref_conv is not None:
+ self.ref_conv = operations.Conv2d(in_dim_ref_conv, dim, kernel_size=patch_size[1:], stride=patch_size[1:], device=operation_settings.get("device"), dtype=operation_settings.get("dtype"))
+ else:
+ self.ref_conv = None
+
def forward_orig(
self,
x,
@@ -526,6 +532,13 @@ class WanModel(torch.nn.Module):
e = e.reshape(t.shape[0], -1, e.shape[-1])
e0 = self.time_projection(e).unflatten(2, (6, self.dim))
+ full_ref = None
+ if self.ref_conv is not None:
+ full_ref = kwargs.get("reference_latent", None)
+ if full_ref is not None:
+ full_ref = self.ref_conv(full_ref).flatten(2).transpose(1, 2)
+ x = torch.concat((full_ref, x), dim=1)
+
# context
context = self.text_embedding(context)
@@ -552,6 +565,9 @@ class WanModel(torch.nn.Module):
# head
x = self.head(x, e)
+ if full_ref is not None:
+ x = x[:, full_ref.shape[1]:]
+
# unpatchify
x = self.unpatchify(x, grid_sizes)
return x
@@ -570,6 +586,9 @@ class WanModel(torch.nn.Module):
x = torch.cat([x, time_dim_concat], dim=2)
t_len = ((x.shape[2] + (patch_size[0] // 2)) // patch_size[0])
+ if self.ref_conv is not None and "reference_latent" in kwargs:
+ t_len += 1
+
img_ids = torch.zeros((t_len, h_len, w_len, 3), device=x.device, dtype=x.dtype)
img_ids[:, :, :, 0] = img_ids[:, :, :, 0] + torch.linspace(0, t_len - 1, steps=t_len, device=x.device, dtype=x.dtype).reshape(-1, 1, 1)
img_ids[:, :, :, 1] = img_ids[:, :, :, 1] + torch.linspace(0, h_len - 1, steps=h_len, device=x.device, dtype=x.dtype).reshape(1, -1, 1)
@@ -749,7 +768,12 @@ class CameraWanModel(WanModel):
operations=None,
):
- super().__init__(model_type='i2v', patch_size=patch_size, text_len=text_len, in_dim=in_dim, dim=dim, ffn_dim=ffn_dim, freq_dim=freq_dim, text_dim=text_dim, out_dim=out_dim, num_heads=num_heads, num_layers=num_layers, window_size=window_size, qk_norm=qk_norm, cross_attn_norm=cross_attn_norm, eps=eps, flf_pos_embed_token_number=flf_pos_embed_token_number, image_model=image_model, device=device, dtype=dtype, operations=operations)
+ if model_type == 'camera':
+ model_type = 'i2v'
+ else:
+ model_type = 't2v'
+
+ super().__init__(model_type=model_type, patch_size=patch_size, text_len=text_len, in_dim=in_dim, dim=dim, ffn_dim=ffn_dim, freq_dim=freq_dim, text_dim=text_dim, out_dim=out_dim, num_heads=num_heads, num_layers=num_layers, window_size=window_size, qk_norm=qk_norm, cross_attn_norm=cross_attn_norm, eps=eps, flf_pos_embed_token_number=flf_pos_embed_token_number, image_model=image_model, device=device, dtype=dtype, operations=operations)
operation_settings = {"operations": operations, "device": device, "dtype": dtype}
self.control_adapter = WanCamAdapter(in_dim_control_adapter, dim, kernel_size=patch_size[1:], stride=patch_size[1:], operation_settings=operation_settings)
diff --git a/comfy/lora.py b/comfy/lora.py
index 387d5c52a..00358884b 100644
--- a/comfy/lora.py
+++ b/comfy/lora.py
@@ -293,6 +293,16 @@ def model_lora_keys_unet(model, key_map={}):
key_lora = k[len("diffusion_model."):-len(".weight")]
key_map["{}".format(key_lora)] = k
+ if isinstance(model, comfy.model_base.QwenImage):
+ for k in sdk:
+ if k.startswith("diffusion_model.") and k.endswith(".weight"): #QwenImage lora format
+ key_lora = k[len("diffusion_model."):-len(".weight")]
+ # Direct mapping for transformer_blocks format (QwenImage LoRA format)
+ key_map["{}".format(key_lora)] = k
+ # Support transformer prefix format
+ key_map["transformer.{}".format(key_lora)] = k
+ key_map["lycoris_{}".format(key_lora.replace(".", "_"))] = k #SimpleTuner lycoris format
+
return key_map
diff --git a/comfy/model_base.py b/comfy/model_base.py
index a06686436..6c861b15e 100644
--- a/comfy/model_base.py
+++ b/comfy/model_base.py
@@ -42,6 +42,7 @@ import comfy.ldm.hidream.model
import comfy.ldm.chroma.model
import comfy.ldm.ace.model
import comfy.ldm.omnigen.omnigen2
+import comfy.ldm.qwen_image.model
import comfy.model_management
import comfy.patcher_extension
@@ -889,6 +890,10 @@ class Flux(BaseModel):
for lat in ref_latents:
latents.append(self.process_latent_in(lat))
out['ref_latents'] = comfy.conds.CONDList(latents)
+
+ ref_latents_method = kwargs.get("reference_latents_method", None)
+ if ref_latents_method is not None:
+ out['ref_latents_method'] = comfy.conds.CONDConstant(ref_latents_method)
return out
def extra_conds_shapes(self, **kwargs):
@@ -1123,7 +1128,11 @@ class WAN21(BaseModel):
mask = mask.repeat(1, 4, 1, 1, 1)
mask = utils.resize_to_batch_size(mask, noise.shape[0])
- return torch.cat((mask, image), dim=1)
+ concat_mask_index = kwargs.get("concat_mask_index", 0)
+ if concat_mask_index != 0:
+ return torch.cat((image[:, :concat_mask_index], mask, image[:, concat_mask_index:]), dim=1)
+ else:
+ return torch.cat((mask, image), dim=1)
def extra_conds(self, **kwargs):
out = super().extra_conds(**kwargs)
@@ -1139,6 +1148,10 @@ class WAN21(BaseModel):
if time_dim_concat is not None:
out['time_dim_concat'] = comfy.conds.CONDRegular(self.process_latent_in(time_dim_concat))
+ reference_latents = kwargs.get("reference_latents", None)
+ if reference_latents is not None:
+ out['reference_latent'] = comfy.conds.CONDRegular(self.process_latent_in(reference_latents[-1])[:, :, 0])
+
return out
@@ -1308,3 +1321,32 @@ class Omnigen2(BaseModel):
if ref_latents is not None:
out['ref_latents'] = list([1, 16, sum(map(lambda a: math.prod(a.size()), ref_latents)) // 16])
return out
+
+class QwenImage(BaseModel):
+ def __init__(self, model_config, model_type=ModelType.FLUX, device=None):
+ super().__init__(model_config, model_type, device=device, unet_model=comfy.ldm.qwen_image.model.QwenImageTransformer2DModel)
+ self.memory_usage_factor_conds = ("ref_latents",)
+
+ def extra_conds(self, **kwargs):
+ out = super().extra_conds(**kwargs)
+ cross_attn = kwargs.get("cross_attn", None)
+ if cross_attn is not None:
+ out['c_crossattn'] = comfy.conds.CONDRegular(cross_attn)
+ ref_latents = kwargs.get("reference_latents", None)
+ if ref_latents is not None:
+ latents = []
+ for lat in ref_latents:
+ latents.append(self.process_latent_in(lat))
+ out['ref_latents'] = comfy.conds.CONDList(latents)
+
+ ref_latents_method = kwargs.get("reference_latents_method", None)
+ if ref_latents_method is not None:
+ out['ref_latents_method'] = comfy.conds.CONDConstant(ref_latents_method)
+ return out
+
+ def extra_conds_shapes(self, **kwargs):
+ out = {}
+ ref_latents = kwargs.get("reference_latents", None)
+ if ref_latents is not None:
+ out['ref_latents'] = list([1, 16, sum(map(lambda a: math.prod(a.size()), ref_latents)) // 16])
+ return out
diff --git a/comfy/model_detection.py b/comfy/model_detection.py
index 9fc1f42de..2bec0541e 100644
--- a/comfy/model_detection.py
+++ b/comfy/model_detection.py
@@ -364,7 +364,10 @@ def detect_unet_config(state_dict, key_prefix, metadata=None):
dit_config["vace_in_dim"] = state_dict['{}vace_patch_embedding.weight'.format(key_prefix)].shape[1]
dit_config["vace_layers"] = count_blocks(state_dict_keys, '{}vace_blocks.'.format(key_prefix) + '{}.')
elif '{}control_adapter.conv.weight'.format(key_prefix) in state_dict_keys:
- dit_config["model_type"] = "camera"
+ if '{}img_emb.proj.0.bias'.format(key_prefix) in state_dict_keys:
+ dit_config["model_type"] = "camera"
+ else:
+ dit_config["model_type"] = "camera_2.2"
else:
if '{}img_emb.proj.0.bias'.format(key_prefix) in state_dict_keys:
dit_config["model_type"] = "i2v"
@@ -373,6 +376,11 @@ def detect_unet_config(state_dict, key_prefix, metadata=None):
flf_weight = state_dict.get('{}img_emb.emb_pos'.format(key_prefix))
if flf_weight is not None:
dit_config["flf_pos_embed_token_number"] = flf_weight.shape[1]
+
+ ref_conv_weight = state_dict.get('{}ref_conv.weight'.format(key_prefix))
+ if ref_conv_weight is not None:
+ dit_config["in_dim_ref_conv"] = ref_conv_weight.shape[1]
+
return dit_config
if '{}latent_in.weight'.format(key_prefix) in state_dict_keys: # Hunyuan 3D
@@ -481,6 +489,11 @@ def detect_unet_config(state_dict, key_prefix, metadata=None):
dit_config["timestep_scale"] = 1000.0
return dit_config
+ if '{}txt_norm.weight'.format(key_prefix) in state_dict_keys: # Qwen Image
+ dit_config = {}
+ dit_config["image_model"] = "qwen_image"
+ return dit_config
+
if '{}input_blocks.0.0.weight'.format(key_prefix) not in state_dict_keys:
return None
@@ -867,7 +880,7 @@ def convert_diffusers_mmdit(state_dict, output_prefix=""):
depth_single_blocks = count_blocks(state_dict, 'single_transformer_blocks.{}.')
hidden_size = state_dict["x_embedder.bias"].shape[0]
sd_map = comfy.utils.flux_to_diffusers({"depth": depth, "depth_single_blocks": depth_single_blocks, "hidden_size": hidden_size}, output_prefix=output_prefix)
- elif 'transformer_blocks.0.attn.add_q_proj.weight' in state_dict: #SD3
+ elif 'transformer_blocks.0.attn.add_q_proj.weight' in state_dict and 'pos_embed.proj.weight' in state_dict: #SD3
num_blocks = count_blocks(state_dict, 'transformer_blocks.{}.')
depth = state_dict["pos_embed.proj.weight"].shape[0] // 64
sd_map = comfy.utils.mmdit_to_diffusers({"depth": depth, "num_blocks": num_blocks}, output_prefix=output_prefix)
diff --git a/comfy/model_management.py b/comfy/model_management.py
index 9e6149d60..d08aee1fe 100644
--- a/comfy/model_management.py
+++ b/comfy/model_management.py
@@ -78,7 +78,6 @@ try:
torch_version = torch.version.__version__
temp = torch_version.split(".")
torch_version_numeric = (int(temp[0]), int(temp[1]))
- xpu_available = (torch_version_numeric[0] < 2 or (torch_version_numeric[0] == 2 and torch_version_numeric[1] <= 4)) and torch.xpu.is_available()
except:
pass
@@ -102,10 +101,14 @@ if args.directml is not None:
try:
import intel_extension_for_pytorch as ipex # noqa: F401
- _ = torch.xpu.device_count()
- xpu_available = xpu_available or torch.xpu.is_available()
except:
- xpu_available = xpu_available or (hasattr(torch, "xpu") and torch.xpu.is_available())
+ pass
+
+try:
+ _ = torch.xpu.device_count()
+ xpu_available = torch.xpu.is_available()
+except:
+ xpu_available = False
try:
if torch.backends.mps.is_available():
@@ -321,9 +324,9 @@ try:
if torch_version_numeric >= (2, 7): # works on 2.6 but doesn't actually seem to improve much
if any((a in arch) for a in ["gfx90a", "gfx942", "gfx1100", "gfx1101", "gfx1151"]): # TODO: more arches, TODO: gfx950
ENABLE_PYTORCH_ATTENTION = True
- if torch_version_numeric >= (2, 8):
- if any((a in arch) for a in ["gfx1201"]):
- ENABLE_PYTORCH_ATTENTION = True
+# if torch_version_numeric >= (2, 8):
+# if any((a in arch) for a in ["gfx1201"]):
+# ENABLE_PYTORCH_ATTENTION = True
if torch_version_numeric >= (2, 7) and rocm_version >= (6, 4):
if any((a in arch) for a in ["gfx1201", "gfx942", "gfx950"]): # TODO: more arches
SUPPORT_FP8_OPS = True
@@ -340,7 +343,7 @@ if ENABLE_PYTORCH_ATTENTION:
PRIORITIZE_FP16 = False # TODO: remove and replace with something that shows exactly which dtype is faster than the other
try:
- if is_nvidia() and PerformanceFeature.Fp16Accumulation in args.fast:
+ if (is_nvidia() or is_amd()) and PerformanceFeature.Fp16Accumulation in args.fast:
torch.backends.cuda.matmul.allow_fp16_accumulation = True
PRIORITIZE_FP16 = True # TODO: limit to cards where it actually boosts performance
logging.info("Enabled fp16 accumulation.")
@@ -590,7 +593,13 @@ def load_models_gpu(models, memory_required=0, force_patch_weights=False, minimu
else:
minimum_memory_required = max(inference_memory, minimum_memory_required + extra_reserved_memory())
- models = set(models)
+ models_temp = set()
+ for m in models:
+ models_temp.add(m)
+ for mm in m.model_patches_models():
+ models_temp.add(mm)
+
+ models = models_temp
models_to_load = []
@@ -946,10 +955,12 @@ def pick_weight_dtype(dtype, fallback_dtype, device=None):
return dtype
def device_supports_non_blocking(device):
+ if args.force_non_blocking:
+ return True
if is_device_mps(device):
return False #pytorch bug? mps doesn't support non blocking
- if is_intel_xpu():
- return True
+ if is_intel_xpu(): #xpu does support non blocking but it is slower on iGPUs for some reason so disable by default until situation changes
+ return False
if args.deterministic: #TODO: figure out why deterministic breaks non blocking from gpu to cpu (previews)
return False
if directml_enabled:
@@ -1282,10 +1293,10 @@ def should_use_bf16(device=None, model_params=0, prioritize_performance=True, ma
return False
if is_intel_xpu():
- if torch_version_numeric < (2, 6):
+ if torch_version_numeric < (2, 3):
return True
else:
- return torch.xpu.get_device_capability(device)['has_bfloat16_conversions']
+ return torch.xpu.is_bf16_supported()
if is_ascend_npu():
return True
diff --git a/comfy/model_patcher.py b/comfy/model_patcher.py
index 52e76b5f3..a944cb421 100644
--- a/comfy/model_patcher.py
+++ b/comfy/model_patcher.py
@@ -430,6 +430,9 @@ class ModelPatcher:
def set_model_forward_timestep_embed_patch(self, patch):
self.set_model_patch(patch, "forward_timestep_embed_patch")
+ def set_model_double_block_patch(self, patch):
+ self.set_model_patch(patch, "double_block")
+
def add_object_patch(self, name, obj):
self.object_patches[name] = obj
@@ -486,6 +489,30 @@ class ModelPatcher:
if hasattr(wrap_func, "to"):
self.model_options["model_function_wrapper"] = wrap_func.to(device)
+ def model_patches_models(self):
+ to = self.model_options["transformer_options"]
+ models = []
+ if "patches" in to:
+ patches = to["patches"]
+ for name in patches:
+ patch_list = patches[name]
+ for i in range(len(patch_list)):
+ if hasattr(patch_list[i], "models"):
+ models += patch_list[i].models()
+ if "patches_replace" in to:
+ patches = to["patches_replace"]
+ for name in patches:
+ patch_list = patches[name]
+ for k in patch_list:
+ if hasattr(patch_list[k], "models"):
+ models += patch_list[k].models()
+ if "model_function_wrapper" in self.model_options:
+ wrap_func = self.model_options["model_function_wrapper"]
+ if hasattr(wrap_func, "models"):
+ models += wrap_func.models()
+
+ return models
+
def model_dtype(self):
if hasattr(self.model, "get_dtype"):
return self.model.get_dtype()
diff --git a/comfy/ops.py b/comfy/ops.py
index 2cc9bbc27..18e7db705 100644
--- a/comfy/ops.py
+++ b/comfy/ops.py
@@ -24,6 +24,32 @@ import comfy.float
import comfy.rmsnorm
import contextlib
+
+def scaled_dot_product_attention(q, k, v, *args, **kwargs):
+ return torch.nn.functional.scaled_dot_product_attention(q, k, v, *args, **kwargs)
+
+
+try:
+ if torch.cuda.is_available():
+ from torch.nn.attention import SDPBackend, sdpa_kernel
+ import inspect
+ if "set_priority" in inspect.signature(sdpa_kernel).parameters:
+ SDPA_BACKEND_PRIORITY = [
+ SDPBackend.FLASH_ATTENTION,
+ SDPBackend.EFFICIENT_ATTENTION,
+ SDPBackend.MATH,
+ ]
+
+ SDPA_BACKEND_PRIORITY.insert(0, SDPBackend.CUDNN_ATTENTION)
+
+ def scaled_dot_product_attention(q, k, v, *args, **kwargs):
+ with sdpa_kernel(SDPA_BACKEND_PRIORITY, set_priority=True):
+ return torch.nn.functional.scaled_dot_product_attention(q, k, v, *args, **kwargs)
+ else:
+ logging.warning("Torch version too old to set sdpa backend priority.")
+except (ModuleNotFoundError, TypeError):
+ logging.warning("Could not set sdpa backend priority.")
+
cast_to = comfy.model_management.cast_to #TODO: remove once no more references
def cast_to_input(weight, input, non_blocking=False, copy=True):
diff --git a/comfy/rmsnorm.py b/comfy/rmsnorm.py
index 66ae8321d..555542a46 100644
--- a/comfy/rmsnorm.py
+++ b/comfy/rmsnorm.py
@@ -1,6 +1,7 @@
import torch
import comfy.model_management
import numbers
+import logging
RMSNorm = None
@@ -9,6 +10,7 @@ try:
RMSNorm = torch.nn.RMSNorm
except:
rms_norm_torch = None
+ logging.warning("Please update pytorch to use native RMSNorm")
def rms_norm(x, weight=None, eps=1e-6):
diff --git a/comfy/sampler_helpers.py b/comfy/sampler_helpers.py
index 8dbc41455..e46971afb 100644
--- a/comfy/sampler_helpers.py
+++ b/comfy/sampler_helpers.py
@@ -149,7 +149,7 @@ def cleanup_models(conds, models):
cleanup_additional_models(set(control_cleanup))
-def prepare_model_patcher(model: 'ModelPatcher', conds, model_options: dict):
+def prepare_model_patcher(model: ModelPatcher, conds, model_options: dict):
'''
Registers hooks from conds.
'''
@@ -158,8 +158,8 @@ def prepare_model_patcher(model: 'ModelPatcher', conds, model_options: dict):
for k in conds:
get_hooks_from_cond(conds[k], hooks)
# add wrappers and callbacks from ModelPatcher to transformer_options
- model_options["transformer_options"]["wrappers"] = comfy.patcher_extension.copy_nested_dicts(model.wrappers)
- model_options["transformer_options"]["callbacks"] = comfy.patcher_extension.copy_nested_dicts(model.callbacks)
+ comfy.patcher_extension.merge_nested_dicts(model_options["transformer_options"].setdefault("wrappers", {}), model.wrappers, copy_dict1=False)
+ comfy.patcher_extension.merge_nested_dicts(model_options["transformer_options"].setdefault("callbacks", {}), model.callbacks, copy_dict1=False)
# begin registering hooks
registered = comfy.hooks.HookGroup()
target_dict = comfy.hooks.create_target_dict(comfy.hooks.EnumWeightTarget.Model)
diff --git a/comfy/samplers.py b/comfy/samplers.py
index ad2f40cdc..d5390d64e 100644
--- a/comfy/samplers.py
+++ b/comfy/samplers.py
@@ -16,6 +16,7 @@ import comfy.sampler_helpers
import comfy.model_patcher
import comfy.patcher_extension
import comfy.hooks
+import comfy.context_windows
import scipy.stats
import numpy
@@ -198,14 +199,20 @@ def finalize_default_conds(model: 'BaseModel', hooked_to_run: dict[comfy.hooks.H
hooked_to_run.setdefault(p.hooks, list())
hooked_to_run[p.hooks] += [(p, i)]
-def calc_cond_batch(model: 'BaseModel', conds: list[list[dict]], x_in: torch.Tensor, timestep, model_options):
+def calc_cond_batch(model: BaseModel, conds: list[list[dict]], x_in: torch.Tensor, timestep, model_options: dict[str]):
+ handler: comfy.context_windows.ContextHandlerABC = model_options.get("context_handler", None)
+ if handler is None or not handler.should_use_context(model, conds, x_in, timestep, model_options):
+ return _calc_cond_batch_outer(model, conds, x_in, timestep, model_options)
+ return handler.execute(_calc_cond_batch_outer, model, conds, x_in, timestep, model_options)
+
+def _calc_cond_batch_outer(model: BaseModel, conds: list[list[dict]], x_in: torch.Tensor, timestep, model_options):
executor = comfy.patcher_extension.WrapperExecutor.new_executor(
_calc_cond_batch,
comfy.patcher_extension.get_all_wrappers(comfy.patcher_extension.WrappersMP.CALC_COND_BATCH, model_options, is_model_options=True)
)
return executor.execute(model, conds, x_in, timestep, model_options)
-def _calc_cond_batch(model: 'BaseModel', conds: list[list[dict]], x_in: torch.Tensor, timestep, model_options):
+def _calc_cond_batch(model: BaseModel, conds: list[list[dict]], x_in: torch.Tensor, timestep, model_options):
out_conds = []
out_counts = []
# separate conds by matching hooks
diff --git a/comfy/sd.py b/comfy/sd.py
index e0498e585..bb5d61fb3 100644
--- a/comfy/sd.py
+++ b/comfy/sd.py
@@ -47,6 +47,7 @@ import comfy.text_encoders.wan
import comfy.text_encoders.hidream
import comfy.text_encoders.ace
import comfy.text_encoders.omnigen2
+import comfy.text_encoders.qwen_image
import comfy.model_patcher
import comfy.lora
@@ -771,6 +772,7 @@ class CLIPType(Enum):
CHROMA = 15
ACE = 16
OMNIGEN2 = 17
+ QWEN_IMAGE = 18
def load_clip(ckpt_paths, embedding_directory=None, clip_type=CLIPType.STABLE_DIFFUSION, model_options={}):
@@ -791,6 +793,7 @@ class TEModel(Enum):
T5_XXL_OLD = 8
GEMMA_2_2B = 9
QWEN25_3B = 10
+ QWEN25_7B = 11
def detect_te_model(sd):
if "text_model.encoder.layers.30.mlp.fc1.weight" in sd:
@@ -812,7 +815,11 @@ def detect_te_model(sd):
if 'model.layers.0.post_feedforward_layernorm.weight' in sd:
return TEModel.GEMMA_2_2B
if 'model.layers.0.self_attn.k_proj.bias' in sd:
- return TEModel.QWEN25_3B
+ weight = sd['model.layers.0.self_attn.k_proj.bias']
+ if weight.shape[0] == 256:
+ return TEModel.QWEN25_3B
+ if weight.shape[0] == 512:
+ return TEModel.QWEN25_7B
if "model.layers.0.post_attention_layernorm.weight" in sd:
return TEModel.LLAMA3_8
return None
@@ -917,6 +924,9 @@ def load_text_encoder_state_dicts(state_dicts=[], embedding_directory=None, clip
elif te_model == TEModel.QWEN25_3B:
clip_target.clip = comfy.text_encoders.omnigen2.te(**llama_detect(clip_data))
clip_target.tokenizer = comfy.text_encoders.omnigen2.Omnigen2Tokenizer
+ elif te_model == TEModel.QWEN25_7B:
+ clip_target.clip = comfy.text_encoders.qwen_image.te(**llama_detect(clip_data))
+ clip_target.tokenizer = comfy.text_encoders.qwen_image.QwenImageTokenizer
else:
# clip_l
if clip_type == CLIPType.SD3:
diff --git a/comfy/sd1_clip.py b/comfy/sd1_clip.py
index ade340fd1..f8a7c2a1b 100644
--- a/comfy/sd1_clip.py
+++ b/comfy/sd1_clip.py
@@ -204,17 +204,19 @@ class SDClipModel(torch.nn.Module, ClipTokenWeightEncoder):
tokens_embed = self.transformer.get_input_embeddings()(tokens_embed, out_dtype=torch.float32)
index = 0
pad_extra = 0
+ embeds_info = []
for o in other_embeds:
emb = o[1]
if torch.is_tensor(emb):
emb = {"type": "embedding", "data": emb}
+ extra = None
emb_type = emb.get("type", None)
if emb_type == "embedding":
emb = emb.get("data", None)
else:
if hasattr(self.transformer, "preprocess_embed"):
- emb = self.transformer.preprocess_embed(emb, device=device)
+ emb, extra = self.transformer.preprocess_embed(emb, device=device)
else:
emb = None
@@ -229,6 +231,7 @@ class SDClipModel(torch.nn.Module, ClipTokenWeightEncoder):
tokens_embed = torch.cat([tokens_embed[:, :ind], emb, tokens_embed[:, ind:]], dim=1)
attention_mask = attention_mask[:ind] + [1] * emb_shape + attention_mask[ind:]
index += emb_shape - 1
+ embeds_info.append({"type": emb_type, "index": ind, "size": emb_shape, "extra": extra})
else:
index += -1
pad_extra += emb_shape
@@ -243,11 +246,11 @@ class SDClipModel(torch.nn.Module, ClipTokenWeightEncoder):
attention_masks.append(attention_mask)
num_tokens.append(sum(attention_mask))
- return torch.cat(embeds_out), torch.tensor(attention_masks, device=device, dtype=torch.long), num_tokens
+ return torch.cat(embeds_out), torch.tensor(attention_masks, device=device, dtype=torch.long), num_tokens, embeds_info
def forward(self, tokens):
device = self.transformer.get_input_embeddings().weight.device
- embeds, attention_mask, num_tokens = self.process_tokens(tokens, device)
+ embeds, attention_mask, num_tokens, embeds_info = self.process_tokens(tokens, device)
attention_mask_model = None
if self.enable_attention_masks:
@@ -258,7 +261,7 @@ class SDClipModel(torch.nn.Module, ClipTokenWeightEncoder):
else:
intermediate_output = self.layer_idx
- outputs = self.transformer(None, attention_mask_model, embeds=embeds, num_tokens=num_tokens, intermediate_output=intermediate_output, final_layer_norm_intermediate=self.layer_norm_hidden_state, dtype=torch.float32)
+ outputs = self.transformer(None, attention_mask_model, embeds=embeds, num_tokens=num_tokens, intermediate_output=intermediate_output, final_layer_norm_intermediate=self.layer_norm_hidden_state, dtype=torch.float32, embeds_info=embeds_info)
if self.layer == "last":
z = outputs[0].float()
@@ -531,7 +534,10 @@ class SDTokenizer:
min_padding = tokenizer_options.get("{}_min_padding".format(self.embedding_key), self.min_padding)
text = escape_important(text)
- parsed_weights = token_weights(text, 1.0)
+ if kwargs.get("disable_weights", False):
+ parsed_weights = [(text, 1.0)]
+ else:
+ parsed_weights = token_weights(text, 1.0)
# tokenize words
tokens = []
diff --git a/comfy/supported_models.py b/comfy/supported_models.py
index 8f3f4652d..7ed6dfd69 100644
--- a/comfy/supported_models.py
+++ b/comfy/supported_models.py
@@ -19,6 +19,7 @@ import comfy.text_encoders.lumina2
import comfy.text_encoders.wan
import comfy.text_encoders.ace
import comfy.text_encoders.omnigen2
+import comfy.text_encoders.qwen_image
from . import supported_models_base
from . import latent_formats
@@ -1045,6 +1046,18 @@ class WAN21_Camera(WAN21_T2V):
def get_model(self, state_dict, prefix="", device=None):
out = model_base.WAN21_Camera(self, image_to_video=False, device=device)
return out
+
+class WAN22_Camera(WAN21_T2V):
+ unet_config = {
+ "image_model": "wan2.1",
+ "model_type": "camera_2.2",
+ "in_dim": 36,
+ }
+
+ def get_model(self, state_dict, prefix="", device=None):
+ out = model_base.WAN21_Camera(self, image_to_video=False, device=device)
+ return out
+
class WAN21_Vace(WAN21_T2V):
unet_config = {
"image_model": "wan2.1",
@@ -1229,7 +1242,36 @@ class Omnigen2(supported_models_base.BASE):
hunyuan_detect = comfy.text_encoders.hunyuan_video.llama_detect(state_dict, "{}qwen25_3b.transformer.".format(pref))
return supported_models_base.ClipTarget(comfy.text_encoders.omnigen2.Omnigen2Tokenizer, comfy.text_encoders.omnigen2.te(**hunyuan_detect))
+class QwenImage(supported_models_base.BASE):
+ unet_config = {
+ "image_model": "qwen_image",
+ }
-models = [LotusD, Stable_Zero123, SD15_instructpix2pix, SD15, SD20, SD21UnclipL, SD21UnclipH, SDXL_instructpix2pix, SDXLRefiner, SDXL, SSD1B, KOALA_700M, KOALA_1B, Segmind_Vega, SD_X4Upscaler, Stable_Cascade_C, Stable_Cascade_B, SV3D_u, SV3D_p, SD3, StableAudio, AuraFlow, PixArtAlpha, PixArtSigma, HunyuanDiT, HunyuanDiT1, FluxInpaint, Flux, FluxSchnell, GenmoMochi, LTXV, HunyuanVideoSkyreelsI2V, HunyuanVideoI2V, HunyuanVideo, CosmosT2V, CosmosI2V, CosmosT2IPredict2, CosmosI2VPredict2, Lumina2, WAN22_T2V, WAN21_T2V, WAN21_I2V, WAN21_FunControl2V, WAN21_Vace, WAN21_Camera, Hunyuan3Dv2mini, Hunyuan3Dv2, HiDream, Chroma, ACEStep, Omnigen2]
+ sampling_settings = {
+ "multiplier": 1.0,
+ "shift": 1.15,
+ }
+
+ memory_usage_factor = 1.8 #TODO
+
+ unet_extra_config = {}
+ latent_format = latent_formats.Wan21
+
+ supported_inference_dtypes = [torch.bfloat16, torch.float32]
+
+ vae_key_prefix = ["vae."]
+ text_encoder_key_prefix = ["text_encoders."]
+
+ def get_model(self, state_dict, prefix="", device=None):
+ out = model_base.QwenImage(self, device=device)
+ return out
+
+ def clip_target(self, state_dict={}):
+ pref = self.text_encoder_key_prefix[0]
+ hunyuan_detect = comfy.text_encoders.hunyuan_video.llama_detect(state_dict, "{}qwen25_7b.transformer.".format(pref))
+ return supported_models_base.ClipTarget(comfy.text_encoders.qwen_image.QwenImageTokenizer, comfy.text_encoders.qwen_image.te(**hunyuan_detect))
+
+
+models = [LotusD, Stable_Zero123, SD15_instructpix2pix, SD15, SD20, SD21UnclipL, SD21UnclipH, SDXL_instructpix2pix, SDXLRefiner, SDXL, SSD1B, KOALA_700M, KOALA_1B, Segmind_Vega, SD_X4Upscaler, Stable_Cascade_C, Stable_Cascade_B, SV3D_u, SV3D_p, SD3, StableAudio, AuraFlow, PixArtAlpha, PixArtSigma, HunyuanDiT, HunyuanDiT1, FluxInpaint, Flux, FluxSchnell, GenmoMochi, LTXV, HunyuanVideoSkyreelsI2V, HunyuanVideoI2V, HunyuanVideo, CosmosT2V, CosmosI2V, CosmosT2IPredict2, CosmosI2VPredict2, Lumina2, WAN22_T2V, WAN21_T2V, WAN21_I2V, WAN21_FunControl2V, WAN21_Vace, WAN21_Camera, WAN22_Camera, Hunyuan3Dv2mini, Hunyuan3Dv2, HiDream, Chroma, ACEStep, Omnigen2, QwenImage]
models += [SVD_img2vid]
diff --git a/comfy/text_encoders/bert.py b/comfy/text_encoders/bert.py
index 551b03162..ed4638a9a 100644
--- a/comfy/text_encoders/bert.py
+++ b/comfy/text_encoders/bert.py
@@ -116,7 +116,7 @@ class BertModel_(torch.nn.Module):
self.embeddings = BertEmbeddings(config_dict["vocab_size"], config_dict["max_position_embeddings"], config_dict["type_vocab_size"], config_dict["pad_token_id"], embed_dim, layer_norm_eps, dtype, device, operations)
self.encoder = BertEncoder(config_dict["num_hidden_layers"], embed_dim, config_dict["intermediate_size"], config_dict["num_attention_heads"], layer_norm_eps, dtype, device, operations)
- def forward(self, input_tokens, attention_mask=None, embeds=None, num_tokens=None, intermediate_output=None, final_layer_norm_intermediate=True, dtype=None):
+ def forward(self, input_tokens, attention_mask=None, embeds=None, num_tokens=None, intermediate_output=None, final_layer_norm_intermediate=True, dtype=None, embeds_info=[]):
x = self.embeddings(input_tokens, embeds=embeds, dtype=dtype)
mask = None
if attention_mask is not None:
diff --git a/comfy/text_encoders/llama.py b/comfy/text_encoders/llama.py
index 7fbd0f604..4c976058f 100644
--- a/comfy/text_encoders/llama.py
+++ b/comfy/text_encoders/llama.py
@@ -2,12 +2,14 @@ import torch
import torch.nn as nn
from dataclasses import dataclass
from typing import Optional, Any
+import math
from comfy.ldm.modules.attention import optimized_attention_for_device
import comfy.model_management
import comfy.ldm.common_dit
import comfy.model_management
+from . import qwen_vl
@dataclass
class Llama2Config:
@@ -25,6 +27,7 @@ class Llama2Config:
rms_norm_add = False
mlp_activation = "silu"
qkv_bias = False
+ rope_dims = None
@dataclass
class Qwen25_3BConfig:
@@ -42,6 +45,25 @@ class Qwen25_3BConfig:
rms_norm_add = False
mlp_activation = "silu"
qkv_bias = True
+ rope_dims = None
+
+@dataclass
+class Qwen25_7BVLI_Config:
+ vocab_size: int = 152064
+ hidden_size: int = 3584
+ intermediate_size: int = 18944
+ num_hidden_layers: int = 28
+ num_attention_heads: int = 28
+ num_key_value_heads: int = 4
+ max_position_embeddings: int = 128000
+ rms_norm_eps: float = 1e-6
+ rope_theta: float = 1000000.0
+ transformer_type: str = "llama"
+ head_dim = 128
+ rms_norm_add = False
+ mlp_activation = "silu"
+ qkv_bias = True
+ rope_dims = [16, 24, 24]
@dataclass
class Gemma2_2B_Config:
@@ -59,6 +81,7 @@ class Gemma2_2B_Config:
rms_norm_add = True
mlp_activation = "gelu_pytorch_tanh"
qkv_bias = False
+ rope_dims = None
class RMSNorm(nn.Module):
def __init__(self, dim: int, eps: float = 1e-5, add=False, device=None, dtype=None):
@@ -83,24 +106,30 @@ def rotate_half(x):
return torch.cat((-x2, x1), dim=-1)
-def precompute_freqs_cis(head_dim, seq_len, theta, device=None):
+def precompute_freqs_cis(head_dim, position_ids, theta, rope_dims=None, device=None):
theta_numerator = torch.arange(0, head_dim, 2, device=device).float()
inv_freq = 1.0 / (theta ** (theta_numerator / head_dim))
- position_ids = torch.arange(0, seq_len, device=device).unsqueeze(0)
-
inv_freq_expanded = inv_freq[None, :, None].float().expand(position_ids.shape[0], -1, 1)
position_ids_expanded = position_ids[:, None, :].float()
freqs = (inv_freq_expanded.float() @ position_ids_expanded.float()).transpose(1, 2)
emb = torch.cat((freqs, freqs), dim=-1)
cos = emb.cos()
sin = emb.sin()
+ if rope_dims is not None and position_ids.shape[0] > 1:
+ mrope_section = rope_dims * 2
+ cos = torch.cat([m[i % 3] for i, m in enumerate(cos.split(mrope_section, dim=-1))], dim=-1).unsqueeze(0)
+ sin = torch.cat([m[i % 3] for i, m in enumerate(sin.split(mrope_section, dim=-1))], dim=-1).unsqueeze(0)
+ else:
+ cos = cos.unsqueeze(1)
+ sin = sin.unsqueeze(1)
+
return (cos, sin)
def apply_rope(xq, xk, freqs_cis):
- cos = freqs_cis[0].unsqueeze(1)
- sin = freqs_cis[1].unsqueeze(1)
+ cos = freqs_cis[0]
+ sin = freqs_cis[1]
q_embed = (xq * cos) + (rotate_half(xq) * sin)
k_embed = (xk * cos) + (rotate_half(xk) * sin)
return q_embed, k_embed
@@ -260,7 +289,7 @@ class Llama2_(nn.Module):
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps, add=config.rms_norm_add, device=device, dtype=dtype)
# self.lm_head = ops.Linear(config.hidden_size, config.vocab_size, bias=False, device=device, dtype=dtype)
- def forward(self, x, attention_mask=None, embeds=None, num_tokens=None, intermediate_output=None, final_layer_norm_intermediate=True, dtype=None):
+ def forward(self, x, attention_mask=None, embeds=None, num_tokens=None, intermediate_output=None, final_layer_norm_intermediate=True, dtype=None, position_ids=None, embeds_info=[]):
if embeds is not None:
x = embeds
else:
@@ -269,9 +298,13 @@ class Llama2_(nn.Module):
if self.normalize_in:
x *= self.config.hidden_size ** 0.5
+ if position_ids is None:
+ position_ids = torch.arange(0, x.shape[1], device=x.device).unsqueeze(0)
+
freqs_cis = precompute_freqs_cis(self.config.head_dim,
- x.shape[1],
+ position_ids,
self.config.rope_theta,
+ self.config.rope_dims,
device=x.device)
mask = None
@@ -348,6 +381,45 @@ class Qwen25_3B(BaseLlama, torch.nn.Module):
self.model = Llama2_(config, device=device, dtype=dtype, ops=operations)
self.dtype = dtype
+class Qwen25_7BVLI(BaseLlama, torch.nn.Module):
+ def __init__(self, config_dict, dtype, device, operations):
+ super().__init__()
+ config = Qwen25_7BVLI_Config(**config_dict)
+ self.num_layers = config.num_hidden_layers
+
+ self.model = Llama2_(config, device=device, dtype=dtype, ops=operations)
+ self.visual = qwen_vl.Qwen2VLVisionTransformer(hidden_size=1280, output_hidden_size=config.hidden_size, device=device, dtype=dtype, ops=operations)
+ self.dtype = dtype
+
+ def preprocess_embed(self, embed, device):
+ if embed["type"] == "image":
+ image, grid = qwen_vl.process_qwen2vl_images(embed["data"])
+ return self.visual(image.to(device, dtype=torch.float32), grid), grid
+ return None, None
+
+ def forward(self, x, attention_mask=None, embeds=None, num_tokens=None, intermediate_output=None, final_layer_norm_intermediate=True, dtype=None, embeds_info=[]):
+ grid = None
+ for e in embeds_info:
+ if e.get("type") == "image":
+ grid = e.get("extra", None)
+ position_ids = torch.zeros((3, embeds.shape[1]), device=embeds.device)
+ start = e.get("index")
+ position_ids[:, :start] = torch.arange(0, start, device=embeds.device)
+ end = e.get("size") + start
+ len_max = int(grid.max()) // 2
+ start_next = len_max + start
+ position_ids[:, end:] = torch.arange(start_next, start_next + (embeds.shape[1] - end), device=embeds.device)
+ position_ids[0, start:end] = start
+ max_d = int(grid[0][1]) // 2
+ position_ids[1, start:end] = torch.arange(start, start + max_d, device=embeds.device).unsqueeze(1).repeat(1, math.ceil((end - start) / max_d)).flatten(0)[:end - start]
+ max_d = int(grid[0][2]) // 2
+ position_ids[2, start:end] = torch.arange(start, start + max_d, device=embeds.device).unsqueeze(0).repeat(math.ceil((end - start) / max_d), 1).flatten(0)[:end - start]
+
+ if grid is None:
+ position_ids = None
+
+ return super().forward(x, attention_mask=attention_mask, embeds=embeds, num_tokens=num_tokens, intermediate_output=intermediate_output, final_layer_norm_intermediate=final_layer_norm_intermediate, dtype=dtype, position_ids=position_ids)
+
class Gemma2_2B(BaseLlama, torch.nn.Module):
def __init__(self, config_dict, dtype, device, operations):
super().__init__()
diff --git a/comfy/text_encoders/qwen_image.py b/comfy/text_encoders/qwen_image.py
new file mode 100644
index 000000000..6646b1003
--- /dev/null
+++ b/comfy/text_encoders/qwen_image.py
@@ -0,0 +1,85 @@
+from transformers import Qwen2Tokenizer
+from comfy import sd1_clip
+import comfy.text_encoders.llama
+import os
+import torch
+import numbers
+
+class Qwen25_7BVLITokenizer(sd1_clip.SDTokenizer):
+ def __init__(self, embedding_directory=None, tokenizer_data={}):
+ tokenizer_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "qwen25_tokenizer")
+ super().__init__(tokenizer_path, pad_with_end=False, embedding_size=3584, embedding_key='qwen25_7b', tokenizer_class=Qwen2Tokenizer, has_start_token=False, has_end_token=False, pad_to_max_length=False, max_length=99999999, min_length=1, pad_token=151643, tokenizer_data=tokenizer_data)
+
+
+class QwenImageTokenizer(sd1_clip.SD1Tokenizer):
+ def __init__(self, embedding_directory=None, tokenizer_data={}):
+ super().__init__(embedding_directory=embedding_directory, tokenizer_data=tokenizer_data, name="qwen25_7b", tokenizer=Qwen25_7BVLITokenizer)
+ self.llama_template = "<|im_start|>system\nDescribe the image by detailing the color, shape, size, texture, quantity, text, spatial relationships of the objects and background:<|im_end|>\n<|im_start|>user\n{}<|im_end|>\n<|im_start|>assistant\n"
+ self.llama_template_images = "<|im_start|>system\nDescribe the key features of the input image (color, shape, size, texture, objects, background), then explain how the user's text instruction should alter or modify the image. Generate a new image that meets the user's requirements while maintaining consistency with the original input where appropriate.<|im_end|>\n<|im_start|>user\n<|vision_start|><|image_pad|><|vision_end|>{}<|im_end|>\n<|im_start|>assistant\n"
+
+ def tokenize_with_weights(self, text, return_word_ids=False, llama_template=None, images=[], **kwargs):
+ if llama_template is None:
+ if len(images) > 0:
+ llama_text = self.llama_template_images.format(text)
+ else:
+ llama_text = self.llama_template.format(text)
+ else:
+ llama_text = llama_template.format(text)
+ tokens = super().tokenize_with_weights(llama_text, return_word_ids=return_word_ids, disable_weights=True, **kwargs)
+ key_name = next(iter(tokens))
+ embed_count = 0
+ qwen_tokens = tokens[key_name]
+ for r in qwen_tokens:
+ for i in range(len(r)):
+ if r[i][0] == 151655:
+ if len(images) > embed_count:
+ r[i] = ({"type": "image", "data": images[embed_count], "original_type": "image"},) + r[i][1:]
+ embed_count += 1
+ return tokens
+
+
+class Qwen25_7BVLIModel(sd1_clip.SDClipModel):
+ def __init__(self, device="cpu", layer="last", layer_idx=None, dtype=None, attention_mask=True, model_options={}):
+ super().__init__(device=device, layer=layer, layer_idx=layer_idx, textmodel_json_config={}, dtype=dtype, special_tokens={"pad": 151643}, layer_norm_hidden_state=False, model_class=comfy.text_encoders.llama.Qwen25_7BVLI, enable_attention_masks=attention_mask, return_attention_masks=attention_mask, model_options=model_options)
+
+
+class QwenImageTEModel(sd1_clip.SD1ClipModel):
+ def __init__(self, device="cpu", dtype=None, model_options={}):
+ super().__init__(device=device, dtype=dtype, name="qwen25_7b", clip_model=Qwen25_7BVLIModel, model_options=model_options)
+
+ def encode_token_weights(self, token_weight_pairs):
+ out, pooled, extra = super().encode_token_weights(token_weight_pairs)
+ tok_pairs = token_weight_pairs["qwen25_7b"][0]
+ count_im_start = 0
+ for i, v in enumerate(tok_pairs):
+ elem = v[0]
+ if not torch.is_tensor(elem):
+ if isinstance(elem, numbers.Integral):
+ if elem == 151644 and count_im_start < 2:
+ template_end = i
+ count_im_start += 1
+
+ if out.shape[1] > (template_end + 3):
+ if tok_pairs[template_end + 1][0] == 872:
+ if tok_pairs[template_end + 2][0] == 198:
+ template_end += 3
+
+ out = out[:, template_end:]
+
+ extra["attention_mask"] = extra["attention_mask"][:, template_end:]
+ if extra["attention_mask"].sum() == torch.numel(extra["attention_mask"]):
+ extra.pop("attention_mask") # attention mask is useless if no masked elements
+
+ return out, pooled, extra
+
+
+def te(dtype_llama=None, llama_scaled_fp8=None):
+ class QwenImageTEModel_(QwenImageTEModel):
+ def __init__(self, device="cpu", dtype=None, model_options={}):
+ if llama_scaled_fp8 is not None and "scaled_fp8" not in model_options:
+ model_options = model_options.copy()
+ model_options["scaled_fp8"] = llama_scaled_fp8
+ if dtype_llama is not None:
+ dtype = dtype_llama
+ super().__init__(device=device, dtype=dtype, model_options=model_options)
+ return QwenImageTEModel_
diff --git a/comfy/text_encoders/qwen_vl.py b/comfy/text_encoders/qwen_vl.py
new file mode 100644
index 000000000..3b18ce730
--- /dev/null
+++ b/comfy/text_encoders/qwen_vl.py
@@ -0,0 +1,428 @@
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from typing import Optional, Tuple
+import math
+from comfy.ldm.modules.attention import optimized_attention_for_device
+
+
+def process_qwen2vl_images(
+ images: torch.Tensor,
+ min_pixels: int = 3136,
+ max_pixels: int = 12845056,
+ patch_size: int = 14,
+ temporal_patch_size: int = 2,
+ merge_size: int = 2,
+ image_mean: list = None,
+ image_std: list = None,
+):
+ if image_mean is None:
+ image_mean = [0.48145466, 0.4578275, 0.40821073]
+ if image_std is None:
+ image_std = [0.26862954, 0.26130258, 0.27577711]
+
+ batch_size, height, width, channels = images.shape
+ device = images.device
+ # dtype = images.dtype
+
+ images = images.permute(0, 3, 1, 2)
+
+ grid_thw_list = []
+ img = images[0]
+
+ factor = patch_size * merge_size
+
+ h_bar = round(height / factor) * factor
+ w_bar = round(width / factor) * factor
+
+ if h_bar * w_bar > max_pixels:
+ beta = math.sqrt((height * width) / max_pixels)
+ h_bar = max(factor, math.floor(height / beta / factor) * factor)
+ w_bar = max(factor, math.floor(width / beta / factor) * factor)
+ elif h_bar * w_bar < min_pixels:
+ beta = math.sqrt(min_pixels / (height * width))
+ h_bar = math.ceil(height * beta / factor) * factor
+ w_bar = math.ceil(width * beta / factor) * factor
+
+ img_resized = F.interpolate(
+ img.unsqueeze(0),
+ size=(h_bar, w_bar),
+ mode='bilinear',
+ align_corners=False
+ ).squeeze(0)
+
+ normalized = img_resized.clone()
+ for c in range(3):
+ normalized[c] = (img_resized[c] - image_mean[c]) / image_std[c]
+
+ grid_h = h_bar // patch_size
+ grid_w = w_bar // patch_size
+ grid_thw = torch.tensor([1, grid_h, grid_w], device=device, dtype=torch.long)
+
+ pixel_values = normalized
+ grid_thw_list.append(grid_thw)
+ image_grid_thw = torch.stack(grid_thw_list)
+
+ grid_t = 1
+ channel = pixel_values.shape[0]
+ pixel_values = pixel_values.unsqueeze(0).repeat(2, 1, 1, 1)
+
+ patches = pixel_values.reshape(
+ grid_t,
+ temporal_patch_size,
+ channel,
+ grid_h // merge_size,
+ merge_size,
+ patch_size,
+ grid_w // merge_size,
+ merge_size,
+ patch_size,
+ )
+
+ patches = patches.permute(0, 3, 6, 4, 7, 2, 1, 5, 8)
+ flatten_patches = patches.reshape(
+ grid_t * grid_h * grid_w,
+ channel * temporal_patch_size * patch_size * patch_size
+ )
+
+ return flatten_patches, image_grid_thw
+
+
+class VisionPatchEmbed(nn.Module):
+ def __init__(
+ self,
+ patch_size: int = 14,
+ temporal_patch_size: int = 2,
+ in_channels: int = 3,
+ embed_dim: int = 3584,
+ device=None,
+ dtype=None,
+ ops=None,
+ ):
+ super().__init__()
+ self.patch_size = patch_size
+ self.temporal_patch_size = temporal_patch_size
+ self.in_channels = in_channels
+ self.embed_dim = embed_dim
+
+ kernel_size = [temporal_patch_size, patch_size, patch_size]
+ self.proj = ops.Conv3d(
+ in_channels,
+ embed_dim,
+ kernel_size=kernel_size,
+ stride=kernel_size,
+ bias=False,
+ device=device,
+ dtype=dtype
+ )
+
+ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
+ hidden_states = hidden_states.view(
+ -1, self.in_channels, self.temporal_patch_size, self.patch_size, self.patch_size
+ )
+ hidden_states = self.proj(hidden_states)
+ return hidden_states.view(-1, self.embed_dim)
+
+
+def rotate_half(x):
+ x1 = x[..., : x.shape[-1] // 2]
+ x2 = x[..., x.shape[-1] // 2 :]
+ return torch.cat((-x2, x1), dim=-1)
+
+
+def apply_rotary_pos_emb_vision(q, k, cos, sin):
+ cos, sin = cos.unsqueeze(-2).float(), sin.unsqueeze(-2).float()
+ q_embed = (q * cos) + (rotate_half(q) * sin)
+ k_embed = (k * cos) + (rotate_half(k) * sin)
+ return q_embed, k_embed
+
+
+class VisionRotaryEmbedding(nn.Module):
+ def __init__(self, dim: int, theta: float = 10000.0):
+ super().__init__()
+ self.dim = dim
+ self.theta = theta
+
+ def forward(self, seqlen: int, device) -> torch.Tensor:
+ inv_freq = 1.0 / (self.theta ** (torch.arange(0, self.dim, 2, dtype=torch.float, device=device) / self.dim))
+ seq = torch.arange(seqlen, device=inv_freq.device, dtype=inv_freq.dtype)
+ freqs = torch.outer(seq, inv_freq)
+ return freqs
+
+
+class PatchMerger(nn.Module):
+ def __init__(self, dim: int, context_dim: int, spatial_merge_size: int = 2, device=None, dtype=None, ops=None):
+ super().__init__()
+ self.hidden_size = context_dim * (spatial_merge_size ** 2)
+ self.ln_q = ops.RMSNorm(context_dim, eps=1e-6, device=device, dtype=dtype)
+ self.mlp = nn.Sequential(
+ ops.Linear(self.hidden_size, self.hidden_size, device=device, dtype=dtype),
+ nn.GELU(),
+ ops.Linear(self.hidden_size, dim, device=device, dtype=dtype),
+ )
+
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
+ x = self.ln_q(x).reshape(-1, self.hidden_size)
+ x = self.mlp(x)
+ return x
+
+
+class VisionAttention(nn.Module):
+ def __init__(self, hidden_size: int, num_heads: int, device=None, dtype=None, ops=None):
+ super().__init__()
+ self.hidden_size = hidden_size
+ self.num_heads = num_heads
+ self.head_dim = hidden_size // num_heads
+ self.scaling = self.head_dim ** -0.5
+
+ self.qkv = ops.Linear(hidden_size, hidden_size * 3, bias=True, device=device, dtype=dtype)
+ self.proj = ops.Linear(hidden_size, hidden_size, bias=True, device=device, dtype=dtype)
+
+ def forward(
+ self,
+ hidden_states: torch.Tensor,
+ position_embeddings: Optional[Tuple[torch.Tensor, torch.Tensor]] = None,
+ cu_seqlens=None,
+ optimized_attention=None,
+ ) -> torch.Tensor:
+ if hidden_states.dim() == 2:
+ seq_length, _ = hidden_states.shape
+ batch_size = 1
+ hidden_states = hidden_states.unsqueeze(0)
+ else:
+ batch_size, seq_length, _ = hidden_states.shape
+
+ qkv = self.qkv(hidden_states)
+ qkv = qkv.reshape(batch_size, seq_length, 3, self.num_heads, self.head_dim)
+ query_states, key_states, value_states = qkv.reshape(seq_length, 3, self.num_heads, -1).permute(1, 0, 2, 3).unbind(0)
+
+ if position_embeddings is not None:
+ cos, sin = position_embeddings
+ query_states, key_states = apply_rotary_pos_emb_vision(query_states, key_states, cos, sin)
+
+ query_states = query_states.transpose(0, 1).unsqueeze(0)
+ key_states = key_states.transpose(0, 1).unsqueeze(0)
+ value_states = value_states.transpose(0, 1).unsqueeze(0)
+
+ lengths = cu_seqlens[1:] - cu_seqlens[:-1]
+ splits = [
+ torch.split(tensor, lengths.tolist(), dim=2) for tensor in (query_states, key_states, value_states)
+ ]
+
+ attn_outputs = [
+ optimized_attention(q, k, v, self.num_heads, skip_reshape=True)
+ for q, k, v in zip(*splits)
+ ]
+ attn_output = torch.cat(attn_outputs, dim=1)
+ attn_output = attn_output.reshape(seq_length, -1)
+ attn_output = self.proj(attn_output)
+
+ return attn_output
+
+
+class VisionMLP(nn.Module):
+ def __init__(self, hidden_size: int, intermediate_size: int, device=None, dtype=None, ops=None):
+ super().__init__()
+ self.gate_proj = ops.Linear(hidden_size, intermediate_size, bias=True, device=device, dtype=dtype)
+ self.up_proj = ops.Linear(hidden_size, intermediate_size, bias=True, device=device, dtype=dtype)
+ self.down_proj = ops.Linear(intermediate_size, hidden_size, bias=True, device=device, dtype=dtype)
+ self.act_fn = nn.SiLU()
+
+ def forward(self, hidden_state):
+ return self.down_proj(self.act_fn(self.gate_proj(hidden_state)) * self.up_proj(hidden_state))
+
+
+class VisionBlock(nn.Module):
+ def __init__(self, hidden_size: int, intermediate_size: int, num_heads: int, device=None, dtype=None, ops=None):
+ super().__init__()
+ self.norm1 = ops.RMSNorm(hidden_size, eps=1e-6, device=device, dtype=dtype)
+ self.norm2 = ops.RMSNorm(hidden_size, eps=1e-6, device=device, dtype=dtype)
+ self.attn = VisionAttention(hidden_size, num_heads, device=device, dtype=dtype, ops=ops)
+ self.mlp = VisionMLP(hidden_size, intermediate_size, device=device, dtype=dtype, ops=ops)
+
+ def forward(
+ self,
+ hidden_states: torch.Tensor,
+ position_embeddings: Optional[Tuple[torch.Tensor, torch.Tensor]] = None,
+ cu_seqlens=None,
+ optimized_attention=None,
+ ) -> torch.Tensor:
+ residual = hidden_states
+ hidden_states = self.norm1(hidden_states)
+ hidden_states = self.attn(hidden_states, position_embeddings, cu_seqlens, optimized_attention)
+ hidden_states = residual + hidden_states
+
+ residual = hidden_states
+ hidden_states = self.norm2(hidden_states)
+ hidden_states = self.mlp(hidden_states)
+ hidden_states = residual + hidden_states
+
+ return hidden_states
+
+
+class Qwen2VLVisionTransformer(nn.Module):
+ def __init__(
+ self,
+ hidden_size: int = 3584,
+ output_hidden_size: int = 3584,
+ intermediate_size: int = 3420,
+ num_heads: int = 16,
+ num_layers: int = 32,
+ patch_size: int = 14,
+ temporal_patch_size: int = 2,
+ spatial_merge_size: int = 2,
+ window_size: int = 112,
+ device=None,
+ dtype=None,
+ ops=None
+ ):
+ super().__init__()
+ self.hidden_size = hidden_size
+ self.patch_size = patch_size
+ self.spatial_merge_size = spatial_merge_size
+ self.window_size = window_size
+ self.fullatt_block_indexes = [7, 15, 23, 31]
+
+ self.patch_embed = VisionPatchEmbed(
+ patch_size=patch_size,
+ temporal_patch_size=temporal_patch_size,
+ in_channels=3,
+ embed_dim=hidden_size,
+ device=device,
+ dtype=dtype,
+ ops=ops,
+ )
+
+ head_dim = hidden_size // num_heads
+ self.rotary_pos_emb = VisionRotaryEmbedding(head_dim // 2)
+
+ self.blocks = nn.ModuleList([
+ VisionBlock(hidden_size, intermediate_size, num_heads, device, dtype, ops)
+ for _ in range(num_layers)
+ ])
+
+ self.merger = PatchMerger(
+ dim=output_hidden_size,
+ context_dim=hidden_size,
+ spatial_merge_size=spatial_merge_size,
+ device=device,
+ dtype=dtype,
+ ops=ops,
+ )
+
+ def get_window_index(self, grid_thw):
+ window_index = []
+ cu_window_seqlens = [0]
+ window_index_id = 0
+ vit_merger_window_size = self.window_size // self.spatial_merge_size // self.patch_size
+
+ for grid_t, grid_h, grid_w in grid_thw:
+ llm_grid_h = grid_h // self.spatial_merge_size
+ llm_grid_w = grid_w // self.spatial_merge_size
+
+ index = torch.arange(grid_t * llm_grid_h * llm_grid_w).reshape(grid_t, llm_grid_h, llm_grid_w)
+
+ pad_h = vit_merger_window_size - llm_grid_h % vit_merger_window_size
+ pad_w = vit_merger_window_size - llm_grid_w % vit_merger_window_size
+ num_windows_h = (llm_grid_h + pad_h) // vit_merger_window_size
+ num_windows_w = (llm_grid_w + pad_w) // vit_merger_window_size
+
+ index_padded = F.pad(index, (0, pad_w, 0, pad_h), "constant", -100)
+ index_padded = index_padded.reshape(
+ grid_t,
+ num_windows_h,
+ vit_merger_window_size,
+ num_windows_w,
+ vit_merger_window_size,
+ )
+ index_padded = index_padded.permute(0, 1, 3, 2, 4).reshape(
+ grid_t,
+ num_windows_h * num_windows_w,
+ vit_merger_window_size,
+ vit_merger_window_size,
+ )
+
+ seqlens = (index_padded != -100).sum([2, 3]).reshape(-1)
+ index_padded = index_padded.reshape(-1)
+ index_new = index_padded[index_padded != -100]
+ window_index.append(index_new + window_index_id)
+
+ cu_seqlens_tmp = seqlens.cumsum(0) * self.spatial_merge_size * self.spatial_merge_size + cu_window_seqlens[-1]
+ cu_window_seqlens.extend(cu_seqlens_tmp.tolist())
+ window_index_id += (grid_t * llm_grid_h * llm_grid_w).item()
+
+ window_index = torch.cat(window_index, dim=0)
+ return window_index, cu_window_seqlens
+
+ def get_position_embeddings(self, grid_thw, device):
+ pos_ids = []
+
+ for t, h, w in grid_thw:
+ hpos_ids = torch.arange(h, device=device).unsqueeze(1).expand(-1, w)
+ hpos_ids = hpos_ids.reshape(
+ h // self.spatial_merge_size,
+ self.spatial_merge_size,
+ w // self.spatial_merge_size,
+ self.spatial_merge_size,
+ )
+ hpos_ids = hpos_ids.permute(0, 2, 1, 3).flatten()
+
+ wpos_ids = torch.arange(w, device=device).unsqueeze(0).expand(h, -1)
+ wpos_ids = wpos_ids.reshape(
+ h // self.spatial_merge_size,
+ self.spatial_merge_size,
+ w // self.spatial_merge_size,
+ self.spatial_merge_size,
+ )
+ wpos_ids = wpos_ids.permute(0, 2, 1, 3).flatten()
+
+ pos_ids.append(torch.stack([hpos_ids, wpos_ids], dim=-1).repeat(t, 1))
+
+ pos_ids = torch.cat(pos_ids, dim=0)
+ max_grid_size = grid_thw[:, 1:].max()
+ rotary_pos_emb_full = self.rotary_pos_emb(max_grid_size, device)
+ return rotary_pos_emb_full[pos_ids].flatten(1)
+
+ def forward(
+ self,
+ pixel_values: torch.Tensor,
+ image_grid_thw: Optional[torch.Tensor] = None,
+ ) -> torch.Tensor:
+ optimized_attention = optimized_attention_for_device(pixel_values.device, mask=False, small_input=True)
+
+ hidden_states = self.patch_embed(pixel_values)
+
+ window_index, cu_window_seqlens = self.get_window_index(image_grid_thw)
+ cu_window_seqlens = torch.tensor(cu_window_seqlens, device=hidden_states.device)
+ cu_window_seqlens = torch.unique_consecutive(cu_window_seqlens)
+
+ position_embeddings = self.get_position_embeddings(image_grid_thw, hidden_states.device)
+
+ seq_len, _ = hidden_states.size()
+ spatial_merge_unit = self.spatial_merge_size * self.spatial_merge_size
+
+ hidden_states = hidden_states.reshape(seq_len // spatial_merge_unit, spatial_merge_unit, -1)
+ hidden_states = hidden_states[window_index, :, :]
+ hidden_states = hidden_states.reshape(seq_len, -1)
+
+ position_embeddings = position_embeddings.reshape(seq_len // spatial_merge_unit, spatial_merge_unit, -1)
+ position_embeddings = position_embeddings[window_index, :, :]
+ position_embeddings = position_embeddings.reshape(seq_len, -1)
+ position_embeddings = torch.cat((position_embeddings, position_embeddings), dim=-1)
+ position_embeddings = (position_embeddings.cos(), position_embeddings.sin())
+
+ cu_seqlens = torch.repeat_interleave(image_grid_thw[:, 1] * image_grid_thw[:, 2], image_grid_thw[:, 0]).cumsum(
+ dim=0,
+ dtype=torch.int32,
+ )
+ cu_seqlens = F.pad(cu_seqlens, (1, 0), value=0)
+
+ for i, block in enumerate(self.blocks):
+ if i in self.fullatt_block_indexes:
+ cu_seqlens_now = cu_seqlens
+ else:
+ cu_seqlens_now = cu_window_seqlens
+ hidden_states = block(hidden_states, position_embeddings, cu_seqlens_now, optimized_attention=optimized_attention)
+
+ hidden_states = self.merger(hidden_states)
+ return hidden_states
diff --git a/comfy/text_encoders/t5.py b/comfy/text_encoders/t5.py
index 36bf35309..e8588992a 100644
--- a/comfy/text_encoders/t5.py
+++ b/comfy/text_encoders/t5.py
@@ -199,7 +199,7 @@ class T5Stack(torch.nn.Module):
self.final_layer_norm = T5LayerNorm(model_dim, dtype=dtype, device=device, operations=operations)
# self.dropout = nn.Dropout(config.dropout_rate)
- def forward(self, x, attention_mask=None, intermediate_output=None, final_layer_norm_intermediate=True, dtype=None):
+ def forward(self, x, attention_mask=None, intermediate_output=None, final_layer_norm_intermediate=True, dtype=None, embeds_info=[]):
mask = None
if attention_mask is not None:
mask = 1.0 - attention_mask.to(x.dtype).reshape((attention_mask.shape[0], 1, -1, attention_mask.shape[-1])).expand(attention_mask.shape[0], 1, attention_mask.shape[-1], attention_mask.shape[-1])
diff --git a/comfy/weight_adapter/lora.py b/comfy/weight_adapter/lora.py
index 729dbd9e6..47aa17d13 100644
--- a/comfy/weight_adapter/lora.py
+++ b/comfy/weight_adapter/lora.py
@@ -96,6 +96,7 @@ class LoRAAdapter(WeightAdapterBase):
diffusers3_lora = "{}.lora.up.weight".format(x)
mochi_lora = "{}.lora_B".format(x)
transformers_lora = "{}.lora_linear_layer.up.weight".format(x)
+ qwen_default_lora = "{}.lora_B.default.weight".format(x)
A_name = None
if regular_lora in lora.keys():
@@ -122,6 +123,10 @@ class LoRAAdapter(WeightAdapterBase):
A_name = transformers_lora
B_name = "{}.lora_linear_layer.down.weight".format(x)
mid_name = None
+ elif qwen_default_lora in lora.keys():
+ A_name = qwen_default_lora
+ B_name = "{}.lora_A.default.weight".format(x)
+ mid_name = None
if A_name is not None:
mid = None
diff --git a/comfy_api/latest/_io.py b/comfy_api/latest/_io.py
index ec1efb51d..a3a21facc 100644
--- a/comfy_api/latest/_io.py
+++ b/comfy_api/latest/_io.py
@@ -726,6 +726,10 @@ class SEGS(ComfyTypeIO):
class AnyType(ComfyTypeIO):
Type = Any
+@comfytype(io_type="MODEL_PATCH")
+class MODEL_PATCH(ComfyTypeIO):
+ Type = Any
+
@comfytype(io_type="COMFY_MULTITYPED_V3")
class MultiType:
Type = Any
diff --git a/comfy_api/latest/_ui.py b/comfy_api/latest/_ui.py
index 6b8a39d58..26a55615f 100644
--- a/comfy_api/latest/_ui.py
+++ b/comfy_api/latest/_ui.py
@@ -9,7 +9,11 @@ from typing import Type
import av
import numpy as np
import torch
-import torchaudio
+try:
+ import torchaudio
+ TORCH_AUDIO_AVAILABLE = True
+except:
+ TORCH_AUDIO_AVAILABLE = False
from PIL import Image as PILImage
from PIL.PngImagePlugin import PngInfo
@@ -302,6 +306,8 @@ class AudioSaveHelper:
# Resample if necessary
if sample_rate != audio["sample_rate"]:
+ if not TORCH_AUDIO_AVAILABLE:
+ raise Exception("torchaudio is not available; cannot resample audio.")
waveform = torchaudio.functional.resample(waveform, audio["sample_rate"], sample_rate)
# Create output with specified format
diff --git a/comfy_api_nodes/apinode_utils.py b/comfy_api_nodes/apinode_utils.py
index 788e2803f..f953f86df 100644
--- a/comfy_api_nodes/apinode_utils.py
+++ b/comfy_api_nodes/apinode_utils.py
@@ -1,4 +1,5 @@
from __future__ import annotations
+import aiohttp
import io
import logging
import mimetypes
@@ -21,7 +22,6 @@ from server import PromptServer
import numpy as np
from PIL import Image
-import requests
import torch
import math
import base64
@@ -30,7 +30,7 @@ from io import BytesIO
import av
-def download_url_to_video_output(video_url: str, timeout: int = None) -> VideoFromFile:
+async def download_url_to_video_output(video_url: str, timeout: int = None) -> VideoFromFile:
"""Downloads a video from a URL and returns a `VIDEO` output.
Args:
@@ -39,7 +39,7 @@ def download_url_to_video_output(video_url: str, timeout: int = None) -> VideoFr
Returns:
A Comfy node `VIDEO` output.
"""
- video_io = download_url_to_bytesio(video_url, timeout)
+ video_io = await download_url_to_bytesio(video_url, timeout)
if video_io is None:
error_msg = f"Failed to download video from {video_url}"
logging.error(error_msg)
@@ -62,7 +62,7 @@ def downscale_image_tensor(image, total_pixels=1536 * 1024) -> torch.Tensor:
return s
-def validate_and_cast_response(
+async def validate_and_cast_response(
response, timeout: int = None, node_id: Union[str, None] = None
) -> torch.Tensor:
"""Validates and casts a response to a torch.Tensor.
@@ -86,35 +86,24 @@ def validate_and_cast_response(
image_tensors: list[torch.Tensor] = []
# Process each image in the data array
- for image_data in data:
- image_url = image_data.url
- b64_data = image_data.b64_json
+ async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=timeout)) as session:
+ for img_data in data:
+ img_bytes: bytes
+ if img_data.b64_json:
+ img_bytes = base64.b64decode(img_data.b64_json)
+ elif img_data.url:
+ if node_id:
+ PromptServer.instance.send_progress_text(f"Result URL: {img_data.url}", node_id)
+ async with session.get(img_data.url) as resp:
+ if resp.status != 200:
+ raise ValueError("Failed to download generated image")
+ img_bytes = await resp.read()
+ else:
+ raise ValueError("Invalid image payload – neither URL nor base64 data present.")
- if not image_url and not b64_data:
- raise ValueError("No image was generated in the response")
-
- if b64_data:
- img_data = base64.b64decode(b64_data)
- img = Image.open(io.BytesIO(img_data))
-
- elif image_url:
- if node_id:
- PromptServer.instance.send_progress_text(
- f"Result URL: {image_url}", node_id
- )
- img_response = requests.get(image_url, timeout=timeout)
- if img_response.status_code != 200:
- raise ValueError("Failed to download the image")
- img = Image.open(io.BytesIO(img_response.content))
-
- img = img.convert("RGBA")
-
- # Convert to numpy array, normalize to float32 between 0 and 1
- img_array = np.array(img).astype(np.float32) / 255.0
- img_tensor = torch.from_numpy(img_array)
-
- # Add to list of tensors
- image_tensors.append(img_tensor)
+ pil_img = Image.open(BytesIO(img_bytes)).convert("RGBA")
+ arr = np.asarray(pil_img).astype(np.float32) / 255.0
+ image_tensors.append(torch.from_numpy(arr))
return torch.stack(image_tensors, dim=0)
@@ -175,7 +164,7 @@ def mimetype_to_extension(mime_type: str) -> str:
return mime_type.split("/")[-1].lower()
-def download_url_to_bytesio(url: str, timeout: int = None) -> BytesIO:
+async def download_url_to_bytesio(url: str, timeout: int = None) -> BytesIO:
"""Downloads content from a URL using requests and returns it as BytesIO.
Args:
@@ -185,9 +174,11 @@ def download_url_to_bytesio(url: str, timeout: int = None) -> BytesIO:
Returns:
BytesIO object containing the downloaded content.
"""
- response = requests.get(url, stream=True, timeout=timeout)
- response.raise_for_status() # Raises HTTPError for bad responses (4XX or 5XX)
- return BytesIO(response.content)
+ timeout_cfg = aiohttp.ClientTimeout(total=timeout) if timeout else None
+ async with aiohttp.ClientSession(timeout=timeout_cfg) as session:
+ async with session.get(url) as resp:
+ resp.raise_for_status() # Raises HTTPError for bad responses (4XX or 5XX)
+ return BytesIO(await resp.read())
def bytesio_to_image_tensor(image_bytesio: BytesIO, mode: str = "RGBA") -> torch.Tensor:
@@ -210,15 +201,15 @@ def bytesio_to_image_tensor(image_bytesio: BytesIO, mode: str = "RGBA") -> torch
return torch.from_numpy(image_array).unsqueeze(0)
-def download_url_to_image_tensor(url: str, timeout: int = None) -> torch.Tensor:
+async def download_url_to_image_tensor(url: str, timeout: int = None) -> torch.Tensor:
"""Downloads an image from a URL and returns a [B, H, W, C] tensor."""
- image_bytesio = download_url_to_bytesio(url, timeout)
+ image_bytesio = await download_url_to_bytesio(url, timeout)
return bytesio_to_image_tensor(image_bytesio)
-def process_image_response(response: requests.Response) -> torch.Tensor:
+def process_image_response(response_content: bytes | str) -> torch.Tensor:
"""Uses content from a Response object and converts it to a torch.Tensor"""
- return bytesio_to_image_tensor(BytesIO(response.content))
+ return bytesio_to_image_tensor(BytesIO(response_content))
def _tensor_to_pil(image: torch.Tensor, total_pixels: int = 2048 * 2048) -> Image.Image:
@@ -336,10 +327,10 @@ def text_filepath_to_data_uri(filepath: str) -> str:
return f"data:{mime_type};base64,{base64_string}"
-def upload_file_to_comfyapi(
+async def upload_file_to_comfyapi(
file_bytes_io: BytesIO,
filename: str,
- upload_mime_type: str,
+ upload_mime_type: Optional[str],
auth_kwargs: Optional[dict[str, str]] = None,
) -> str:
"""
@@ -354,7 +345,10 @@ def upload_file_to_comfyapi(
Returns:
The download URL for the uploaded file.
"""
- request_object = UploadRequest(file_name=filename, content_type=upload_mime_type)
+ if upload_mime_type is None:
+ request_object = UploadRequest(file_name=filename)
+ else:
+ request_object = UploadRequest(file_name=filename, content_type=upload_mime_type)
operation = SynchronousOperation(
endpoint=ApiEndpoint(
path="/customers/storage",
@@ -366,12 +360,8 @@ def upload_file_to_comfyapi(
auth_kwargs=auth_kwargs,
)
- response: UploadResponse = operation.execute()
- upload_response = ApiClient.upload_file(
- response.upload_url, file_bytes_io, content_type=upload_mime_type
- )
- upload_response.raise_for_status()
-
+ response: UploadResponse = await operation.execute()
+ await ApiClient.upload_file(response.upload_url, file_bytes_io, content_type=upload_mime_type)
return response.download_url
@@ -399,7 +389,7 @@ def video_to_base64_string(
return base64.b64encode(video_bytes_io.getvalue()).decode("utf-8")
-def upload_video_to_comfyapi(
+async def upload_video_to_comfyapi(
video: VideoInput,
auth_kwargs: Optional[dict[str, str]] = None,
container: VideoContainer = VideoContainer.MP4,
@@ -439,9 +429,7 @@ def upload_video_to_comfyapi(
video.save_to(video_bytes_io, format=container, codec=codec)
video_bytes_io.seek(0)
- return upload_file_to_comfyapi(
- video_bytes_io, filename, upload_mime_type, auth_kwargs
- )
+ return await upload_file_to_comfyapi(video_bytes_io, filename, upload_mime_type, auth_kwargs)
def audio_tensor_to_contiguous_ndarray(waveform: torch.Tensor) -> np.ndarray:
@@ -501,7 +489,7 @@ def audio_ndarray_to_bytesio(
return audio_bytes_io
-def upload_audio_to_comfyapi(
+async def upload_audio_to_comfyapi(
audio: AudioInput,
auth_kwargs: Optional[dict[str, str]] = None,
container_format: str = "mp4",
@@ -527,7 +515,7 @@ def upload_audio_to_comfyapi(
audio_data_np, sample_rate, container_format, codec_name
)
- return upload_file_to_comfyapi(audio_bytes_io, filename, mime_type, auth_kwargs)
+ return await upload_file_to_comfyapi(audio_bytes_io, filename, mime_type, auth_kwargs)
def audio_to_base64_string(
@@ -544,7 +532,7 @@ def audio_to_base64_string(
return base64.b64encode(audio_bytes).decode("utf-8")
-def upload_images_to_comfyapi(
+async def upload_images_to_comfyapi(
image: torch.Tensor,
max_images=8,
auth_kwargs: Optional[dict[str, str]] = None,
@@ -561,55 +549,15 @@ def upload_images_to_comfyapi(
mime_type: Optional MIME type for the image.
"""
# if batch, try to upload each file if max_images is greater than 0
- idx_image = 0
download_urls: list[str] = []
is_batch = len(image.shape) > 3
- batch_length = 1
- if is_batch:
- batch_length = image.shape[0]
- while True:
- curr_image = image
- if len(image.shape) > 3:
- curr_image = image[idx_image]
- # get BytesIO version of image
- img_binary = tensor_to_bytesio(curr_image, mime_type=mime_type)
- # first, request upload/download urls from comfy API
- if not mime_type:
- request_object = UploadRequest(file_name=img_binary.name)
- else:
- request_object = UploadRequest(
- file_name=img_binary.name, content_type=mime_type
- )
- operation = SynchronousOperation(
- endpoint=ApiEndpoint(
- path="/customers/storage",
- method=HttpMethod.POST,
- request_model=UploadRequest,
- response_model=UploadResponse,
- ),
- request=request_object,
- auth_kwargs=auth_kwargs,
- )
- response = operation.execute()
+ batch_len = image.shape[0] if is_batch else 1
- upload_response = ApiClient.upload_file(
- response.upload_url, img_binary, content_type=mime_type
- )
- # verify success
- try:
- upload_response.raise_for_status()
- except requests.exceptions.HTTPError as e:
- raise ValueError(f"Could not upload one or more images: {e}") from e
- # add download_url to list
- download_urls.append(response.download_url)
-
- idx_image += 1
- # stop uploading additional files if done
- if is_batch and max_images > 0:
- if idx_image >= max_images:
- break
- if idx_image >= batch_length:
- break
+ for idx in range(min(batch_len, max_images)):
+ tensor = image[idx] if is_batch else image
+ img_io = tensor_to_bytesio(tensor, mime_type=mime_type)
+ url = await upload_file_to_comfyapi(img_io, img_io.name, mime_type, auth_kwargs)
+ download_urls.append(url)
return download_urls
diff --git a/comfy_api_nodes/apis/__init__.py b/comfy_api_nodes/apis/__init__.py
index 086028abe..7a09df55b 100644
--- a/comfy_api_nodes/apis/__init__.py
+++ b/comfy_api_nodes/apis/__init__.py
@@ -1,6 +1,6 @@
# generated by datamodel-codegen:
# filename: filtered-openapi.yaml
-# timestamp: 2025-07-06T09:47:31+00:00
+# timestamp: 2025-07-30T08:54:00+00:00
from __future__ import annotations
@@ -37,6 +37,99 @@ class AuditLog(BaseModel):
)
+class BFLAsyncResponse(BaseModel):
+ id: str = Field(..., title='Id')
+ polling_url: str = Field(..., title='Polling Url')
+
+
+class BFLAsyncWebhookResponse(BaseModel):
+ id: str = Field(..., title='Id')
+ status: str = Field(..., title='Status')
+ webhook_url: str = Field(..., title='Webhook Url')
+
+
+class CannyHighThreshold(RootModel[int]):
+ root: int = Field(
+ ...,
+ description='High threshold for Canny edge detection',
+ ge=0,
+ le=500,
+ title='Canny High Threshold',
+ )
+
+
+class CannyLowThreshold(RootModel[int]):
+ root: int = Field(
+ ...,
+ description='Low threshold for Canny edge detection',
+ ge=0,
+ le=500,
+ title='Canny Low Threshold',
+ )
+
+
+class Guidance(RootModel[float]):
+ root: float = Field(
+ ...,
+ description='Guidance strength for the image generation process',
+ ge=1.0,
+ le=100.0,
+ title='Guidance',
+ )
+
+
+class Steps(RootModel[int]):
+ root: int = Field(
+ ...,
+ description='Number of steps for the image generation process',
+ ge=15,
+ le=50,
+ title='Steps',
+ )
+
+
+class WebhookUrl(RootModel[AnyUrl]):
+ root: AnyUrl = Field(
+ ..., description='URL to receive webhook notifications', title='Webhook Url'
+ )
+
+
+class BFLFluxKontextMaxGenerateRequest(BaseModel):
+ guidance: Optional[float] = Field(
+ 3, description='The guidance scale for generation', ge=1.0, le=20.0
+ )
+ input_image: str = Field(..., description='Base64 encoded image to be edited')
+ prompt: str = Field(
+ ..., description='The text prompt describing what to edit on the image'
+ )
+ steps: Optional[int] = Field(
+ 50, description='Number of inference steps', ge=1, le=50
+ )
+
+
+class BFLFluxKontextMaxGenerateResponse(BaseModel):
+ id: str = Field(..., description='Job ID for tracking')
+ polling_url: str = Field(..., description='URL to poll for results')
+
+
+class BFLFluxKontextProGenerateRequest(BaseModel):
+ guidance: Optional[float] = Field(
+ 3, description='The guidance scale for generation', ge=1.0, le=20.0
+ )
+ input_image: str = Field(..., description='Base64 encoded image to be edited')
+ prompt: str = Field(
+ ..., description='The text prompt describing what to edit on the image'
+ )
+ steps: Optional[int] = Field(
+ 50, description='Number of inference steps', ge=1, le=50
+ )
+
+
+class BFLFluxKontextProGenerateResponse(BaseModel):
+ id: str = Field(..., description='Job ID for tracking')
+ polling_url: str = Field(..., description='URL to poll for results')
+
+
class OutputFormat(str, Enum):
jpeg = 'jpeg'
png = 'png'
@@ -68,6 +161,67 @@ class BFLFluxPro11GenerateResponse(BaseModel):
polling_url: str = Field(..., description='URL to poll for results')
+class Bottom(RootModel[int]):
+ root: int = Field(
+ ...,
+ description='Number of pixels to expand at the bottom of the image',
+ ge=0,
+ le=2048,
+ title='Bottom',
+ )
+
+
+class Guidance2(RootModel[float]):
+ root: float = Field(
+ ...,
+ description='Guidance strength for the image generation process',
+ ge=1.5,
+ le=100.0,
+ title='Guidance',
+ )
+
+
+class Left(RootModel[int]):
+ root: int = Field(
+ ...,
+ description='Number of pixels to expand on the left side of the image',
+ ge=0,
+ le=2048,
+ title='Left',
+ )
+
+
+class Right(RootModel[int]):
+ root: int = Field(
+ ...,
+ description='Number of pixels to expand on the right side of the image',
+ ge=0,
+ le=2048,
+ title='Right',
+ )
+
+
+class Steps2(RootModel[int]):
+ root: int = Field(
+ ...,
+ description='Number of steps for the image generation process',
+ examples=[50],
+ ge=15,
+ le=50,
+ title='Steps',
+ )
+
+
+class Top(RootModel[int]):
+ root: int = Field(
+ ...,
+ description='Number of pixels to expand at the top of the image',
+ ge=0,
+ le=2048,
+ title='Top',
+ )
+
+
class BFLFluxProGenerateRequest(BaseModel):
guidance_scale: Optional[float] = Field(
None, description='The guidance scale for generation.', ge=1.0, le=20.0
@@ -96,7 +250,71 @@ class BFLFluxProGenerateResponse(BaseModel):
polling_url: str = Field(..., description='URL to poll for the generation result.')
+class BFLOutputFormat(str, Enum):
+ jpeg = 'jpeg'
+ png = 'png'
+
+
+class BFLValidationError(BaseModel):
+ loc: List[Union[str, int]] = Field(..., title='Location')
+ msg: str = Field(..., title='Message')
+ type: str = Field(..., title='Error Type')
+
+
class Status(str, Enum):
+ success = 'success'
+ not_found = 'not_found'
+ error = 'error'
+
+
+class ClaimMyNodeRequest(BaseModel):
+ GH_TOKEN: str = Field(
+ ..., description='GitHub token to verify if the user owns the repo of the node'
+ )
+
+
+class ComfyNode(BaseModel):
+ category: Optional[str] = Field(
+ None,
+ description='UI category where the node is listed, used for grouping nodes.',
+ )
+ comfy_node_name: Optional[str] = Field(
+ None, description='Unique identifier for the node'
+ )
+ deprecated: Optional[bool] = Field(
+ None,
+ description='Indicates if the node is deprecated. Deprecated nodes are hidden in the UI.',
+ )
+ description: Optional[str] = Field(
+ None, description="Brief description of the node's functionality or purpose."
+ )
+ experimental: Optional[bool] = Field(
+ None,
+ description='Indicates if the node is experimental, subject to changes or removal.',
+ )
+ function: Optional[str] = Field(
+ None, description='Name of the entry-point function to execute the node.'
+ )
+ input_types: Optional[str] = Field(None, description='Defines input parameters')
+ output_is_list: Optional[List[bool]] = Field(
+ None, description='Boolean values indicating if each output is a list.'
+ )
+ return_names: Optional[str] = Field(
+ None, description='Names of the outputs for clarity in workflows.'
+ )
+ return_types: Optional[str] = Field(
+ None, description='Specifies the types of outputs produced by the node.'
+ )
+
+
+class ComfyNodeCloudBuildInfo(BaseModel):
+ build_id: Optional[str] = None
+ location: Optional[str] = None
+ project_id: Optional[str] = None
+ project_number: Optional[str] = None
+
+
+class Status1(str, Enum):
in_progress = 'in_progress'
completed = 'completed'
incomplete = 'incomplete'
@@ -113,7 +331,7 @@ class ComputerToolCall(BaseModel):
description='An identifier used when responding to the tool call with output.\n',
)
id: str = Field(..., description='The unique ID of the computer call.')
- status: Status = Field(
+ status: Status1 = Field(
...,
description='The status of the item. One of `in_progress`, `completed`, or\n`incomplete`. Populated when items are returned via API.\n',
)
@@ -156,6 +374,7 @@ class Customer(BaseModel):
None, description='The date and time the user was created'
)
email: Optional[str] = Field(None, description='The email address for this user')
+ has_fund: Optional[bool] = Field(None, description='Whether the user has funds')
id: str = Field(..., description='The firebase UID of the user')
is_admin: Optional[bool] = Field(None, description='Whether the user is an admin')
metronome_id: Optional[str] = Field(None, description='The Metronome customer ID')
@@ -194,6 +413,16 @@ class Type2(str, Enum):
message = 'message'
+class Error(BaseModel):
+ details: Optional[List[str]] = Field(
+ None,
+ description='Optional detailed information about the error or hints for resolving it.',
+ )
+ message: Optional[str] = Field(
+ None, description='A clear and concise description of the error.'
+ )
+
+
class ErrorResponse(BaseModel):
error: str
message: str
@@ -221,7 +450,7 @@ class Result(BaseModel):
)
-class Status1(str, Enum):
+class Status2(str, Enum):
in_progress = 'in_progress'
searching = 'searching'
completed = 'completed'
@@ -241,7 +470,7 @@ class FileSearchToolCall(BaseModel):
results: Optional[List[Result]] = Field(
None, description='The results of the file search tool call.\n'
)
- status: Status1 = Field(
+ status: Status2 = Field(
...,
description='The status of the file search tool call. One of `in_progress`, \n`searching`, `incomplete` or `failed`,\n',
)
@@ -266,7 +495,7 @@ class FunctionTool(BaseModel):
type: Literal['FunctionTool'] = Field(..., description='The type of tool')
-class Status2(str, Enum):
+class Status3(str, Enum):
in_progress = 'in_progress'
completed = 'completed'
incomplete = 'incomplete'
@@ -288,7 +517,7 @@ class FunctionToolCall(BaseModel):
None, description='The unique ID of the function tool call.\n'
)
name: str = Field(..., description='The name of the function to run.\n')
- status: Optional[Status2] = Field(
+ status: Optional[Status3] = Field(
None,
description='The status of the item. One of `in_progress`, `completed`, or\n`incomplete`. Populated when items are returned via API.\n',
)
@@ -442,6 +671,95 @@ class GeminiVideoMetadata(BaseModel):
startOffset: Optional[GeminiOffset] = None
+class GitCommitSummary(BaseModel):
+ author: Optional[str] = Field(None, description='The author of the commit')
+ branch_name: Optional[str] = Field(
+ None, description='The branch where the commit was made'
+ )
+ commit_hash: Optional[str] = Field(None, description='The hash of the commit')
+ commit_name: Optional[str] = Field(None, description='The name of the commit')
+ status_summary: Optional[Dict[str, str]] = Field(
+ None, description='A map of operating system to status pairs'
+ )
+ timestamp: Optional[datetime] = Field(
+ None, description='The timestamp when the commit was made'
+ )
+
+
+class GithubEnterprise(BaseModel):
+ avatar_url: str = Field(..., description='URL to the enterprise avatar')
+ created_at: datetime = Field(..., description='When the enterprise was created')
+ description: Optional[str] = Field(None, description='The enterprise description')
+ html_url: str = Field(..., description='The HTML URL of the enterprise')
+ id: int = Field(..., description='The enterprise ID')
+ name: str = Field(..., description='The enterprise name')
+ node_id: str = Field(..., description='The enterprise node ID')
+ slug: str = Field(..., description='The enterprise slug')
+ updated_at: datetime = Field(
+ ..., description='When the enterprise was last updated'
+ )
+ website_url: Optional[str] = Field(None, description='The enterprise website URL')
+
+
+class RepositorySelection(str, Enum):
+ selected = 'selected'
+ all = 'all'
+
+
+class GithubOrganization(BaseModel):
+ avatar_url: str = Field(..., description="URL to the organization's avatar")
+ description: Optional[str] = Field(None, description='The organization description')
+ events_url: str = Field(..., description="The API URL of the organization's events")
+ hooks_url: str = Field(..., description="The API URL of the organization's hooks")
+ id: int = Field(..., description='The organization ID')
+ issues_url: str = Field(..., description="The API URL of the organization's issues")
+ login: str = Field(..., description="The organization's login name")
+ members_url: str = Field(
+ ..., description="The API URL of the organization's members"
+ )
+ node_id: str = Field(..., description='The organization node ID')
+ public_members_url: str = Field(
+ ..., description="The API URL of the organization's public members"
+ )
+ repos_url: str = Field(
+ ..., description="The API URL of the organization's repositories"
+ )
+ url: str = Field(..., description='The API URL of the organization')
+
+
+class State(str, Enum):
+ uploaded = 'uploaded'
+ open = 'open'
+
+
+class Action(str, Enum):
+ published = 'published'
+ unpublished = 'unpublished'
+ created = 'created'
+ edited = 'edited'
+ deleted = 'deleted'
+ prereleased = 'prereleased'
+ released = 'released'
+
+
+class Type7(str, Enum):
+ Bot = 'Bot'
+ User = 'User'
+ Organization = 'Organization'
+
+
+class GithubUser(BaseModel):
+ avatar_url: str = Field(..., description="URL to the user's avatar")
+ gravatar_id: Optional[str] = Field(None, description="The user's gravatar ID")
+ html_url: str = Field(..., description='The HTML URL of the user')
+ id: int = Field(..., description="The user's ID")
+ login: str = Field(..., description="The user's login name")
+ node_id: str = Field(..., description="The user's node ID")
+ site_admin: bool = Field(..., description='Whether the user is a site admin')
+ type: Type7 = Field(..., description='The type of user')
+ url: str = Field(..., description='The API URL of the user')
+
+
class IdeogramColorPalette1(BaseModel):
name: str = Field(..., description='Name of the preset color palette')
@@ -689,7 +1007,7 @@ class Includable(str, Enum):
computer_call_output_output_image_url = 'computer_call_output.output.image_url'
-class Type7(str, Enum):
+class Type8(str, Enum):
input_file = 'input_file'
@@ -703,7 +1021,7 @@ class InputFileContent(BaseModel):
filename: Optional[str] = Field(
None, description='The name of the file to be sent to the model.'
)
- type: Type7 = Field(
+ type: Type8 = Field(
..., description='The type of the input item. Always `input_file`.'
)
@@ -714,7 +1032,7 @@ class Detail(str, Enum):
auto = 'auto'
-class Type8(str, Enum):
+class Type9(str, Enum):
input_image = 'input_image'
@@ -730,7 +1048,7 @@ class InputImageContent(BaseModel):
None,
description='The URL of the image to be sent to the model. A fully qualified URL or base64 encoded image in a data URL.',
)
- type: Type8 = Field(
+ type: Type9 = Field(
..., description='The type of the input item. Always `input_image`.'
)
@@ -741,17 +1059,17 @@ class Role3(str, Enum):
developer = 'developer'
-class Type9(str, Enum):
+class Type10(str, Enum):
message = 'message'
-class Type10(str, Enum):
+class Type11(str, Enum):
input_text = 'input_text'
class InputTextContent(BaseModel):
text: str = Field(..., description='The text input to the model.')
- type: Type10 = Field(
+ type: Type11 = Field(
..., description='The type of the input item. Always `input_text`.'
)
@@ -923,7 +1241,7 @@ class ResourcePackType(str, Enum):
constant_period = 'constant_period'
-class Status4(str, Enum):
+class Status5(str, Enum):
toBeOnline = 'toBeOnline'
online = 'online'
expired = 'expired'
@@ -949,7 +1267,7 @@ class ResourcePackSubscribeInfo(BaseModel):
None,
description='Resource package type (decreasing_total=decreasing total, constant_period=constant periodicity)',
)
- status: Optional[Status4] = Field(None, description='Resource Package Status')
+ status: Optional[Status5] = Field(None, description='Resource Package Status')
total_quantity: Optional[float] = Field(None, description='Total quantity')
@@ -997,6 +1315,7 @@ class KlingTaskStatus(str, Enum):
class KlingTextToVideoModelName(str, Enum):
kling_v1 = 'kling-v1'
kling_v1_6 = 'kling-v1-6'
+ kling_v2_1_master = 'kling-v2-1-master'
class KlingVideoGenAspectRatio(str, Enum):
@@ -1029,6 +1348,8 @@ class KlingVideoGenModelName(str, Enum):
kling_v1_5 = 'kling-v1-5'
kling_v1_6 = 'kling-v1-6'
kling_v2_master = 'kling-v2-master'
+ kling_v2_1 = 'kling-v2-1'
+ kling_v2_1_master = 'kling-v2-1-master'
class KlingVideoResult(BaseModel):
@@ -1113,7 +1434,7 @@ class LumaError(BaseModel):
detail: Optional[str] = Field(None, description='The error message')
-class Type11(str, Enum):
+class Type12(str, Enum):
generation = 'generation'
@@ -1153,7 +1474,7 @@ class LumaImageRef(BaseModel):
)
-class Type12(str, Enum):
+class Type13(str, Enum):
image = 'image'
@@ -1223,6 +1544,36 @@ class LumaVideoModelOutputResolution(
root: Union[LumaVideoModelOutputResolution1, str]
+class MachineStats(BaseModel):
+ cpu_capacity: Optional[str] = Field(None, description='Total CPU on the machine.')
+ disk_capacity: Optional[str] = Field(
+ None, description='Total disk capacity on the machine.'
+ )
+ gpu_type: Optional[str] = Field(
+ None, description='The GPU type. eg. NVIDIA Tesla K80'
+ )
+ initial_cpu: Optional[str] = Field(
+ None, description='Initial CPU available before the job starts.'
+ )
+ initial_disk: Optional[str] = Field(
+ None, description='Initial disk available before the job starts.'
+ )
+ initial_ram: Optional[str] = Field(
+ None, description='Initial RAM available before the job starts.'
+ )
+ machine_name: Optional[str] = Field(None, description='Name of the machine.')
+ memory_capacity: Optional[str] = Field(
+ None, description='Total memory on the machine.'
+ )
+ os_version: Optional[str] = Field(
+ None, description='The operating system version. eg. Ubuntu Linux 20.04'
+ )
+ pip_freeze: Optional[str] = Field(None, description='The pip freeze output')
+ vram_time_series: Optional[Dict[str, Any]] = Field(
+ None, description='Time series of VRAM usage.'
+ )
+
+
class MinimaxBaseResponse(BaseModel):
status_code: int = Field(
...,
@@ -1251,7 +1602,7 @@ class MinimaxFileRetrieveResponse(BaseModel):
file: File
-class Status5(str, Enum):
+class Status6(str, Enum):
Queueing = 'Queueing'
Preparing = 'Preparing'
Processing = 'Processing'
@@ -1265,20 +1616,21 @@ class MinimaxTaskResultResponse(BaseModel):
None,
description='After the task status changes to Success, this field returns the file ID corresponding to the generated video.',
)
- status: Status5 = Field(
+ status: Status6 = Field(
...,
description="Task status: 'Queueing' (in queue), 'Preparing' (task is preparing), 'Processing' (generating), 'Success' (task completed successfully), or 'Fail' (task failed).",
)
task_id: str = Field(..., description='The task ID being queried.')
-class Model(str, Enum):
+class MiniMaxModel(str, Enum):
T2V_01_Director = 'T2V-01-Director'
I2V_01_Director = 'I2V-01-Director'
S2V_01 = 'S2V-01'
I2V_01 = 'I2V-01'
I2V_01_live = 'I2V-01-live'
T2V_01 = 'T2V-01'
+ Hailuo_02 = 'MiniMax-Hailuo-02'
class SubjectReferenceItem(BaseModel):
@@ -1300,7 +1652,7 @@ class MinimaxVideoGenerationRequest(BaseModel):
None,
description='URL or base64 encoding of the first frame image. Required when model is I2V-01, I2V-01-Director, or I2V-01-live.',
)
- model: Model = Field(
+ model: MiniMaxModel = Field(
...,
description='Required. ID of model. Options: T2V-01-Director, I2V-01-Director, S2V-01, I2V-01, I2V-01-live, T2V-01',
)
@@ -1317,6 +1669,14 @@ class MinimaxVideoGenerationRequest(BaseModel):
None,
description='Only available when model is S2V-01. The model will generate a video based on the subject uploaded through this parameter.',
)
+ duration: Optional[int] = Field(
+ None,
+ description="The length of the output video in seconds."
+ )
+ resolution: Optional[str] = Field(
+ None,
+ description="The dimensions of the video display. 1080p corresponds to 1920 x 1080 pixels, 768p corresponds to 1366 x 768 pixels."
+ )
class MinimaxVideoGenerationResponse(BaseModel):
@@ -1326,6 +1686,22 @@ class MinimaxVideoGenerationResponse(BaseModel):
)
+class Modality(str, Enum):
+ MODALITY_UNSPECIFIED = 'MODALITY_UNSPECIFIED'
+ TEXT = 'TEXT'
+ IMAGE = 'IMAGE'
+ VIDEO = 'VIDEO'
+ AUDIO = 'AUDIO'
+ DOCUMENT = 'DOCUMENT'
+
+
+class ModalityTokenCount(BaseModel):
+ modality: Optional[Modality] = None
+ tokenCount: Optional[int] = Field(
+ None, description='Number of tokens for the given modality.'
+ )
+
+
class Truncation(str, Enum):
disabled = 'disabled'
auto = 'auto'
@@ -1391,13 +1767,13 @@ class MoonvalleyTextToVideoInferenceParams(BaseModel):
0, description='Index of the conditioning frame'
)
cooldown_steps: Optional[int] = Field(
- None, description='Number of cooldown steps (calculated based on num_frames)'
+ 75, description='Number of cooldown steps (calculated based on num_frames)'
)
fps: Optional[int] = Field(
24, description='Frames per second of the generated video'
)
guidance_scale: Optional[float] = Field(
- 12.5, description='Guidance scale for generation control'
+ 10, description='Guidance scale for generation control'
)
height: Optional[int] = Field(
1080, description='Height of the generated video in pixels'
@@ -1421,7 +1797,7 @@ class MoonvalleyTextToVideoInferenceParams(BaseModel):
True, description='Whether to use timestep transformation'
)
warmup_steps: Optional[int] = Field(
- None, description='Number of warmup steps (calculated based on num_frames)'
+ 0, description='Number of warmup steps (calculated based on num_frames)'
)
width: Optional[int] = Field(
1920, description='Width of the generated video in pixels'
@@ -1463,10 +1839,10 @@ class MoonvalleyVideoToVideoInferenceParams(BaseModel):
0, description='Index of the conditioning frame'
)
cooldown_steps: Optional[int] = Field(
- None, description='Number of cooldown steps (calculated based on num_frames)'
+ 36, description='Number of cooldown steps (calculated based on num_frames)'
)
guidance_scale: Optional[float] = Field(
- 12.5, description='Guidance scale for generation control'
+ 15, description='Guidance scale for generation control'
)
negative_prompt: Optional[str] = Field(None, description='Negative prompt text')
seed: Optional[int] = Field(
@@ -1486,7 +1862,7 @@ class MoonvalleyVideoToVideoInferenceParams(BaseModel):
True, description='Whether to use timestep transformation'
)
warmup_steps: Optional[int] = Field(
- None, description='Number of warmup steps (calculated based on num_frames)'
+ 24, description='Number of warmup steps (calculated based on num_frames)'
)
@@ -1507,6 +1883,34 @@ class MoonvalleyVideoToVideoRequest(BaseModel):
)
+class NodeStatus(str, Enum):
+ NodeStatusActive = 'NodeStatusActive'
+ NodeStatusDeleted = 'NodeStatusDeleted'
+ NodeStatusBanned = 'NodeStatusBanned'
+
+
+class NodeVersionIdentifier(BaseModel):
+ node_id: str = Field(..., description='The unique identifier of the node')
+ version: str = Field(..., description='The version of the node')
+
+
+class NodeVersionStatus(str, Enum):
+ NodeVersionStatusActive = 'NodeVersionStatusActive'
+ NodeVersionStatusDeleted = 'NodeVersionStatusDeleted'
+ NodeVersionStatusBanned = 'NodeVersionStatusBanned'
+ NodeVersionStatusPending = 'NodeVersionStatusPending'
+ NodeVersionStatusFlagged = 'NodeVersionStatusFlagged'
+
+
+class NodeVersionUpdateRequest(BaseModel):
+ changelog: Optional[str] = Field(
+ None, description='The changelog describing the version changes.'
+ )
+ deprecated: Optional[bool] = Field(
+ None, description='Whether the version is deprecated.'
+ )
+
+
class Moderation(str, Enum):
low = 'low'
auto = 'auto'
@@ -1723,38 +2127,57 @@ class Object(str, Enum):
response = 'response'
-class Status6(str, Enum):
+class Status7(str, Enum):
completed = 'completed'
failed = 'failed'
in_progress = 'in_progress'
incomplete = 'incomplete'
-class Type13(str, Enum):
+class Type14(str, Enum):
output_audio = 'output_audio'
class OutputAudioContent(BaseModel):
data: str = Field(..., description='Base64-encoded audio data')
transcript: str = Field(..., description='Transcript of the audio')
- type: Type13 = Field(..., description='The type of output content')
+ type: Type14 = Field(..., description='The type of output content')
class Role4(str, Enum):
assistant = 'assistant'
-class Type14(str, Enum):
+class Type15(str, Enum):
message = 'message'
-class Type15(str, Enum):
+class Type16(str, Enum):
output_text = 'output_text'
class OutputTextContent(BaseModel):
text: str = Field(..., description='The text content')
- type: Type15 = Field(..., description='The type of output content')
+ type: Type16 = Field(..., description='The type of output content')
+
+
+class PersonalAccessToken(BaseModel):
+ createdAt: Optional[datetime] = Field(
+ None, description='[Output Only]The date and time the token was created.'
+ )
+ description: Optional[str] = Field(
+ None,
+ description="Optional. A more detailed description of the token's intended use.",
+ )
+ id: Optional[UUID] = Field(None, description='Unique identifier for the GitCommit')
+ name: Optional[str] = Field(
+ None,
+ description='Required. The name of the token. Can be a simple description.',
+ )
+ token: Optional[str] = Field(
+ None,
+ description='[Output Only]. The personal access token. Only returned during creation.',
+ )
class AspectRatio1(RootModel[float]):
@@ -1961,7 +2384,7 @@ class PixverseVideoResponse(BaseModel):
Resp: Optional[Resp1] = None
-class Status7(int, Enum):
+class Status8(int, Enum):
integer_1 = 1
integer_5 = 5
integer_6 = 6
@@ -1980,7 +2403,7 @@ class Resp2(BaseModel):
resolution_ratio: Optional[int] = None
seed: Optional[int] = None
size: Optional[int] = None
- status: Optional[Status7] = Field(
+ status: Optional[Status8] = Field(
None,
description='Video generation status codes:\n* 1 - Generation successful\n* 5 - Generating\n* 6 - Deleted\n* 7 - Contents moderation failed\n* 8 - Generation failed\n',
)
@@ -1994,6 +2417,17 @@ class PixverseVideoResultResponse(BaseModel):
Resp: Optional[Resp2] = None
+class PublisherStatus(str, Enum):
+ PublisherStatusActive = 'PublisherStatusActive'
+ PublisherStatusBanned = 'PublisherStatusBanned'
+
+
+class PublisherUser(BaseModel):
+ email: Optional[str] = Field(None, description='The email address for this user.')
+ id: Optional[str] = Field(None, description='The unique id for this user.')
+ name: Optional[str] = Field(None, description='The name for this user.')
+
+
class RgbItem(RootModel[int]):
root: int = Field(..., ge=0, le=255)
@@ -2020,13 +2454,13 @@ class ReasoningEffort(str, Enum):
high = 'high'
-class Status8(str, Enum):
+class Status9(str, Enum):
in_progress = 'in_progress'
completed = 'completed'
incomplete = 'incomplete'
-class Type16(str, Enum):
+class Type17(str, Enum):
summary_text = 'summary_text'
@@ -2035,12 +2469,12 @@ class SummaryItem(BaseModel):
...,
description='A short summary of the reasoning used by the model when generating\nthe response.\n',
)
- type: Type16 = Field(
+ type: Type17 = Field(
..., description='The type of the object. Always `summary_text`.\n'
)
-class Type17(str, Enum):
+class Type18(str, Enum):
reasoning = 'reasoning'
@@ -2048,16 +2482,31 @@ class ReasoningItem(BaseModel):
id: str = Field(
..., description='The unique identifier of the reasoning content.\n'
)
- status: Optional[Status8] = Field(
+ status: Optional[Status9] = Field(
None,
description='The status of the item. One of `in_progress`, `completed`, or\n`incomplete`. Populated when items are returned via API.\n',
)
summary: List[SummaryItem] = Field(..., description='Reasoning text contents.\n')
- type: Type17 = Field(
+ type: Type18 = Field(
..., description='The type of the object. Always `reasoning`.\n'
)
+class RecraftImageColor(BaseModel):
+ rgb: Optional[List[int]] = None
+ std: Optional[List[float]] = None
+ weight: Optional[float] = None
+
+
+class RecraftImageFeatures(BaseModel):
+ nsfw_score: Optional[float] = None
+
+
+class RecraftImageFormat(str, Enum):
+ webp = 'webp'
+ png = 'png'
+
+
class Controls(BaseModel):
artistic_level: Optional[int] = Field(
None,
@@ -2111,12 +2560,143 @@ class RecraftImageGenerationResponse(BaseModel):
data: List[Datum3] = Field(..., description='Array of generated image information')
+class RecraftImageStyle(str, Enum):
+ digital_illustration = 'digital_illustration'
+ icon = 'icon'
+ realistic_image = 'realistic_image'
+ vector_illustration = 'vector_illustration'
+
+
+class RecraftImageSubStyle(str, Enum):
+ field_2d_art_poster = '2d_art_poster'
+ field_3d = '3d'
+ field_80s = '80s'
+ glow = 'glow'
+ grain = 'grain'
+ hand_drawn = 'hand_drawn'
+ infantile_sketch = 'infantile_sketch'
+ kawaii = 'kawaii'
+ pixel_art = 'pixel_art'
+ psychedelic = 'psychedelic'
+ seamless = 'seamless'
+ voxel = 'voxel'
+ watercolor = 'watercolor'
+ broken_line = 'broken_line'
+ colored_outline = 'colored_outline'
+ colored_shapes = 'colored_shapes'
+ colored_shapes_gradient = 'colored_shapes_gradient'
+ doodle_fill = 'doodle_fill'
+ doodle_offset_fill = 'doodle_offset_fill'
+ offset_fill = 'offset_fill'
+ outline = 'outline'
+ outline_gradient = 'outline_gradient'
+ uneven_fill = 'uneven_fill'
+ field_70s = '70s'
+ cartoon = 'cartoon'
+ doodle_line_art = 'doodle_line_art'
+ engraving = 'engraving'
+ flat_2 = 'flat_2'
+ kawaii_1 = 'kawaii'
+ line_art = 'line_art'
+ linocut = 'linocut'
+ seamless_1 = 'seamless'
+ b_and_w = 'b_and_w'
+ enterprise = 'enterprise'
+ hard_flash = 'hard_flash'
+ hdr = 'hdr'
+ motion_blur = 'motion_blur'
+ natural_light = 'natural_light'
+ studio_portrait = 'studio_portrait'
+ line_circuit = 'line_circuit'
+ field_2d_art_poster_2 = '2d_art_poster_2'
+ engraving_color = 'engraving_color'
+ flat_air_art = 'flat_air_art'
+ hand_drawn_outline = 'hand_drawn_outline'
+ handmade_3d = 'handmade_3d'
+ stickers_drawings = 'stickers_drawings'
+ plastic = 'plastic'
+ pictogram = 'pictogram'
+
+
+class RecraftResponseFormat(str, Enum):
+ url = 'url'
+ b64_json = 'b64_json'
+
+
+class RecraftTextLayoutItem(BaseModel):
+ bbox: List[List[float]]
+ text: str
+
+
+class RecraftTransformModel(str, Enum):
+ refm1 = 'refm1'
+ recraft20b = 'recraft20b'
+ recraftv2 = 'recraftv2'
+ recraftv3 = 'recraftv3'
+ flux1_1pro = 'flux1_1pro'
+ flux1dev = 'flux1dev'
+ imagen3 = 'imagen3'
+ hidream_i1_dev = 'hidream_i1_dev'
+
+
+class RecraftUserControls(BaseModel):
+ artistic_level: Optional[int] = None
+ background_color: Optional[RecraftImageColor] = None
+ colors: Optional[List[RecraftImageColor]] = None
+ no_text: Optional[bool] = None
+
+
+class Attention(str, Enum):
+ low = 'low'
+ medium = 'medium'
+ high = 'high'
+
+
+class Project(str, Enum):
+ comfyui = 'comfyui'
+ comfyui_frontend = 'comfyui_frontend'
+ desktop = 'desktop'
+
+
+class ReleaseNote(BaseModel):
+ attention: Attention = Field(
+ ..., description='The attention level for this release'
+ )
+ content: str = Field(
+ ..., description='The content of the release note in markdown format'
+ )
+ id: int = Field(..., description='Unique identifier for the release note')
+ project: Project = Field(
+ ..., description='The project this release note belongs to'
+ )
+ published_at: datetime = Field(
+ ..., description='When the release note was published'
+ )
+ version: str = Field(..., description='The version of the release')
+
+
class RenderingSpeed(str, Enum):
BALANCED = 'BALANCED'
TURBO = 'TURBO'
QUALITY = 'QUALITY'
+class Type19(str, Enum):
+ response_completed = 'response.completed'
+
+
+class Type20(str, Enum):
+ response_content_part_added = 'response.content_part.added'
+
+
+class Type21(str, Enum):
+ response_content_part_done = 'response.content_part.done'
+
+
+class Type22(str, Enum):
+ response_created = 'response.created'
+
+
class ResponseErrorCode(str, Enum):
server_error = 'server_error'
rate_limit_exceeded = 'rate_limit_exceeded'
@@ -2138,12 +2718,27 @@ class ResponseErrorCode(str, Enum):
image_file_not_found = 'image_file_not_found'
-class Type18(str, Enum):
+class Type23(str, Enum):
+ error = 'error'
+
+
+class ResponseErrorEvent(BaseModel):
+ code: str = Field(..., description='The error code.\n')
+ message: str = Field(..., description='The error message.\n')
+ param: str = Field(..., description='The error parameter.\n')
+ type: Type23 = Field(..., description='The type of the event. Always `error`.\n')
+
+
+class Type24(str, Enum):
+ response_failed = 'response.failed'
+
+
+class Type25(str, Enum):
json_object = 'json_object'
class ResponseFormatJsonObject(BaseModel):
- type: Type18 = Field(
+ type: Type25 = Field(
...,
description='The type of response format being defined. Always `json_object`.',
)
@@ -2156,16 +2751,32 @@ class ResponseFormatJsonSchemaSchema(BaseModel):
)
-class Type19(str, Enum):
+class Type26(str, Enum):
text = 'text'
class ResponseFormatText(BaseModel):
- type: Type19 = Field(
+ type: Type26 = Field(
..., description='The type of response format being defined. Always `text`.'
)
+class Type27(str, Enum):
+ response_in_progress = 'response.in_progress'
+
+
+class Type28(str, Enum):
+ response_incomplete = 'response.incomplete'
+
+
+class Type29(str, Enum):
+ response_output_item_added = 'response.output_item.added'
+
+
+class Type30(str, Enum):
+ response_output_item_done = 'response.output_item.done'
+
+
class Truncation1(str, Enum):
auto = 'auto'
disabled = 'disabled'
@@ -2200,10 +2811,6 @@ class Rodin3DCheckStatusRequest(BaseModel):
)
-class Rodin3DCheckStatusResponse(BaseModel):
- pass
-
-
class Rodin3DDownloadRequest(BaseModel):
task_uuid: str = Field(..., description='Task UUID')
@@ -2235,6 +2842,13 @@ class RodinResourceItem(BaseModel):
url: Optional[str] = Field(None, description='Download url')
+class RodinStatusOptions(str, Enum):
+ Done = 'Done'
+ Failed = 'Failed'
+ Generating = 'Generating'
+ Waiting = 'Waiting'
+
+
class RodinTierType(str, Enum):
Regular = 'Regular'
Sketch = 'Sketch'
@@ -2325,6 +2939,7 @@ class RunwayTextToImageAspectRatioEnum(str, Enum):
field_1808_768 = '1808:768'
field_2112_912 = '2112:912'
+
class Model4(str, Enum):
gen4_image = 'gen4_image'
@@ -2350,6 +2965,38 @@ class RunwayTextToImageResponse(BaseModel):
id: Optional[str] = Field(None, description='Task ID')
+class Name(str, Enum):
+ content_moderation = 'content_moderation'
+
+
+class StabilityContentModerationResponse(BaseModel):
+ errors: List[str] = Field(
+ ...,
+ description='One or more error messages indicating what went wrong.',
+ examples=[['some-field: is required']],
+ min_length=1,
+ )
+ id: str = Field(
+ ...,
+ description='A unique identifier associated with this error. Please include this in any [support tickets](https://kb.stability.ai/knowledge-base/kb-tickets/new) you file, as it will greatly assist us in diagnosing the root cause of the problem.',
+ examples=['a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4'],
+ min_length=1,
+ )
+ name: Name = Field(
+ ...,
+ description='Our content moderation system has flagged some part of your request and subsequently denied it. You were not charged for this request. While this may at times be frustrating, it is necessary to maintain the integrity of our platform and ensure a safe experience for all users. If you would like to provide feedback, please use the [Support Form](https://kb.stability.ai/knowledge-base/kb-tickets/new).',
+ )
+
+
+class StabilityCreativity(RootModel[float]):
+ root: float = Field(
+ ...,
+ description='Controls the likelihood of creating additional details not heavily conditioned by the init image.',
+ ge=0.2,
+ le=0.5,
+ )
+
+
class StabilityError(BaseModel):
errors: List[str] = Field(
...,
@@ -2371,7 +3018,17 @@ class StabilityError(BaseModel):
)
-class Status9(str, Enum):
+class StabilityGenerationID(RootModel[str]):
+ root: str = Field(
+ ...,
+ description='The `id` of a generation, typically used for async generations, that can be used to check the status of the generation or retrieve the result.',
+ examples=['a6dc6c6e20acda010fe14d71f180658f2896ed9b4ec25aa99a6ff06c796987c4'],
+ max_length=64,
+ min_length=64,
+ )
+
+
+class Status10(str, Enum):
in_progress = 'in-progress'
@@ -2379,10 +3036,860 @@ class StabilityGetResultResponse202(BaseModel):
id: Optional[str] = Field(
None, description='The ID of the generation result.', examples=[1234567890]
)
- status: Optional[Status9] = None
+ status: Optional[Status10] = None
-class Type20(str, Enum):
+class AspectRatio3(str, Enum):
+ field_21_9 = '21:9'
+ field_16_9 = '16:9'
+ field_3_2 = '3:2'
+ field_5_4 = '5:4'
+ field_1_1 = '1:1'
+ field_4_5 = '4:5'
+ field_2_3 = '2:3'
+ field_9_16 = '9:16'
+ field_9_21 = '9:21'
+
+
+class Mode(str, Enum):
+ text_to_image = 'text-to-image'
+ image_to_image = 'image-to-image'
+
+
+class Model5(str, Enum):
+ sd3_5_large = 'sd3.5-large'
+ sd3_5_large_turbo = 'sd3.5-large-turbo'
+ sd3_5_medium = 'sd3.5-medium'
+
+
+class OutputFormat3(str, Enum):
+ png = 'png'
+ jpeg = 'jpeg'
+
+
+class StylePreset(str, Enum):
+ enhance = 'enhance'
+ anime = 'anime'
+ photographic = 'photographic'
+ digital_art = 'digital-art'
+ comic_book = 'comic-book'
+ fantasy_art = 'fantasy-art'
+ line_art = 'line-art'
+ analog_film = 'analog-film'
+ neon_punk = 'neon-punk'
+ isometric = 'isometric'
+ low_poly = 'low-poly'
+ origami = 'origami'
+ modeling_compound = 'modeling-compound'
+ cinematic = 'cinematic'
+ field_3d_model = '3d-model'
+ pixel_art = 'pixel-art'
+ tile_texture = 'tile-texture'
+
+
+class StabilityImageGenerationSD3Request(BaseModel):
+ aspect_ratio: Optional[AspectRatio3] = Field(
+ '1:1',
+ description='Controls the aspect ratio of the generated image. Defaults to 1:1.\n\n> **Important:** This parameter is only valid for **text-to-image** requests.',
+ )
+ cfg_scale: Optional[float] = Field(
+ None,
+ description='How strictly the diffusion process adheres to the prompt text (higher values keep your image closer to your prompt). The _Large_ and _Medium_ models use a default of `4`. The _Turbo_ model uses a default of `1`.',
+ ge=1.0,
+ le=10.0,
+ )
+ image: Optional[StrictBytes] = Field(
+ None,
+ description='The image to use as the starting point for the generation.\n\nSupported formats:\n\n\n\n - jpeg\n - png\n - webp\n\nSupported dimensions:\n\n\n\n - Every side must be at least 64 pixels\n\n> **Important:** This parameter is only valid for **image-to-image** requests.',
+ )
+ mode: Optional[Mode] = Field(
+ 'text-to-image',
+ description='Controls whether this is a text-to-image or image-to-image generation, which affects which parameters are required:\n- **text-to-image** requires only the `prompt` parameter\n- **image-to-image** requires the `prompt`, `image`, and `strength` parameters',
+ title='GenerationMode',
+ )
+ model: Optional[Model5] = Field(
+ 'sd3.5-large',
+ description='The model to use for generation.\n\n- `sd3.5-large` requires 6.5 credits per generation\n- `sd3.5-large-turbo` requires 4 credits per generation\n- `sd3.5-medium` requires 3.5 credits per generation\n- As of the April 17, 2025, `sd3-large`, `sd3-large-turbo` and `sd3-medium`\n\n\n\n are re-routed to their `sd3.5-[model version]` equivalent, at the same price.',
+ )
+ negative_prompt: Optional[str] = Field(
+ None,
+ description='Keywords of what you **do not** wish to see in the output image.\nThis is an advanced feature.',
+ max_length=10000,
+ )
+ output_format: Optional[OutputFormat3] = Field(
+ 'png', description='Dictates the `content-type` of the generated image.'
+ )
+ prompt: str = Field(
+ ...,
+ description='What you wish to see in the output image. A strong, descriptive prompt that clearly defines\nelements, colors, and subjects will lead to better results.',
+ max_length=10000,
+ min_length=1,
+ )
+ seed: Optional[float] = Field(
+ 0,
+ description="A specific value that is used to guide the 'randomness' of the generation. (Omit this parameter or pass `0` to use a random seed.)",
+ ge=0.0,
+ le=4294967294.0,
+ )
+ strength: Optional[float] = Field(
+ None,
+ description='Sometimes referred to as _denoising_, this parameter controls how much influence the\n`image` parameter has on the generated image. A value of 0 would yield an image that\nis identical to the input. A value of 1 would be as if you passed in no image at all.\n\n> **Important:** This parameter is only valid for **image-to-image** requests.',
+ ge=0.0,
+ le=1.0,
+ )
+ style_preset: Optional[StylePreset] = Field(
+ None, description='Guides the image model towards a particular style.'
+ )
+
+
+class FinishReason(str, Enum):
+ SUCCESS = 'SUCCESS'
+ CONTENT_FILTERED = 'CONTENT_FILTERED'
+
+
+class StabilityImageGenrationSD3Response200(BaseModel):
+ finish_reason: FinishReason = Field(
+ ...,
+ description='The reason the generation finished.\n\n- `SUCCESS` = successful generation.\n- `CONTENT_FILTERED` = successful generation, however the output violated our content moderation\npolicy and has been blurred as a result.',
+ examples=['SUCCESS'],
+ )
+ image: str = Field(
+ ...,
+ description='The generated image, encoded to base64.',
+ examples=['AAAAIGZ0eXBpc29tAAACAGlzb21pc28yYXZjMW1...'],
+ )
+ seed: Optional[float] = Field(
+ 0,
+ description='The seed used as random noise for this generation.',
+ examples=[343940597],
+ ge=0.0,
+ le=4294967294.0,
+ )
+
+
+class StabilityImageGenrationSD3Response400(BaseModel):
+ errors: List[str] = Field(
+ ...,
+ description='One or more error messages indicating what went wrong.',
+ examples=[['some-field: is required']],
+ min_length=1,
+ )
+ id: str = Field(
+ ...,
+ description='A unique identifier associated with this error. Please include this in any [support tickets](https://kb.stability.ai/knowledge-base/kb-tickets/new)\nyou file, as it will greatly assist us in diagnosing the root cause of the problem.',
+ examples=['a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4'],
+ min_length=1,
+ )
+ name: str = Field(
+ ...,
+ description='Short-hand name for an error, useful for discriminating between errors with the same status code.',
+ examples=['bad_request'],
+ min_length=1,
+ )
+
+
+class StabilityImageGenrationSD3Response413(BaseModel):
+ errors: List[str] = Field(
+ ...,
+ description='One or more error messages indicating what went wrong.',
+ examples=[['some-field: is required']],
+ min_length=1,
+ )
+ id: str = Field(
+ ...,
+ description='A unique identifier associated with this error. Please include this in any [support tickets](https://kb.stability.ai/knowledge-base/kb-tickets/new)\nyou file, as it will greatly assist us in diagnosing the root cause of the problem.',
+ examples=['a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4'],
+ min_length=1,
+ )
+ name: str = Field(
+ ...,
+ description='Short-hand name for an error, useful for discriminating between errors with the same status code.',
+ examples=['bad_request'],
+ min_length=1,
+ )
+
+
+class StabilityImageGenrationSD3Response422(BaseModel):
+ errors: List[str] = Field(
+ ...,
+ description='One or more error messages indicating what went wrong.',
+ examples=[['some-field: is required']],
+ min_length=1,
+ )
+ id: str = Field(
+ ...,
+ description='A unique identifier associated with this error. Please include this in any [support tickets](https://kb.stability.ai/knowledge-base/kb-tickets/new)\nyou file, as it will greatly assist us in diagnosing the root cause of the problem.',
+ examples=['a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4'],
+ min_length=1,
+ )
+ name: str = Field(
+ ...,
+ description='Short-hand name for an error, useful for discriminating between errors with the same status code.',
+ examples=['bad_request'],
+ min_length=1,
+ )
+
+
+class StabilityImageGenrationSD3Response429(BaseModel):
+ errors: List[str] = Field(
+ ...,
+ description='One or more error messages indicating what went wrong.',
+ examples=[['some-field: is required']],
+ min_length=1,
+ )
+ id: str = Field(
+ ...,
+ description='A unique identifier associated with this error. Please include this in any [support tickets](https://kb.stability.ai/knowledge-base/kb-tickets/new)\nyou file, as it will greatly assist us in diagnosing the root cause of the problem.',
+ examples=['a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4'],
+ min_length=1,
+ )
+ name: str = Field(
+ ...,
+ description='Short-hand name for an error, useful for discriminating between errors with the same status code.',
+ examples=['bad_request'],
+ min_length=1,
+ )
+
+
+class StabilityImageGenrationSD3Response500(BaseModel):
+ errors: List[str] = Field(
+ ...,
+ description='One or more error messages indicating what went wrong.',
+ examples=[['some-field: is required']],
+ min_length=1,
+ )
+ id: str = Field(
+ ...,
+ description='A unique identifier associated with this error. Please include this in any [support tickets](https://kb.stability.ai/knowledge-base/kb-tickets/new)\nyou file, as it will greatly assist us in diagnosing the root cause of the problem.',
+ examples=['a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4'],
+ min_length=1,
+ )
+ name: str = Field(
+ ...,
+ description='Short-hand name for an error, useful for discriminating between errors with the same status code.',
+ examples=['bad_request'],
+ min_length=1,
+ )
+
+
+class OutputFormat4(str, Enum):
+ jpeg = 'jpeg'
+ png = 'png'
+ webp = 'webp'
+
+
+class StabilityImageGenrationUpscaleConservativeRequest(BaseModel):
+ creativity: Optional[StabilityCreativity] = Field(
+ default_factory=lambda: StabilityCreativity.model_validate(0.35)
+ )
+ image: StrictBytes = Field(
+ ...,
+ description='The image you wish to upscale.\n\nSupported Formats:\n- jpeg\n- png\n- webp\n\nValidation Rules:\n- Every side must be at least 64 pixels\n- Total pixel count must be between 4,096 and 9,437,184 pixels\n- The aspect ratio must be between 1:2.5 and 2.5:1',
+ examples=['./some/image.png'],
+ )
+ negative_prompt: Optional[str] = Field(
+ None,
+ description='A blurb of text describing what you **do not** wish to see in the output image.\nThis is an advanced feature.',
+ max_length=10000,
+ )
+ output_format: Optional[OutputFormat4] = Field(
+ 'png', description='Dictates the `content-type` of the generated image.'
+ )
+ prompt: str = Field(
+ ...,
+ description="What you wish to see in the output image. A strong, descriptive prompt that clearly defines\nelements, colors, and subjects will lead to better results.\n\nTo control the weight of a given word use the format `(word:weight)`,\nwhere `word` is the word you'd like to control the weight of and `weight`\nis a value between 0 and 1. For example: `The sky was a crisp (blue:0.3) and (green:0.8)`\nwould convey a sky that was blue and green, but more green than blue.",
+ max_length=10000,
+ min_length=1,
+ )
+ seed: Optional[float] = Field(
+ 0,
+ description="A specific value that is used to guide the 'randomness' of the generation. (Omit this parameter or pass `0` to use a random seed.)",
+ ge=0.0,
+ le=4294967294.0,
+ )
+
+
+class StabilityImageGenrationUpscaleConservativeResponse200(BaseModel):
+ finish_reason: FinishReason = Field(
+ ...,
+ description='The reason the generation finished.\n\n- `SUCCESS` = successful generation.\n- `CONTENT_FILTERED` = successful generation, however the output violated our content moderation\npolicy and has been blurred as a result.',
+ examples=['SUCCESS'],
+ )
+ image: str = Field(
+ ...,
+ description='The generated image, encoded to base64.',
+ examples=['AAAAIGZ0eXBpc29tAAACAGlzb21pc28yYXZjMW1...'],
+ )
+ seed: Optional[float] = Field(
+ 0,
+ description='The seed used as random noise for this generation.',
+ examples=[343940597],
+ ge=0.0,
+ le=4294967294.0,
+ )
+
+
+class StabilityImageGenrationUpscaleConservativeResponse400(BaseModel):
+ errors: List[str] = Field(
+ ...,
+ description='One or more error messages indicating what went wrong.',
+ examples=[['some-field: is required']],
+ min_length=1,
+ )
+ id: str = Field(
+ ...,
+ description='A unique identifier associated with this error. Please include this in any [support tickets](https://kb.stability.ai/knowledge-base/kb-tickets/new)\nyou file, as it will greatly assist us in diagnosing the root cause of the problem.',
+ examples=['a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4'],
+ min_length=1,
+ )
+ name: str = Field(
+ ...,
+ description='Short-hand name for an error, useful for discriminating between errors with the same status code.',
+ examples=['bad_request'],
+ min_length=1,
+ )
+
+
+class StabilityImageGenrationUpscaleConservativeResponse413(BaseModel):
+ errors: List[str] = Field(
+ ...,
+ description='One or more error messages indicating what went wrong.',
+ examples=[['some-field: is required']],
+ min_length=1,
+ )
+ id: str = Field(
+ ...,
+ description='A unique identifier associated with this error. Please include this in any [support tickets](https://kb.stability.ai/knowledge-base/kb-tickets/new)\nyou file, as it will greatly assist us in diagnosing the root cause of the problem.',
+ examples=['a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4'],
+ min_length=1,
+ )
+ name: str = Field(
+ ...,
+ description='Short-hand name for an error, useful for discriminating between errors with the same status code.',
+ examples=['bad_request'],
+ min_length=1,
+ )
+
+
+class StabilityImageGenrationUpscaleConservativeResponse422(BaseModel):
+ errors: List[str] = Field(
+ ...,
+ description='One or more error messages indicating what went wrong.',
+ examples=[['some-field: is required']],
+ min_length=1,
+ )
+ id: str = Field(
+ ...,
+ description='A unique identifier associated with this error. Please include this in any [support tickets](https://kb.stability.ai/knowledge-base/kb-tickets/new)\nyou file, as it will greatly assist us in diagnosing the root cause of the problem.',
+ examples=['a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4'],
+ min_length=1,
+ )
+ name: str = Field(
+ ...,
+ description='Short-hand name for an error, useful for discriminating between errors with the same status code.',
+ examples=['bad_request'],
+ min_length=1,
+ )
+
+
+class StabilityImageGenrationUpscaleConservativeResponse429(BaseModel):
+ errors: List[str] = Field(
+ ...,
+ description='One or more error messages indicating what went wrong.',
+ examples=[['some-field: is required']],
+ min_length=1,
+ )
+ id: str = Field(
+ ...,
+ description='A unique identifier associated with this error. Please include this in any [support tickets](https://kb.stability.ai/knowledge-base/kb-tickets/new)\nyou file, as it will greatly assist us in diagnosing the root cause of the problem.',
+ examples=['a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4'],
+ min_length=1,
+ )
+ name: str = Field(
+ ...,
+ description='Short-hand name for an error, useful for discriminating between errors with the same status code.',
+ examples=['bad_request'],
+ min_length=1,
+ )
+
+
+class StabilityImageGenrationUpscaleConservativeResponse500(BaseModel):
+ errors: List[str] = Field(
+ ...,
+ description='One or more error messages indicating what went wrong.',
+ examples=[['some-field: is required']],
+ min_length=1,
+ )
+ id: str = Field(
+ ...,
+ description='A unique identifier associated with this error. Please include this in any [support tickets](https://kb.stability.ai/knowledge-base/kb-tickets/new)\nyou file, as it will greatly assist us in diagnosing the root cause of the problem.',
+ examples=['a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4'],
+ min_length=1,
+ )
+ name: str = Field(
+ ...,
+ description='Short-hand name for an error, useful for discriminating between errors with the same status code.',
+ examples=['bad_request'],
+ min_length=1,
+ )
+
+
+class StabilityImageGenrationUpscaleCreativeRequest(BaseModel):
+ creativity: Optional[float] = Field(
+ 0.3,
+ description='Indicates how creative the model should be when upscaling an image.\nHigher values will result in more details being added to the image during upscaling.',
+ ge=0.1,
+ le=0.5,
+ )
+ image: StrictBytes = Field(
+ ...,
+ description='The image you wish to upscale.\n\nSupported Formats:\n- jpeg\n- png\n- webp\n\nValidation Rules:\n- Every side must be at least 64 pixels\n- Total pixel count must be between 4,096 and 1,048,576 pixels',
+ examples=['./some/image.png'],
+ )
+ negative_prompt: Optional[str] = Field(
+ None,
+ description='A blurb of text describing what you **do not** wish to see in the output image.\nThis is an advanced feature.',
+ max_length=10000,
+ )
+ output_format: Optional[OutputFormat4] = Field(
+ 'png', description='Dictates the `content-type` of the generated image.'
+ )
+ prompt: str = Field(
+ ...,
+ description="What you wish to see in the output image. A strong, descriptive prompt that clearly defines\nelements, colors, and subjects will lead to better results.\n\nTo control the weight of a given word use the format `(word:weight)`,\nwhere `word` is the word you'd like to control the weight of and `weight`\nis a value between 0 and 1. For example: `The sky was a crisp (blue:0.3) and (green:0.8)`\nwould convey a sky that was blue and green, but more green than blue.",
+ max_length=10000,
+ min_length=1,
+ )
+ seed: Optional[float] = Field(
+ 0,
+ description="A specific value that is used to guide the 'randomness' of the generation. (Omit this parameter or pass `0` to use a random seed.)",
+ ge=0.0,
+ le=4294967294.0,
+ )
+ style_preset: Optional[StylePreset] = Field(
+ None, description='Guides the image model towards a particular style.'
+ )
+
+
+class StabilityImageGenrationUpscaleCreativeResponse200(BaseModel):
+ id: StabilityGenerationID
+
+
+class StabilityImageGenrationUpscaleCreativeResponse400(BaseModel):
+ errors: List[str] = Field(
+ ...,
+ description='One or more error messages indicating what went wrong.',
+ examples=[['some-field: is required']],
+ min_length=1,
+ )
+ id: str = Field(
+ ...,
+ description='A unique identifier associated with this error. Please include this in any [support tickets](https://kb.stability.ai/knowledge-base/kb-tickets/new)\nyou file, as it will greatly assist us in diagnosing the root cause of the problem.',
+ examples=['a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4'],
+ min_length=1,
+ )
+ name: str = Field(
+ ...,
+ description='Short-hand name for an error, useful for discriminating between errors with the same status code.',
+ examples=['bad_request'],
+ min_length=1,
+ )
+
+
+class StabilityImageGenrationUpscaleCreativeResponse413(BaseModel):
+ errors: List[str] = Field(
+ ...,
+ description='One or more error messages indicating what went wrong.',
+ examples=[['some-field: is required']],
+ min_length=1,
+ )
+ id: str = Field(
+ ...,
+ description='A unique identifier associated with this error. Please include this in any [support tickets](https://kb.stability.ai/knowledge-base/kb-tickets/new)\nyou file, as it will greatly assist us in diagnosing the root cause of the problem.',
+ examples=['a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4'],
+ min_length=1,
+ )
+ name: str = Field(
+ ...,
+ description='Short-hand name for an error, useful for discriminating between errors with the same status code.',
+ examples=['bad_request'],
+ min_length=1,
+ )
+
+
+class StabilityImageGenrationUpscaleCreativeResponse422(BaseModel):
+ errors: List[str] = Field(
+ ...,
+ description='One or more error messages indicating what went wrong.',
+ examples=[['some-field: is required']],
+ min_length=1,
+ )
+ id: str = Field(
+ ...,
+ description='A unique identifier associated with this error. Please include this in any [support tickets](https://kb.stability.ai/knowledge-base/kb-tickets/new)\nyou file, as it will greatly assist us in diagnosing the root cause of the problem.',
+ examples=['a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4'],
+ min_length=1,
+ )
+ name: str = Field(
+ ...,
+ description='Short-hand name for an error, useful for discriminating between errors with the same status code.',
+ examples=['bad_request'],
+ min_length=1,
+ )
+
+
+class StabilityImageGenrationUpscaleCreativeResponse429(BaseModel):
+ errors: List[str] = Field(
+ ...,
+ description='One or more error messages indicating what went wrong.',
+ examples=[['some-field: is required']],
+ min_length=1,
+ )
+ id: str = Field(
+ ...,
+ description='A unique identifier associated with this error. Please include this in any [support tickets](https://kb.stability.ai/knowledge-base/kb-tickets/new)\nyou file, as it will greatly assist us in diagnosing the root cause of the problem.',
+ examples=['a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4'],
+ min_length=1,
+ )
+ name: str = Field(
+ ...,
+ description='Short-hand name for an error, useful for discriminating between errors with the same status code.',
+ examples=['bad_request'],
+ min_length=1,
+ )
+
+
+class StabilityImageGenrationUpscaleCreativeResponse500(BaseModel):
+ errors: List[str] = Field(
+ ...,
+ description='One or more error messages indicating what went wrong.',
+ examples=[['some-field: is required']],
+ min_length=1,
+ )
+ id: str = Field(
+ ...,
+ description='A unique identifier associated with this error. Please include this in any [support tickets](https://kb.stability.ai/knowledge-base/kb-tickets/new)\nyou file, as it will greatly assist us in diagnosing the root cause of the problem.',
+ examples=['a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4'],
+ min_length=1,
+ )
+ name: str = Field(
+ ...,
+ description='Short-hand name for an error, useful for discriminating between errors with the same status code.',
+ examples=['bad_request'],
+ min_length=1,
+ )
+
+
+class StabilityImageGenrationUpscaleFastRequest(BaseModel):
+ image: StrictBytes = Field(
+ ...,
+ description='The image you wish to upscale.\n\nSupported Formats:\n- jpeg\n- png\n- webp\n\nValidation Rules:\n- Width must be between 32 and 1,536 pixels\n- Height must be between 32 and 1,536 pixels\n- Total pixel count must be between 1,024 and 1,048,576 pixels',
+ examples=['./some/image.png'],
+ )
+ output_format: Optional[OutputFormat4] = Field(
+ 'png', description='Dictates the `content-type` of the generated image.'
+ )
+
+
+class StabilityImageGenrationUpscaleFastResponse200(BaseModel):
+ finish_reason: FinishReason = Field(
+ ...,
+ description='The reason the generation finished.\n\n- `SUCCESS` = successful generation.\n- `CONTENT_FILTERED` = successful generation, however the output violated our content moderation\npolicy and has been blurred as a result.',
+ examples=['SUCCESS'],
+ )
+ image: str = Field(
+ ...,
+ description='The generated image, encoded to base64.',
+ examples=['AAAAIGZ0eXBpc29tAAACAGlzb21pc28yYXZjMW1...'],
+ )
+ seed: Optional[float] = Field(
+ 0,
+ description='The seed used as random noise for this generation.',
+ examples=[343940597],
+ ge=0.0,
+ le=4294967294.0,
+ )
+
+
+class StabilityImageGenrationUpscaleFastResponse400(BaseModel):
+ errors: List[str] = Field(
+ ...,
+ description='One or more error messages indicating what went wrong.',
+ examples=[['some-field: is required']],
+ min_length=1,
+ )
+ id: str = Field(
+ ...,
+ description='A unique identifier associated with this error. Please include this in any [support tickets](https://kb.stability.ai/knowledge-base/kb-tickets/new)\nyou file, as it will greatly assist us in diagnosing the root cause of the problem.',
+ examples=['a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4'],
+ min_length=1,
+ )
+ name: str = Field(
+ ...,
+ description='Short-hand name for an error, useful for discriminating between errors with the same status code.',
+ examples=['bad_request'],
+ min_length=1,
+ )
+
+
+class StabilityImageGenrationUpscaleFastResponse413(BaseModel):
+ errors: List[str] = Field(
+ ...,
+ description='One or more error messages indicating what went wrong.',
+ examples=[['some-field: is required']],
+ min_length=1,
+ )
+ id: str = Field(
+ ...,
+ description='A unique identifier associated with this error. Please include this in any [support tickets](https://kb.stability.ai/knowledge-base/kb-tickets/new)\nyou file, as it will greatly assist us in diagnosing the root cause of the problem.',
+ examples=['a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4'],
+ min_length=1,
+ )
+ name: str = Field(
+ ...,
+ description='Short-hand name for an error, useful for discriminating between errors with the same status code.',
+ examples=['bad_request'],
+ min_length=1,
+ )
+
+
+class StabilityImageGenrationUpscaleFastResponse422(BaseModel):
+ errors: List[str] = Field(
+ ...,
+ description='One or more error messages indicating what went wrong.',
+ examples=[['some-field: is required']],
+ min_length=1,
+ )
+ id: str = Field(
+ ...,
+ description='A unique identifier associated with this error. Please include this in any [support tickets](https://kb.stability.ai/knowledge-base/kb-tickets/new)\nyou file, as it will greatly assist us in diagnosing the root cause of the problem.',
+ examples=['a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4'],
+ min_length=1,
+ )
+ name: str = Field(
+ ...,
+ description='Short-hand name for an error, useful for discriminating between errors with the same status code.',
+ examples=['bad_request'],
+ min_length=1,
+ )
+
+
+class StabilityImageGenrationUpscaleFastResponse429(BaseModel):
+ errors: List[str] = Field(
+ ...,
+ description='One or more error messages indicating what went wrong.',
+ examples=[['some-field: is required']],
+ min_length=1,
+ )
+ id: str = Field(
+ ...,
+ description='A unique identifier associated with this error. Please include this in any [support tickets](https://kb.stability.ai/knowledge-base/kb-tickets/new)\nyou file, as it will greatly assist us in diagnosing the root cause of the problem.',
+ examples=['a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4'],
+ min_length=1,
+ )
+ name: str = Field(
+ ...,
+ description='Short-hand name for an error, useful for discriminating between errors with the same status code.',
+ examples=['bad_request'],
+ min_length=1,
+ )
+
+
+class StabilityImageGenrationUpscaleFastResponse500(BaseModel):
+ errors: List[str] = Field(
+ ...,
+ description='One or more error messages indicating what went wrong.',
+ examples=[['some-field: is required']],
+ min_length=1,
+ )
+ id: str = Field(
+ ...,
+ description='A unique identifier associated with this error. Please include this in any [support tickets](https://kb.stability.ai/knowledge-base/kb-tickets/new)\nyou file, as it will greatly assist us in diagnosing the root cause of the problem.',
+ examples=['a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4'],
+ min_length=1,
+ )
+ name: str = Field(
+ ...,
+ description='Short-hand name for an error, useful for discriminating between errors with the same status code.',
+ examples=['bad_request'],
+ min_length=1,
+ )
+
+
+class StabilityStabilityClientID(RootModel[str]):
+ root: str = Field(
+ ...,
+ description='The name of your application, used to help us communicate app-specific debugging or moderation issues to you.',
+ examples=['my-awesome-app'],
+ max_length=256,
+ )
+
+
+class StabilityStabilityClientUserID(RootModel[str]):
+ root: str = Field(
+ ...,
+ description='A unique identifier for your end user. Used to help us communicate user-specific debugging or moderation issues to you. Feel free to obfuscate this value to protect user privacy.',
+ examples=['DiscordUser#9999'],
+ max_length=256,
+ )
+
+
+class StabilityStabilityClientVersion(RootModel[str]):
+ root: str = Field(
+ ...,
+ description='The version of your application, used to help us communicate version-specific debugging or moderation issues to you.',
+ examples=['1.2.1'],
+ max_length=256,
+ )
+
+
+class StorageFile(BaseModel):
+ file_path: Optional[str] = Field(None, description='Path to the file in storage')
+ id: Optional[UUID] = Field(
+ None, description='Unique identifier for the storage file'
+ )
+ public_url: Optional[str] = Field(None, description='Public URL')
+
+
+class StripeAddress(BaseModel):
+ city: Optional[str] = None
+ country: Optional[str] = None
+ line1: Optional[str] = None
+ line2: Optional[str] = None
+ postal_code: Optional[str] = None
+ state: Optional[str] = None
+
+
+class StripeAmountDetails(BaseModel):
+ tip: Optional[Dict[str, Any]] = None
+
+
+class StripeBillingDetails(BaseModel):
+ address: Optional[StripeAddress] = None
+ email: Optional[str] = None
+ name: Optional[str] = None
+ phone: Optional[str] = None
+ tax_id: Optional[Any] = None
+
+
+class Checks(BaseModel):
+ address_line1_check: Optional[Any] = None
+ address_postal_code_check: Optional[Any] = None
+ cvc_check: Optional[str] = None
+
+
+class ExtendedAuthorization(BaseModel):
+ status: Optional[str] = None
+
+
+class IncrementalAuthorization(BaseModel):
+ status: Optional[str] = None
+
+
+class Multicapture(BaseModel):
+ status: Optional[str] = None
+
+
+class NetworkToken(BaseModel):
+ used: Optional[bool] = None
+
+
+class Overcapture(BaseModel):
+ maximum_amount_capturable: Optional[int] = None
+ status: Optional[str] = None
+
+
+class StripeCardDetails(BaseModel):
+ amount_authorized: Optional[int] = None
+ authorization_code: Optional[Any] = None
+ brand: Optional[str] = None
+ checks: Optional[Checks] = None
+ country: Optional[str] = None
+ exp_month: Optional[int] = None
+ exp_year: Optional[int] = None
+ extended_authorization: Optional[ExtendedAuthorization] = None
+ fingerprint: Optional[str] = None
+ funding: Optional[str] = None
+ incremental_authorization: Optional[IncrementalAuthorization] = None
+ installments: Optional[Any] = None
+ last4: Optional[str] = None
+ mandate: Optional[Any] = None
+ multicapture: Optional[Multicapture] = None
+ network: Optional[str] = None
+ network_token: Optional[NetworkToken] = None
+ network_transaction_id: Optional[str] = None
+ overcapture: Optional[Overcapture] = None
+ regulated_status: Optional[str] = None
+ three_d_secure: Optional[Any] = None
+ wallet: Optional[Any] = None
+
+
+class Object1(str, Enum):
+ charge = 'charge'
+
+
+class Object2(str, Enum):
+ event = 'event'
+
+
+class Type31(str, Enum):
+ payment_intent_succeeded = 'payment_intent.succeeded'
+
+
+class StripeOutcome(BaseModel):
+ advice_code: Optional[Any] = None
+ network_advice_code: Optional[Any] = None
+ network_decline_code: Optional[Any] = None
+ network_status: Optional[str] = None
+ reason: Optional[Any] = None
+ risk_level: Optional[str] = None
+ risk_score: Optional[int] = None
+ seller_message: Optional[str] = None
+ type: Optional[str] = None
+
+
+class Object3(str, Enum):
+ payment_intent = 'payment_intent'
+
+
+class StripePaymentMethodDetails(BaseModel):
+ card: Optional[StripeCardDetails] = None
+ type: Optional[str] = None
+
+
+class Card(BaseModel):
+ installments: Optional[Any] = None
+ mandate_options: Optional[Any] = None
+ network: Optional[Any] = None
+ request_three_d_secure: Optional[str] = None
+
+
+class StripePaymentMethodOptions(BaseModel):
+ card: Optional[Card] = None
+
+
+class StripeRefundList(BaseModel):
+ data: Optional[List[Dict[str, Any]]] = None
+ has_more: Optional[bool] = None
+ object: Optional[str] = None
+ total_count: Optional[int] = None
+ url: Optional[str] = None
+
+
+class StripeRequestInfo(BaseModel):
+ id: Optional[str] = None
+ idempotency_key: Optional[str] = None
+
+
+class StripeShipping(BaseModel):
+ address: Optional[StripeAddress] = None
+ carrier: Optional[str] = None
+ name: Optional[str] = None
+ phone: Optional[str] = None
+ tracking_number: Optional[str] = None
+
+
+class Type32(str, Enum):
json_schema = 'json_schema'
@@ -2400,19 +3907,19 @@ class TextResponseFormatJsonSchema(BaseModel):
False,
description='Whether to enable strict schema adherence when generating the output.\nIf set to true, the model will always follow the exact schema defined\nin the `schema` field. Only a subset of JSON Schema is supported when\n`strict` is `true`. To learn more, read the [Structured Outputs\nguide](/docs/guides/structured-outputs).\n',
)
- type: Type20 = Field(
+ type: Type32 = Field(
...,
description='The type of response format being defined. Always `json_schema`.',
)
-class Type21(str, Enum):
+class Type33(str, Enum):
function = 'function'
class ToolChoiceFunction(BaseModel):
name: str = Field(..., description='The name of the function to call.')
- type: Type21 = Field(
+ type: Type33 = Field(
..., description='For function calling, the type is always `function`.'
)
@@ -2423,7 +3930,7 @@ class ToolChoiceOptions(str, Enum):
required = 'required'
-class Type22(str, Enum):
+class Type34(str, Enum):
file_search = 'file_search'
web_search_preview = 'web_search_preview'
computer_use_preview = 'computer_use_preview'
@@ -2431,7 +3938,7 @@ class Type22(str, Enum):
class ToolChoiceTypes(BaseModel):
- type: Type22 = Field(
+ type: Type34 = Field(
...,
description='The type of hosted tool the model should to use. Learn more about\n[built-in tools](/docs/guides/tools).\n\nAllowed values are:\n- `file_search`\n- `web_search_preview`\n- `computer_use_preview`\n',
)
@@ -2499,9 +4006,9 @@ class TripoModelStyle(str, Enum):
class TripoModelVersion(str, Enum):
- V2_5 = 'v2.5-20250123'
- V2_0 = 'v2.0-20240919'
- V1_4 = 'v1.4-20240625'
+ v2_5_20250123 = 'v2.5-20250123'
+ v2_0_20240919 = 'v2.0-20240919'
+ v1_4_20240625 = 'v1.4-20240625'
class TripoMultiviewMode(str, Enum):
@@ -2547,13 +4054,13 @@ class Code1(int, Enum):
integer_0 = 0
-class Data8(BaseModel):
+class Data9(BaseModel):
task_id: str = Field(..., description='used for getTask')
class TripoSuccessTask(BaseModel):
code: Code1
- data: Data8
+ data: Data9
class Topology(str, Enum):
@@ -2570,7 +4077,7 @@ class Output(BaseModel):
topology: Optional[Topology] = None
-class Status10(str, Enum):
+class Status11(str, Enum):
queued = 'queued'
running = 'running'
success = 'success'
@@ -2586,7 +4093,7 @@ class TripoTask(BaseModel):
input: Dict[str, Any]
output: Output
progress: int = Field(..., ge=0, le=100)
- status: Status10
+ status: Status11
task_id: str
type: str
@@ -2650,6 +4157,18 @@ class TripoTypeTextureModel(str, Enum):
texture_model = 'texture_model'
+class User(BaseModel):
+ email: Optional[str] = Field(None, description='The email address for this user.')
+ id: Optional[str] = Field(None, description='The unique id for this user.')
+ isAdmin: Optional[bool] = Field(
+ None, description='Indicates if the user has admin privileges.'
+ )
+ isApproved: Optional[bool] = Field(
+ None, description='Indicates if the user is approved.'
+ )
+ name: Optional[str] = Field(None, description='The name for this user.')
+
+
class Veo2GenVidPollRequest(BaseModel):
operationName: str = Field(
...,
@@ -2660,7 +4179,7 @@ class Veo2GenVidPollRequest(BaseModel):
)
-class Error(BaseModel):
+class Error1(BaseModel):
code: Optional[int] = Field(None, description='Error code')
message: Optional[str] = Field(None, description='Error message')
@@ -2692,7 +4211,7 @@ class Response(BaseModel):
class Veo2GenVidPollResponse(BaseModel):
done: Optional[bool] = None
- error: Optional[Error] = Field(
+ error: Optional[Error1] = Field(
None, description='Error details if operation failed'
)
name: Optional[str] = None
@@ -2753,13 +4272,102 @@ class Veo2GenVidResponse(BaseModel):
)
+class VeoGenVidPollRequest(BaseModel):
+ operationName: str = Field(
+ ...,
+ description='Full operation name (from predict response)',
+ examples=[
+ 'projects/PROJECT_ID/locations/us-central1/publishers/google/models/MODEL_ID/operations/OPERATION_ID'
+ ],
+ )
+
+
+class Response1(BaseModel):
+ field_type: Optional[str] = Field(
+ None,
+ alias='@type',
+ examples=[
+ 'type.googleapis.com/cloud.ai.large_models.vision.GenerateVideoResponse'
+ ],
+ )
+ raiMediaFilteredCount: Optional[int] = Field(
+ None, description='Count of media filtered by responsible AI policies'
+ )
+ raiMediaFilteredReasons: Optional[List[str]] = Field(
+ None, description='Reasons why media was filtered by responsible AI policies'
+ )
+ videos: Optional[List[Video]] = None
+
+
+class VeoGenVidPollResponse(BaseModel):
+ done: Optional[bool] = None
+ error: Optional[Error1] = Field(
+ None, description='Error details if operation failed'
+ )
+ name: Optional[str] = None
+ response: Optional[Response1] = Field(
+ None, description='The actual prediction response if done is true'
+ )
+
+
+class Image2(BaseModel):
+ bytesBase64Encoded: str
+ gcsUri: Optional[str] = None
+ mimeType: Optional[str] = None
+
+
+class Image3(BaseModel):
+ bytesBase64Encoded: Optional[str] = None
+ gcsUri: str
+ mimeType: Optional[str] = None
+
+
+class Instance1(BaseModel):
+ image: Optional[Union[Image2, Image3]] = Field(
+ None, description='Optional image to guide video generation'
+ )
+ prompt: str = Field(..., description='Text description of the video')
+
+
+class Parameters1(BaseModel):
+ aspectRatio: Optional[str] = Field(None, examples=['16:9'])
+ durationSeconds: Optional[int] = None
+ enhancePrompt: Optional[bool] = None
+ generateAudio: Optional[bool] = Field(
+ None,
+ description='Generate audio for the video. Only supported by veo 3 models.',
+ )
+ negativePrompt: Optional[str] = None
+ personGeneration: Optional[PersonGeneration1] = None
+ sampleCount: Optional[int] = None
+ seed: Optional[int] = None
+ storageUri: Optional[str] = Field(
+ None, description='Optional Cloud Storage URI to upload the video'
+ )
+
+
+class VeoGenVidRequest(BaseModel):
+ instances: Optional[List[Instance1]] = None
+ parameters: Optional[Parameters1] = None
+
+
+class VeoGenVidResponse(BaseModel):
+ name: str = Field(
+ ...,
+ description='Operation resource name',
+ examples=[
+ 'projects/PROJECT_ID/locations/us-central1/publishers/google/models/MODEL_ID/operations/a1b07c8e-7b5a-4aba-bb34-3e1ccb8afcc8'
+ ],
+ )
+
+
class SearchContextSize(str, Enum):
low = 'low'
medium = 'medium'
high = 'high'
-class Type23(str, Enum):
+class Type35(str, Enum):
web_search_preview = 'web_search_preview'
web_search_preview_2025_03_11 = 'web_search_preview_2025_03_11'
@@ -2775,30 +4383,348 @@ class WebSearchPreviewTool(BaseModel):
)
-class Status11(str, Enum):
+class Status12(str, Enum):
in_progress = 'in_progress'
searching = 'searching'
completed = 'completed'
failed = 'failed'
-class Type24(str, Enum):
+class Type36(str, Enum):
web_search_call = 'web_search_call'
class WebSearchToolCall(BaseModel):
id: str = Field(..., description='The unique ID of the web search tool call.\n')
- status: Status11 = Field(
+ status: Status12 = Field(
..., description='The status of the web search tool call.\n'
)
- type: Type24 = Field(
+ type: Type36 = Field(
...,
description='The type of the web search tool call. Always `web_search_call`.\n',
)
-class CreateModelResponseProperties(ModelResponseProperties):
- pass
+class WorkflowRunStatus(str, Enum):
+ WorkflowRunStatusStarted = 'WorkflowRunStatusStarted'
+ WorkflowRunStatusFailed = 'WorkflowRunStatusFailed'
+ WorkflowRunStatusCompleted = 'WorkflowRunStatusCompleted'
+
+
+class ActionJobResult(BaseModel):
+ action_job_id: Optional[str] = Field(
+ None, description='Identifier of the job this result belongs to'
+ )
+ action_run_id: Optional[str] = Field(
+ None, description='Identifier of the run this result belongs to'
+ )
+ author: Optional[str] = Field(None, description='The author of the commit')
+ avg_vram: Optional[int] = Field(
+ None, description='The average VRAM used by the job'
+ )
+ branch_name: Optional[str] = Field(
+ None, description='Name of the relevant git branch'
+ )
+ comfy_run_flags: Optional[str] = Field(
+ None, description='The comfy run flags. E.g. `--low-vram`'
+ )
+ commit_hash: Optional[str] = Field(None, description='The hash of the commit')
+ commit_id: Optional[str] = Field(None, description='The ID of the commit')
+ commit_message: Optional[str] = Field(None, description='The message of the commit')
+ commit_time: Optional[int] = Field(
+ None, description='The Unix timestamp when the commit was made'
+ )
+ cuda_version: Optional[str] = Field(None, description='CUDA version used')
+ end_time: Optional[int] = Field(
+ None, description='The end time of the job as a Unix timestamp.'
+ )
+ git_repo: Optional[str] = Field(None, description='The repository name')
+ id: Optional[UUID] = Field(None, description='Unique identifier for the job result')
+ job_trigger_user: Optional[str] = Field(
+ None, description='The user who triggered the job.'
+ )
+ machine_stats: Optional[MachineStats] = None
+ operating_system: Optional[str] = Field(None, description='Operating system used')
+ peak_vram: Optional[int] = Field(None, description='The peak VRAM used by the job')
+ pr_number: Optional[str] = Field(None, description='The pull request number')
+ python_version: Optional[str] = Field(None, description='PyTorch version used')
+ pytorch_version: Optional[str] = Field(None, description='PyTorch version used')
+ start_time: Optional[int] = Field(
+ None, description='The start time of the job as a Unix timestamp.'
+ )
+ status: Optional[WorkflowRunStatus] = None
+ storage_file: Optional[StorageFile] = None
+ workflow_name: Optional[str] = Field(None, description='Name of the workflow')
+
+
+class BFLCannyInputs(BaseModel):
+ canny_high_threshold: Optional[CannyHighThreshold] = Field(
+ default_factory=lambda: CannyHighThreshold.model_validate(200),
+ description='High threshold for Canny edge detection',
+ title='Canny High Threshold',
+ )
+ canny_low_threshold: Optional[CannyLowThreshold] = Field(
+ default_factory=lambda: CannyLowThreshold.model_validate(50),
+ description='Low threshold for Canny edge detection',
+ title='Canny Low Threshold',
+ )
+ control_image: Optional[str] = Field(
+ None,
+ description='Base64 encoded image to use as control input if no preprocessed image is provided',
+ title='Control Image',
+ )
+ guidance: Optional[Guidance] = Field(
+ default_factory=lambda: Guidance.model_validate(30),
+ description='Guidance strength for the image generation process',
+ title='Guidance',
+ )
+ output_format: Optional[BFLOutputFormat] = Field(
+ 'jpeg',
+ description="Output format for the generated image. Can be 'jpeg' or 'png'.",
+ )
+ preprocessed_image: Optional[str] = Field(
+ None,
+ description='Optional pre-processed image that will bypass the control preprocessing step',
+ title='Preprocessed Image',
+ )
+ prompt: str = Field(
+ ...,
+ description='Text prompt for image generation',
+ examples=['ein fantastisches bild'],
+ title='Prompt',
+ )
+ prompt_upsampling: Optional[bool] = Field(
+ False,
+ description='Whether to perform upsampling on the prompt',
+ title='Prompt Upsampling',
+ )
+ safety_tolerance: Optional[int] = Field(
+ 2,
+ description='Tolerance level for input and output moderation. Between 0 and 6, 0 being most strict, 6 being least strict.',
+ ge=0,
+ le=6,
+ title='Safety Tolerance',
+ )
+ seed: Optional[int] = Field(
+ None,
+ description='Optional seed for reproducibility',
+ examples=[42],
+ title='Seed',
+ )
+ steps: Optional[Steps] = Field(
+ default_factory=lambda: Steps.model_validate(50),
+ description='Number of steps for the image generation process',
+ title='Steps',
+ )
+ webhook_secret: Optional[str] = Field(
+ None,
+ description='Optional secret for webhook signature verification',
+ title='Webhook Secret',
+ )
+ webhook_url: Optional[WebhookUrl] = Field(
+ None, description='URL to receive webhook notifications', title='Webhook Url'
+ )
+
+
+class BFLDepthInputs(BaseModel):
+ control_image: Optional[str] = Field(
+ None,
+ description='Base64 encoded image to use as control input',
+ title='Control Image',
+ )
+ guidance: Optional[Guidance] = Field(
+ default_factory=lambda: Guidance.model_validate(15),
+ description='Guidance strength for the image generation process',
+ title='Guidance',
+ )
+ output_format: Optional[BFLOutputFormat] = Field(
+ 'jpeg',
+ description="Output format for the generated image. Can be 'jpeg' or 'png'.",
+ )
+ preprocessed_image: Optional[str] = Field(
+ None,
+ description='Optional pre-processed image that will bypass the control preprocessing step',
+ title='Preprocessed Image',
+ )
+ prompt: str = Field(
+ ...,
+ description='Text prompt for image generation',
+ examples=['ein fantastisches bild'],
+ title='Prompt',
+ )
+ prompt_upsampling: Optional[bool] = Field(
+ False,
+ description='Whether to perform upsampling on the prompt',
+ title='Prompt Upsampling',
+ )
+ safety_tolerance: Optional[int] = Field(
+ 2,
+ description='Tolerance level for input and output moderation. Between 0 and 6, 0 being most strict, 6 being least strict.',
+ ge=0,
+ le=6,
+ title='Safety Tolerance',
+ )
+ seed: Optional[int] = Field(
+ None,
+ description='Optional seed for reproducibility',
+ examples=[42],
+ title='Seed',
+ )
+ steps: Optional[Steps] = Field(
+ default_factory=lambda: Steps.model_validate(50),
+ description='Number of steps for the image generation process',
+ title='Steps',
+ )
+ webhook_secret: Optional[str] = Field(
+ None,
+ description='Optional secret for webhook signature verification',
+ title='Webhook Secret',
+ )
+ webhook_url: Optional[WebhookUrl] = Field(
+ None, description='URL to receive webhook notifications', title='Webhook Url'
+ )
+
+
+class BFLFluxProExpandInputs(BaseModel):
+ bottom: Optional[Bottom] = Field(
+ 0,
+ description='Number of pixels to expand at the bottom of the image',
+ title='Bottom',
+ )
+ guidance: Optional[Guidance2] = Field(
+ default_factory=lambda: Guidance2.model_validate(60),
+ description='Guidance strength for the image generation process',
+ title='Guidance',
+ )
+ image: str = Field(
+ ...,
+ description='A Base64-encoded string representing the image you wish to expand.',
+ title='Image',
+ )
+ left: Optional[Left] = Field(
+ 0,
+ description='Number of pixels to expand on the left side of the image',
+ title='Left',
+ )
+ output_format: Optional[BFLOutputFormat] = Field(
+ 'jpeg',
+ description="Output format for the generated image. Can be 'jpeg' or 'png'.",
+ )
+ prompt: Optional[str] = Field(
+ '',
+ description='The description of the changes you want to make. This text guides the expansion process, allowing you to specify features, styles, or modifications for the expanded areas.',
+ examples=['ein fantastisches bild'],
+ title='Prompt',
+ )
+ prompt_upsampling: Optional[bool] = Field(
+ False,
+ description='Whether to perform upsampling on the prompt. If active, automatically modifies the prompt for more creative generation',
+ title='Prompt Upsampling',
+ )
+ right: Optional[Right] = Field(
+ 0,
+ description='Number of pixels to expand on the right side of the image',
+ title='Right',
+ )
+ safety_tolerance: Optional[int] = Field(
+ 2,
+ description='Tolerance level for input and output moderation. Between 0 and 6, 0 being most strict, 6 being least strict.',
+ examples=[2],
+ ge=0,
+ le=6,
+ title='Safety Tolerance',
+ )
+ seed: Optional[int] = Field(
+ None, description='Optional seed for reproducibility', title='Seed'
+ )
+ steps: Optional[Steps2] = Field(
+ default_factory=lambda: Steps2.model_validate(50),
+ description='Number of steps for the image generation process',
+ examples=[50],
+ title='Steps',
+ )
+ top: Optional[Top] = Field(
+ 0, description='Number of pixels to expand at the top of the image', title='Top'
+ )
+ webhook_secret: Optional[str] = Field(
+ None,
+ description='Optional secret for webhook signature verification',
+ title='Webhook Secret',
+ )
+ webhook_url: Optional[WebhookUrl] = Field(
+ None, description='URL to receive webhook notifications', title='Webhook Url'
+ )
+
+
+class BFLFluxProFillInputs(BaseModel):
+ guidance: Optional[Guidance2] = Field(
+ default_factory=lambda: Guidance2.model_validate(60),
+ description='Guidance strength for the image generation process',
+ title='Guidance',
+ )
+ image: str = Field(
+ ...,
+ description='A Base64-encoded string representing the image you wish to modify. Can contain alpha mask if desired.',
+ title='Image',
+ )
+ mask: Optional[str] = Field(
+ None,
+ description='A Base64-encoded string representing a mask for the areas you want to modify in the image. The mask should be the same dimensions as the image and in black and white. Black areas (0%) indicate no modification, while white areas (100%) specify areas for inpainting. Optional if you provide an alpha mask in the original image. Validation: The endpoint verifies that the dimensions of the mask match the original image.',
+ title='Mask',
+ )
+ output_format: Optional[BFLOutputFormat] = Field(
+ 'jpeg',
+ description="Output format for the generated image. Can be 'jpeg' or 'png'.",
+ )
+ prompt: Optional[str] = Field(
+ '',
+ description='The description of the changes you want to make. This text guides the inpainting process, allowing you to specify features, styles, or modifications for the masked area.',
+ examples=['ein fantastisches bild'],
+ title='Prompt',
+ )
+ prompt_upsampling: Optional[bool] = Field(
+ False,
+ description='Whether to perform upsampling on the prompt. If active, automatically modifies the prompt for more creative generation',
+ title='Prompt Upsampling',
+ )
+ safety_tolerance: Optional[int] = Field(
+ 2,
+ description='Tolerance level for input and output moderation. Between 0 and 6, 0 being most strict, 6 being least strict.',
+ examples=[2],
+ ge=0,
+ le=6,
+ title='Safety Tolerance',
+ )
+ seed: Optional[int] = Field(
+ None, description='Optional seed for reproducibility', title='Seed'
+ )
+ steps: Optional[Steps2] = Field(
+ default_factory=lambda: Steps2.model_validate(50),
+ description='Number of steps for the image generation process',
+ examples=[50],
+ title='Steps',
+ )
+ webhook_secret: Optional[str] = Field(
+ None,
+ description='Optional secret for webhook signature verification',
+ title='Webhook Secret',
+ )
+ webhook_url: Optional[WebhookUrl] = Field(
+ None, description='URL to receive webhook notifications', title='Webhook Url'
+ )
+
+
+class BFLHTTPValidationError(BaseModel):
+ detail: Optional[List[BFLValidationError]] = Field(None, title='Detail')
+
+
+class BulkNodeVersionsRequest(BaseModel):
+ node_versions: List[NodeVersionIdentifier] = Field(
+ ..., description='List of node ID and version pairs to retrieve'
+ )
+
+
+CreateModelResponseProperties = ModelResponseProperties
class GeminiInlineData(BaseModel):
@@ -2841,6 +4767,125 @@ class GeminiSystemInstructionContent(BaseModel):
)
+class GeminiUsageMetadata(BaseModel):
+ cachedContentTokenCount: Optional[int] = Field(
+ None,
+ description='Output only. Number of tokens in the cached part in the input (the cached content).',
+ )
+ candidatesTokenCount: Optional[int] = Field(
+ None, description='Number of tokens in the response(s).'
+ )
+ candidatesTokensDetails: Optional[List[ModalityTokenCount]] = Field(
+ None, description='Breakdown of candidate tokens by modality.'
+ )
+ promptTokenCount: Optional[int] = Field(
+ None,
+ description='Number of tokens in the request. When cachedContent is set, this is still the total effective prompt size meaning this includes the number of tokens in the cached content.',
+ )
+ promptTokensDetails: Optional[List[ModalityTokenCount]] = Field(
+ None, description='Breakdown of prompt tokens by modality.'
+ )
+ thoughtsTokenCount: Optional[int] = Field(
+ None, description='Number of tokens present in thoughts output.'
+ )
+ toolUsePromptTokenCount: Optional[int] = Field(
+ None, description='Number of tokens present in tool-use prompt(s).'
+ )
+
+
+class GithubInstallation(BaseModel):
+ access_tokens_url: str = Field(..., description='The API URL for access tokens')
+ account: GithubUser
+ app_id: int = Field(..., description='The GitHub App ID')
+ created_at: datetime = Field(..., description='When the installation was created')
+ events: List[str] = Field(
+ ..., description='The events the installation subscribes to'
+ )
+ html_url: str = Field(..., description='The HTML URL of the installation')
+ id: int = Field(..., description='The installation ID')
+ permissions: Dict[str, Any] = Field(..., description='The installation permissions')
+ repositories_url: str = Field(..., description='The API URL for repositories')
+ repository_selection: RepositorySelection = Field(
+ ..., description='Repository selection for the installation'
+ )
+ single_file_name: Optional[str] = Field(
+ None, description='The single file name if applicable'
+ )
+ target_id: int = Field(..., description='The target ID')
+ target_type: str = Field(..., description='The target type')
+ updated_at: datetime = Field(
+ ..., description='When the installation was last updated'
+ )
+
+
+class GithubReleaseAsset(BaseModel):
+ browser_download_url: str = Field(..., description='The browser download URL')
+ content_type: str = Field(..., description='The content type of the asset')
+ created_at: datetime = Field(..., description='When the asset was created')
+ download_count: int = Field(..., description='The number of downloads')
+ id: int = Field(..., description='The asset ID')
+ label: Optional[str] = Field(None, description='The label of the asset')
+ name: str = Field(..., description='The name of the asset')
+ node_id: str = Field(..., description='The asset node ID')
+ size: int = Field(..., description='The size of the asset in bytes')
+ state: State = Field(..., description='The state of the asset')
+ updated_at: datetime = Field(..., description='When the asset was last updated')
+ uploader: GithubUser
+
+
+class Release(BaseModel):
+ assets: List[GithubReleaseAsset] = Field(..., description='Array of release assets')
+ assets_url: Optional[str] = Field(None, description='The URL to the release assets')
+ author: GithubUser
+ body: Optional[str] = Field(None, description='The release notes/body')
+ created_at: datetime = Field(..., description='When the release was created')
+ draft: bool = Field(..., description='Whether the release is a draft')
+ html_url: str = Field(..., description='The HTML URL of the release')
+ id: int = Field(..., description='The ID of the release')
+ name: Optional[str] = Field(None, description='The name of the release')
+ node_id: str = Field(..., description='The node ID of the release')
+ prerelease: bool = Field(..., description='Whether the release is a prerelease')
+ published_at: Optional[datetime] = Field(
+ None, description='When the release was published'
+ )
+ tag_name: str = Field(..., description='The tag name of the release')
+ tarball_url: str = Field(..., description='URL to the tarball')
+ target_commitish: str = Field(
+ ..., description='The branch or commit the release was created from'
+ )
+ upload_url: Optional[str] = Field(
+ None, description='The URL to upload release assets'
+ )
+ url: str = Field(..., description='The API URL of the release')
+ zipball_url: str = Field(..., description='URL to the zipball')
+
+
+class GithubRepository(BaseModel):
+ clone_url: str = Field(..., description='The clone URL of the repository')
+ created_at: datetime = Field(..., description='When the repository was created')
+ default_branch: str = Field(..., description='The default branch of the repository')
+ description: Optional[str] = Field(None, description='The repository description')
+ fork: bool = Field(..., description='Whether the repository is a fork')
+ full_name: str = Field(
+ ..., description='The full name of the repository (owner/repo)'
+ )
+ git_url: str = Field(..., description='The git URL of the repository')
+ html_url: str = Field(..., description='The HTML URL of the repository')
+ id: int = Field(..., description='The repository ID')
+ name: str = Field(..., description='The name of the repository')
+ node_id: str = Field(..., description='The repository node ID')
+ owner: GithubUser
+ private: bool = Field(..., description='Whether the repository is private')
+ pushed_at: datetime = Field(
+ ..., description='When the repository was last pushed to'
+ )
+ ssh_url: str = Field(..., description='The SSH URL of the repository')
+ updated_at: datetime = Field(
+ ..., description='When the repository was last updated'
+ )
+ url: str = Field(..., description='The API URL of the repository')
+
+
class IdeogramV3EditRequest(BaseModel):
color_palette: Optional[IdeogramColorPalette] = None
image: Optional[StrictBytes] = Field(
@@ -3276,6 +5321,52 @@ class MoonvalleyTextToImageRequest(BaseModel):
webhook_url: Optional[str] = None
+class NodeVersion(BaseModel):
+ changelog: Optional[str] = Field(
+ None, description='Summary of changes made in this version'
+ )
+ comfy_node_extract_status: Optional[str] = Field(
+ None, description='The status of comfy node extraction process.'
+ )
+ createdAt: Optional[datetime] = Field(
+ None, description='The date and time the version was created.'
+ )
+ dependencies: Optional[List[str]] = Field(
+ None, description='A list of pip dependencies required by the node.'
+ )
+ deprecated: Optional[bool] = Field(
+ None, description='Indicates if this version is deprecated.'
+ )
+ downloadUrl: Optional[str] = Field(
+ None, description='[Output Only] URL to download this version of the node'
+ )
+ id: Optional[str] = None
+ node_id: Optional[str] = Field(
+ None, description='The unique identifier of the node.'
+ )
+ status: Optional[NodeVersionStatus] = None
+ status_reason: Optional[str] = Field(
+ None, description='The reason for the status change.'
+ )
+ supported_accelerators: Optional[List[str]] = Field(
+ None,
+ description='List of accelerators (e.g. CUDA, DirectML, ROCm) that this node supports',
+ )
+ supported_comfyui_frontend_version: Optional[str] = Field(
+ None, description='Supported versions of ComfyUI frontend'
+ )
+ supported_comfyui_version: Optional[str] = Field(
+ None, description='Supported versions of ComfyUI'
+ )
+ supported_os: Optional[List[str]] = Field(
+ None, description='List of operating systems that this node supports'
+ )
+ version: Optional[str] = Field(
+ None,
+ description='The version identifier, following semantic versioning. Must be unique for the node.',
+ )
+
+
class OutputContent(RootModel[Union[OutputTextContent, OutputAudioContent]]):
root: Union[OutputTextContent, OutputAudioContent]
@@ -3283,7 +5374,7 @@ class OutputContent(RootModel[Union[OutputTextContent, OutputAudioContent]]):
class OutputMessage(BaseModel):
content: List[OutputContent] = Field(..., description='The content of the message')
role: Role4 = Field(..., description='The role of the message')
- type: Type14 = Field(..., description='The type of output item')
+ type: Type15 = Field(..., description='The type of output item')
class PikaBodyGenerate22I2vGenerate22I2vPost(BaseModel):
@@ -3333,6 +5424,16 @@ class PikaHTTPValidationError(BaseModel):
detail: Optional[List[PikaValidationError]] = Field(None, title='Detail')
+class PublisherMember(BaseModel):
+ id: Optional[str] = Field(
+ None, description='The unique identifier for the publisher member.'
+ )
+ role: Optional[str] = Field(
+ None, description='The role of the user in the publisher.'
+ )
+ user: Optional[PublisherUser] = None
+
+
class Reasoning(BaseModel):
effort: Optional[ReasoningEffort] = 'medium'
generate_summary: Optional[GenerateSummary] = Field(
@@ -3345,13 +5446,88 @@ class Reasoning(BaseModel):
)
+class RecraftImage(BaseModel):
+ b64_json: Optional[str] = None
+ features: Optional[RecraftImageFeatures] = None
+ image_id: UUID
+ revised_prompt: Optional[str] = None
+ url: Optional[str] = None
+
+
+class RecraftProcessImageRequest(BaseModel):
+ image: StrictBytes
+ image_format: Optional[RecraftImageFormat] = None
+ response_format: Optional[RecraftResponseFormat] = None
+
+
+class RecraftProcessImageResponse(BaseModel):
+ created: int
+ credits: int
+ image: RecraftImage
+
+
+class RecraftTextLayout(RootModel[List[RecraftTextLayoutItem]]):
+ root: List[RecraftTextLayoutItem]
+
+
+class RecraftTransformImageWithMaskRequest(BaseModel):
+ block_nsfw: Optional[bool] = None
+ calculate_features: Optional[bool] = None
+ image: StrictBytes
+ image_format: Optional[RecraftImageFormat] = None
+ mask: StrictBytes
+ model: Optional[RecraftTransformModel] = None
+ n: Optional[int] = None
+ negative_prompt: Optional[str] = None
+ prompt: str
+ response_format: Optional[RecraftResponseFormat] = None
+ style: Optional[RecraftImageStyle] = None
+ style_id: Optional[UUID] = None
+ substyle: Optional[RecraftImageSubStyle] = None
+ text_layout: Optional[RecraftTextLayout] = None
+
+
+class ResponseContentPartAddedEvent(BaseModel):
+ content_index: int = Field(
+ ..., description='The index of the content part that was added.'
+ )
+ item_id: str = Field(
+ ..., description='The ID of the output item that the content part was added to.'
+ )
+ output_index: int = Field(
+ ...,
+ description='The index of the output item that the content part was added to.',
+ )
+ part: OutputContent
+ type: Type20 = Field(
+ ..., description='The type of the event. Always `response.content_part.added`.'
+ )
+
+
+class ResponseContentPartDoneEvent(BaseModel):
+ content_index: int = Field(
+ ..., description='The index of the content part that is done.'
+ )
+ item_id: str = Field(
+ ..., description='The ID of the output item that the content part was added to.'
+ )
+ output_index: int = Field(
+ ...,
+ description='The index of the output item that the content part was added to.',
+ )
+ part: OutputContent
+ type: Type21 = Field(
+ ..., description='The type of the event. Always `response.content_part.done`.'
+ )
+
+
class ResponseError(BaseModel):
code: ResponseErrorCode
message: str = Field(..., description='A human-readable description of the error.')
class Rodin3DDownloadResponse(BaseModel):
- list: Optional[RodinResourceItem] = None
+ list: Optional[List[RodinResourceItem]] = None
class Rodin3DGenerateRequest(BaseModel):
@@ -3371,6 +5547,11 @@ class Rodin3DGenerateResponse(BaseModel):
uuid: Optional[str] = Field(None, description='Task UUID')
+class RodinCheckStatusJobItem(BaseModel):
+ status: Optional[RodinStatusOptions] = None
+ uuid: Optional[str] = Field(None, description='sub uuid')
+
+
class RunwayImageToVideoRequest(BaseModel):
duration: RunwayDurationEnum
model: RunwayModelEnum
@@ -3384,6 +5565,109 @@ class RunwayImageToVideoRequest(BaseModel):
)
+class StripeCharge(BaseModel):
+ amount: Optional[int] = None
+ amount_captured: Optional[int] = None
+ amount_refunded: Optional[int] = None
+ application: Optional[str] = None
+ application_fee: Optional[str] = None
+ application_fee_amount: Optional[int] = None
+ balance_transaction: Optional[str] = None
+ billing_details: Optional[StripeBillingDetails] = None
+ calculated_statement_descriptor: Optional[str] = None
+ captured: Optional[bool] = None
+ created: Optional[int] = None
+ currency: Optional[str] = None
+ customer: Optional[str] = None
+ description: Optional[str] = None
+ destination: Optional[Any] = None
+ dispute: Optional[Any] = None
+ disputed: Optional[bool] = None
+ failure_balance_transaction: Optional[Any] = None
+ failure_code: Optional[Any] = None
+ failure_message: Optional[Any] = None
+ fraud_details: Optional[Dict[str, Any]] = None
+ id: Optional[str] = None
+ invoice: Optional[Any] = None
+ livemode: Optional[bool] = None
+ metadata: Optional[Dict[str, Any]] = None
+ object: Optional[Object1] = None
+ on_behalf_of: Optional[Any] = None
+ order: Optional[Any] = None
+ outcome: Optional[StripeOutcome] = None
+ paid: Optional[bool] = None
+ payment_intent: Optional[str] = None
+ payment_method: Optional[str] = None
+ payment_method_details: Optional[StripePaymentMethodDetails] = None
+ radar_options: Optional[Dict[str, Any]] = None
+ receipt_email: Optional[str] = None
+ receipt_number: Optional[str] = None
+ receipt_url: Optional[str] = None
+ refunded: Optional[bool] = None
+ refunds: Optional[StripeRefundList] = None
+ review: Optional[Any] = None
+ shipping: Optional[StripeShipping] = None
+ source: Optional[Any] = None
+ source_transfer: Optional[Any] = None
+ statement_descriptor: Optional[Any] = None
+ statement_descriptor_suffix: Optional[Any] = None
+ status: Optional[str] = None
+ transfer_data: Optional[Any] = None
+ transfer_group: Optional[Any] = None
+
+
+class StripeChargeList(BaseModel):
+ data: Optional[List[StripeCharge]] = None
+ has_more: Optional[bool] = None
+ object: Optional[str] = None
+ total_count: Optional[int] = None
+ url: Optional[str] = None
+
+
+class StripePaymentIntent(BaseModel):
+ amount: Optional[int] = None
+ amount_capturable: Optional[int] = None
+ amount_details: Optional[StripeAmountDetails] = None
+ amount_received: Optional[int] = None
+ application: Optional[str] = None
+ application_fee_amount: Optional[int] = None
+ automatic_payment_methods: Optional[Any] = None
+ canceled_at: Optional[int] = None
+ cancellation_reason: Optional[str] = None
+ capture_method: Optional[str] = None
+ charges: Optional[StripeChargeList] = None
+ client_secret: Optional[str] = None
+ confirmation_method: Optional[str] = None
+ created: Optional[int] = None
+ currency: Optional[str] = None
+ customer: Optional[str] = None
+ description: Optional[str] = None
+ id: Optional[str] = None
+ invoice: Optional[str] = None
+ last_payment_error: Optional[Any] = None
+ latest_charge: Optional[str] = None
+ livemode: Optional[bool] = None
+ metadata: Optional[Dict[str, Any]] = None
+ next_action: Optional[Any] = None
+ object: Optional[Object3] = None
+ on_behalf_of: Optional[Any] = None
+ payment_method: Optional[str] = None
+ payment_method_configuration_details: Optional[Any] = None
+ payment_method_options: Optional[StripePaymentMethodOptions] = None
+ payment_method_types: Optional[List[str]] = None
+ processing: Optional[Any] = None
+ receipt_email: Optional[str] = None
+ review: Optional[Any] = None
+ setup_future_usage: Optional[Any] = None
+ shipping: Optional[StripeShipping] = None
+ source: Optional[Any] = None
+ statement_descriptor: Optional[Any] = None
+ statement_descriptor_suffix: Optional[Any] = None
+ status: Optional[str] = None
+ transfer_data: Optional[Any] = None
+ transfer_group: Optional[Any] = None
+
+
class TextResponseFormatConfiguration(
RootModel[
Union[
@@ -3411,6 +5695,22 @@ class Tool(
] = Field(..., discriminator='type')
+class BulkNodeVersionResult(BaseModel):
+ error_message: Optional[str] = Field(
+ None,
+ description='Error message if retrieval failed (only present if status is error)',
+ )
+ identifier: NodeVersionIdentifier
+ node_version: Optional[NodeVersion] = None
+ status: Status = Field(..., description='Status of the retrieval operation')
+
+
+class BulkNodeVersionsResponse(BaseModel):
+ node_versions: List[BulkNodeVersionResult] = Field(
+ ..., description='List of retrieved node versions with their status'
+ )
+
+
class EasyInputMessage(BaseModel):
content: Union[str, InputMessageContentList] = Field(
...,
@@ -3439,6 +5739,16 @@ class GeminiGenerateContentRequest(BaseModel):
videoMetadata: Optional[GeminiVideoMetadata] = None
+class GithubReleaseWebhook(BaseModel):
+ action: Action = Field(..., description='The action performed on the release')
+ enterprise: Optional[GithubEnterprise] = None
+ installation: Optional[GithubInstallation] = None
+ organization: Optional[GithubOrganization] = None
+ release: Release = Field(..., description='The release object')
+ repository: GithubRepository
+ sender: GithubUser
+
+
class ImagenGenerateImageRequest(BaseModel):
instances: List[ImagenImageGenerationInstance]
parameters: ImagenImageGenerationParameters
@@ -3447,8 +5757,8 @@ class ImagenGenerateImageRequest(BaseModel):
class InputMessage(BaseModel):
content: Optional[InputMessageContentList] = None
role: Optional[Role3] = None
- status: Optional[Status2] = None
- type: Optional[Type9] = None
+ status: Optional[Status3] = None
+ type: Optional[Type10] = None
class Item(
@@ -3519,6 +5829,70 @@ class OutputItem(
]
+class Publisher(BaseModel):
+ createdAt: Optional[datetime] = Field(
+ None, description='The date and time the publisher was created.'
+ )
+ description: Optional[str] = None
+ id: Optional[str] = Field(
+ None,
+ description="The unique identifier for the publisher. It's akin to a username. Should be lowercase.",
+ )
+ logo: Optional[str] = Field(None, description="URL to the publisher's logo.")
+ members: Optional[List[PublisherMember]] = Field(
+ None, description='A list of members in the publisher.'
+ )
+ name: Optional[str] = None
+ source_code_repo: Optional[str] = None
+ status: Optional[PublisherStatus] = None
+ support: Optional[str] = None
+ website: Optional[str] = None
+
+
+class RecraftGenerateImageResponse(BaseModel):
+ created: int
+ credits: int
+ data: List[RecraftImage]
+
+
+class RecraftImageToImageRequest(BaseModel):
+ block_nsfw: Optional[bool] = None
+ calculate_features: Optional[bool] = None
+ controls: Optional[RecraftUserControls] = None
+ image: StrictBytes
+ image_format: Optional[RecraftImageFormat] = None
+ model: Optional[RecraftTransformModel] = None
+ n: Optional[int] = None
+ negative_prompt: Optional[str] = None
+ prompt: str
+ response_format: Optional[RecraftResponseFormat] = None
+ strength: float
+ style: Optional[RecraftImageStyle] = None
+ style_id: Optional[UUID] = None
+ substyle: Optional[RecraftImageSubStyle] = None
+ text_layout: Optional[RecraftTextLayout] = None
+
+
+class ResponseOutputItemAddedEvent(BaseModel):
+ item: OutputItem
+ output_index: int = Field(
+ ..., description='The index of the output item that was added.\n'
+ )
+ type: Type29 = Field(
+ ..., description='The type of the event. Always `response.output_item.added`.\n'
+ )
+
+
+class ResponseOutputItemDoneEvent(BaseModel):
+ item: OutputItem
+ output_index: int = Field(
+ ..., description='The index of the output item that was marked done.\n'
+ )
+ type: Type30 = Field(
+ ..., description='The type of the event. Always `response.output_item.done`.\n'
+ )
+
+
class Text(BaseModel):
format: Optional[TextResponseFormatConfiguration] = None
@@ -3552,6 +5926,28 @@ class ResponseProperties(BaseModel):
)
+class Rodin3DCheckStatusResponse(BaseModel):
+ jobs: Optional[List[RodinCheckStatusJobItem]] = Field(
+ None, description='Details for the generation status.'
+ )
+
+
+class Data8(BaseModel):
+ object: Optional[StripePaymentIntent] = None
+
+
+class StripeEvent(BaseModel):
+ api_version: Optional[str] = None
+ created: Optional[int] = None
+ data: Data8
+ id: str
+ livemode: Optional[bool] = None
+ object: Object2
+ pending_webhooks: Optional[int] = None
+ request: Optional[StripeRequestInfo] = None
+ type: Type31
+
+
class GeminiCandidate(BaseModel):
citationMetadata: Optional[GeminiCitationMetadata] = None
content: Optional[GeminiContent] = None
@@ -3562,12 +5958,67 @@ class GeminiCandidate(BaseModel):
class GeminiGenerateContentResponse(BaseModel):
candidates: Optional[List[GeminiCandidate]] = None
promptFeedback: Optional[GeminiPromptFeedback] = None
+ usageMetadata: Optional[GeminiUsageMetadata] = None
class InputItem(RootModel[Union[EasyInputMessage, Item]]):
root: Union[EasyInputMessage, Item]
+class Node(BaseModel):
+ author: Optional[str] = None
+ banner_url: Optional[str] = Field(None, description="URL to the node's banner.")
+ category: Optional[str] = Field(None, description='The category of the node.')
+ created_at: Optional[datetime] = Field(
+ None, description='The date and time when the node was created'
+ )
+ description: Optional[str] = None
+ downloads: Optional[int] = Field(
+ None, description='The number of downloads of the node.'
+ )
+ github_stars: Optional[int] = Field(
+ None, description='Number of stars on the GitHub repository.'
+ )
+ icon: Optional[str] = Field(None, description="URL to the node's icon.")
+ id: Optional[str] = Field(None, description='The unique identifier of the node.')
+ latest_version: Optional[NodeVersion] = None
+ license: Optional[str] = Field(
+ None, description="The path to the LICENSE file in the node's repository."
+ )
+ name: Optional[str] = Field(None, description='The display name of the node.')
+ preempted_comfy_node_names: Optional[List[str]] = Field(
+ None, description='A list of Comfy node names that are preempted by this node.'
+ )
+ publisher: Optional[Publisher] = None
+ rating: Optional[float] = Field(None, description='The average rating of the node.')
+ repository: Optional[str] = Field(None, description="URL to the node's repository.")
+ search_ranking: Optional[int] = Field(
+ None,
+ description="A numerical value representing the node's search ranking, used for sorting search results.",
+ )
+ status: Optional[NodeStatus] = None
+ status_detail: Optional[str] = Field(
+ None, description='The status detail of the node.'
+ )
+ supported_accelerators: Optional[List[str]] = Field(
+ None,
+ description='List of accelerators (e.g. CUDA, DirectML, ROCm) that this node supports',
+ )
+ supported_comfyui_frontend_version: Optional[str] = Field(
+ None, description='Supported versions of ComfyUI frontend'
+ )
+ supported_comfyui_version: Optional[str] = Field(
+ None, description='Supported versions of ComfyUI'
+ )
+ supported_os: Optional[List[str]] = Field(
+ None, description='List of operating systems that this node supports'
+ )
+ tags: Optional[List[str]] = None
+ translations: Optional[Dict[str, Dict[str, Any]]] = Field(
+ None, description='Translations of node metadata in different languages.'
+ )
+
+
class OpenAICreateResponse(CreateModelResponseProperties, ResponseProperties):
include: Optional[List[Includable]] = Field(
None,
@@ -3615,8 +6066,73 @@ class OpenAIResponse(ModelResponseProperties, ResponseProperties):
parallel_tool_calls: Optional[bool] = Field(
True, description='Whether to allow the model to run tool calls in parallel.\n'
)
- status: Optional[Status6] = Field(
+ status: Optional[Status7] = Field(
None,
description='The status of the response generation. One of `completed`, `failed`, `in_progress`, or `incomplete`.',
)
usage: Optional[ResponseUsage] = None
+
+
+class ResponseCompletedEvent(BaseModel):
+ response: OpenAIResponse
+ type: Type19 = Field(
+ ..., description='The type of the event. Always `response.completed`.'
+ )
+
+
+class ResponseCreatedEvent(BaseModel):
+ response: OpenAIResponse
+ type: Type22 = Field(
+ ..., description='The type of the event. Always `response.created`.'
+ )
+
+
+class ResponseFailedEvent(BaseModel):
+ response: OpenAIResponse
+ type: Type24 = Field(
+ ..., description='The type of the event. Always `response.failed`.\n'
+ )
+
+
+class ResponseInProgressEvent(BaseModel):
+ response: OpenAIResponse
+ type: Type27 = Field(
+ ..., description='The type of the event. Always `response.in_progress`.\n'
+ )
+
+
+class ResponseIncompleteEvent(BaseModel):
+ response: OpenAIResponse
+ type: Type28 = Field(
+ ..., description='The type of the event. Always `response.incomplete`.\n'
+ )
+
+
+class OpenAIResponseStreamEvent(
+ RootModel[
+ Union[
+ ResponseCreatedEvent,
+ ResponseInProgressEvent,
+ ResponseCompletedEvent,
+ ResponseFailedEvent,
+ ResponseIncompleteEvent,
+ ResponseOutputItemAddedEvent,
+ ResponseOutputItemDoneEvent,
+ ResponseContentPartAddedEvent,
+ ResponseContentPartDoneEvent,
+ ResponseErrorEvent,
+ ]
+ ]
+):
+ root: Union[
+ ResponseCreatedEvent,
+ ResponseInProgressEvent,
+ ResponseCompletedEvent,
+ ResponseFailedEvent,
+ ResponseIncompleteEvent,
+ ResponseOutputItemAddedEvent,
+ ResponseOutputItemDoneEvent,
+ ResponseContentPartAddedEvent,
+ ResponseContentPartDoneEvent,
+ ResponseErrorEvent,
+ ] = Field(..., description='Events that can be emitted during response streaming')
diff --git a/comfy_api_nodes/apis/client.py b/comfy_api_nodes/apis/client.py
index 2a4bac88b..4ad0b783b 100644
--- a/comfy_api_nodes/apis/client.py
+++ b/comfy_api_nodes/apis/client.py
@@ -43,7 +43,7 @@ operation = ApiOperation(
endpoint=user_info_endpoint,
request=request
)
-user_profile = operation.execute(client=api_client) # Returns immediately with the result
+user_profile = await operation.execute(client=api_client) # Returns immediately with the result
# Example 2: Asynchronous API Operation with Polling
@@ -87,18 +87,19 @@ operation = PollingOperation(
)
# This will make the initial request and then poll until completion
-result = operation.execute(client=api_client) # Returns the final ImageGenerationResult when done
+result = await operation.execute(client=api_client) # Returns the final ImageGenerationResult when done
"""
from __future__ import annotations
+import aiohttp
+import asyncio
import logging
-import time
import io
import socket
+from aiohttp.client_exceptions import ClientError, ClientResponseError
from typing import Dict, Type, Optional, Any, TypeVar, Generic, Callable, Tuple
from enum import Enum
import json
-import requests
from urllib.parse import urljoin, urlparse
from pydantic import BaseModel, Field
import uuid # For generating unique operation IDs
@@ -174,6 +175,7 @@ class ApiClient:
retry_delay: float = 1.0,
retry_backoff_factor: float = 2.0,
retry_status_codes: Optional[Tuple[int, ...]] = None,
+ session: Optional[aiohttp.ClientSession] = None,
):
self.base_url = base_url
self.auth_token = auth_token
@@ -186,13 +188,16 @@ class ApiClient:
# Default retry status codes: 408 (Request Timeout), 429 (Too Many Requests),
# 500, 502, 503, 504 (Server Errors)
self.retry_status_codes = retry_status_codes or (408, 429, 500, 502, 503, 504)
+ self._session: Optional[aiohttp.ClientSession] = session
+ self._owns_session = session is None # Track if we have to close it
- def _generate_operation_id(self, path: str) -> str:
+ @staticmethod
+ def _generate_operation_id(path: str) -> str:
"""Generates a unique operation ID for logging."""
return f"{path.strip('/').replace('/', '_')}_{uuid.uuid4().hex[:8]}"
+ @staticmethod
def _create_json_payload_args(
- self,
data: Optional[Dict[str, Any]] = None,
headers: Optional[Dict[str, str]] = None,
) -> Dict[str, Any]:
@@ -203,31 +208,53 @@ class ApiClient:
def _create_form_data_args(
self,
- data: Dict[str, Any],
- files: Dict[str, Any],
+ data: Dict[str, Any] | None,
+ files: Dict[str, Any] | None,
headers: Optional[Dict[str, str]] = None,
- multipart_parser = None,
+ multipart_parser: Callable | None = None,
) -> Dict[str, Any]:
if headers and "Content-Type" in headers:
del headers["Content-Type"]
- if multipart_parser:
+ if multipart_parser and data:
data = multipart_parser(data)
- return {
- "data": data,
- "files": files,
- "headers": headers,
- }
+ form = aiohttp.FormData(default_to_multipart=True)
+ if data: # regular text fields
+ for k, v in data.items():
+ if v is None:
+ continue # aiohttp fails to serialize "None" values
+ # aiohttp expects strings or bytes; convert enums etc.
+ form.add_field(k, str(v) if not isinstance(v, (bytes, bytearray)) else v)
+ if files:
+ file_iter = files if isinstance(files, list) else files.items()
+ for field_name, file_obj in file_iter:
+ if file_obj is None:
+ continue # aiohttp fails to serialize "None" values
+ # file_obj can be (filename, bytes/io.BytesIO, content_type) tuple
+ if isinstance(file_obj, tuple):
+ filename, file_value, content_type = self._unpack_tuple(file_obj)
+ else:
+ file_value = file_obj
+ filename = getattr(file_obj, "name", field_name)
+ content_type = "application/octet-stream"
+
+ form.add_field(
+ name=field_name,
+ value=file_value,
+ filename=filename,
+ content_type=content_type,
+ )
+ return {"data": form, "headers": headers or {}}
+
+ @staticmethod
def _create_urlencoded_form_data_args(
- self,
data: Dict[str, Any],
headers: Optional[Dict[str, str]] = None,
) -> Dict[str, Any]:
headers = headers or {}
headers["Content-Type"] = "application/x-www-form-urlencoded"
-
return {
"data": data,
"headers": headers,
@@ -244,7 +271,7 @@ class ApiClient:
return headers
- def _check_connectivity(self, target_url: str) -> Dict[str, bool]:
+ async def _check_connectivity(self, target_url: str) -> Dict[str, bool]:
"""
Check connectivity to determine if network issues are local or server-related.
@@ -258,52 +285,39 @@ class ApiClient:
"internet_accessible": False,
"api_accessible": False,
"is_local_issue": False,
- "is_api_issue": False
+ "is_api_issue": False,
}
+ timeout = aiohttp.ClientTimeout(total=5.0)
+ async with aiohttp.ClientSession(timeout=timeout) as session:
+ try:
+ async with session.get("https://www.google.com", ssl=self.verify_ssl) as resp:
+ results["internet_accessible"] = resp.status < 500
+ except (ClientError, asyncio.TimeoutError, socket.gaierror):
+ results["is_local_issue"] = True
+ return results # cannot reach the internet – early exit
- # First check basic internet connectivity using a reliable external site
- try:
- # Use a reliable external domain for checking basic connectivity
- check_response = requests.get("https://www.google.com",
- timeout=5.0,
- verify=self.verify_ssl)
- if check_response.status_code < 500:
- results["internet_accessible"] = True
- except (requests.RequestException, socket.error):
- results["internet_accessible"] = False
- results["is_local_issue"] = True
- return results
-
- # Now check API server connectivity
- try:
- # Extract domain from the target URL to do a simpler health check
- parsed_url = urlparse(target_url)
- api_base = f"{parsed_url.scheme}://{parsed_url.netloc}"
-
- # Try to reach the API domain
- api_response = requests.get(f"{api_base}/health", timeout=5.0, verify=self.verify_ssl)
- if api_response.status_code < 500:
- results["api_accessible"] = True
- else:
- results["api_accessible"] = False
- results["is_api_issue"] = True
- except requests.RequestException:
- results["api_accessible"] = False
- # If we can reach the internet but not the API, it's an API issue
- results["is_api_issue"] = True
+ # Now check API health endpoint
+ parsed = urlparse(target_url)
+ health_url = f"{parsed.scheme}://{parsed.netloc}/health"
+ try:
+ async with session.get(health_url, ssl=self.verify_ssl) as resp:
+ results["api_accessible"] = resp.status < 500
+ except ClientError:
+ pass # leave as False
+ results["is_api_issue"] = results["internet_accessible"] and not results["api_accessible"]
return results
- def request(
+ async def request(
self,
method: str,
path: str,
params: Optional[Dict[str, Any]] = None,
data: Optional[Dict[str, Any]] = None,
- files: Optional[Dict[str, Any]] = None,
+ files: Optional[Dict[str, Any] | list[tuple[str, Any]]] = None,
headers: Optional[Dict[str, str]] = None,
content_type: str = "application/json",
- multipart_parser: Callable = None,
+ multipart_parser: Callable | None = None,
retry_count: int = 0, # Used internally for tracking retries
) -> Dict[str, Any]:
"""
@@ -327,18 +341,19 @@ class ApiClient:
ApiServerError: If the API server is unreachable but internet is working
Exception: For other request failures
"""
- # Use urljoin but ensure path is relative to avoid absolute path behavior
- relative_path = path.lstrip('/')
+
+ # Build full URL and merge headers
+ relative_path = path.lstrip("/")
url = urljoin(self.base_url, relative_path)
- self.check_auth(self.auth_token, self.comfy_api_key)
- # Combine default headers with any provided headers
+ self._check_auth(self.auth_token, self.comfy_api_key)
+
request_headers = self.get_headers()
if headers:
request_headers.update(headers)
-
- # Let requests handle the content type when files are present.
if files:
- del request_headers["Content-Type"]
+ request_headers.pop("Content-Type", None)
+ if params:
+ params = {k: v for k, v in params.items() if v is not None} # aiohttp fails to serialize None values
logging.debug(f"[DEBUG] Request Headers: {request_headers}")
logging.debug(f"[DEBUG] Files: {files}")
@@ -346,11 +361,9 @@ class ApiClient:
logging.debug(f"[DEBUG] Data: {data}")
if content_type == "application/x-www-form-urlencoded":
- payload_args = self._create_urlencoded_form_data_args(data, request_headers)
+ payload_args = self._create_urlencoded_form_data_args(data or {}, request_headers)
elif content_type == "multipart/form-data":
- payload_args = self._create_form_data_args(
- data, files, request_headers, multipart_parser
- )
+ payload_args = self._create_form_data_args(data, files, request_headers, multipart_parser)
else:
payload_args = self._create_json_payload_args(data, request_headers)
@@ -361,220 +374,67 @@ class ApiClient:
request_url=url,
request_headers=request_headers,
request_params=params,
- request_data=data if content_type == "application/json" else "[form-data or other]"
+ request_data=data if content_type == "application/json" else "[form-data or other]",
)
+ session = await self._get_session()
try:
- response = requests.request(
- method=method,
- url=url,
+ async with session.request(
+ method,
+ url,
params=params,
- timeout=self.timeout,
- verify=self.verify_ssl,
+ ssl=self.verify_ssl,
**payload_args,
- )
+ ) as resp:
+ if resp.status >= 400:
+ try:
+ error_data = await resp.json()
+ except (aiohttp.ContentTypeError, json.JSONDecodeError):
+ error_data = await resp.text()
- # Check if we should retry based on status code
- if (response.status_code in self.retry_status_codes and
- retry_count < self.max_retries):
+ return await self._handle_http_error(
+ ClientResponseError(resp.request_info, resp.history, status=resp.status, message=error_data),
+ operation_id,
+ method,
+ url,
+ params,
+ data,
+ files,
+ headers,
+ content_type,
+ multipart_parser,
+ retry_count=retry_count,
+ response_content=error_data,
+ )
- # Calculate delay with exponential backoff
- delay = self.retry_delay * (self.retry_backoff_factor ** retry_count)
-
- logging.warning(
- f"Request failed with status {response.status_code}. "
- f"Retrying in {delay:.2f}s ({retry_count + 1}/{self.max_retries})"
- )
-
- time.sleep(delay)
- return self.request(
- method=method,
- path=path,
- params=params,
- data=data,
- files=files,
- headers=headers,
- content_type=content_type,
- multipart_parser=multipart_parser,
- retry_count=retry_count + 1,
- )
-
- # Raise exception for error status codes
- response.raise_for_status()
-
- # Log successful response
- response_content_to_log = response.content
- try:
- # Attempt to parse JSON for prettier logging, fallback to raw content
- response_content_to_log = response.json()
- except json.JSONDecodeError:
- pass # Keep as bytes/str if not JSON
-
- request_logger.log_request_response(
- operation_id=operation_id,
- request_method=method, # Pass request details again for context in log
- request_url=url,
- response_status_code=response.status_code,
- response_headers=dict(response.headers),
- response_content=response_content_to_log
- )
-
- except requests.ConnectionError as e:
- error_message = f"ConnectionError: {str(e)}"
- request_logger.log_request_response(
- operation_id=operation_id,
- request_method=method,
- request_url=url,
- error_message=error_message
- )
- # Only perform connectivity check if we've exhausted all retries
- if retry_count >= self.max_retries:
- # Check connectivity to determine if it's a local or API issue
- connectivity = self._check_connectivity(self.base_url)
-
- if connectivity["is_local_issue"]:
- raise LocalNetworkError(
- "Unable to connect to the API server due to local network issues. "
- "Please check your internet connection and try again."
- ) from e
- elif connectivity["is_api_issue"]:
- raise ApiServerError(
- f"The API server at {self.base_url} is currently unreachable. "
- f"The service may be experiencing issues. Please try again later."
- ) from e
-
- # If we haven't exhausted retries yet, retry the request
- if retry_count < self.max_retries:
- delay = self.retry_delay * (self.retry_backoff_factor ** retry_count)
- logging.warning(
- f"Connection error: {str(e)}. "
- f"Retrying in {delay:.2f}s ({retry_count + 1}/{self.max_retries})"
- )
- time.sleep(delay)
- return self.request(
- method=method,
- path=path,
- params=params,
- data=data,
- files=files,
- headers=headers,
- content_type=content_type,
- multipart_parser=multipart_parser,
- retry_count=retry_count + 1,
- )
-
- # If we've exhausted retries and didn't identify the specific issue,
- # raise a generic exception
- final_error_message = (
- f"Unable to connect to the API server after {self.max_retries} attempts. "
- f"Please check your internet connection or try again later."
- )
- request_logger.log_request_response( # Log final failure
- operation_id=operation_id,
- request_method=method, request_url=url,
- error_message=final_error_message
- )
- raise Exception(final_error_message) from e
-
- except requests.Timeout as e:
- error_message = f"Timeout: {str(e)}"
- request_logger.log_request_response(
- operation_id=operation_id,
- request_method=method, request_url=url,
- error_message=error_message
- )
- # Retry timeouts if we haven't exhausted retries
- if retry_count < self.max_retries:
- delay = self.retry_delay * (self.retry_backoff_factor ** retry_count)
- logging.warning(
- f"Request timed out. "
- f"Retrying in {delay:.2f}s ({retry_count + 1}/{self.max_retries})"
- )
- time.sleep(delay)
- return self.request(
- method=method,
- path=path,
- params=params,
- data=data,
- files=files,
- headers=headers,
- content_type=content_type,
- multipart_parser=multipart_parser,
- retry_count=retry_count + 1,
- )
- final_error_message = (
- f"Request timed out after {self.timeout} seconds and {self.max_retries} retry attempts. "
- f"The server might be experiencing high load or the operation is taking longer than expected."
- )
- request_logger.log_request_response( # Log final failure
- operation_id=operation_id,
- request_method=method, request_url=url,
- error_message=final_error_message
- )
- raise Exception(final_error_message) from e
-
- except requests.HTTPError as e:
- status_code = e.response.status_code if hasattr(e, "response") else None
- original_error_message = f"HTTP Error: {str(e)}"
- error_content_for_log = None
- if hasattr(e, "response") and e.response is not None:
- error_content_for_log = e.response.content
+ # Success – parse JSON (safely) and log
try:
- error_content_for_log = e.response.json()
- except json.JSONDecodeError:
- pass
+ payload = await resp.json()
+ response_content_to_log = payload
+ except (aiohttp.ContentTypeError, json.JSONDecodeError):
+ payload = {}
+ response_content_to_log = await resp.text()
-
- # Try to extract detailed error message from JSON response for user display
- # but log the full error content.
- user_display_error_message = original_error_message
-
- try:
- if hasattr(e, "response") and e.response is not None and e.response.content:
- error_json = e.response.json()
- if "error" in error_json and "message" in error_json["error"]:
- user_display_error_message = f"API Error: {error_json['error']['message']}"
- if "type" in error_json["error"]:
- user_display_error_message += f" (Type: {error_json['error']['type']})"
- elif isinstance(error_json, dict): # Handle cases where error is just a JSON dict
- user_display_error_message = f"API Error: {json.dumps(error_json)}"
- else: # Non-dict JSON error
- user_display_error_message = f"API Error: {str(error_json)}"
- except json.JSONDecodeError:
- # If not JSON, use the raw content if it's not too long, or a summary
- if hasattr(e, "response") and e.response is not None and e.response.content:
- raw_content = e.response.content.decode(errors='ignore')
- if len(raw_content) < 200: # Arbitrary limit for display
- user_display_error_message = f"API Error (raw): {raw_content}"
- else:
- user_display_error_message = f"API Error (raw, status {status_code})"
-
- request_logger.log_request_response(
- operation_id=operation_id,
- request_method=method, request_url=url,
- response_status_code=status_code,
- response_headers=dict(e.response.headers) if hasattr(e, "response") and e.response is not None else None,
- response_content=error_content_for_log,
- error_message=original_error_message # Log the original exception string as error
- )
-
- logging.debug(f"[DEBUG] API Error: {user_display_error_message} (Status: {status_code})")
- if hasattr(e, "response") and e.response is not None and e.response.content:
- logging.debug(f"[DEBUG] Response content: {e.response.content}")
-
- # Retry if the status code is in our retry list and we haven't exhausted retries
- if (status_code in self.retry_status_codes and
- retry_count < self.max_retries):
-
- delay = self.retry_delay * (self.retry_backoff_factor ** retry_count)
- logging.warning(
- f"HTTP error {status_code}. "
- f"Retrying in {delay:.2f}s ({retry_count + 1}/{self.max_retries})"
+ request_logger.log_request_response(
+ operation_id=operation_id,
+ request_method=method,
+ request_url=url,
+ response_status_code=resp.status,
+ response_headers=dict(resp.headers),
+ response_content=response_content_to_log,
)
- time.sleep(delay)
- return self.request(
- method=method,
- path=path,
+ return payload
+
+ except (ClientError, asyncio.TimeoutError, socket.gaierror) as e:
+ # Treat as *connection* problem – optionally retry, else escalate
+ if retry_count < self.max_retries:
+ delay = self.retry_delay * (self.retry_backoff_factor ** retry_count)
+ logging.warning("Connection error. Retrying in %.2fs (%s/%s): %s", delay, retry_count + 1,
+ self.max_retries, str(e))
+ await asyncio.sleep(delay)
+ return await self.request(
+ method,
+ path,
params=params,
data=data,
files=files,
@@ -583,40 +443,34 @@ class ApiClient:
multipart_parser=multipart_parser,
retry_count=retry_count + 1,
)
+ # One final connectivity check for diagnostics
+ connectivity = await self._check_connectivity(self.base_url)
+ if connectivity["is_local_issue"]:
+ raise LocalNetworkError(
+ "Unable to connect to the API server due to local network issues. "
+ "Please check your internet connection and try again."
+ ) from e
+ raise ApiServerError(
+ f"The API server at {self.base_url} is currently unreachable. "
+ f"The service may be experiencing issues. Please try again later."
+ ) from e
- # Specific error messages for common status codes for user display
- if status_code == 401:
- user_display_error_message = "Unauthorized: Please login first to use this node."
- elif status_code == 402:
- user_display_error_message = "Payment Required: Please add credits to your account to use this node."
- elif status_code == 409:
- user_display_error_message = "There is a problem with your account. Please contact support@comfy.org."
- elif status_code == 429:
- user_display_error_message = "Rate Limit Exceeded: Please try again later."
- # else, user_display_error_message remains as parsed from response or original HTTPError string
-
- raise Exception(user_display_error_message) # Raise with the user-friendly message
-
- # Parse and return JSON response
- if response.content:
- return response.json()
- return {}
-
- def check_auth(self, auth_token, comfy_api_key):
+ @staticmethod
+ def _check_auth(auth_token, comfy_api_key):
"""Verify that an auth token is present or comfy_api_key is present"""
if auth_token is None and comfy_api_key is None:
raise Exception("Unauthorized: Please login first to use this node.")
return auth_token or comfy_api_key
@staticmethod
- def upload_file(
+ async def upload_file(
upload_url: str,
file: io.BytesIO | str,
content_type: str | None = None,
max_retries: int = 3,
retry_delay: float = 1.0,
retry_backoff_factor: float = 2.0,
- ):
+ ) -> aiohttp.ClientResponse:
"""Upload a file to the API with retry logic.
Args:
@@ -627,112 +481,167 @@ class ApiClient:
retry_delay: Initial delay between retries in seconds
retry_backoff_factor: Multiplier for the delay after each retry
"""
- headers = {}
+ headers: Dict[str, str] = {}
+ skip_auto_headers: set[str] = set()
if content_type:
headers["Content-Type"] = content_type
+ else:
+ # tell aiohttp not to add Content-Type that will break the request signature and result in a 403 status.
+ skip_auto_headers.add("Content-Type")
- # Prepare the file data
+ # Extract file bytes
if isinstance(file, io.BytesIO):
- file.seek(0) # Ensure we're at the start of the file
+ file.seek(0)
data = file.read()
elif isinstance(file, str):
with open(file, "rb") as f:
data = f.read()
else:
- raise ValueError("File must be either a BytesIO object or a file path string")
+ raise ValueError("File must be BytesIO or str path")
- # Try the upload with retries
- last_exception = None
- operation_id = f"upload_{upload_url.split('/')[-1]}_{uuid.uuid4().hex[:8]}" # Simplified ID for uploads
-
- # Log initial attempt (without full file data for brevity)
+ operation_id = f"upload_{upload_url.split('/')[-1]}_{uuid.uuid4().hex[:8]}"
request_logger.log_request_response(
operation_id=operation_id,
request_method="PUT",
request_url=upload_url,
request_headers=headers,
- request_data=f"[File data of type {content_type or 'unknown'}, size {len(data)} bytes]"
+ request_data=f"[File data {len(data)} bytes]",
)
- for retry_attempt in range(max_retries + 1):
+ delay = retry_delay
+ for attempt in range(max_retries + 1):
try:
- response = requests.put(upload_url, data=data, headers=headers)
- response.raise_for_status()
+ timeout = aiohttp.ClientTimeout(total=None) # honour server side timeouts
+ async with aiohttp.ClientSession(timeout=timeout) as session:
+ async with session.put(
+ upload_url, data=data, headers=headers, skip_auto_headers=skip_auto_headers,
+ ) as resp:
+ resp.raise_for_status()
+ request_logger.log_request_response(
+ operation_id=operation_id,
+ request_method="PUT",
+ request_url=upload_url,
+ response_status_code=resp.status,
+ response_headers=dict(resp.headers),
+ response_content="File uploaded successfully.",
+ )
+ return resp
+ except (ClientError, asyncio.TimeoutError) as e:
request_logger.log_request_response(
operation_id=operation_id,
- request_method="PUT", request_url=upload_url, # For context
- response_status_code=response.status_code,
- response_headers=dict(response.headers),
- response_content="File uploaded successfully." # Or response.text if available
+ request_method="PUT",
+ request_url=upload_url,
+ response_status_code=e.status if hasattr(e, "status") else None,
+ response_headers=dict(e.headers) if getattr(e, "headers") else None,
+ response_content=None,
+ error_message=f"{type(e).__name__}: {str(e)}",
)
- return response
-
- except (requests.ConnectionError, requests.Timeout, requests.HTTPError) as e:
- last_exception = e
- error_message_for_log = f"{type(e).__name__}: {str(e)}"
- response_content_for_log = None
- status_code_for_log = None
- headers_for_log = None
-
- if hasattr(e, 'response') and e.response is not None:
- status_code_for_log = e.response.status_code
- headers_for_log = dict(e.response.headers)
- try:
- response_content_for_log = e.response.json()
- except json.JSONDecodeError:
- response_content_for_log = e.response.content
-
-
- request_logger.log_request_response(
- operation_id=operation_id,
- request_method="PUT", request_url=upload_url,
- response_status_code=status_code_for_log,
- response_headers=headers_for_log,
- response_content=response_content_for_log,
- error_message=error_message_for_log
- )
-
- if retry_attempt < max_retries:
- delay = retry_delay * (retry_backoff_factor ** retry_attempt)
+ if attempt < max_retries:
logging.warning(
- f"File upload failed: {str(e)}. "
- f"Retrying in {delay:.2f}s ({retry_attempt + 1}/{max_retries})"
+ "Upload failed (%s/%s). Retrying in %.2fs. %s", attempt + 1, max_retries, delay, str(e)
)
- time.sleep(delay)
+ await asyncio.sleep(delay)
+ delay *= retry_backoff_factor
else:
- break # Max retries reached
+ raise NetworkError(f"Failed to upload file after {max_retries + 1} attempts: {e}") from e
- # If we've exhausted all retries, determine the final error type and raise
- final_error_message = f"Failed to upload file after {max_retries + 1} attempts. Error: {str(last_exception)}"
- try:
- # Check basic internet connectivity
- check_response = requests.get("https://www.google.com", timeout=5.0, verify=True) # Assuming verify=True is desired
- if check_response.status_code >= 500: # Google itself has an issue (rare)
- final_error_message = (f"Failed to upload file. Internet connectivity check to Google failed "
- f"(status {check_response.status_code}). Original error: {str(last_exception)}")
- # Not raising LocalNetworkError here as Google itself might be down.
- # If Google is reachable, the issue is likely with the upload server or a more specific local problem
- # not caught by a simple Google ping (e.g., DNS for the specific upload URL, firewall).
- # The original last_exception is probably most relevant.
+ async def _handle_http_error(
+ self,
+ exc: ClientResponseError,
+ operation_id: str,
+ *req_meta,
+ retry_count: int,
+ response_content: dict | str = "",
+ ) -> Dict[str, Any]:
+ status_code = exc.status
+ if status_code == 401:
+ user_friendly = "Unauthorized: Please login first to use this node."
+ elif status_code == 402:
+ user_friendly = "Payment Required: Please add credits to your account to use this node."
+ elif status_code == 409:
+ user_friendly = "There is a problem with your account. Please contact support@comfy.org."
+ elif status_code == 429:
+ user_friendly = "Rate Limit Exceeded: Please try again later."
+ else:
+ if isinstance(response_content, dict):
+ if "error" in response_content and "message" in response_content["error"]:
+ user_friendly = f"API Error: {response_content['error']['message']}"
+ if "type" in response_content["error"]:
+ user_friendly += f" (Type: {response_content['error']['type']})"
+ else: # Handle cases where error is just a JSON dict with unknown format
+ user_friendly = f"API Error: {json.dumps(response_content)}"
+ else:
+ if len(response_content) < 200: # Arbitrary limit for display
+ user_friendly = f"API Error (raw): {response_content}"
+ else:
+ user_friendly = f"API Error (raw, status {response_content})"
- except (requests.RequestException, socket.error) as conn_check_exc:
- # Could not reach Google, likely a local network issue
- final_error_message = (f"Failed to upload file due to network connectivity issues "
- f"(cannot reach Google: {str(conn_check_exc)}). "
- f"Original upload error: {str(last_exception)}")
- request_logger.log_request_response( # Log final failure reason
- operation_id=operation_id,
- request_method="PUT", request_url=upload_url,
- error_message=final_error_message
- )
- raise LocalNetworkError(final_error_message) from last_exception
-
- request_logger.log_request_response( # Log final failure reason if not LocalNetworkError
+ request_logger.log_request_response(
operation_id=operation_id,
- request_method="PUT", request_url=upload_url,
- error_message=final_error_message
+ request_method=req_meta[0],
+ request_url=req_meta[1],
+ response_status_code=exc.status,
+ response_headers=dict(req_meta[5]) if req_meta[5] else None,
+ response_content=response_content,
+ error_message=f"HTTP Error {exc.status}",
)
- raise Exception(final_error_message) from last_exception
+
+ logging.debug(f"[DEBUG] API Error: {user_friendly} (Status: {status_code})")
+ if response_content:
+ logging.debug(f"[DEBUG] Response content: {response_content}")
+
+ # Retry if eligible
+ if status_code in self.retry_status_codes and retry_count < self.max_retries:
+ delay = self.retry_delay * (self.retry_backoff_factor ** retry_count)
+ logging.warning(
+ "HTTP error %s. Retrying in %.2fs (%s/%s)",
+ status_code,
+ delay,
+ retry_count + 1,
+ self.max_retries,
+ )
+ await asyncio.sleep(delay)
+ return await self.request(
+ req_meta[0], # method
+ req_meta[1].replace(self.base_url, ""), # path
+ params=req_meta[2],
+ data=req_meta[3],
+ files=req_meta[4],
+ headers=req_meta[5],
+ content_type=req_meta[6],
+ multipart_parser=req_meta[7],
+ retry_count=retry_count + 1,
+ )
+
+ raise Exception(user_friendly) from exc
+
+ @staticmethod
+ def _unpack_tuple(t):
+ """Helper to normalise (filename, file, content_type) tuples."""
+ if len(t) == 3:
+ return t
+ elif len(t) == 2:
+ return t[0], t[1], "application/octet-stream"
+ else:
+ raise ValueError("files tuple must be (filename, file[, content_type])")
+
+ async def _get_session(self) -> aiohttp.ClientSession:
+ if self._session is None or self._session.closed:
+ timeout = aiohttp.ClientTimeout(total=self.timeout)
+ self._session = aiohttp.ClientSession(timeout=timeout)
+ self._owns_session = True
+ return self._session
+
+ async def close(self) -> None:
+ if self._owns_session and self._session and not self._session.closed:
+ await self._session.close()
+
+ async def __aenter__(self) -> "ApiClient":
+ """Allow usage as async‑context‑manager – ensures clean teardown"""
+ return self
+
+ async def __aexit__(self, exc_type, exc, tb):
+ await self.close()
class ApiEndpoint(Generic[T, R]):
@@ -763,31 +672,28 @@ class ApiEndpoint(Generic[T, R]):
class SynchronousOperation(Generic[T, R]):
- """
- Represents a single synchronous API operation.
- """
+ """Represents a single synchronous API operation."""
def __init__(
self,
endpoint: ApiEndpoint[T, R],
request: T,
- files: Optional[Dict[str, Any]] = None,
+ files: Optional[Dict[str, Any] | list[tuple[str, Any]]] = None,
api_base: str | None = None,
auth_token: Optional[str] = None,
comfy_api_key: Optional[str] = None,
- auth_kwargs: Optional[Dict[str,str]] = None,
+ auth_kwargs: Optional[Dict[str, str]] = None,
timeout: float = 604800.0,
verify_ssl: bool = True,
content_type: str = "application/json",
- multipart_parser: Callable = None,
+ multipart_parser: Callable | None = None,
max_retries: int = 3,
retry_delay: float = 1.0,
retry_backoff_factor: float = 2.0,
- ):
+ ) -> None:
self.endpoint = endpoint
self.request = request
- self.response = None
- self.error = None
+ self.files = files
self.api_base: str = api_base or args.comfy_api_base
self.auth_token = auth_token
self.comfy_api_key = comfy_api_key
@@ -796,91 +702,64 @@ class SynchronousOperation(Generic[T, R]):
self.comfy_api_key = auth_kwargs.get("comfy_api_key", self.comfy_api_key)
self.timeout = timeout
self.verify_ssl = verify_ssl
- self.files = files
self.content_type = content_type
self.multipart_parser = multipart_parser
self.max_retries = max_retries
self.retry_delay = retry_delay
self.retry_backoff_factor = retry_backoff_factor
- def execute(self, client: Optional[ApiClient] = None) -> R:
- """Execute the API operation using the provided client or create one with retry support"""
- try:
- # Create client if not provided
- if client is None:
- client = ApiClient(
- base_url=self.api_base,
- auth_token=self.auth_token,
- comfy_api_key=self.comfy_api_key,
- timeout=self.timeout,
- verify_ssl=self.verify_ssl,
- max_retries=self.max_retries,
- retry_delay=self.retry_delay,
- retry_backoff_factor=self.retry_backoff_factor,
- )
-
- # Convert request model to dict, but use None for EmptyRequest
- request_dict = (
- None
- if isinstance(self.request, EmptyRequest)
- else self.request.model_dump(exclude_none=True)
+ async def execute(self, client: Optional[ApiClient] = None) -> R:
+ owns_client = client is None
+ if owns_client:
+ client = ApiClient(
+ base_url=self.api_base,
+ auth_token=self.auth_token,
+ comfy_api_key=self.comfy_api_key,
+ timeout=self.timeout,
+ verify_ssl=self.verify_ssl,
+ max_retries=self.max_retries,
+ retry_delay=self.retry_delay,
+ retry_backoff_factor=self.retry_backoff_factor,
)
- if request_dict:
- for key, value in request_dict.items():
- if isinstance(value, Enum):
- request_dict[key] = value.value
- # Debug log for request
+ try:
+ request_dict: Optional[Dict[str, Any]]
+ if isinstance(self.request, EmptyRequest):
+ request_dict = None
+ else:
+ request_dict = self.request.model_dump(exclude_none=True)
+ for k, v in list(request_dict.items()):
+ if isinstance(v, Enum):
+ request_dict[k] = v.value
+
logging.debug(
f"[DEBUG] API Request: {self.endpoint.method.value} {self.endpoint.path}"
)
logging.debug(f"[DEBUG] Request Data: {json.dumps(request_dict, indent=2)}")
logging.debug(f"[DEBUG] Query Params: {self.endpoint.query_params}")
- # Make the request with built-in retry
- resp = client.request(
- method=self.endpoint.method.value,
- path=self.endpoint.path,
- data=request_dict,
+ response_json = await client.request(
+ self.endpoint.method.value,
+ self.endpoint.path,
params=self.endpoint.query_params,
+ data=request_dict,
files=self.files,
content_type=self.content_type,
- multipart_parser=self.multipart_parser
+ multipart_parser=self.multipart_parser,
)
- # Debug log for response
logging.debug("=" * 50)
logging.debug("[DEBUG] RESPONSE DETAILS:")
logging.debug("[DEBUG] Status Code: 200 (Success)")
- logging.debug(f"[DEBUG] Response Body: {json.dumps(resp, indent=2)}")
+ logging.debug(f"[DEBUG] Response Body: {json.dumps(response_json, indent=2)}")
logging.debug("=" * 50)
- # Parse and return the response
- return self._parse_response(resp)
-
- except LocalNetworkError as e:
- # Propagate specific network error types
- logging.error(f"[ERROR] Local network error: {str(e)}")
- raise
-
- except ApiServerError as e:
- # Propagate API server errors
- logging.error(f"[ERROR] API server error: {str(e)}")
- raise
-
- except Exception as e:
- logging.error(f"[ERROR] API Exception: {str(e)}")
- raise Exception(str(e))
-
- def _parse_response(self, resp):
- """Parse response data - can be overridden by subclasses"""
- # The response is already the complete object, don't extract just the "data" field
- # as that would lose the outer structure (created timestamp, etc.)
-
- # Parse response using the provided model
- self.response = self.endpoint.response_model.model_validate(resp)
- logging.debug(f"[DEBUG] Parsed Response: {self.response}")
- return self.response
+ parsed_response = self.endpoint.response_model.model_validate(response_json)
+ logging.debug(f"[DEBUG] Parsed Response: {parsed_response}")
+ return parsed_response
+ finally:
+ if owns_client:
+ await client.close()
class TaskStatus(str, Enum):
@@ -892,23 +771,21 @@ class TaskStatus(str, Enum):
class PollingOperation(Generic[T, R]):
- """
- Represents an asynchronous API operation that requires polling for completion.
- """
+ """Represents an asynchronous API operation that requires polling for completion."""
def __init__(
self,
poll_endpoint: ApiEndpoint[EmptyRequest, R],
- completed_statuses: list,
- failed_statuses: list,
+ completed_statuses: list[str],
+ failed_statuses: list[str],
status_extractor: Callable[[R], str],
- progress_extractor: Callable[[R], float] = None,
- result_url_extractor: Callable[[R], str] = None,
+ progress_extractor: Callable[[R], float] | None = None,
+ result_url_extractor: Callable[[R], str] | None = None,
request: Optional[T] = None,
api_base: str | None = None,
auth_token: Optional[str] = None,
comfy_api_key: Optional[str] = None,
- auth_kwargs: Optional[Dict[str,str]] = None,
+ auth_kwargs: Optional[Dict[str, str]] = None,
poll_interval: float = 5.0,
max_poll_attempts: int = 120, # Default max polling attempts (10 minutes with 5s interval)
max_retries: int = 3, # Max retries per individual API call
@@ -916,7 +793,7 @@ class PollingOperation(Generic[T, R]):
retry_backoff_factor: float = 2.0,
estimated_duration: Optional[float] = None,
node_id: Optional[str] = None,
- ):
+ ) -> None:
self.poll_endpoint = poll_endpoint
self.request = request
self.api_base: str = api_base or args.comfy_api_base
@@ -931,100 +808,73 @@ class PollingOperation(Generic[T, R]):
self.retry_delay = retry_delay
self.retry_backoff_factor = retry_backoff_factor
self.estimated_duration = estimated_duration
-
- # Polling configuration
- self.status_extractor = status_extractor or (
- lambda x: getattr(x, "status", None)
- )
+ self.status_extractor = status_extractor or (lambda x: getattr(x, "status", None))
self.progress_extractor = progress_extractor
self.result_url_extractor = result_url_extractor
self.node_id = node_id
self.completed_statuses = completed_statuses
self.failed_statuses = failed_statuses
+ self.final_response: Optional[R] = None
- # For storing response data
- self.final_response = None
- self.error = None
-
- def execute(self, client: Optional[ApiClient] = None) -> R:
- """Execute the polling operation using the provided client. If failed, raise an exception."""
+ async def execute(self, client: Optional[ApiClient] = None) -> R:
+ owns_client = client is None
+ if owns_client:
+ client = ApiClient(
+ base_url=self.api_base,
+ auth_token=self.auth_token,
+ comfy_api_key=self.comfy_api_key,
+ max_retries=self.max_retries,
+ retry_delay=self.retry_delay,
+ retry_backoff_factor=self.retry_backoff_factor,
+ )
try:
- if client is None:
- client = ApiClient(
- base_url=self.api_base,
- auth_token=self.auth_token,
- comfy_api_key=self.comfy_api_key,
- max_retries=self.max_retries,
- retry_delay=self.retry_delay,
- retry_backoff_factor=self.retry_backoff_factor,
- )
- return self._poll_until_complete(client)
- except LocalNetworkError as e:
- # Provide clear message for local network issues
- raise Exception(
- f"Polling failed due to local network issues. Please check your internet connection. "
- f"Details: {str(e)}"
- ) from e
- except ApiServerError as e:
- # Provide clear message for API server issues
- raise Exception(
- f"Polling failed due to API server issues. The service may be experiencing problems. "
- f"Please try again later. Details: {str(e)}"
- ) from e
- except Exception as e:
- raise Exception(f"Error during polling: {str(e)}")
+ return await self._poll_until_complete(client)
+ finally:
+ if owns_client:
+ await client.close()
def _display_text_on_node(self, text: str):
- """Sends text to the client which will be displayed on the node in the UI"""
if not self.node_id:
return
-
PromptServer.instance.send_progress_text(text, self.node_id)
- def _display_time_progress_on_node(self, time_completed: int):
+ def _display_time_progress_on_node(self, time_completed: int | float):
if not self.node_id:
return
-
if self.estimated_duration is not None:
- estimated_time_remaining = max(
- 0, int(self.estimated_duration) - int(time_completed)
- )
- message = f"Task in progress: {time_completed:.0f}s (~{estimated_time_remaining:.0f}s remaining)"
+ remaining = max(0, int(self.estimated_duration) - time_completed)
+ message = f"Task in progress: {time_completed}s (~{remaining}s remaining)"
else:
- message = f"Task in progress: {time_completed:.0f}s"
+ message = f"Task in progress: {time_completed}s"
self._display_text_on_node(message)
def _check_task_status(self, response: R) -> TaskStatus:
- """Check task status using the status extractor function"""
try:
status = self.status_extractor(response)
if status in self.completed_statuses:
return TaskStatus.COMPLETED
- elif status in self.failed_statuses:
+ if status in self.failed_statuses:
return TaskStatus.FAILED
return TaskStatus.PENDING
except Exception as e:
- logging.error(f"Error extracting status: {e}")
+ logging.error("Error extracting status: %s", e)
return TaskStatus.PENDING
- def _poll_until_complete(self, client: ApiClient) -> R:
+ async def _poll_until_complete(self, client: ApiClient) -> R:
"""Poll until the task is complete"""
- poll_count = 0
consecutive_errors = 0
max_consecutive_errors = min(5, self.max_retries * 2) # Limit consecutive errors
if self.progress_extractor:
progress = utils.ProgressBar(PROGRESS_BAR_MAX)
- while poll_count < self.max_poll_attempts:
+ status = TaskStatus.PENDING
+ for poll_count in range(1, self.max_poll_attempts + 1):
try:
- poll_count += 1
logging.debug(f"[DEBUG] Polling attempt #{poll_count}")
request_dict = (
- self.request.model_dump(exclude_none=True)
- if self.request is not None
- else None
+ None if self.request is None else self.request.model_dump(exclude_none=True)
)
if poll_count == 1:
@@ -1036,18 +886,14 @@ class PollingOperation(Generic[T, R]):
)
# Query task status
- resp = client.request(
- method=self.poll_endpoint.method.value,
- path=self.poll_endpoint.path,
+ resp = await client.request(
+ self.poll_endpoint.method.value,
+ self.poll_endpoint.path,
params=self.poll_endpoint.query_params,
data=request_dict,
)
-
- # Successfully got a response, reset consecutive error count
- consecutive_errors = 0
-
- # Parse response
- response_obj = self.poll_endpoint.response_model.model_validate(resp)
+ consecutive_errors = 0 # reset on success
+ response_obj: R = self.poll_endpoint.response_model.model_validate(resp)
# Check if task is complete
status = self._check_task_status(response_obj)
@@ -1065,45 +911,30 @@ class PollingOperation(Generic[T, R]):
result_url = self.result_url_extractor(response_obj)
if result_url:
message = f"Result URL: {result_url}"
- else:
- message = "Task completed successfully!"
logging.debug(f"[DEBUG] {message}")
self._display_text_on_node(message)
self.final_response = response_obj
if self.progress_extractor:
progress.update(100)
return self.final_response
- elif status == TaskStatus.FAILED:
+ if status == TaskStatus.FAILED:
message = f"Task failed: {json.dumps(resp)}"
logging.error(f"[DEBUG] {message}")
raise Exception(message)
- else:
- logging.debug("[DEBUG] Task still pending, continuing to poll...")
-
- # Wait before polling again
- logging.debug(
- f"[DEBUG] Waiting {self.poll_interval} seconds before next poll"
- )
+ logging.debug("[DEBUG] Task still pending, continuing to poll...")
+ # Task pending – wait
for i in range(int(self.poll_interval)):
- time_completed = (poll_count * self.poll_interval) + i
- self._display_time_progress_on_node(time_completed)
- time.sleep(1)
+ self._display_time_progress_on_node((poll_count - 1) * self.poll_interval + i)
+ await asyncio.sleep(1)
- except (LocalNetworkError, ApiServerError) as e:
- # For network-related errors, increment error count and potentially abort
+ except (LocalNetworkError, ApiServerError, NetworkError) as e:
consecutive_errors += 1
if consecutive_errors >= max_consecutive_errors:
raise Exception(
- f"Polling aborted after {consecutive_errors} consecutive network errors: {str(e)}"
+ f"Polling aborted after {consecutive_errors} network errors: {str(e)}"
) from e
-
- # Log the error but continue polling
- logging.warning(
- f"Network error during polling (attempt {poll_count}/{self.max_poll_attempts}): {str(e)}. "
- f"Will retry in {self.poll_interval} seconds."
- )
- time.sleep(self.poll_interval)
-
+ logging.warning("Network error (%s/%s): %s", consecutive_errors, max_consecutive_errors, str(e))
+ await asyncio.sleep(self.poll_interval)
except Exception as e:
# For other errors, increment count and potentially abort
consecutive_errors += 1
@@ -1117,10 +948,10 @@ class PollingOperation(Generic[T, R]):
f"Error during polling (attempt {poll_count}/{self.max_poll_attempts}): {str(e)}. "
f"Will retry in {self.poll_interval} seconds."
)
- time.sleep(self.poll_interval)
+ await asyncio.sleep(self.poll_interval)
# If we've exhausted all polling attempts
raise Exception(
- f"Polling timed out after {poll_count} attempts ({poll_count * self.poll_interval} seconds). "
- f"The operation may still be running on the server but is taking longer than expected."
+ f"Polling timed out after {self.max_poll_attempts} attempts (" f"{self.max_poll_attempts * self.poll_interval} seconds). "
+ "The operation may still be running on the server but is taking longer than expected."
)
diff --git a/comfy_api_nodes/apis/tripo_api.py b/comfy_api_nodes/apis/tripo_api.py
index 626e8d277..9f43d4d09 100644
--- a/comfy_api_nodes/apis/tripo_api.py
+++ b/comfy_api_nodes/apis/tripo_api.py
@@ -127,7 +127,7 @@ class TripoTextToModelRequest(BaseModel):
type: TripoTaskType = Field(TripoTaskType.TEXT_TO_MODEL, description='Type of task')
prompt: str = Field(..., description='The text prompt describing the model to generate', max_length=1024)
negative_prompt: Optional[str] = Field(None, description='The negative text prompt', max_length=1024)
- model_version: Optional[TripoModelVersion] = TripoModelVersion.V2_5
+ model_version: Optional[TripoModelVersion] = TripoModelVersion.v2_5_20250123
face_limit: Optional[int] = Field(None, description='The number of faces to limit the generation to')
texture: Optional[bool] = Field(True, description='Whether to apply texture to the generated model')
pbr: Optional[bool] = Field(True, description='Whether to apply PBR to the generated model')
diff --git a/comfy_api_nodes/nodes_bfl.py b/comfy_api_nodes/nodes_bfl.py
index d93fbd778..c09be8d5b 100644
--- a/comfy_api_nodes/nodes_bfl.py
+++ b/comfy_api_nodes/nodes_bfl.py
@@ -1,3 +1,4 @@
+import asyncio
import io
from inspect import cleandoc
from typing import Union, Optional
@@ -28,7 +29,7 @@ from comfy_api_nodes.apinode_utils import (
import numpy as np
from PIL import Image
-import requests
+import aiohttp
import torch
import base64
import time
@@ -44,18 +45,18 @@ def convert_mask_to_image(mask: torch.Tensor):
return mask
-def handle_bfl_synchronous_operation(
+async def handle_bfl_synchronous_operation(
operation: SynchronousOperation,
timeout_bfl_calls=360,
node_id: Union[str, None] = None,
):
- response_api: BFLFluxProGenerateResponse = operation.execute()
- return _poll_until_generated(
+ response_api: BFLFluxProGenerateResponse = await operation.execute()
+ return await _poll_until_generated(
response_api.polling_url, timeout=timeout_bfl_calls, node_id=node_id
)
-def _poll_until_generated(
+async def _poll_until_generated(
polling_url: str, timeout=360, node_id: Union[str, None] = None
):
# used bfl-comfy-nodes to verify code implementation:
@@ -66,55 +67,56 @@ def _poll_until_generated(
retry_404_seconds = 2
retry_202_seconds = 2
retry_pending_seconds = 1
- request = requests.Request(method=HttpMethod.GET, url=polling_url)
- # NOTE: should True loop be replaced with checking if workflow has been interrupted?
- while True:
- if node_id:
- time_elapsed = time.time() - start_time
- PromptServer.instance.send_progress_text(
- f"Generating ({time_elapsed:.0f}s)", node_id
- )
- response = requests.Session().send(request.prepare())
- if response.status_code == 200:
- result = response.json()
- if result["status"] == BFLStatus.ready:
- img_url = result["result"]["sample"]
- if node_id:
- PromptServer.instance.send_progress_text(
- f"Result URL: {img_url}", node_id
- )
- img_response = requests.get(img_url)
- return process_image_response(img_response)
- elif result["status"] in [
- BFLStatus.request_moderated,
- BFLStatus.content_moderated,
- ]:
- status = result["status"]
- raise Exception(
- f"BFL API did not return an image due to: {status}."
+ async with aiohttp.ClientSession() as session:
+ # NOTE: should True loop be replaced with checking if workflow has been interrupted?
+ while True:
+ if node_id:
+ time_elapsed = time.time() - start_time
+ PromptServer.instance.send_progress_text(
+ f"Generating ({time_elapsed:.0f}s)", node_id
)
- elif result["status"] == BFLStatus.error:
- raise Exception(f"BFL API encountered an error: {result}.")
- elif result["status"] == BFLStatus.pending:
- time.sleep(retry_pending_seconds)
- continue
- elif response.status_code == 404:
- if retries_404 < max_retries_404:
- retries_404 += 1
- time.sleep(retry_404_seconds)
- continue
- raise Exception(
- f"BFL API could not find task after {max_retries_404} tries."
- )
- elif response.status_code == 202:
- time.sleep(retry_202_seconds)
- elif time.time() - start_time > timeout:
- raise Exception(
- f"BFL API experienced a timeout; could not return request under {timeout} seconds."
- )
- else:
- raise Exception(f"BFL API encountered an error: {response.json()}")
+
+ async with session.get(polling_url) as response:
+ if response.status == 200:
+ result = await response.json()
+ if result["status"] == BFLStatus.ready:
+ img_url = result["result"]["sample"]
+ if node_id:
+ PromptServer.instance.send_progress_text(
+ f"Result URL: {img_url}", node_id
+ )
+ async with session.get(img_url) as img_resp:
+ return process_image_response(await img_resp.content.read())
+ elif result["status"] in [
+ BFLStatus.request_moderated,
+ BFLStatus.content_moderated,
+ ]:
+ status = result["status"]
+ raise Exception(
+ f"BFL API did not return an image due to: {status}."
+ )
+ elif result["status"] == BFLStatus.error:
+ raise Exception(f"BFL API encountered an error: {result}.")
+ elif result["status"] == BFLStatus.pending:
+ await asyncio.sleep(retry_pending_seconds)
+ continue
+ elif response.status == 404:
+ if retries_404 < max_retries_404:
+ retries_404 += 1
+ await asyncio.sleep(retry_404_seconds)
+ continue
+ raise Exception(
+ f"BFL API could not find task after {max_retries_404} tries."
+ )
+ elif response.status == 202:
+ await asyncio.sleep(retry_202_seconds)
+ elif time.time() - start_time > timeout:
+ raise Exception(
+ f"BFL API experienced a timeout; could not return request under {timeout} seconds."
+ )
+ else:
+ raise Exception(f"BFL API encountered an error: {response.json()}")
def convert_image_to_base64(image: torch.Tensor):
scaled_image = downscale_image_tensor(image, total_pixels=2048 * 2048)
@@ -222,7 +224,7 @@ class FluxProUltraImageNode(ComfyNodeABC):
API_NODE = True
CATEGORY = "api node/image/BFL"
- def api_call(
+ async def api_call(
self,
prompt: str,
aspect_ratio: str,
@@ -266,7 +268,7 @@ class FluxProUltraImageNode(ComfyNodeABC):
),
auth_kwargs=kwargs,
)
- output_image = handle_bfl_synchronous_operation(operation, node_id=unique_id)
+ output_image = await handle_bfl_synchronous_operation(operation, node_id=unique_id)
return (output_image,)
@@ -354,7 +356,7 @@ class FluxKontextProImageNode(ComfyNodeABC):
BFL_PATH = "/proxy/bfl/flux-kontext-pro/generate"
- def api_call(
+ async def api_call(
self,
prompt: str,
aspect_ratio: str,
@@ -397,7 +399,7 @@ class FluxKontextProImageNode(ComfyNodeABC):
),
auth_kwargs=kwargs,
)
- output_image = handle_bfl_synchronous_operation(operation, node_id=unique_id)
+ output_image = await handle_bfl_synchronous_operation(operation, node_id=unique_id)
return (output_image,)
@@ -489,7 +491,7 @@ class FluxProImageNode(ComfyNodeABC):
API_NODE = True
CATEGORY = "api node/image/BFL"
- def api_call(
+ async def api_call(
self,
prompt: str,
prompt_upsampling,
@@ -524,7 +526,7 @@ class FluxProImageNode(ComfyNodeABC):
),
auth_kwargs=kwargs,
)
- output_image = handle_bfl_synchronous_operation(operation, node_id=unique_id)
+ output_image = await handle_bfl_synchronous_operation(operation, node_id=unique_id)
return (output_image,)
@@ -632,7 +634,7 @@ class FluxProExpandNode(ComfyNodeABC):
API_NODE = True
CATEGORY = "api node/image/BFL"
- def api_call(
+ async def api_call(
self,
image: torch.Tensor,
prompt: str,
@@ -670,7 +672,7 @@ class FluxProExpandNode(ComfyNodeABC):
),
auth_kwargs=kwargs,
)
- output_image = handle_bfl_synchronous_operation(operation, node_id=unique_id)
+ output_image = await handle_bfl_synchronous_operation(operation, node_id=unique_id)
return (output_image,)
@@ -744,7 +746,7 @@ class FluxProFillNode(ComfyNodeABC):
API_NODE = True
CATEGORY = "api node/image/BFL"
- def api_call(
+ async def api_call(
self,
image: torch.Tensor,
mask: torch.Tensor,
@@ -780,7 +782,7 @@ class FluxProFillNode(ComfyNodeABC):
),
auth_kwargs=kwargs,
)
- output_image = handle_bfl_synchronous_operation(operation, node_id=unique_id)
+ output_image = await handle_bfl_synchronous_operation(operation, node_id=unique_id)
return (output_image,)
@@ -879,7 +881,7 @@ class FluxProCannyNode(ComfyNodeABC):
API_NODE = True
CATEGORY = "api node/image/BFL"
- def api_call(
+ async def api_call(
self,
control_image: torch.Tensor,
prompt: str,
@@ -929,7 +931,7 @@ class FluxProCannyNode(ComfyNodeABC):
),
auth_kwargs=kwargs,
)
- output_image = handle_bfl_synchronous_operation(operation, node_id=unique_id)
+ output_image = await handle_bfl_synchronous_operation(operation, node_id=unique_id)
return (output_image,)
@@ -1008,7 +1010,7 @@ class FluxProDepthNode(ComfyNodeABC):
API_NODE = True
CATEGORY = "api node/image/BFL"
- def api_call(
+ async def api_call(
self,
control_image: torch.Tensor,
prompt: str,
@@ -1045,7 +1047,7 @@ class FluxProDepthNode(ComfyNodeABC):
),
auth_kwargs=kwargs,
)
- output_image = handle_bfl_synchronous_operation(operation, node_id=unique_id)
+ output_image = await handle_bfl_synchronous_operation(operation, node_id=unique_id)
return (output_image,)
diff --git a/comfy_api_nodes/nodes_gemini.py b/comfy_api_nodes/nodes_gemini.py
index af33279d5..ba4167a50 100644
--- a/comfy_api_nodes/nodes_gemini.py
+++ b/comfy_api_nodes/nodes_gemini.py
@@ -46,6 +46,8 @@ class GeminiModel(str, Enum):
gemini_2_5_pro_preview_05_06 = "gemini-2.5-pro-preview-05-06"
gemini_2_5_flash_preview_04_17 = "gemini-2.5-flash-preview-04-17"
+ gemini_2_5_pro = "gemini-2.5-pro"
+ gemini_2_5_flash = "gemini-2.5-flash"
def get_gemini_endpoint(
@@ -97,7 +99,7 @@ class GeminiNode(ComfyNodeABC):
{
"tooltip": "The Gemini model to use for generating responses.",
"options": [model.value for model in GeminiModel],
- "default": GeminiModel.gemini_2_5_pro_preview_05_06.value,
+ "default": GeminiModel.gemini_2_5_pro.value,
},
),
"seed": (
@@ -303,7 +305,7 @@ class GeminiNode(ComfyNodeABC):
"""
return GeminiPart(text=text)
- def api_call(
+ async def api_call(
self,
prompt: str,
model: GeminiModel,
@@ -332,7 +334,7 @@ class GeminiNode(ComfyNodeABC):
parts.extend(files)
# Create response
- response = SynchronousOperation(
+ response = await SynchronousOperation(
endpoint=get_gemini_endpoint(model),
request=GeminiGenerateContentRequest(
contents=[
diff --git a/comfy_api_nodes/nodes_ideogram.py b/comfy_api_nodes/nodes_ideogram.py
index b8487355f..db24e6da4 100644
--- a/comfy_api_nodes/nodes_ideogram.py
+++ b/comfy_api_nodes/nodes_ideogram.py
@@ -212,7 +212,7 @@ V3_RESOLUTIONS= [
"1536x640"
]
-def download_and_process_images(image_urls):
+async def download_and_process_images(image_urls):
"""Helper function to download and process multiple images from URLs"""
# Initialize list to store image tensors
@@ -220,7 +220,7 @@ def download_and_process_images(image_urls):
for image_url in image_urls:
# Using functions from apinode_utils.py to handle downloading and processing
- image_bytesio = download_url_to_bytesio(image_url) # Download image content to BytesIO
+ image_bytesio = await download_url_to_bytesio(image_url) # Download image content to BytesIO
img_tensor = bytesio_to_image_tensor(image_bytesio, mode="RGB") # Convert to torch.Tensor with RGB mode
image_tensors.append(img_tensor)
@@ -328,7 +328,7 @@ class IdeogramV1(ComfyNodeABC):
DESCRIPTION = cleandoc(__doc__ or "")
API_NODE = True
- def api_call(
+ async def api_call(
self,
prompt,
turbo=False,
@@ -367,7 +367,7 @@ class IdeogramV1(ComfyNodeABC):
auth_kwargs=kwargs,
)
- response = operation.execute()
+ response = await operation.execute()
if not response.data or len(response.data) == 0:
raise Exception("No images were generated in the response")
@@ -378,7 +378,7 @@ class IdeogramV1(ComfyNodeABC):
raise Exception("No image URLs were generated in the response")
display_image_urls_on_node(image_urls, unique_id)
- return (download_and_process_images(image_urls),)
+ return (await download_and_process_images(image_urls),)
class IdeogramV2(ComfyNodeABC):
@@ -487,7 +487,7 @@ class IdeogramV2(ComfyNodeABC):
DESCRIPTION = cleandoc(__doc__ or "")
API_NODE = True
- def api_call(
+ async def api_call(
self,
prompt,
turbo=False,
@@ -543,7 +543,7 @@ class IdeogramV2(ComfyNodeABC):
auth_kwargs=kwargs,
)
- response = operation.execute()
+ response = await operation.execute()
if not response.data or len(response.data) == 0:
raise Exception("No images were generated in the response")
@@ -554,7 +554,7 @@ class IdeogramV2(ComfyNodeABC):
raise Exception("No image URLs were generated in the response")
display_image_urls_on_node(image_urls, unique_id)
- return (download_and_process_images(image_urls),)
+ return (await download_and_process_images(image_urls),)
class IdeogramV3(ComfyNodeABC):
"""
@@ -653,7 +653,7 @@ class IdeogramV3(ComfyNodeABC):
DESCRIPTION = cleandoc(__doc__ or "")
API_NODE = True
- def api_call(
+ async def api_call(
self,
prompt,
image=None,
@@ -774,7 +774,7 @@ class IdeogramV3(ComfyNodeABC):
)
# Execute the operation and process response
- response = operation.execute()
+ response = await operation.execute()
if not response.data or len(response.data) == 0:
raise Exception("No images were generated in the response")
@@ -785,7 +785,7 @@ class IdeogramV3(ComfyNodeABC):
raise Exception("No image URLs were generated in the response")
display_image_urls_on_node(image_urls, unique_id)
- return (download_and_process_images(image_urls),)
+ return (await download_and_process_images(image_urls),)
NODE_CLASS_MAPPINGS = {
diff --git a/comfy_api_nodes/nodes_kling.py b/comfy_api_nodes/nodes_kling.py
index 69e9e5cf0..9fa390985 100644
--- a/comfy_api_nodes/nodes_kling.py
+++ b/comfy_api_nodes/nodes_kling.py
@@ -109,7 +109,7 @@ class KlingApiError(Exception):
pass
-def poll_until_finished(
+async def poll_until_finished(
auth_kwargs: dict[str, str],
api_endpoint: ApiEndpoint[Any, R],
result_url_extractor: Optional[Callable[[R], str]] = None,
@@ -117,7 +117,7 @@ def poll_until_finished(
node_id: Optional[str] = None,
) -> R:
"""Polls the Kling API endpoint until the task reaches a terminal state, then returns the response."""
- return PollingOperation(
+ return await PollingOperation(
poll_endpoint=api_endpoint,
completed_statuses=[
KlingTaskStatus.succeed.value,
@@ -278,18 +278,18 @@ def get_images_urls_from_response(response) -> Optional[str]:
return None
-def video_result_to_node_output(
+async def video_result_to_node_output(
video: KlingVideoResult,
) -> tuple[VideoFromFile, str, str]:
"""Converts a KlingVideoResult to a tuple of (VideoFromFile, str, str) to be used as a ComfyUI node output."""
return (
- download_url_to_video_output(video.url),
+ await download_url_to_video_output(str(video.url)),
str(video.id),
str(video.duration),
)
-def image_result_to_node_output(
+async def image_result_to_node_output(
images: list[KlingImageResult],
) -> torch.Tensor:
"""
@@ -297,9 +297,9 @@ def image_result_to_node_output(
If multiple images are returned, they will be stacked along the batch dimension.
"""
if len(images) == 1:
- return download_url_to_image_tensor(images[0].url)
+ return await download_url_to_image_tensor(str(images[0].url))
else:
- return torch.cat([download_url_to_image_tensor(image.url) for image in images])
+ return torch.cat([await download_url_to_image_tensor(str(image.url)) for image in images])
class KlingNodeBase(ComfyNodeABC):
@@ -421,6 +421,8 @@ class KlingTextToVideoNode(KlingNodeBase):
"pro mode / 10s duration / kling-v2-master": ("pro", "10", "kling-v2-master"),
"standard mode / 5s duration / kling-v2-master": ("std", "5", "kling-v2-master"),
"standard mode / 10s duration / kling-v2-master": ("std", "10", "kling-v2-master"),
+ "pro mode / 5s duration / kling-v2-1-master": ("pro", "5", "kling-v2-1-master"),
+ "pro mode / 10s duration / kling-v2-1-master": ("pro", "10", "kling-v2-1-master"),
}
@classmethod
@@ -467,10 +469,10 @@ class KlingTextToVideoNode(KlingNodeBase):
RETURN_NAMES = ("VIDEO", "video_id", "duration")
DESCRIPTION = "Kling Text to Video Node"
- def get_response(
+ async def get_response(
self, task_id: str, auth_kwargs: dict[str, str], node_id: Optional[str] = None
) -> KlingText2VideoResponse:
- return poll_until_finished(
+ return await poll_until_finished(
auth_kwargs,
ApiEndpoint(
path=f"{PATH_TEXT_TO_VIDEO}/{task_id}",
@@ -483,7 +485,7 @@ class KlingTextToVideoNode(KlingNodeBase):
node_id=node_id,
)
- def api_call(
+ async def api_call(
self,
prompt: str,
negative_prompt: str,
@@ -519,17 +521,17 @@ class KlingTextToVideoNode(KlingNodeBase):
auth_kwargs=kwargs,
)
- task_creation_response = initial_operation.execute()
+ task_creation_response = await initial_operation.execute()
validate_task_creation_response(task_creation_response)
task_id = task_creation_response.data.task_id
- final_response = self.get_response(
+ final_response = await self.get_response(
task_id, auth_kwargs=kwargs, node_id=unique_id
)
validate_video_result_response(final_response)
video = get_video_from_response(final_response)
- return video_result_to_node_output(video)
+ return await video_result_to_node_output(video)
class KlingCameraControlT2VNode(KlingTextToVideoNode):
@@ -581,7 +583,7 @@ class KlingCameraControlT2VNode(KlingTextToVideoNode):
DESCRIPTION = "Transform text into cinematic videos with professional camera movements that simulate real-world cinematography. Control virtual camera actions including zoom, rotation, pan, tilt, and first-person view, while maintaining focus on your original text."
- def api_call(
+ async def api_call(
self,
prompt: str,
negative_prompt: str,
@@ -591,7 +593,7 @@ class KlingCameraControlT2VNode(KlingTextToVideoNode):
unique_id: Optional[str] = None,
**kwargs,
):
- return super().api_call(
+ return await super().api_call(
model_name=KlingVideoGenModelName.kling_v1,
cfg_scale=cfg_scale,
mode=KlingVideoGenMode.std,
@@ -670,10 +672,10 @@ class KlingImage2VideoNode(KlingNodeBase):
RETURN_NAMES = ("VIDEO", "video_id", "duration")
DESCRIPTION = "Kling Image to Video Node"
- def get_response(
+ async def get_response(
self, task_id: str, auth_kwargs: dict[str, str], node_id: Optional[str] = None
) -> KlingImage2VideoResponse:
- return poll_until_finished(
+ return await poll_until_finished(
auth_kwargs,
ApiEndpoint(
path=f"{PATH_IMAGE_TO_VIDEO}/{task_id}",
@@ -686,7 +688,7 @@ class KlingImage2VideoNode(KlingNodeBase):
node_id=node_id,
)
- def api_call(
+ async def api_call(
self,
start_frame: torch.Tensor,
prompt: str,
@@ -733,17 +735,17 @@ class KlingImage2VideoNode(KlingNodeBase):
auth_kwargs=kwargs,
)
- task_creation_response = initial_operation.execute()
+ task_creation_response = await initial_operation.execute()
validate_task_creation_response(task_creation_response)
task_id = task_creation_response.data.task_id
- final_response = self.get_response(
+ final_response = await self.get_response(
task_id, auth_kwargs=kwargs, node_id=unique_id
)
validate_video_result_response(final_response)
video = get_video_from_response(final_response)
- return video_result_to_node_output(video)
+ return await video_result_to_node_output(video)
class KlingCameraControlI2VNode(KlingImage2VideoNode):
@@ -798,7 +800,7 @@ class KlingCameraControlI2VNode(KlingImage2VideoNode):
DESCRIPTION = "Transform still images into cinematic videos with professional camera movements that simulate real-world cinematography. Control virtual camera actions including zoom, rotation, pan, tilt, and first-person view, while maintaining focus on your original image."
- def api_call(
+ async def api_call(
self,
start_frame: torch.Tensor,
prompt: str,
@@ -809,7 +811,7 @@ class KlingCameraControlI2VNode(KlingImage2VideoNode):
unique_id: Optional[str] = None,
**kwargs,
):
- return super().api_call(
+ return await super().api_call(
model_name=KlingVideoGenModelName.kling_v1_5,
start_frame=start_frame,
cfg_scale=cfg_scale,
@@ -897,7 +899,7 @@ class KlingStartEndFrameNode(KlingImage2VideoNode):
DESCRIPTION = "Generate a video sequence that transitions between your provided start and end images. The node creates all frames in between, producing a smooth transformation from the first frame to the last."
- def api_call(
+ async def api_call(
self,
start_frame: torch.Tensor,
end_frame: torch.Tensor,
@@ -912,7 +914,7 @@ class KlingStartEndFrameNode(KlingImage2VideoNode):
mode, duration, model_name = KlingStartEndFrameNode.get_mode_string_mapping()[
mode
]
- return super().api_call(
+ return await super().api_call(
prompt=prompt,
negative_prompt=negative_prompt,
model_name=model_name,
@@ -964,10 +966,10 @@ class KlingVideoExtendNode(KlingNodeBase):
RETURN_NAMES = ("VIDEO", "video_id", "duration")
DESCRIPTION = "Kling Video Extend Node. Extend videos made by other Kling nodes. The video_id is created by using other Kling Nodes."
- def get_response(
+ async def get_response(
self, task_id: str, auth_kwargs: dict[str, str], node_id: Optional[str] = None
) -> KlingVideoExtendResponse:
- return poll_until_finished(
+ return await poll_until_finished(
auth_kwargs,
ApiEndpoint(
path=f"{PATH_VIDEO_EXTEND}/{task_id}",
@@ -980,7 +982,7 @@ class KlingVideoExtendNode(KlingNodeBase):
node_id=node_id,
)
- def api_call(
+ async def api_call(
self,
prompt: str,
negative_prompt: str,
@@ -1006,17 +1008,17 @@ class KlingVideoExtendNode(KlingNodeBase):
auth_kwargs=kwargs,
)
- task_creation_response = initial_operation.execute()
+ task_creation_response = await initial_operation.execute()
validate_task_creation_response(task_creation_response)
task_id = task_creation_response.data.task_id
- final_response = self.get_response(
+ final_response = await self.get_response(
task_id, auth_kwargs=kwargs, node_id=unique_id
)
validate_video_result_response(final_response)
video = get_video_from_response(final_response)
- return video_result_to_node_output(video)
+ return await video_result_to_node_output(video)
class KlingVideoEffectsBase(KlingNodeBase):
@@ -1025,10 +1027,10 @@ class KlingVideoEffectsBase(KlingNodeBase):
RETURN_TYPES = ("VIDEO", "STRING", "STRING")
RETURN_NAMES = ("VIDEO", "video_id", "duration")
- def get_response(
+ async def get_response(
self, task_id: str, auth_kwargs: dict[str, str], node_id: Optional[str] = None
) -> KlingVideoEffectsResponse:
- return poll_until_finished(
+ return await poll_until_finished(
auth_kwargs,
ApiEndpoint(
path=f"{PATH_VIDEO_EFFECTS}/{task_id}",
@@ -1041,7 +1043,7 @@ class KlingVideoEffectsBase(KlingNodeBase):
node_id=node_id,
)
- def api_call(
+ async def api_call(
self,
dual_character: bool,
effect_scene: KlingDualCharacterEffectsScene | KlingSingleImageEffectsScene,
@@ -1084,17 +1086,17 @@ class KlingVideoEffectsBase(KlingNodeBase):
auth_kwargs=kwargs,
)
- task_creation_response = initial_operation.execute()
+ task_creation_response = await initial_operation.execute()
validate_task_creation_response(task_creation_response)
task_id = task_creation_response.data.task_id
- final_response = self.get_response(
+ final_response = await self.get_response(
task_id, auth_kwargs=kwargs, node_id=unique_id
)
validate_video_result_response(final_response)
video = get_video_from_response(final_response)
- return video_result_to_node_output(video)
+ return await video_result_to_node_output(video)
class KlingDualCharacterVideoEffectNode(KlingVideoEffectsBase):
@@ -1142,7 +1144,7 @@ class KlingDualCharacterVideoEffectNode(KlingVideoEffectsBase):
RETURN_TYPES = ("VIDEO", "STRING")
RETURN_NAMES = ("VIDEO", "duration")
- def api_call(
+ async def api_call(
self,
image_left: torch.Tensor,
image_right: torch.Tensor,
@@ -1153,7 +1155,7 @@ class KlingDualCharacterVideoEffectNode(KlingVideoEffectsBase):
unique_id: Optional[str] = None,
**kwargs,
):
- video, _, duration = super().api_call(
+ video, _, duration = await super().api_call(
dual_character=True,
effect_scene=effect_scene,
model_name=model_name,
@@ -1208,7 +1210,7 @@ class KlingSingleImageVideoEffectNode(KlingVideoEffectsBase):
DESCRIPTION = "Achieve different special effects when generating a video based on the effect_scene."
- def api_call(
+ async def api_call(
self,
image: torch.Tensor,
effect_scene: KlingSingleImageEffectsScene,
@@ -1217,7 +1219,7 @@ class KlingSingleImageVideoEffectNode(KlingVideoEffectsBase):
unique_id: Optional[str] = None,
**kwargs,
):
- return super().api_call(
+ return await super().api_call(
dual_character=False,
effect_scene=effect_scene,
model_name=model_name,
@@ -1253,11 +1255,11 @@ class KlingLipSyncBase(KlingNodeBase):
f"Text is too long. Maximum length is {MAX_PROMPT_LENGTH_LIP_SYNC} characters."
)
- def get_response(
+ async def get_response(
self, task_id: str, auth_kwargs: dict[str, str], node_id: Optional[str] = None
) -> KlingLipSyncResponse:
"""Polls the Kling API endpoint until the task reaches a terminal state."""
- return poll_until_finished(
+ return await poll_until_finished(
auth_kwargs,
ApiEndpoint(
path=f"{PATH_LIP_SYNC}/{task_id}",
@@ -1270,7 +1272,7 @@ class KlingLipSyncBase(KlingNodeBase):
node_id=node_id,
)
- def api_call(
+ async def api_call(
self,
video: VideoInput,
audio: Optional[AudioInput] = None,
@@ -1287,12 +1289,12 @@ class KlingLipSyncBase(KlingNodeBase):
self.validate_lip_sync_video(video)
# Upload video to Comfy API and get download URL
- video_url = upload_video_to_comfyapi(video, auth_kwargs=kwargs)
+ video_url = await upload_video_to_comfyapi(video, auth_kwargs=kwargs)
logging.info("Uploaded video to Comfy API. URL: %s", video_url)
# Upload the audio file to Comfy API and get download URL
if audio:
- audio_url = upload_audio_to_comfyapi(audio, auth_kwargs=kwargs)
+ audio_url = await upload_audio_to_comfyapi(audio, auth_kwargs=kwargs)
logging.info("Uploaded audio to Comfy API. URL: %s", audio_url)
else:
audio_url = None
@@ -1319,17 +1321,17 @@ class KlingLipSyncBase(KlingNodeBase):
auth_kwargs=kwargs,
)
- task_creation_response = initial_operation.execute()
+ task_creation_response = await initial_operation.execute()
validate_task_creation_response(task_creation_response)
task_id = task_creation_response.data.task_id
- final_response = self.get_response(
+ final_response = await self.get_response(
task_id, auth_kwargs=kwargs, node_id=unique_id
)
validate_video_result_response(final_response)
video = get_video_from_response(final_response)
- return video_result_to_node_output(video)
+ return await video_result_to_node_output(video)
class KlingLipSyncAudioToVideoNode(KlingLipSyncBase):
@@ -1357,7 +1359,7 @@ class KlingLipSyncAudioToVideoNode(KlingLipSyncBase):
DESCRIPTION = "Kling Lip Sync Audio to Video Node. Syncs mouth movements in a video file to the audio content of an audio file. When using, ensure that the audio contains clearly distinguishable vocals and that the video contains a distinct face. The audio file should not be larger than 5MB. The video file should not be larger than 100MB, should have height/width between 720px and 1920px, and should be between 2s and 10s in length."
- def api_call(
+ async def api_call(
self,
video: VideoInput,
audio: AudioInput,
@@ -1365,7 +1367,7 @@ class KlingLipSyncAudioToVideoNode(KlingLipSyncBase):
unique_id: Optional[str] = None,
**kwargs,
):
- return super().api_call(
+ return await super().api_call(
video=video,
audio=audio,
voice_language=voice_language,
@@ -1469,7 +1471,7 @@ class KlingLipSyncTextToVideoNode(KlingLipSyncBase):
DESCRIPTION = "Kling Lip Sync Text to Video Node. Syncs mouth movements in a video file to a text prompt. The video file should not be larger than 100MB, should have height/width between 720px and 1920px, and should be between 2s and 10s in length."
- def api_call(
+ async def api_call(
self,
video: VideoInput,
text: str,
@@ -1479,7 +1481,7 @@ class KlingLipSyncTextToVideoNode(KlingLipSyncBase):
**kwargs,
):
voice_id, voice_language = KlingLipSyncTextToVideoNode.get_voice_config()[voice]
- return super().api_call(
+ return await super().api_call(
video=video,
text=text,
voice_language=voice_language,
@@ -1533,10 +1535,10 @@ class KlingVirtualTryOnNode(KlingImageGenerationBase):
DESCRIPTION = "Kling Virtual Try On Node. Input a human image and a cloth image to try on the cloth on the human. You can merge multiple clothing item pictures into one image with a white background."
- def get_response(
+ async def get_response(
self, task_id: str, auth_kwargs: dict[str, str], node_id: Optional[str] = None
) -> KlingVirtualTryOnResponse:
- return poll_until_finished(
+ return await poll_until_finished(
auth_kwargs,
ApiEndpoint(
path=f"{PATH_VIRTUAL_TRY_ON}/{task_id}",
@@ -1549,7 +1551,7 @@ class KlingVirtualTryOnNode(KlingImageGenerationBase):
node_id=node_id,
)
- def api_call(
+ async def api_call(
self,
human_image: torch.Tensor,
cloth_image: torch.Tensor,
@@ -1572,17 +1574,17 @@ class KlingVirtualTryOnNode(KlingImageGenerationBase):
auth_kwargs=kwargs,
)
- task_creation_response = initial_operation.execute()
+ task_creation_response = await initial_operation.execute()
validate_task_creation_response(task_creation_response)
task_id = task_creation_response.data.task_id
- final_response = self.get_response(
+ final_response = await self.get_response(
task_id, auth_kwargs=kwargs, node_id=unique_id
)
validate_image_result_response(final_response)
images = get_images_from_response(final_response)
- return (image_result_to_node_output(images),)
+ return (await image_result_to_node_output(images),)
class KlingImageGenerationNode(KlingImageGenerationBase):
@@ -1655,13 +1657,13 @@ class KlingImageGenerationNode(KlingImageGenerationBase):
DESCRIPTION = "Kling Image Generation Node. Generate an image from a text prompt with an optional reference image."
- def get_response(
+ async def get_response(
self,
task_id: str,
auth_kwargs: Optional[dict[str, str]],
node_id: Optional[str] = None,
) -> KlingImageGenerationsResponse:
- return poll_until_finished(
+ return await poll_until_finished(
auth_kwargs,
ApiEndpoint(
path=f"{PATH_IMAGE_GENERATIONS}/{task_id}",
@@ -1674,7 +1676,7 @@ class KlingImageGenerationNode(KlingImageGenerationBase):
node_id=node_id,
)
- def api_call(
+ async def api_call(
self,
model_name: KlingImageGenModelName,
prompt: str,
@@ -1690,7 +1692,11 @@ class KlingImageGenerationNode(KlingImageGenerationBase):
):
self.validate_prompt(prompt, negative_prompt)
- if image is not None:
+ if image is None:
+ image_type = None
+ elif model_name == KlingImageGenModelName.kling_v1:
+ raise ValueError(f"The model {KlingImageGenModelName.kling_v1.value} does not support reference images.")
+ else:
image = tensor_to_base64_string(image)
initial_operation = SynchronousOperation(
@@ -1714,17 +1720,17 @@ class KlingImageGenerationNode(KlingImageGenerationBase):
auth_kwargs=kwargs,
)
- task_creation_response = initial_operation.execute()
+ task_creation_response = await initial_operation.execute()
validate_task_creation_response(task_creation_response)
task_id = task_creation_response.data.task_id
- final_response = self.get_response(
+ final_response = await self.get_response(
task_id, auth_kwargs=kwargs, node_id=unique_id
)
validate_image_result_response(final_response)
images = get_images_from_response(final_response)
- return (image_result_to_node_output(images),)
+ return (await image_result_to_node_output(images),)
NODE_CLASS_MAPPINGS = {
diff --git a/comfy_api_nodes/nodes_luma.py b/comfy_api_nodes/nodes_luma.py
index 525dc38e6..b3c32bed5 100644
--- a/comfy_api_nodes/nodes_luma.py
+++ b/comfy_api_nodes/nodes_luma.py
@@ -38,7 +38,7 @@ from comfy_api_nodes.apinode_utils import (
)
from server import PromptServer
-import requests
+import aiohttp
import torch
from io import BytesIO
@@ -217,7 +217,7 @@ class LumaImageGenerationNode(ComfyNodeABC):
},
}
- def api_call(
+ async def api_call(
self,
prompt: str,
model: str,
@@ -234,19 +234,19 @@ class LumaImageGenerationNode(ComfyNodeABC):
# handle image_luma_ref
api_image_ref = None
if image_luma_ref is not None:
- api_image_ref = self._convert_luma_refs(
+ api_image_ref = await self._convert_luma_refs(
image_luma_ref, max_refs=4, auth_kwargs=kwargs,
)
# handle style_luma_ref
api_style_ref = None
if style_image is not None:
- api_style_ref = self._convert_style_image(
+ api_style_ref = await self._convert_style_image(
style_image, weight=style_image_weight, auth_kwargs=kwargs,
)
# handle character_ref images
character_ref = None
if character_image is not None:
- download_urls = upload_images_to_comfyapi(
+ download_urls = await upload_images_to_comfyapi(
character_image, max_images=4, auth_kwargs=kwargs,
)
character_ref = LumaCharacterRef(
@@ -270,7 +270,7 @@ class LumaImageGenerationNode(ComfyNodeABC):
),
auth_kwargs=kwargs,
)
- response_api: LumaGeneration = operation.execute()
+ response_api: LumaGeneration = await operation.execute()
operation = PollingOperation(
poll_endpoint=ApiEndpoint(
@@ -286,19 +286,20 @@ class LumaImageGenerationNode(ComfyNodeABC):
node_id=unique_id,
auth_kwargs=kwargs,
)
- response_poll = operation.execute()
+ response_poll = await operation.execute()
- img_response = requests.get(response_poll.assets.image)
- img = process_image_response(img_response)
+ async with aiohttp.ClientSession() as session:
+ async with session.get(response_poll.assets.image) as img_response:
+ img = process_image_response(await img_response.content.read())
return (img,)
- def _convert_luma_refs(
+ async def _convert_luma_refs(
self, luma_ref: LumaReferenceChain, max_refs: int, auth_kwargs: Optional[dict[str,str]] = None
):
luma_urls = []
ref_count = 0
for ref in luma_ref.refs:
- download_urls = upload_images_to_comfyapi(
+ download_urls = await upload_images_to_comfyapi(
ref.image, max_images=1, auth_kwargs=auth_kwargs
)
luma_urls.append(download_urls[0])
@@ -307,13 +308,13 @@ class LumaImageGenerationNode(ComfyNodeABC):
break
return luma_ref.create_api_model(download_urls=luma_urls, max_refs=max_refs)
- def _convert_style_image(
+ async def _convert_style_image(
self, style_image: torch.Tensor, weight: float, auth_kwargs: Optional[dict[str,str]] = None
):
chain = LumaReferenceChain(
first_ref=LumaReference(image=style_image, weight=weight)
)
- return self._convert_luma_refs(chain, max_refs=1, auth_kwargs=auth_kwargs)
+ return await self._convert_luma_refs(chain, max_refs=1, auth_kwargs=auth_kwargs)
class LumaImageModifyNode(ComfyNodeABC):
@@ -370,7 +371,7 @@ class LumaImageModifyNode(ComfyNodeABC):
},
}
- def api_call(
+ async def api_call(
self,
prompt: str,
model: str,
@@ -381,7 +382,7 @@ class LumaImageModifyNode(ComfyNodeABC):
**kwargs,
):
# first, upload image
- download_urls = upload_images_to_comfyapi(
+ download_urls = await upload_images_to_comfyapi(
image, max_images=1, auth_kwargs=kwargs,
)
image_url = download_urls[0]
@@ -402,7 +403,7 @@ class LumaImageModifyNode(ComfyNodeABC):
),
auth_kwargs=kwargs,
)
- response_api: LumaGeneration = operation.execute()
+ response_api: LumaGeneration = await operation.execute()
operation = PollingOperation(
poll_endpoint=ApiEndpoint(
@@ -418,10 +419,11 @@ class LumaImageModifyNode(ComfyNodeABC):
node_id=unique_id,
auth_kwargs=kwargs,
)
- response_poll = operation.execute()
+ response_poll = await operation.execute()
- img_response = requests.get(response_poll.assets.image)
- img = process_image_response(img_response)
+ async with aiohttp.ClientSession() as session:
+ async with session.get(response_poll.assets.image) as img_response:
+ img = process_image_response(await img_response.content.read())
return (img,)
@@ -494,7 +496,7 @@ class LumaTextToVideoGenerationNode(ComfyNodeABC):
},
}
- def api_call(
+ async def api_call(
self,
prompt: str,
model: str,
@@ -529,7 +531,7 @@ class LumaTextToVideoGenerationNode(ComfyNodeABC):
),
auth_kwargs=kwargs,
)
- response_api: LumaGeneration = operation.execute()
+ response_api: LumaGeneration = await operation.execute()
if unique_id:
PromptServer.instance.send_progress_text(f"Luma video generation started: {response_api.id}", unique_id)
@@ -549,10 +551,11 @@ class LumaTextToVideoGenerationNode(ComfyNodeABC):
estimated_duration=LUMA_T2V_AVERAGE_DURATION,
auth_kwargs=kwargs,
)
- response_poll = operation.execute()
+ response_poll = await operation.execute()
- vid_response = requests.get(response_poll.assets.video)
- return (VideoFromFile(BytesIO(vid_response.content)),)
+ async with aiohttp.ClientSession() as session:
+ async with session.get(response_poll.assets.video) as vid_response:
+ return (VideoFromFile(BytesIO(await vid_response.content.read())),)
class LumaImageToVideoGenerationNode(ComfyNodeABC):
@@ -626,7 +629,7 @@ class LumaImageToVideoGenerationNode(ComfyNodeABC):
},
}
- def api_call(
+ async def api_call(
self,
prompt: str,
model: str,
@@ -644,7 +647,7 @@ class LumaImageToVideoGenerationNode(ComfyNodeABC):
raise Exception(
"At least one of first_image and last_image requires an input."
)
- keyframes = self._convert_to_keyframes(first_image, last_image, auth_kwargs=kwargs)
+ keyframes = await self._convert_to_keyframes(first_image, last_image, auth_kwargs=kwargs)
duration = duration if model != LumaVideoModel.ray_1_6 else None
resolution = resolution if model != LumaVideoModel.ray_1_6 else None
@@ -667,7 +670,7 @@ class LumaImageToVideoGenerationNode(ComfyNodeABC):
),
auth_kwargs=kwargs,
)
- response_api: LumaGeneration = operation.execute()
+ response_api: LumaGeneration = await operation.execute()
if unique_id:
PromptServer.instance.send_progress_text(f"Luma video generation started: {response_api.id}", unique_id)
@@ -687,12 +690,13 @@ class LumaImageToVideoGenerationNode(ComfyNodeABC):
estimated_duration=LUMA_I2V_AVERAGE_DURATION,
auth_kwargs=kwargs,
)
- response_poll = operation.execute()
+ response_poll = await operation.execute()
- vid_response = requests.get(response_poll.assets.video)
- return (VideoFromFile(BytesIO(vid_response.content)),)
+ async with aiohttp.ClientSession() as session:
+ async with session.get(response_poll.assets.video) as vid_response:
+ return (VideoFromFile(BytesIO(await vid_response.content.read())),)
- def _convert_to_keyframes(
+ async def _convert_to_keyframes(
self,
first_image: torch.Tensor = None,
last_image: torch.Tensor = None,
@@ -703,12 +707,12 @@ class LumaImageToVideoGenerationNode(ComfyNodeABC):
frame0 = None
frame1 = None
if first_image is not None:
- download_urls = upload_images_to_comfyapi(
+ download_urls = await upload_images_to_comfyapi(
first_image, max_images=1, auth_kwargs=auth_kwargs,
)
frame0 = LumaImageReference(type="image", url=download_urls[0])
if last_image is not None:
- download_urls = upload_images_to_comfyapi(
+ download_urls = await upload_images_to_comfyapi(
last_image, max_images=1, auth_kwargs=auth_kwargs,
)
frame1 = LumaImageReference(type="image", url=download_urls[0])
diff --git a/comfy_api_nodes/nodes_minimax.py b/comfy_api_nodes/nodes_minimax.py
index 9b46636db..bb3c9e710 100644
--- a/comfy_api_nodes/nodes_minimax.py
+++ b/comfy_api_nodes/nodes_minimax.py
@@ -1,3 +1,4 @@
+from inspect import cleandoc
from typing import Union
import logging
import torch
@@ -10,7 +11,7 @@ from comfy_api_nodes.apis import (
MinimaxFileRetrieveResponse,
MinimaxTaskResultResponse,
SubjectReferenceItem,
- Model
+ MiniMaxModel
)
from comfy_api_nodes.apis.client import (
ApiEndpoint,
@@ -84,9 +85,8 @@ class MinimaxTextToVideoNode:
FUNCTION = "generate_video"
CATEGORY = "api node/video/MiniMax"
API_NODE = True
- OUTPUT_NODE = True
- def generate_video(
+ async def generate_video(
self,
prompt_text,
seed=0,
@@ -104,12 +104,12 @@ class MinimaxTextToVideoNode:
# upload image, if passed in
image_url = None
if image is not None:
- image_url = upload_images_to_comfyapi(image, max_images=1, auth_kwargs=kwargs)[0]
+ image_url = (await upload_images_to_comfyapi(image, max_images=1, auth_kwargs=kwargs))[0]
# TODO: figure out how to deal with subject properly, API returns invalid params when using S2V-01 model
subject_reference = None
if subject is not None:
- subject_url = upload_images_to_comfyapi(subject, max_images=1, auth_kwargs=kwargs)[0]
+ subject_url = (await upload_images_to_comfyapi(subject, max_images=1, auth_kwargs=kwargs))[0]
subject_reference = [SubjectReferenceItem(image=subject_url)]
@@ -121,7 +121,7 @@ class MinimaxTextToVideoNode:
response_model=MinimaxVideoGenerationResponse,
),
request=MinimaxVideoGenerationRequest(
- model=Model(model),
+ model=MiniMaxModel(model),
prompt=prompt_text,
callback_url=None,
first_frame_image=image_url,
@@ -130,7 +130,7 @@ class MinimaxTextToVideoNode:
),
auth_kwargs=kwargs,
)
- response = video_generate_operation.execute()
+ response = await video_generate_operation.execute()
task_id = response.task_id
if not task_id:
@@ -151,7 +151,7 @@ class MinimaxTextToVideoNode:
node_id=unique_id,
auth_kwargs=kwargs,
)
- task_result = video_generate_operation.execute()
+ task_result = await video_generate_operation.execute()
file_id = task_result.file_id
if file_id is None:
@@ -167,7 +167,7 @@ class MinimaxTextToVideoNode:
request=EmptyRequest(),
auth_kwargs=kwargs,
)
- file_result = file_retrieve_operation.execute()
+ file_result = await file_retrieve_operation.execute()
file_url = file_result.file.download_url
if file_url is None:
@@ -182,7 +182,7 @@ class MinimaxTextToVideoNode:
message = f"Result URL: {file_url}"
PromptServer.instance.send_progress_text(message, unique_id)
- video_io = download_url_to_bytesio(file_url)
+ video_io = await download_url_to_bytesio(file_url)
if video_io is None:
error_msg = f"Failed to download video from {file_url}"
logging.error(error_msg)
@@ -251,7 +251,6 @@ class MinimaxImageToVideoNode(MinimaxTextToVideoNode):
FUNCTION = "generate_video"
CATEGORY = "api node/video/MiniMax"
API_NODE = True
- OUTPUT_NODE = True
class MinimaxSubjectToVideoNode(MinimaxTextToVideoNode):
@@ -313,7 +312,181 @@ class MinimaxSubjectToVideoNode(MinimaxTextToVideoNode):
FUNCTION = "generate_video"
CATEGORY = "api node/video/MiniMax"
API_NODE = True
- OUTPUT_NODE = True
+
+
+class MinimaxHailuoVideoNode:
+ """Generates videos from prompt, with optional start frame using the new MiniMax Hailuo-02 model."""
+
+ @classmethod
+ def INPUT_TYPES(s):
+ return {
+ "required": {
+ "prompt_text": (
+ "STRING",
+ {
+ "multiline": True,
+ "default": "",
+ "tooltip": "Text prompt to guide the video generation.",
+ },
+ ),
+ },
+ "optional": {
+ "seed": (
+ IO.INT,
+ {
+ "default": 0,
+ "min": 0,
+ "max": 0xFFFFFFFFFFFFFFFF,
+ "control_after_generate": True,
+ "tooltip": "The random seed used for creating the noise.",
+ },
+ ),
+ "first_frame_image": (
+ IO.IMAGE,
+ {
+ "tooltip": "Optional image to use as the first frame to generate a video."
+ },
+ ),
+ "prompt_optimizer": (
+ IO.BOOLEAN,
+ {
+ "tooltip": "Optimize prompt to improve generation quality when needed.",
+ "default": True,
+ },
+ ),
+ "duration": (
+ IO.COMBO,
+ {
+ "tooltip": "The length of the output video in seconds.",
+ "default": 6,
+ "options": [6, 10],
+ },
+ ),
+ "resolution": (
+ IO.COMBO,
+ {
+ "tooltip": "The dimensions of the video display. "
+ "1080p corresponds to 1920 x 1080 pixels, 768p corresponds to 1366 x 768 pixels.",
+ "default": "768P",
+ "options": ["768P", "1080P"],
+ },
+ ),
+ },
+ "hidden": {
+ "auth_token": "AUTH_TOKEN_COMFY_ORG",
+ "comfy_api_key": "API_KEY_COMFY_ORG",
+ "unique_id": "UNIQUE_ID",
+ },
+ }
+
+ RETURN_TYPES = ("VIDEO",)
+ DESCRIPTION = cleandoc(__doc__ or "")
+ FUNCTION = "generate_video"
+ CATEGORY = "api node/video/MiniMax"
+ API_NODE = True
+
+ async def generate_video(
+ self,
+ prompt_text,
+ seed=0,
+ first_frame_image: torch.Tensor=None, # used for ImageToVideo
+ prompt_optimizer=True,
+ duration=6,
+ resolution="768P",
+ model="MiniMax-Hailuo-02",
+ unique_id: Union[str, None]=None,
+ **kwargs,
+ ):
+ if first_frame_image is None:
+ validate_string(prompt_text, field_name="prompt_text")
+
+ if model == "MiniMax-Hailuo-02" and resolution.upper() == "1080P" and duration != 6:
+ raise Exception(
+ "When model is MiniMax-Hailuo-02 and resolution is 1080P, duration is limited to 6 seconds."
+ )
+
+ # upload image, if passed in
+ image_url = None
+ if first_frame_image is not None:
+ image_url = (await upload_images_to_comfyapi(first_frame_image, max_images=1, auth_kwargs=kwargs))[0]
+
+ video_generate_operation = SynchronousOperation(
+ endpoint=ApiEndpoint(
+ path="/proxy/minimax/video_generation",
+ method=HttpMethod.POST,
+ request_model=MinimaxVideoGenerationRequest,
+ response_model=MinimaxVideoGenerationResponse,
+ ),
+ request=MinimaxVideoGenerationRequest(
+ model=MiniMaxModel(model),
+ prompt=prompt_text,
+ callback_url=None,
+ first_frame_image=image_url,
+ prompt_optimizer=prompt_optimizer,
+ duration=duration,
+ resolution=resolution,
+ ),
+ auth_kwargs=kwargs,
+ )
+ response = await video_generate_operation.execute()
+
+ task_id = response.task_id
+ if not task_id:
+ raise Exception(f"MiniMax generation failed: {response.base_resp}")
+
+ average_duration = 120 if resolution == "768P" else 240
+ video_generate_operation = PollingOperation(
+ poll_endpoint=ApiEndpoint(
+ path="/proxy/minimax/query/video_generation",
+ method=HttpMethod.GET,
+ request_model=EmptyRequest,
+ response_model=MinimaxTaskResultResponse,
+ query_params={"task_id": task_id},
+ ),
+ completed_statuses=["Success"],
+ failed_statuses=["Fail"],
+ status_extractor=lambda x: x.status.value,
+ estimated_duration=average_duration,
+ node_id=unique_id,
+ auth_kwargs=kwargs,
+ )
+ task_result = await video_generate_operation.execute()
+
+ file_id = task_result.file_id
+ if file_id is None:
+ raise Exception("Request was not successful. Missing file ID.")
+ file_retrieve_operation = SynchronousOperation(
+ endpoint=ApiEndpoint(
+ path="/proxy/minimax/files/retrieve",
+ method=HttpMethod.GET,
+ request_model=EmptyRequest,
+ response_model=MinimaxFileRetrieveResponse,
+ query_params={"file_id": int(file_id)},
+ ),
+ request=EmptyRequest(),
+ auth_kwargs=kwargs,
+ )
+ file_result = await file_retrieve_operation.execute()
+
+ file_url = file_result.file.download_url
+ if file_url is None:
+ raise Exception(
+ f"No video was found in the response. Full response: {file_result.model_dump()}"
+ )
+ logging.info(f"Generated video URL: {file_url}")
+ if unique_id:
+ if hasattr(file_result.file, "backup_download_url"):
+ message = f"Result URL: {file_url}\nBackup URL: {file_result.file.backup_download_url}"
+ else:
+ message = f"Result URL: {file_url}"
+ PromptServer.instance.send_progress_text(message, unique_id)
+
+ video_io = await download_url_to_bytesio(file_url)
+ if video_io is None:
+ error_msg = f"Failed to download video from {file_url}"
+ logging.error(error_msg)
+ raise Exception(error_msg)
+ return (VideoFromFile(video_io),)
# A dictionary that contains all nodes you want to export with their names
@@ -322,6 +495,7 @@ NODE_CLASS_MAPPINGS = {
"MinimaxTextToVideoNode": MinimaxTextToVideoNode,
"MinimaxImageToVideoNode": MinimaxImageToVideoNode,
# "MinimaxSubjectToVideoNode": MinimaxSubjectToVideoNode,
+ "MinimaxHailuoVideoNode": MinimaxHailuoVideoNode,
}
# A dictionary that contains the friendly/humanly readable titles for the nodes
@@ -329,4 +503,5 @@ NODE_DISPLAY_NAME_MAPPINGS = {
"MinimaxTextToVideoNode": "MiniMax Text to Video",
"MinimaxImageToVideoNode": "MiniMax Image to Video",
"MinimaxSubjectToVideoNode": "MiniMax Subject to Video",
+ "MinimaxHailuoVideoNode": "MiniMax Hailuo Video",
}
diff --git a/comfy_api_nodes/nodes_moonvalley.py b/comfy_api_nodes/nodes_moonvalley.py
index 789fcef02..806a70e06 100644
--- a/comfy_api_nodes/nodes_moonvalley.py
+++ b/comfy_api_nodes/nodes_moonvalley.py
@@ -1,6 +1,5 @@
import logging
from typing import Any, Callable, Optional, TypeVar
-import random
import torch
from comfy_api_nodes.util.validation_utils import (
get_image_dimensions,
@@ -95,14 +94,14 @@ def get_video_url_from_response(response) -> Optional[str]:
return None
-def poll_until_finished(
+async def poll_until_finished(
auth_kwargs: dict[str, str],
api_endpoint: ApiEndpoint[Any, R],
result_url_extractor: Optional[Callable[[R], str]] = None,
node_id: Optional[str] = None,
) -> R:
"""Polls the Moonvalley API endpoint until the task reaches a terminal state, then returns the response."""
- return PollingOperation(
+ return await PollingOperation(
poll_endpoint=api_endpoint,
completed_statuses=[
"completed",
@@ -208,20 +207,29 @@ def _get_video_dimensions(video: VideoInput) -> tuple[int, int]:
def _validate_video_dimensions(width: int, height: int) -> None:
"""Validates video dimensions meet Moonvalley V2V requirements."""
supported_resolutions = {
- (1920, 1080), (1080, 1920), (1152, 1152),
- (1536, 1152), (1152, 1536)
+ (1920, 1080),
+ (1080, 1920),
+ (1152, 1152),
+ (1536, 1152),
+ (1152, 1536),
}
if (width, height) not in supported_resolutions:
- supported_list = ', '.join([f'{w}x{h}' for w, h in sorted(supported_resolutions)])
- raise ValueError(f"Resolution {width}x{height} not supported. Supported: {supported_list}")
+ supported_list = ", ".join(
+ [f"{w}x{h}" for w, h in sorted(supported_resolutions)]
+ )
+ raise ValueError(
+ f"Resolution {width}x{height} not supported. Supported: {supported_list}"
+ )
def _validate_container_format(video: VideoInput) -> None:
"""Validates video container format is MP4."""
container_format = video.get_container_format()
- if container_format not in ['mp4', 'mov,mp4,m4a,3gp,3g2,mj2']:
- raise ValueError(f"Only MP4 container format supported. Got: {container_format}")
+ if container_format not in ["mp4", "mov,mp4,m4a,3gp,3g2,mj2"]:
+ raise ValueError(
+ f"Only MP4 container format supported. Got: {container_format}"
+ )
def _validate_and_trim_duration(video: VideoInput) -> VideoInput:
@@ -244,7 +252,6 @@ def _trim_if_too_long(video: VideoInput, duration: float) -> VideoInput:
return video
-
def trim_video(video: VideoInput, duration_sec: float) -> VideoInput:
"""
Returns a new VideoInput object trimmed from the beginning to the specified duration,
@@ -302,7 +309,9 @@ def trim_video(video: VideoInput, duration_sec: float) -> VideoInput:
# Calculate target frame count that's divisible by 16
fps = input_container.streams.video[0].average_rate
estimated_frames = int(duration_sec * fps)
- target_frames = (estimated_frames // 16) * 16 # Round down to nearest multiple of 16
+ target_frames = (
+ estimated_frames // 16
+ ) * 16 # Round down to nearest multiple of 16
if target_frames == 0:
raise ValueError("Video too short: need at least 16 frames for Moonvalley")
@@ -394,10 +403,10 @@ class BaseMoonvalleyVideoNode:
else:
return control_map["Motion Transfer"]
- def get_response(
+ async def get_response(
self, task_id: str, auth_kwargs: dict[str, str], node_id: Optional[str] = None
) -> MoonvalleyPromptResponse:
- return poll_until_finished(
+ return await poll_until_finished(
auth_kwargs,
ApiEndpoint(
path=f"{API_PROMPTS_ENDPOINT}/{task_id}",
@@ -424,7 +433,7 @@ class BaseMoonvalleyVideoNode:
MoonvalleyTextToVideoInferenceParams,
"negative_prompt",
multiline=True,
- default="low-poly, flat shader, bad rigging, stiff animation, uncanny eyes, low-quality textures, looping glitch, cheap effect, overbloom, bloom spam, default lighting, game asset, stiff face, ugly specular, AI artifacts",
+ default=" gopro, bright, contrast, static, overexposed, vignette, artifacts, still, noise, texture, scanlines, videogame, 360 camera, VR, transition, flare, saturation, distorted, warped, wide angle, saturated, vibrant, glowing, cross dissolve, cheesy, ugly hands, mutated hands, mutant, disfigured, extra fingers, blown out, horrible, blurry, worst quality, bad, dissolve, melt, fade in, fade out, wobbly, weird, low quality, plastic, stock footage, video camera, boring",
),
"resolution": (
IO.COMBO,
@@ -441,12 +450,11 @@ class BaseMoonvalleyVideoNode:
"tooltip": "Resolution of the output video",
},
),
- # "length": (IO.COMBO,{"options":['5s','10s'], "default": '5s'}),
"prompt_adherence": model_field_to_node_input(
IO.FLOAT,
MoonvalleyTextToVideoInferenceParams,
"guidance_scale",
- default=7.0,
+ default=10.0,
step=1,
min=1,
max=20,
@@ -455,13 +463,12 @@ class BaseMoonvalleyVideoNode:
IO.INT,
MoonvalleyTextToVideoInferenceParams,
"seed",
- default=random.randint(0, 2**32 - 1),
+ default=9,
min=0,
max=4294967295,
step=1,
display="number",
tooltip="Random seed value",
- control_after_generate=True,
),
"steps": model_field_to_node_input(
IO.INT,
@@ -507,7 +514,7 @@ class MoonvalleyImg2VideoNode(BaseMoonvalleyVideoNode):
RETURN_NAMES = ("video",)
DESCRIPTION = "Moonvalley Marey Image to Video Node"
- def generate(
+ async def generate(
self, prompt, negative_prompt, unique_id: Optional[str] = None, **kwargs
):
image = kwargs.get("image", None)
@@ -532,8 +539,10 @@ class MoonvalleyImg2VideoNode(BaseMoonvalleyVideoNode):
# Get MIME type from tensor - assuming PNG format for image tensors
mime_type = "image/png"
- image_url = upload_images_to_comfyapi(
- image, max_images=1, auth_kwargs=kwargs, mime_type=mime_type
+ image_url = (
+ await upload_images_to_comfyapi(
+ image, max_images=1, auth_kwargs=kwargs, mime_type=mime_type
+ )
)[0]
request = MoonvalleyTextToVideoRequest(
@@ -549,14 +558,14 @@ class MoonvalleyImg2VideoNode(BaseMoonvalleyVideoNode):
request=request,
auth_kwargs=kwargs,
)
- task_creation_response = initial_operation.execute()
+ task_creation_response = await initial_operation.execute()
validate_task_creation_response(task_creation_response)
task_id = task_creation_response.id
- final_response = self.get_response(
+ final_response = await self.get_response(
task_id, auth_kwargs=kwargs, node_id=unique_id
)
- video = download_url_to_video_output(final_response.output_url)
+ video = await download_url_to_video_output(final_response.output_url)
return (video,)
@@ -570,17 +579,39 @@ class MoonvalleyVideo2VideoNode(BaseMoonvalleyVideoNode):
return {
"required": {
"prompt": model_field_to_node_input(
- IO.STRING, MoonvalleyVideoToVideoRequest, "prompt_text",
- multiline=True
+ IO.STRING,
+ MoonvalleyVideoToVideoRequest,
+ "prompt_text",
+ multiline=True,
),
"negative_prompt": model_field_to_node_input(
IO.STRING,
MoonvalleyVideoToVideoInferenceParams,
"negative_prompt",
multiline=True,
- default="low-poly, flat shader, bad rigging, stiff animation, uncanny eyes, low-quality textures, looping glitch, cheap effect, overbloom, bloom spam, default lighting, game asset, stiff face, ugly specular, AI artifacts"
+ default=" gopro, bright, contrast, static, overexposed, vignette, artifacts, still, noise, texture, scanlines, videogame, 360 camera, VR, transition, flare, saturation, distorted, warped, wide angle, saturated, vibrant, glowing, cross dissolve, cheesy, ugly hands, mutated hands, mutant, disfigured, extra fingers, blown out, horrible, blurry, worst quality, bad, dissolve, melt, fade in, fade out, wobbly, weird, low quality, plastic, stock footage, video camera, boring",
+ ),
+ "seed": model_field_to_node_input(
+ IO.INT,
+ MoonvalleyVideoToVideoInferenceParams,
+ "seed",
+ default=9,
+ min=0,
+ max=4294967295,
+ step=1,
+ display="number",
+ tooltip="Random seed value",
+ control_after_generate=False,
+ ),
+ "prompt_adherence": model_field_to_node_input(
+ IO.FLOAT,
+ MoonvalleyVideoToVideoInferenceParams,
+ "guidance_scale",
+ default=10.0,
+ step=1,
+ min=1,
+ max=20,
),
- "seed": model_field_to_node_input(IO.INT,MoonvalleyVideoToVideoInferenceParams, "seed", default=random.randint(0, 2**32 - 1), min=0, max=4294967295, step=1, display="number", tooltip="Random seed value", control_after_generate=True),
},
"hidden": {
"auth_token": "AUTH_TOKEN_COMFY_ORG",
@@ -588,7 +619,14 @@ class MoonvalleyVideo2VideoNode(BaseMoonvalleyVideoNode):
"unique_id": "UNIQUE_ID",
},
"optional": {
- "video": (IO.VIDEO, {"default": "", "multiline": False, "tooltip": "The reference video used to generate the output video. Must be at least 5 seconds long. Videos longer than 5s will be automatically trimmed. Only MP4 format supported."}),
+ "video": (
+ IO.VIDEO,
+ {
+ "default": "",
+ "multiline": False,
+ "tooltip": "The reference video used to generate the output video. Must be at least 5 seconds long. Videos longer than 5s will be automatically trimmed. Only MP4 format supported.",
+ },
+ ),
"control_type": (
["Motion Transfer", "Pose Transfer"],
{"default": "Motion Transfer"},
@@ -602,17 +640,24 @@ class MoonvalleyVideo2VideoNode(BaseMoonvalleyVideoNode):
"max": 100,
"tooltip": "Only used if control_type is 'Motion Transfer'",
},
- )
- }
+ ),
+ "image": model_field_to_node_input(
+ IO.IMAGE,
+ MoonvalleyTextToVideoRequest,
+ "image_url",
+ tooltip="The reference image used to generate the video",
+ ),
+ },
}
RETURN_TYPES = ("VIDEO",)
RETURN_NAMES = ("video",)
- def generate(
+ async def generate(
self, prompt, negative_prompt, unique_id: Optional[str] = None, **kwargs
):
video = kwargs.get("video")
+ image = kwargs.get("image", None)
if not video:
raise MoonvalleyApiError("video is required")
@@ -620,8 +665,16 @@ class MoonvalleyVideo2VideoNode(BaseMoonvalleyVideoNode):
video_url = ""
if video:
validated_video = validate_video_to_video_input(video)
- video_url = upload_video_to_comfyapi(validated_video, auth_kwargs=kwargs)
+ video_url = await upload_video_to_comfyapi(
+ validated_video, auth_kwargs=kwargs
+ )
+ mime_type = "image/png"
+ if not image is None:
+ validate_input_image(image, with_frame_conditioning=True)
+ image_url = await upload_images_to_comfyapi(
+ image=image, auth_kwargs=kwargs, max_images=1, mime_type=mime_type
+ )
control_type = kwargs.get("control_type")
motion_intensity = kwargs.get("motion_intensity")
@@ -631,12 +684,12 @@ class MoonvalleyVideo2VideoNode(BaseMoonvalleyVideoNode):
# Only include motion_intensity for Motion Transfer
control_params = {}
if control_type == "Motion Transfer" and motion_intensity is not None:
- control_params['motion_intensity'] = motion_intensity
+ control_params["motion_intensity"] = motion_intensity
- inference_params=MoonvalleyVideoToVideoInferenceParams(
+ inference_params = MoonvalleyVideoToVideoInferenceParams(
negative_prompt=negative_prompt,
seed=kwargs.get("seed"),
- control_params=control_params
+ control_params=control_params,
)
control = self.parseControlParameter(control_type)
@@ -647,6 +700,7 @@ class MoonvalleyVideo2VideoNode(BaseMoonvalleyVideoNode):
prompt_text=prompt,
inference_params=inference_params,
)
+ request.image_url = image_url if not image is None else None
initial_operation = SynchronousOperation(
endpoint=ApiEndpoint(
@@ -658,15 +712,15 @@ class MoonvalleyVideo2VideoNode(BaseMoonvalleyVideoNode):
request=request,
auth_kwargs=kwargs,
)
- task_creation_response = initial_operation.execute()
+ task_creation_response = await initial_operation.execute()
validate_task_creation_response(task_creation_response)
task_id = task_creation_response.id
- final_response = self.get_response(
+ final_response = await self.get_response(
task_id, auth_kwargs=kwargs, node_id=unique_id
)
- video = download_url_to_video_output(final_response.output_url)
+ video = await download_url_to_video_output(final_response.output_url)
return (video,)
@@ -688,21 +742,21 @@ class MoonvalleyTxt2VideoNode(BaseMoonvalleyVideoNode):
del input_types["optional"][param]
return input_types
- def generate(
+ async def generate(
self, prompt, negative_prompt, unique_id: Optional[str] = None, **kwargs
):
validate_prompts(prompt, negative_prompt, MOONVALLEY_MAREY_MAX_PROMPT_LENGTH)
width_height = self.parseWidthHeightFromRes(kwargs.get("resolution"))
- inference_params=MoonvalleyTextToVideoInferenceParams(
- negative_prompt=negative_prompt,
- steps=kwargs.get("steps"),
- seed=kwargs.get("seed"),
- guidance_scale=kwargs.get("prompt_adherence"),
- num_frames=128,
- width=width_height.get("width"),
- height=width_height.get("height"),
- )
+ inference_params = MoonvalleyTextToVideoInferenceParams(
+ negative_prompt=negative_prompt,
+ steps=kwargs.get("steps"),
+ seed=kwargs.get("seed"),
+ guidance_scale=kwargs.get("prompt_adherence"),
+ num_frames=128,
+ width=width_height.get("width"),
+ height=width_height.get("height"),
+ )
request = MoonvalleyTextToVideoRequest(
prompt_text=prompt, inference_params=inference_params
)
@@ -717,15 +771,15 @@ class MoonvalleyTxt2VideoNode(BaseMoonvalleyVideoNode):
request=request,
auth_kwargs=kwargs,
)
- task_creation_response = initial_operation.execute()
+ task_creation_response = await initial_operation.execute()
validate_task_creation_response(task_creation_response)
task_id = task_creation_response.id
- final_response = self.get_response(
+ final_response = await self.get_response(
task_id, auth_kwargs=kwargs, node_id=unique_id
)
- video = download_url_to_video_output(final_response.output_url)
+ video = await download_url_to_video_output(final_response.output_url)
return (video,)
diff --git a/comfy_api_nodes/nodes_openai.py b/comfy_api_nodes/nodes_openai.py
index be1d2de4a..674c9ede0 100644
--- a/comfy_api_nodes/nodes_openai.py
+++ b/comfy_api_nodes/nodes_openai.py
@@ -80,6 +80,9 @@ class SupportedOpenAIModel(str, Enum):
gpt_4_1 = "gpt-4.1"
gpt_4_1_mini = "gpt-4.1-mini"
gpt_4_1_nano = "gpt-4.1-nano"
+ gpt_5 = "gpt-5"
+ gpt_5_mini = "gpt-5-mini"
+ gpt_5_nano = "gpt-5-nano"
class OpenAIDalle2(ComfyNodeABC):
@@ -163,7 +166,7 @@ class OpenAIDalle2(ComfyNodeABC):
DESCRIPTION = cleandoc(__doc__ or "")
API_NODE = True
- def api_call(
+ async def api_call(
self,
prompt,
seed=0,
@@ -233,9 +236,9 @@ class OpenAIDalle2(ComfyNodeABC):
auth_kwargs=kwargs,
)
- response = operation.execute()
+ response = await operation.execute()
- img_tensor = validate_and_cast_response(response, node_id=unique_id)
+ img_tensor = await validate_and_cast_response(response, node_id=unique_id)
return (img_tensor,)
@@ -311,7 +314,7 @@ class OpenAIDalle3(ComfyNodeABC):
DESCRIPTION = cleandoc(__doc__ or "")
API_NODE = True
- def api_call(
+ async def api_call(
self,
prompt,
seed=0,
@@ -343,9 +346,9 @@ class OpenAIDalle3(ComfyNodeABC):
auth_kwargs=kwargs,
)
- response = operation.execute()
+ response = await operation.execute()
- img_tensor = validate_and_cast_response(response, node_id=unique_id)
+ img_tensor = await validate_and_cast_response(response, node_id=unique_id)
return (img_tensor,)
@@ -446,7 +449,7 @@ class OpenAIGPTImage1(ComfyNodeABC):
DESCRIPTION = cleandoc(__doc__ or "")
API_NODE = True
- def api_call(
+ async def api_call(
self,
prompt,
seed=0,
@@ -464,8 +467,6 @@ class OpenAIGPTImage1(ComfyNodeABC):
path = "/proxy/openai/images/generations"
content_type = "application/json"
request_class = OpenAIImageGenerationRequest
- img_binaries = []
- mask_binary = None
files = []
if image is not None:
@@ -484,14 +485,11 @@ class OpenAIGPTImage1(ComfyNodeABC):
img_byte_arr = io.BytesIO()
img.save(img_byte_arr, format="PNG")
img_byte_arr.seek(0)
- img_binary = img_byte_arr
- img_binary.name = f"image_{i}.png"
- img_binaries.append(img_binary)
if batch_size == 1:
- files.append(("image", img_binary))
+ files.append(("image", (f"image_{i}.png", img_byte_arr, "image/png")))
else:
- files.append(("image[]", img_binary))
+ files.append(("image[]", (f"image_{i}.png", img_byte_arr, "image/png")))
if mask is not None:
if image is None:
@@ -511,9 +509,7 @@ class OpenAIGPTImage1(ComfyNodeABC):
mask_img_byte_arr = io.BytesIO()
mask_img.save(mask_img_byte_arr, format="PNG")
mask_img_byte_arr.seek(0)
- mask_binary = mask_img_byte_arr
- mask_binary.name = "mask.png"
- files.append(("mask", mask_binary))
+ files.append(("mask", ("mask.png", mask_img_byte_arr, "image/png")))
# Build the operation
operation = SynchronousOperation(
@@ -537,9 +533,9 @@ class OpenAIGPTImage1(ComfyNodeABC):
auth_kwargs=kwargs,
)
- response = operation.execute()
+ response = await operation.execute()
- img_tensor = validate_and_cast_response(response, node_id=unique_id)
+ img_tensor = await validate_and_cast_response(response, node_id=unique_id)
return (img_tensor,)
@@ -623,7 +619,7 @@ class OpenAIChatNode(OpenAITextNode):
DESCRIPTION = "Generate text responses from an OpenAI model."
- def get_result_response(
+ async def get_result_response(
self,
response_id: str,
include: Optional[list[Includable]] = None,
@@ -639,7 +635,7 @@ class OpenAIChatNode(OpenAITextNode):
creation above for more information.
"""
- return PollingOperation(
+ return await PollingOperation(
poll_endpoint=ApiEndpoint(
path=f"{RESPONSES_ENDPOINT}/{response_id}",
method=HttpMethod.GET,
@@ -784,7 +780,7 @@ class OpenAIChatNode(OpenAITextNode):
self.history[session_id] = new_history
- def api_call(
+ async def api_call(
self,
prompt: str,
persist_context: bool,
@@ -815,7 +811,7 @@ class OpenAIChatNode(OpenAITextNode):
previous_response_id = None
# Create response
- create_response = SynchronousOperation(
+ create_response = await SynchronousOperation(
endpoint=ApiEndpoint(
path=RESPONSES_ENDPOINT,
method=HttpMethod.POST,
@@ -848,7 +844,7 @@ class OpenAIChatNode(OpenAITextNode):
response_id = create_response.id
# Get result output
- result_response = self.get_result_response(response_id, auth_kwargs=kwargs)
+ result_response = await self.get_result_response(response_id, auth_kwargs=kwargs)
output_text = self.parse_output_text_from_response(result_response)
# Update history
diff --git a/comfy_api_nodes/nodes_pika.py b/comfy_api_nodes/nodes_pika.py
index 1cc708564..a8dc43cb3 100644
--- a/comfy_api_nodes/nodes_pika.py
+++ b/comfy_api_nodes/nodes_pika.py
@@ -122,7 +122,7 @@ class PikaNodeBase(ComfyNodeABC):
FUNCTION = "api_call"
RETURN_TYPES = ("VIDEO",)
- def poll_for_task_status(
+ async def poll_for_task_status(
self,
task_id: str,
auth_kwargs: Optional[dict[str, str]] = None,
@@ -152,9 +152,9 @@ class PikaNodeBase(ComfyNodeABC):
node_id=node_id,
estimated_duration=60
)
- return polling_operation.execute()
+ return await polling_operation.execute()
- def execute_task(
+ async def execute_task(
self,
initial_operation: SynchronousOperation[R, PikaGenerateResponse],
auth_kwargs: Optional[dict[str, str]] = None,
@@ -169,14 +169,14 @@ class PikaNodeBase(ComfyNodeABC):
Returns:
A tuple containing the video file as a VIDEO output.
"""
- initial_response = initial_operation.execute()
+ initial_response = await initial_operation.execute()
if not is_valid_initial_response(initial_response):
error_msg = f"Pika initial request failed. Code: {initial_response.code}, Message: {initial_response.message}, Data: {initial_response.data}"
logging.error(error_msg)
raise PikaApiError(error_msg)
task_id = initial_response.video_id
- final_response = self.poll_for_task_status(task_id, auth_kwargs)
+ final_response = await self.poll_for_task_status(task_id, auth_kwargs)
if not is_valid_video_response(final_response):
error_msg = (
f"Pika task {task_id} succeeded but no video data found in response."
@@ -187,7 +187,7 @@ class PikaNodeBase(ComfyNodeABC):
video_url = str(final_response.url)
logging.info("Pika task %s succeeded. Video URL: %s", task_id, video_url)
- return (download_url_to_video_output(video_url),)
+ return (await download_url_to_video_output(video_url),)
class PikaImageToVideoV2_2(PikaNodeBase):
@@ -212,7 +212,7 @@ class PikaImageToVideoV2_2(PikaNodeBase):
DESCRIPTION = "Sends an image and prompt to the Pika API v2.2 to generate a video."
- def api_call(
+ async def api_call(
self,
image: torch.Tensor,
prompt_text: str,
@@ -251,7 +251,7 @@ class PikaImageToVideoV2_2(PikaNodeBase):
auth_kwargs=kwargs,
)
- return self.execute_task(initial_operation, auth_kwargs=kwargs, node_id=unique_id)
+ return await self.execute_task(initial_operation, auth_kwargs=kwargs, node_id=unique_id)
class PikaTextToVideoNodeV2_2(PikaNodeBase):
@@ -281,7 +281,7 @@ class PikaTextToVideoNodeV2_2(PikaNodeBase):
DESCRIPTION = "Sends a text prompt to the Pika API v2.2 to generate a video."
- def api_call(
+ async def api_call(
self,
prompt_text: str,
negative_prompt: str,
@@ -311,7 +311,7 @@ class PikaTextToVideoNodeV2_2(PikaNodeBase):
content_type="application/x-www-form-urlencoded",
)
- return self.execute_task(initial_operation, auth_kwargs=kwargs, node_id=unique_id)
+ return await self.execute_task(initial_operation, auth_kwargs=kwargs, node_id=unique_id)
class PikaScenesV2_2(PikaNodeBase):
@@ -361,7 +361,7 @@ class PikaScenesV2_2(PikaNodeBase):
DESCRIPTION = "Combine your images to create a video with the objects in them. Upload multiple images as ingredients and generate a high-quality video that incorporates all of them."
- def api_call(
+ async def api_call(
self,
prompt_text: str,
negative_prompt: str,
@@ -420,7 +420,7 @@ class PikaScenesV2_2(PikaNodeBase):
auth_kwargs=kwargs,
)
- return self.execute_task(initial_operation, auth_kwargs=kwargs, node_id=unique_id)
+ return await self.execute_task(initial_operation, auth_kwargs=kwargs, node_id=unique_id)
class PikAdditionsNode(PikaNodeBase):
@@ -462,7 +462,7 @@ class PikAdditionsNode(PikaNodeBase):
DESCRIPTION = "Add any object or image into your video. Upload a video and specify what you'd like to add to create a seamlessly integrated result."
- def api_call(
+ async def api_call(
self,
video: VideoInput,
image: torch.Tensor,
@@ -481,10 +481,10 @@ class PikAdditionsNode(PikaNodeBase):
image_bytes_io = tensor_to_bytesio(image)
image_bytes_io.seek(0)
- pika_files = [
- ("video", ("video.mp4", video_bytes_io, "video/mp4")),
- ("image", ("image.png", image_bytes_io, "image/png")),
- ]
+ pika_files = {
+ "video": ("video.mp4", video_bytes_io, "video/mp4"),
+ "image": ("image.png", image_bytes_io, "image/png"),
+ }
# Prepare non-file data
pika_request_data = PikaBodyGeneratePikadditionsGeneratePikadditionsPost(
@@ -506,7 +506,7 @@ class PikAdditionsNode(PikaNodeBase):
auth_kwargs=kwargs,
)
- return self.execute_task(initial_operation, auth_kwargs=kwargs, node_id=unique_id)
+ return await self.execute_task(initial_operation, auth_kwargs=kwargs, node_id=unique_id)
class PikaSwapsNode(PikaNodeBase):
@@ -558,7 +558,7 @@ class PikaSwapsNode(PikaNodeBase):
DESCRIPTION = "Swap out any object or region of your video with a new image or object. Define areas to replace either with a mask or coordinates."
RETURN_TYPES = ("VIDEO",)
- def api_call(
+ async def api_call(
self,
video: VideoInput,
image: torch.Tensor,
@@ -587,11 +587,11 @@ class PikaSwapsNode(PikaNodeBase):
image_bytes_io = tensor_to_bytesio(image)
image_bytes_io.seek(0)
- pika_files = [
- ("video", ("video.mp4", video_bytes_io, "video/mp4")),
- ("image", ("image.png", image_bytes_io, "image/png")),
- ("modifyRegionMask", ("mask.png", mask_bytes_io, "image/png")),
- ]
+ pika_files = {
+ "video": ("video.mp4", video_bytes_io, "video/mp4"),
+ "image": ("image.png", image_bytes_io, "image/png"),
+ "modifyRegionMask": ("mask.png", mask_bytes_io, "image/png"),
+ }
# Prepare non-file data
pika_request_data = PikaBodyGeneratePikaswapsGeneratePikaswapsPost(
@@ -613,7 +613,7 @@ class PikaSwapsNode(PikaNodeBase):
auth_kwargs=kwargs,
)
- return self.execute_task(initial_operation, auth_kwargs=kwargs, node_id=unique_id)
+ return await self.execute_task(initial_operation, auth_kwargs=kwargs, node_id=unique_id)
class PikaffectsNode(PikaNodeBase):
@@ -664,7 +664,7 @@ class PikaffectsNode(PikaNodeBase):
DESCRIPTION = "Generate a video with a specific Pikaffect. Supported Pikaffects: Cake-ify, Crumble, Crush, Decapitate, Deflate, Dissolve, Explode, Eye-pop, Inflate, Levitate, Melt, Peel, Poke, Squish, Ta-da, Tear"
- def api_call(
+ async def api_call(
self,
image: torch.Tensor,
pikaffect: str,
@@ -693,7 +693,7 @@ class PikaffectsNode(PikaNodeBase):
auth_kwargs=kwargs,
)
- return self.execute_task(initial_operation, auth_kwargs=kwargs, node_id=unique_id)
+ return await self.execute_task(initial_operation, auth_kwargs=kwargs, node_id=unique_id)
class PikaStartEndFrameNode2_2(PikaNodeBase):
@@ -718,7 +718,7 @@ class PikaStartEndFrameNode2_2(PikaNodeBase):
DESCRIPTION = "Generate a video by combining your first and last frame. Upload two images to define the start and end points, and let the AI create a smooth transition between them."
- def api_call(
+ async def api_call(
self,
image_start: torch.Tensor,
image_end: torch.Tensor,
@@ -732,10 +732,7 @@ class PikaStartEndFrameNode2_2(PikaNodeBase):
) -> tuple[VideoFromFile]:
pika_files = [
- (
- "keyFrames",
- ("image_start.png", tensor_to_bytesio(image_start), "image/png"),
- ),
+ ("keyFrames", ("image_start.png", tensor_to_bytesio(image_start), "image/png")),
("keyFrames", ("image_end.png", tensor_to_bytesio(image_end), "image/png")),
]
@@ -758,7 +755,7 @@ class PikaStartEndFrameNode2_2(PikaNodeBase):
auth_kwargs=kwargs,
)
- return self.execute_task(initial_operation, auth_kwargs=kwargs, node_id=unique_id)
+ return await self.execute_task(initial_operation, auth_kwargs=kwargs, node_id=unique_id)
NODE_CLASS_MAPPINGS = {
diff --git a/comfy_api_nodes/nodes_pixverse.py b/comfy_api_nodes/nodes_pixverse.py
index ef4a9a802..7c5a52feb 100644
--- a/comfy_api_nodes/nodes_pixverse.py
+++ b/comfy_api_nodes/nodes_pixverse.py
@@ -30,7 +30,7 @@ from comfy.comfy_types.node_typing import IO, ComfyNodeABC
from comfy_api.input_impl import VideoFromFile
import torch
-import requests
+import aiohttp
from io import BytesIO
@@ -47,7 +47,7 @@ def get_video_url_from_response(
return str(response.Resp.url)
-def upload_image_to_pixverse(image: torch.Tensor, auth_kwargs=None):
+async def upload_image_to_pixverse(image: torch.Tensor, auth_kwargs=None):
# first, upload image to Pixverse and get image id to use in actual generation call
files = {"image": tensor_to_bytesio(image)}
operation = SynchronousOperation(
@@ -62,7 +62,7 @@ def upload_image_to_pixverse(image: torch.Tensor, auth_kwargs=None):
content_type="multipart/form-data",
auth_kwargs=auth_kwargs,
)
- response_upload: PixverseImageUploadResponse = operation.execute()
+ response_upload: PixverseImageUploadResponse = await operation.execute()
if response_upload.Resp is None:
raise Exception(
@@ -164,7 +164,7 @@ class PixverseTextToVideoNode(ComfyNodeABC):
},
}
- def api_call(
+ async def api_call(
self,
prompt: str,
aspect_ratio: str,
@@ -205,7 +205,7 @@ class PixverseTextToVideoNode(ComfyNodeABC):
),
auth_kwargs=kwargs,
)
- response_api = operation.execute()
+ response_api = await operation.execute()
if response_api.Resp is None:
raise Exception(f"PixVerse request failed: '{response_api.ErrMsg}'")
@@ -229,11 +229,11 @@ class PixverseTextToVideoNode(ComfyNodeABC):
result_url_extractor=get_video_url_from_response,
estimated_duration=AVERAGE_DURATION_T2V,
)
- response_poll = operation.execute()
+ response_poll = await operation.execute()
- vid_response = requests.get(response_poll.Resp.url)
-
- return (VideoFromFile(BytesIO(vid_response.content)),)
+ async with aiohttp.ClientSession() as session:
+ async with session.get(response_poll.Resp.url) as vid_response:
+ return (VideoFromFile(BytesIO(await vid_response.content.read())),)
class PixverseImageToVideoNode(ComfyNodeABC):
@@ -302,7 +302,7 @@ class PixverseImageToVideoNode(ComfyNodeABC):
},
}
- def api_call(
+ async def api_call(
self,
image: torch.Tensor,
prompt: str,
@@ -316,7 +316,7 @@ class PixverseImageToVideoNode(ComfyNodeABC):
**kwargs,
):
validate_string(prompt, strip_whitespace=False)
- img_id = upload_image_to_pixverse(image, auth_kwargs=kwargs)
+ img_id = await upload_image_to_pixverse(image, auth_kwargs=kwargs)
# 1080p is limited to 5 seconds duration
# only normal motion_mode supported for 1080p or for non-5 second duration
@@ -345,7 +345,7 @@ class PixverseImageToVideoNode(ComfyNodeABC):
),
auth_kwargs=kwargs,
)
- response_api = operation.execute()
+ response_api = await operation.execute()
if response_api.Resp is None:
raise Exception(f"PixVerse request failed: '{response_api.ErrMsg}'")
@@ -369,10 +369,11 @@ class PixverseImageToVideoNode(ComfyNodeABC):
result_url_extractor=get_video_url_from_response,
estimated_duration=AVERAGE_DURATION_I2V,
)
- response_poll = operation.execute()
+ response_poll = await operation.execute()
- vid_response = requests.get(response_poll.Resp.url)
- return (VideoFromFile(BytesIO(vid_response.content)),)
+ async with aiohttp.ClientSession() as session:
+ async with session.get(response_poll.Resp.url) as vid_response:
+ return (VideoFromFile(BytesIO(await vid_response.content.read())),)
class PixverseTransitionVideoNode(ComfyNodeABC):
@@ -436,7 +437,7 @@ class PixverseTransitionVideoNode(ComfyNodeABC):
},
}
- def api_call(
+ async def api_call(
self,
first_frame: torch.Tensor,
last_frame: torch.Tensor,
@@ -450,8 +451,8 @@ class PixverseTransitionVideoNode(ComfyNodeABC):
**kwargs,
):
validate_string(prompt, strip_whitespace=False)
- first_frame_id = upload_image_to_pixverse(first_frame, auth_kwargs=kwargs)
- last_frame_id = upload_image_to_pixverse(last_frame, auth_kwargs=kwargs)
+ first_frame_id = await upload_image_to_pixverse(first_frame, auth_kwargs=kwargs)
+ last_frame_id = await upload_image_to_pixverse(last_frame, auth_kwargs=kwargs)
# 1080p is limited to 5 seconds duration
# only normal motion_mode supported for 1080p or for non-5 second duration
@@ -480,7 +481,7 @@ class PixverseTransitionVideoNode(ComfyNodeABC):
),
auth_kwargs=kwargs,
)
- response_api = operation.execute()
+ response_api = await operation.execute()
if response_api.Resp is None:
raise Exception(f"PixVerse request failed: '{response_api.ErrMsg}'")
@@ -504,10 +505,11 @@ class PixverseTransitionVideoNode(ComfyNodeABC):
result_url_extractor=get_video_url_from_response,
estimated_duration=AVERAGE_DURATION_T2V,
)
- response_poll = operation.execute()
+ response_poll = await operation.execute()
- vid_response = requests.get(response_poll.Resp.url)
- return (VideoFromFile(BytesIO(vid_response.content)),)
+ async with aiohttp.ClientSession() as session:
+ async with session.get(response_poll.Resp.url) as vid_response:
+ return (VideoFromFile(BytesIO(await vid_response.content.read())),)
NODE_CLASS_MAPPINGS = {
diff --git a/comfy_api_nodes/nodes_recraft.py b/comfy_api_nodes/nodes_recraft.py
index e369c4b7e..c8516b368 100644
--- a/comfy_api_nodes/nodes_recraft.py
+++ b/comfy_api_nodes/nodes_recraft.py
@@ -37,7 +37,7 @@ from io import BytesIO
from PIL import UnidentifiedImageError
-def handle_recraft_file_request(
+async def handle_recraft_file_request(
image: torch.Tensor,
path: str,
mask: torch.Tensor=None,
@@ -71,13 +71,13 @@ def handle_recraft_file_request(
auth_kwargs=auth_kwargs,
multipart_parser=recraft_multipart_parser,
)
- response: RecraftImageGenerationResponse = operation.execute()
+ response: RecraftImageGenerationResponse = await operation.execute()
all_bytesio = []
if response.image is not None:
- all_bytesio.append(download_url_to_bytesio(response.image.url, timeout=timeout))
+ all_bytesio.append(await download_url_to_bytesio(response.image.url, timeout=timeout))
else:
for data in response.data:
- all_bytesio.append(download_url_to_bytesio(data.url, timeout=timeout))
+ all_bytesio.append(await download_url_to_bytesio(data.url, timeout=timeout))
return all_bytesio
@@ -395,7 +395,7 @@ class RecraftTextToImageNode:
},
}
- def api_call(
+ async def api_call(
self,
prompt: str,
size: str,
@@ -439,7 +439,7 @@ class RecraftTextToImageNode:
),
auth_kwargs=kwargs,
)
- response: RecraftImageGenerationResponse = operation.execute()
+ response: RecraftImageGenerationResponse = await operation.execute()
images = []
urls = []
for data in response.data:
@@ -451,7 +451,7 @@ class RecraftTextToImageNode:
f"Result URL: {urls_string}", unique_id
)
image = bytesio_to_image_tensor(
- download_url_to_bytesio(data.url, timeout=1024)
+ await download_url_to_bytesio(data.url, timeout=1024)
)
if len(image.shape) < 4:
image = image.unsqueeze(0)
@@ -538,7 +538,7 @@ class RecraftImageToImageNode:
},
}
- def api_call(
+ async def api_call(
self,
image: torch.Tensor,
prompt: str,
@@ -578,7 +578,7 @@ class RecraftImageToImageNode:
total = image.shape[0]
pbar = ProgressBar(total)
for i in range(total):
- sub_bytes = handle_recraft_file_request(
+ sub_bytes = await handle_recraft_file_request(
image=image[i],
path="/proxy/recraft/images/imageToImage",
request=request,
@@ -654,7 +654,7 @@ class RecraftImageInpaintingNode:
},
}
- def api_call(
+ async def api_call(
self,
image: torch.Tensor,
mask: torch.Tensor,
@@ -690,7 +690,7 @@ class RecraftImageInpaintingNode:
total = image.shape[0]
pbar = ProgressBar(total)
for i in range(total):
- sub_bytes = handle_recraft_file_request(
+ sub_bytes = await handle_recraft_file_request(
image=image[i],
mask=mask[i:i+1],
path="/proxy/recraft/images/inpaint",
@@ -779,7 +779,7 @@ class RecraftTextToVectorNode:
},
}
- def api_call(
+ async def api_call(
self,
prompt: str,
substyle: str,
@@ -821,7 +821,7 @@ class RecraftTextToVectorNode:
),
auth_kwargs=kwargs,
)
- response: RecraftImageGenerationResponse = operation.execute()
+ response: RecraftImageGenerationResponse = await operation.execute()
svg_data = []
urls = []
for data in response.data:
@@ -831,7 +831,7 @@ class RecraftTextToVectorNode:
PromptServer.instance.send_progress_text(
f"Result URL: {' '.join(urls)}", unique_id
)
- svg_data.append(download_url_to_bytesio(data.url, timeout=1024))
+ svg_data.append(await download_url_to_bytesio(data.url, timeout=1024))
return (SVG(svg_data),)
@@ -861,7 +861,7 @@ class RecraftVectorizeImageNode:
},
}
- def api_call(
+ async def api_call(
self,
image: torch.Tensor,
**kwargs,
@@ -870,7 +870,7 @@ class RecraftVectorizeImageNode:
total = image.shape[0]
pbar = ProgressBar(total)
for i in range(total):
- sub_bytes = handle_recraft_file_request(
+ sub_bytes = await handle_recraft_file_request(
image=image[i],
path="/proxy/recraft/images/vectorize",
auth_kwargs=kwargs,
@@ -942,7 +942,7 @@ class RecraftReplaceBackgroundNode:
},
}
- def api_call(
+ async def api_call(
self,
image: torch.Tensor,
prompt: str,
@@ -973,7 +973,7 @@ class RecraftReplaceBackgroundNode:
total = image.shape[0]
pbar = ProgressBar(total)
for i in range(total):
- sub_bytes = handle_recraft_file_request(
+ sub_bytes = await handle_recraft_file_request(
image=image[i],
path="/proxy/recraft/images/replaceBackground",
request=request,
@@ -1011,7 +1011,7 @@ class RecraftRemoveBackgroundNode:
},
}
- def api_call(
+ async def api_call(
self,
image: torch.Tensor,
**kwargs,
@@ -1020,7 +1020,7 @@ class RecraftRemoveBackgroundNode:
total = image.shape[0]
pbar = ProgressBar(total)
for i in range(total):
- sub_bytes = handle_recraft_file_request(
+ sub_bytes = await handle_recraft_file_request(
image=image[i],
path="/proxy/recraft/images/removeBackground",
auth_kwargs=kwargs,
@@ -1062,7 +1062,7 @@ class RecraftCrispUpscaleNode:
},
}
- def api_call(
+ async def api_call(
self,
image: torch.Tensor,
**kwargs,
@@ -1071,7 +1071,7 @@ class RecraftCrispUpscaleNode:
total = image.shape[0]
pbar = ProgressBar(total)
for i in range(total):
- sub_bytes = handle_recraft_file_request(
+ sub_bytes = await handle_recraft_file_request(
image=image[i],
path=self.RECRAFT_PATH,
auth_kwargs=kwargs,
diff --git a/comfy_api_nodes/nodes_rodin.py b/comfy_api_nodes/nodes_rodin.py
index 67f90478c..c89d087e5 100644
--- a/comfy_api_nodes/nodes_rodin.py
+++ b/comfy_api_nodes/nodes_rodin.py
@@ -9,11 +9,10 @@ from __future__ import annotations
from inspect import cleandoc
from comfy.comfy_types.node_typing import IO
import folder_paths as comfy_paths
-import requests
+import aiohttp
import os
import datetime
-import shutil
-import time
+import asyncio
import io
import logging
import math
@@ -66,7 +65,6 @@ def create_task_error(response: Rodin3DGenerateResponse):
return hasattr(response, "error")
-
class Rodin3DAPI:
"""
Generate 3D Assets using Rodin API
@@ -123,8 +121,8 @@ class Rodin3DAPI:
else:
return "Generating"
- def CreateGenerateTask(self, images=None, seed=1, material="PBR", quality="medium", tier="Regular", mesh_mode="Quad", **kwargs):
- if images == None:
+ async def create_generate_task(self, images=None, seed=1, material="PBR", quality="medium", tier="Regular", mesh_mode="Quad", **kwargs):
+ if images is None:
raise Exception("Rodin 3D generate requires at least 1 image.")
if len(images) >= 5:
raise Exception("Rodin 3D generate requires up to 5 image.")
@@ -155,7 +153,7 @@ class Rodin3DAPI:
auth_kwargs=kwargs,
)
- response = operation.execute()
+ response = await operation.execute()
if create_task_error(response):
error_message = f"Rodin3D Create 3D generate Task Failed. Message: {response.message}, error: {response.error}"
@@ -168,7 +166,7 @@ class Rodin3DAPI:
logging.info(f"[ Rodin3D API - Submit Jobs ] UUID: {task_uuid}")
return task_uuid, subscription_key
- def poll_for_task_status(self, subscription_key, **kwargs) -> Rodin3DCheckStatusResponse:
+ async def poll_for_task_status(self, subscription_key, **kwargs) -> Rodin3DCheckStatusResponse:
path = "/proxy/rodin/api/v2/status"
@@ -191,11 +189,9 @@ class Rodin3DAPI:
logging.info("[ Rodin3D API - CheckStatus ] Generate Start!")
- return poll_operation.execute()
+ return await poll_operation.execute()
-
-
- def GetRodinDownloadList(self, uuid, **kwargs) -> Rodin3DDownloadResponse:
+ async def get_rodin_download_list(self, uuid, **kwargs) -> Rodin3DDownloadResponse:
logging.info("[ Rodin3D API - Downloading ] Generate Successfully!")
path = "/proxy/rodin/api/v2/download"
@@ -212,53 +208,59 @@ class Rodin3DAPI:
auth_kwargs=kwargs
)
- return operation.execute()
+ return await operation.execute()
- def GetQualityAndMode(self, PolyCount):
- if PolyCount == "200K-Triangle":
+ def get_quality_mode(self, poly_count):
+ if poly_count == "200K-Triangle":
mesh_mode = "Raw"
quality = "medium"
else:
mesh_mode = "Quad"
- if PolyCount == "4K-Quad":
+ if poly_count == "4K-Quad":
quality = "extra-low"
- elif PolyCount == "8K-Quad":
+ elif poly_count == "8K-Quad":
quality = "low"
- elif PolyCount == "18K-Quad":
+ elif poly_count == "18K-Quad":
quality = "medium"
- elif PolyCount == "50K-Quad":
+ elif poly_count == "50K-Quad":
quality = "high"
else:
quality = "medium"
return mesh_mode, quality
- def DownLoadFiles(self, Url_List):
- Save_path = os.path.join(comfy_paths.get_output_directory(), "Rodin3D", datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S"))
- os.makedirs(Save_path, exist_ok=True)
+ async def download_files(self, url_list):
+ save_path = os.path.join(comfy_paths.get_output_directory(), "Rodin3D", datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S"))
+ os.makedirs(save_path, exist_ok=True)
model_file_path = None
- for Item in Url_List.list:
- url = Item.url
- file_name = Item.name
- file_path = os.path.join(Save_path, file_name)
- if file_path.endswith(".glb"):
- model_file_path = file_path
- logging.info(f"[ Rodin3D API - download_files ] Downloading file: {file_path}")
- max_retries = 5
- for attempt in range(max_retries):
- try:
- with requests.get(url, stream=True) as r:
- r.raise_for_status()
- with open(file_path, "wb") as f:
- shutil.copyfileobj(r.raw, f)
- break
- except Exception as e:
- logging.info(f"[ Rodin3D API - download_files ] Error downloading {file_path}:{e}")
- if attempt < max_retries - 1:
- logging.info("Retrying...")
- time.sleep(2)
- else:
- logging.info(f"[ Rodin3D API - download_files ] Failed to download {file_path} after {max_retries} attempts.")
+ async with aiohttp.ClientSession() as session:
+ for i in url_list.list:
+ url = i.url
+ file_name = i.name
+ file_path = os.path.join(save_path, file_name)
+ if file_path.endswith(".glb"):
+ model_file_path = file_path
+ logging.info(f"[ Rodin3D API - download_files ] Downloading file: {file_path}")
+ max_retries = 5
+ for attempt in range(max_retries):
+ try:
+ async with session.get(url) as resp:
+ resp.raise_for_status()
+ with open(file_path, "wb") as f:
+ async for chunk in resp.content.iter_chunked(32 * 1024):
+ f.write(chunk)
+ break
+ except Exception as e:
+ logging.info(f"[ Rodin3D API - download_files ] Error downloading {file_path}:{e}")
+ if attempt < max_retries - 1:
+ logging.info("Retrying...")
+ await asyncio.sleep(2)
+ else:
+ logging.info(
+ "[ Rodin3D API - download_files ] Failed to download %s after %s attempts.",
+ file_path,
+ max_retries,
+ )
return model_file_path
@@ -285,7 +287,7 @@ class Rodin3D_Regular(Rodin3DAPI):
},
}
- def api_call(
+ async def api_call(
self,
Images,
Seed,
@@ -298,14 +300,17 @@ class Rodin3D_Regular(Rodin3DAPI):
m_images = []
for i in range(num_images):
m_images.append(Images[i])
- mesh_mode, quality = self.GetQualityAndMode(Polygon_count)
- task_uuid, subscription_key = self.CreateGenerateTask(images=m_images, seed=Seed, material=Material_Type, quality=quality, tier=tier, mesh_mode=mesh_mode, **kwargs)
- self.poll_for_task_status(subscription_key, **kwargs)
- Download_List = self.GetRodinDownloadList(task_uuid, **kwargs)
- model = self.DownLoadFiles(Download_List)
+ mesh_mode, quality = self.get_quality_mode(Polygon_count)
+ task_uuid, subscription_key = await self.create_generate_task(images=m_images, seed=Seed, material=Material_Type,
+ quality=quality, tier=tier, mesh_mode=mesh_mode,
+ **kwargs)
+ await self.poll_for_task_status(subscription_key, **kwargs)
+ download_list = await self.get_rodin_download_list(task_uuid, **kwargs)
+ model = await self.download_files(download_list)
return (model,)
+
class Rodin3D_Detail(Rodin3DAPI):
@classmethod
def INPUT_TYPES(s):
@@ -328,7 +333,7 @@ class Rodin3D_Detail(Rodin3DAPI):
},
}
- def api_call(
+ async def api_call(
self,
Images,
Seed,
@@ -341,14 +346,17 @@ class Rodin3D_Detail(Rodin3DAPI):
m_images = []
for i in range(num_images):
m_images.append(Images[i])
- mesh_mode, quality = self.GetQualityAndMode(Polygon_count)
- task_uuid, subscription_key = self.CreateGenerateTask(images=m_images, seed=Seed, material=Material_Type, quality=quality, tier=tier, mesh_mode=mesh_mode, **kwargs)
- self.poll_for_task_status(subscription_key, **kwargs)
- Download_List = self.GetRodinDownloadList(task_uuid, **kwargs)
- model = self.DownLoadFiles(Download_List)
+ mesh_mode, quality = self.get_quality_mode(Polygon_count)
+ task_uuid, subscription_key = await self.create_generate_task(images=m_images, seed=Seed, material=Material_Type,
+ quality=quality, tier=tier, mesh_mode=mesh_mode,
+ **kwargs)
+ await self.poll_for_task_status(subscription_key, **kwargs)
+ download_list = await self.get_rodin_download_list(task_uuid, **kwargs)
+ model = await self.download_files(download_list)
return (model,)
+
class Rodin3D_Smooth(Rodin3DAPI):
@classmethod
def INPUT_TYPES(s):
@@ -371,7 +379,7 @@ class Rodin3D_Smooth(Rodin3DAPI):
},
}
- def api_call(
+ async def api_call(
self,
Images,
Seed,
@@ -384,14 +392,17 @@ class Rodin3D_Smooth(Rodin3DAPI):
m_images = []
for i in range(num_images):
m_images.append(Images[i])
- mesh_mode, quality = self.GetQualityAndMode(Polygon_count)
- task_uuid, subscription_key = self.CreateGenerateTask(images=m_images, seed=Seed, material=Material_Type, quality=quality, tier=tier, mesh_mode=mesh_mode, **kwargs)
- self.poll_for_task_status(subscription_key, **kwargs)
- Download_List = self.GetRodinDownloadList(task_uuid, **kwargs)
- model = self.DownLoadFiles(Download_List)
+ mesh_mode, quality = self.get_quality_mode(Polygon_count)
+ task_uuid, subscription_key = await self.create_generate_task(images=m_images, seed=Seed, material=Material_Type,
+ quality=quality, tier=tier, mesh_mode=mesh_mode,
+ **kwargs)
+ await self.poll_for_task_status(subscription_key, **kwargs)
+ download_list = await self.get_rodin_download_list(task_uuid, **kwargs)
+ model = await self.download_files(download_list)
return (model,)
+
class Rodin3D_Sketch(Rodin3DAPI):
@classmethod
def INPUT_TYPES(s):
@@ -423,7 +434,7 @@ class Rodin3D_Sketch(Rodin3DAPI):
},
}
- def api_call(
+ async def api_call(
self,
Images,
Seed,
@@ -437,10 +448,12 @@ class Rodin3D_Sketch(Rodin3DAPI):
material_type = "PBR"
quality = "medium"
mesh_mode = "Quad"
- task_uuid, subscription_key = self.CreateGenerateTask(images=m_images, seed=Seed, material=material_type, quality=quality, tier=tier, mesh_mode=mesh_mode, **kwargs)
- self.poll_for_task_status(subscription_key, **kwargs)
- Download_List = self.GetRodinDownloadList(task_uuid, **kwargs)
- model = self.DownLoadFiles(Download_List)
+ task_uuid, subscription_key = await self.create_generate_task(
+ images=m_images, seed=Seed, material=material_type, quality=quality, tier=tier, mesh_mode=mesh_mode, **kwargs
+ )
+ await self.poll_for_task_status(subscription_key, **kwargs)
+ download_list = await self.get_rodin_download_list(task_uuid, **kwargs)
+ model = await self.download_files(download_list)
return (model,)
diff --git a/comfy_api_nodes/nodes_runway.py b/comfy_api_nodes/nodes_runway.py
index af4b321f9..98024a9fa 100644
--- a/comfy_api_nodes/nodes_runway.py
+++ b/comfy_api_nodes/nodes_runway.py
@@ -99,14 +99,14 @@ def validate_input_image(image: torch.Tensor) -> bool:
return image.shape[2] < 8000 and image.shape[1] < 8000
-def poll_until_finished(
+async def poll_until_finished(
auth_kwargs: dict[str, str],
api_endpoint: ApiEndpoint[Any, TaskStatusResponse],
estimated_duration: Optional[int] = None,
node_id: Optional[str] = None,
) -> TaskStatusResponse:
"""Polls the Runway API endpoint until the task reaches a terminal state, then returns the response."""
- return PollingOperation(
+ return await PollingOperation(
poll_endpoint=api_endpoint,
completed_statuses=[
TaskStatus.SUCCEEDED.value,
@@ -115,7 +115,7 @@ def poll_until_finished(
TaskStatus.FAILED.value,
TaskStatus.CANCELLED.value,
],
- status_extractor=lambda response: (response.status.value),
+ status_extractor=lambda response: response.status.value,
auth_kwargs=auth_kwargs,
result_url_extractor=get_video_url_from_task_status,
estimated_duration=estimated_duration,
@@ -167,11 +167,11 @@ class RunwayVideoGenNode(ComfyNodeABC):
)
return True
- def get_response(
+ async def get_response(
self, task_id: str, auth_kwargs: dict[str, str], node_id: Optional[str] = None
) -> RunwayImageToVideoResponse:
"""Poll the task status until it is finished then get the response."""
- return poll_until_finished(
+ return await poll_until_finished(
auth_kwargs,
ApiEndpoint(
path=f"{PATH_GET_TASK_STATUS}/{task_id}",
@@ -183,7 +183,7 @@ class RunwayVideoGenNode(ComfyNodeABC):
node_id=node_id,
)
- def generate_video(
+ async def generate_video(
self,
request: RunwayImageToVideoRequest,
auth_kwargs: dict[str, str],
@@ -200,15 +200,15 @@ class RunwayVideoGenNode(ComfyNodeABC):
auth_kwargs=auth_kwargs,
)
- initial_response = initial_operation.execute()
+ initial_response = await initial_operation.execute()
self.validate_task_created(initial_response)
task_id = initial_response.id
- final_response = self.get_response(task_id, auth_kwargs, node_id)
+ final_response = await self.get_response(task_id, auth_kwargs, node_id)
self.validate_response(final_response)
video_url = get_video_url_from_task_status(final_response)
- return (download_url_to_video_output(video_url),)
+ return (await download_url_to_video_output(video_url),)
class RunwayImageToVideoNodeGen3a(RunwayVideoGenNode):
@@ -250,7 +250,7 @@ class RunwayImageToVideoNodeGen3a(RunwayVideoGenNode):
},
}
- def api_call(
+ async def api_call(
self,
prompt: str,
start_frame: torch.Tensor,
@@ -265,7 +265,7 @@ class RunwayImageToVideoNodeGen3a(RunwayVideoGenNode):
validate_input_image(start_frame)
# Upload image
- download_urls = upload_images_to_comfyapi(
+ download_urls = await upload_images_to_comfyapi(
start_frame,
max_images=1,
mime_type="image/png",
@@ -274,7 +274,7 @@ class RunwayImageToVideoNodeGen3a(RunwayVideoGenNode):
if len(download_urls) != 1:
raise RunwayApiError("Failed to upload one or more images to comfy api.")
- return self.generate_video(
+ return await self.generate_video(
RunwayImageToVideoRequest(
promptText=prompt,
seed=seed,
@@ -333,7 +333,7 @@ class RunwayImageToVideoNodeGen4(RunwayVideoGenNode):
},
}
- def api_call(
+ async def api_call(
self,
prompt: str,
start_frame: torch.Tensor,
@@ -348,7 +348,7 @@ class RunwayImageToVideoNodeGen4(RunwayVideoGenNode):
validate_input_image(start_frame)
# Upload image
- download_urls = upload_images_to_comfyapi(
+ download_urls = await upload_images_to_comfyapi(
start_frame,
max_images=1,
mime_type="image/png",
@@ -357,7 +357,7 @@ class RunwayImageToVideoNodeGen4(RunwayVideoGenNode):
if len(download_urls) != 1:
raise RunwayApiError("Failed to upload one or more images to comfy api.")
- return self.generate_video(
+ return await self.generate_video(
RunwayImageToVideoRequest(
promptText=prompt,
seed=seed,
@@ -382,10 +382,10 @@ class RunwayFirstLastFrameNode(RunwayVideoGenNode):
DESCRIPTION = "Upload first and last keyframes, draft a prompt, and generate a video. More complex transitions, such as cases where the Last frame is completely different from the First frame, may benefit from the longer 10s duration. This would give the generation more time to smoothly transition between the two inputs. Before diving in, review these best practices to ensure that your input selections will set your generation up for success: https://help.runwayml.com/hc/en-us/articles/34170748696595-Creating-with-Keyframes-on-Gen-3."
- def get_response(
+ async def get_response(
self, task_id: str, auth_kwargs: dict[str, str], node_id: Optional[str] = None
) -> RunwayImageToVideoResponse:
- return poll_until_finished(
+ return await poll_until_finished(
auth_kwargs,
ApiEndpoint(
path=f"{PATH_GET_TASK_STATUS}/{task_id}",
@@ -437,7 +437,7 @@ class RunwayFirstLastFrameNode(RunwayVideoGenNode):
},
}
- def api_call(
+ async def api_call(
self,
prompt: str,
start_frame: torch.Tensor,
@@ -455,7 +455,7 @@ class RunwayFirstLastFrameNode(RunwayVideoGenNode):
# Upload images
stacked_input_images = image_tensor_pair_to_batch(start_frame, end_frame)
- download_urls = upload_images_to_comfyapi(
+ download_urls = await upload_images_to_comfyapi(
stacked_input_images,
max_images=2,
mime_type="image/png",
@@ -464,7 +464,7 @@ class RunwayFirstLastFrameNode(RunwayVideoGenNode):
if len(download_urls) != 2:
raise RunwayApiError("Failed to upload one or more images to comfy api.")
- return self.generate_video(
+ return await self.generate_video(
RunwayImageToVideoRequest(
promptText=prompt,
seed=seed,
@@ -543,11 +543,11 @@ class RunwayTextToImageNode(ComfyNodeABC):
)
return True
- def get_response(
+ async def get_response(
self, task_id: str, auth_kwargs: dict[str, str], node_id: Optional[str] = None
) -> TaskStatusResponse:
"""Poll the task status until it is finished then get the response."""
- return poll_until_finished(
+ return await poll_until_finished(
auth_kwargs,
ApiEndpoint(
path=f"{PATH_GET_TASK_STATUS}/{task_id}",
@@ -559,7 +559,7 @@ class RunwayTextToImageNode(ComfyNodeABC):
node_id=node_id,
)
- def api_call(
+ async def api_call(
self,
prompt: str,
ratio: str,
@@ -574,7 +574,7 @@ class RunwayTextToImageNode(ComfyNodeABC):
reference_images = None
if reference_image is not None:
validate_input_image(reference_image)
- download_urls = upload_images_to_comfyapi(
+ download_urls = await upload_images_to_comfyapi(
reference_image,
max_images=1,
mime_type="image/png",
@@ -605,19 +605,19 @@ class RunwayTextToImageNode(ComfyNodeABC):
auth_kwargs=kwargs,
)
- initial_response = initial_operation.execute()
+ initial_response = await initial_operation.execute()
self.validate_task_created(initial_response)
task_id = initial_response.id
# Poll for completion
- final_response = self.get_response(
+ final_response = await self.get_response(
task_id, auth_kwargs=kwargs, node_id=unique_id
)
self.validate_response(final_response)
# Download and return image
image_url = get_image_url_from_task_status(final_response)
- return (download_url_to_image_tensor(image_url),)
+ return (await download_url_to_image_tensor(image_url),)
NODE_CLASS_MAPPINGS = {
diff --git a/comfy_api_nodes/nodes_stability.py b/comfy_api_nodes/nodes_stability.py
index 02e421678..31309d831 100644
--- a/comfy_api_nodes/nodes_stability.py
+++ b/comfy_api_nodes/nodes_stability.py
@@ -124,7 +124,7 @@ class StabilityStableImageUltraNode:
},
}
- def api_call(self, prompt: str, aspect_ratio: str, style_preset: str, seed: int,
+ async def api_call(self, prompt: str, aspect_ratio: str, style_preset: str, seed: int,
negative_prompt: str=None, image: torch.Tensor = None, image_denoise: float=None,
**kwargs):
validate_string(prompt, strip_whitespace=False)
@@ -163,7 +163,7 @@ class StabilityStableImageUltraNode:
content_type="multipart/form-data",
auth_kwargs=kwargs,
)
- response_api = operation.execute()
+ response_api = await operation.execute()
if response_api.finish_reason != "SUCCESS":
raise Exception(f"Stable Image Ultra generation failed: {response_api.finish_reason}.")
@@ -257,7 +257,7 @@ class StabilityStableImageSD_3_5Node:
},
}
- def api_call(self, model: str, prompt: str, aspect_ratio: str, style_preset: str, seed: int, cfg_scale: float,
+ async def api_call(self, model: str, prompt: str, aspect_ratio: str, style_preset: str, seed: int, cfg_scale: float,
negative_prompt: str=None, image: torch.Tensor = None, image_denoise: float=None,
**kwargs):
validate_string(prompt, strip_whitespace=False)
@@ -302,7 +302,7 @@ class StabilityStableImageSD_3_5Node:
content_type="multipart/form-data",
auth_kwargs=kwargs,
)
- response_api = operation.execute()
+ response_api = await operation.execute()
if response_api.finish_reason != "SUCCESS":
raise Exception(f"Stable Diffusion 3.5 Image generation failed: {response_api.finish_reason}.")
@@ -374,7 +374,7 @@ class StabilityUpscaleConservativeNode:
},
}
- def api_call(self, image: torch.Tensor, prompt: str, creativity: float, seed: int, negative_prompt: str=None,
+ async def api_call(self, image: torch.Tensor, prompt: str, creativity: float, seed: int, negative_prompt: str=None,
**kwargs):
validate_string(prompt, strip_whitespace=False)
image_binary = tensor_to_bytesio(image, total_pixels=1024*1024).read()
@@ -403,7 +403,7 @@ class StabilityUpscaleConservativeNode:
content_type="multipart/form-data",
auth_kwargs=kwargs,
)
- response_api = operation.execute()
+ response_api = await operation.execute()
if response_api.finish_reason != "SUCCESS":
raise Exception(f"Stability Upscale Conservative generation failed: {response_api.finish_reason}.")
@@ -480,7 +480,7 @@ class StabilityUpscaleCreativeNode:
},
}
- def api_call(self, image: torch.Tensor, prompt: str, creativity: float, style_preset: str, seed: int, negative_prompt: str=None,
+ async def api_call(self, image: torch.Tensor, prompt: str, creativity: float, style_preset: str, seed: int, negative_prompt: str=None,
**kwargs):
validate_string(prompt, strip_whitespace=False)
image_binary = tensor_to_bytesio(image, total_pixels=1024*1024).read()
@@ -512,7 +512,7 @@ class StabilityUpscaleCreativeNode:
content_type="multipart/form-data",
auth_kwargs=kwargs,
)
- response_api = operation.execute()
+ response_api = await operation.execute()
operation = PollingOperation(
poll_endpoint=ApiEndpoint(
@@ -527,7 +527,7 @@ class StabilityUpscaleCreativeNode:
status_extractor=lambda x: get_async_dummy_status(x),
auth_kwargs=kwargs,
)
- response_poll: StabilityResultsGetResponse = operation.execute()
+ response_poll: StabilityResultsGetResponse = await operation.execute()
if response_poll.finish_reason != "SUCCESS":
raise Exception(f"Stability Upscale Creative generation failed: {response_poll.finish_reason}.")
@@ -563,8 +563,7 @@ class StabilityUpscaleFastNode:
},
}
- def api_call(self, image: torch.Tensor,
- **kwargs):
+ async def api_call(self, image: torch.Tensor, **kwargs):
image_binary = tensor_to_bytesio(image, total_pixels=4096*4096).read()
files = {
@@ -583,7 +582,7 @@ class StabilityUpscaleFastNode:
content_type="multipart/form-data",
auth_kwargs=kwargs,
)
- response_api = operation.execute()
+ response_api = await operation.execute()
if response_api.finish_reason != "SUCCESS":
raise Exception(f"Stability Upscale Fast failed: {response_api.finish_reason}.")
diff --git a/comfy_api_nodes/nodes_tripo.py b/comfy_api_nodes/nodes_tripo.py
index 65f3b21f5..d08cf9007 100644
--- a/comfy_api_nodes/nodes_tripo.py
+++ b/comfy_api_nodes/nodes_tripo.py
@@ -37,8 +37,8 @@ from comfy_api_nodes.apinode_utils import (
)
-def upload_image_to_tripo(image, **kwargs):
- urls = upload_images_to_comfyapi(image, max_images=1, auth_kwargs=kwargs)
+async def upload_image_to_tripo(image, **kwargs):
+ urls = await upload_images_to_comfyapi(image, max_images=1, auth_kwargs=kwargs)
return TripoFileReference(TripoUrlReference(url=urls[0], type="jpeg"))
def get_model_url_from_response(response: TripoTaskResponse) -> str:
@@ -49,7 +49,7 @@ def get_model_url_from_response(response: TripoTaskResponse) -> str:
raise RuntimeError(f"Failed to get model url from response: {response}")
-def poll_until_finished(
+async def poll_until_finished(
kwargs: dict[str, str],
response: TripoTaskResponse,
) -> tuple[str, str]:
@@ -57,7 +57,7 @@ def poll_until_finished(
if response.code != 0:
raise RuntimeError(f"Failed to generate mesh: {response.error}")
task_id = response.data.task_id
- response_poll = PollingOperation(
+ response_poll = await PollingOperation(
poll_endpoint=ApiEndpoint(
path=f"/proxy/tripo/v2/openapi/task/{task_id}",
method=HttpMethod.GET,
@@ -80,7 +80,7 @@ def poll_until_finished(
).execute()
if response_poll.data.status == TripoTaskStatus.SUCCESS:
url = get_model_url_from_response(response_poll)
- bytesio = download_url_to_bytesio(url)
+ bytesio = await download_url_to_bytesio(url)
# Save the downloaded model file
model_file = f"tripo_model_{task_id}.glb"
with open(os.path.join(get_output_directory(), model_file), "wb") as f:
@@ -88,6 +88,7 @@ def poll_until_finished(
return model_file, task_id
raise RuntimeError(f"Failed to generate mesh: {response_poll}")
+
class TripoTextToModelNode:
"""
Generates 3D models synchronously based on a text prompt using Tripo's API.
@@ -126,11 +127,11 @@ class TripoTextToModelNode:
API_NODE = True
OUTPUT_NODE = True
- def generate_mesh(self, prompt, negative_prompt=None, model_version=None, style=None, texture=None, pbr=None, image_seed=None, model_seed=None, texture_seed=None, texture_quality=None, face_limit=None, quad=None, **kwargs):
+ async def generate_mesh(self, prompt, negative_prompt=None, model_version=None, style=None, texture=None, pbr=None, image_seed=None, model_seed=None, texture_seed=None, texture_quality=None, face_limit=None, quad=None, **kwargs):
style_enum = None if style == "None" else style
if not prompt:
raise RuntimeError("Prompt is required")
- response = SynchronousOperation(
+ response = await SynchronousOperation(
endpoint=ApiEndpoint(
path="/proxy/tripo/v2/openapi/task",
method=HttpMethod.POST,
@@ -155,7 +156,8 @@ class TripoTextToModelNode:
),
auth_kwargs=kwargs,
).execute()
- return poll_until_finished(kwargs, response)
+ return await poll_until_finished(kwargs, response)
+
class TripoImageToModelNode:
"""
@@ -195,12 +197,12 @@ class TripoImageToModelNode:
API_NODE = True
OUTPUT_NODE = True
- def generate_mesh(self, image, model_version=None, style=None, texture=None, pbr=None, model_seed=None, orientation=None, texture_alignment=None, texture_seed=None, texture_quality=None, face_limit=None, quad=None, **kwargs):
+ async def generate_mesh(self, image, model_version=None, style=None, texture=None, pbr=None, model_seed=None, orientation=None, texture_alignment=None, texture_seed=None, texture_quality=None, face_limit=None, quad=None, **kwargs):
style_enum = None if style == "None" else style
if image is None:
raise RuntimeError("Image is required")
- tripo_file = upload_image_to_tripo(image, **kwargs)
- response = SynchronousOperation(
+ tripo_file = await upload_image_to_tripo(image, **kwargs)
+ response = await SynchronousOperation(
endpoint=ApiEndpoint(
path="/proxy/tripo/v2/openapi/task",
method=HttpMethod.POST,
@@ -225,7 +227,8 @@ class TripoImageToModelNode:
),
auth_kwargs=kwargs,
).execute()
- return poll_until_finished(kwargs, response)
+ return await poll_until_finished(kwargs, response)
+
class TripoMultiviewToModelNode:
"""
@@ -267,7 +270,7 @@ class TripoMultiviewToModelNode:
API_NODE = True
OUTPUT_NODE = True
- def generate_mesh(self, image, image_left=None, image_back=None, image_right=None, model_version=None, orientation=None, texture=None, pbr=None, model_seed=None, texture_seed=None, texture_quality=None, texture_alignment=None, face_limit=None, quad=None, **kwargs):
+ async def generate_mesh(self, image, image_left=None, image_back=None, image_right=None, model_version=None, orientation=None, texture=None, pbr=None, model_seed=None, texture_seed=None, texture_quality=None, texture_alignment=None, face_limit=None, quad=None, **kwargs):
if image is None:
raise RuntimeError("front image for multiview is required")
images = []
@@ -282,11 +285,11 @@ class TripoMultiviewToModelNode:
for image_name in ["image", "image_left", "image_back", "image_right"]:
image_ = image_dict[image_name]
if image_ is not None:
- tripo_file = upload_image_to_tripo(image_, **kwargs)
+ tripo_file = await upload_image_to_tripo(image_, **kwargs)
images.append(tripo_file)
else:
images.append(TripoFileEmptyReference())
- response = SynchronousOperation(
+ response = await SynchronousOperation(
endpoint=ApiEndpoint(
path="/proxy/tripo/v2/openapi/task",
method=HttpMethod.POST,
@@ -309,7 +312,8 @@ class TripoMultiviewToModelNode:
),
auth_kwargs=kwargs,
).execute()
- return poll_until_finished(kwargs, response)
+ return await poll_until_finished(kwargs, response)
+
class TripoTextureNode:
@classmethod
@@ -340,8 +344,8 @@ class TripoTextureNode:
OUTPUT_NODE = True
AVERAGE_DURATION = 80
- def generate_mesh(self, model_task_id, texture=None, pbr=None, texture_seed=None, texture_quality=None, texture_alignment=None, **kwargs):
- response = SynchronousOperation(
+ async def generate_mesh(self, model_task_id, texture=None, pbr=None, texture_seed=None, texture_quality=None, texture_alignment=None, **kwargs):
+ response = await SynchronousOperation(
endpoint=ApiEndpoint(
path="/proxy/tripo/v2/openapi/task",
method=HttpMethod.POST,
@@ -358,7 +362,7 @@ class TripoTextureNode:
),
auth_kwargs=kwargs,
).execute()
- return poll_until_finished(kwargs, response)
+ return await poll_until_finished(kwargs, response)
class TripoRefineNode:
@@ -387,8 +391,8 @@ class TripoRefineNode:
OUTPUT_NODE = True
AVERAGE_DURATION = 240
- def generate_mesh(self, model_task_id, **kwargs):
- response = SynchronousOperation(
+ async def generate_mesh(self, model_task_id, **kwargs):
+ response = await SynchronousOperation(
endpoint=ApiEndpoint(
path="/proxy/tripo/v2/openapi/task",
method=HttpMethod.POST,
@@ -400,7 +404,7 @@ class TripoRefineNode:
),
auth_kwargs=kwargs,
).execute()
- return poll_until_finished(kwargs, response)
+ return await poll_until_finished(kwargs, response)
class TripoRigNode:
@@ -425,8 +429,8 @@ class TripoRigNode:
OUTPUT_NODE = True
AVERAGE_DURATION = 180
- def generate_mesh(self, original_model_task_id, **kwargs):
- response = SynchronousOperation(
+ async def generate_mesh(self, original_model_task_id, **kwargs):
+ response = await SynchronousOperation(
endpoint=ApiEndpoint(
path="/proxy/tripo/v2/openapi/task",
method=HttpMethod.POST,
@@ -440,7 +444,8 @@ class TripoRigNode:
),
auth_kwargs=kwargs,
).execute()
- return poll_until_finished(kwargs, response)
+ return await poll_until_finished(kwargs, response)
+
class TripoRetargetNode:
@classmethod
@@ -475,8 +480,8 @@ class TripoRetargetNode:
OUTPUT_NODE = True
AVERAGE_DURATION = 30
- def generate_mesh(self, animation, original_model_task_id, **kwargs):
- response = SynchronousOperation(
+ async def generate_mesh(self, animation, original_model_task_id, **kwargs):
+ response = await SynchronousOperation(
endpoint=ApiEndpoint(
path="/proxy/tripo/v2/openapi/task",
method=HttpMethod.POST,
@@ -491,7 +496,8 @@ class TripoRetargetNode:
),
auth_kwargs=kwargs,
).execute()
- return poll_until_finished(kwargs, response)
+ return await poll_until_finished(kwargs, response)
+
class TripoConversionNode:
@classmethod
@@ -529,10 +535,10 @@ class TripoConversionNode:
OUTPUT_NODE = True
AVERAGE_DURATION = 30
- def generate_mesh(self, original_model_task_id, format, quad, face_limit, texture_size, texture_format, **kwargs):
+ async def generate_mesh(self, original_model_task_id, format, quad, face_limit, texture_size, texture_format, **kwargs):
if not original_model_task_id:
raise RuntimeError("original_model_task_id is required")
- response = SynchronousOperation(
+ response = await SynchronousOperation(
endpoint=ApiEndpoint(
path="/proxy/tripo/v2/openapi/task",
method=HttpMethod.POST,
@@ -549,7 +555,8 @@ class TripoConversionNode:
),
auth_kwargs=kwargs,
).execute()
- return poll_until_finished(kwargs, response)
+ return await poll_until_finished(kwargs, response)
+
NODE_CLASS_MAPPINGS = {
"TripoTextToModelNode": TripoTextToModelNode,
diff --git a/comfy_api_nodes/nodes_veo2.py b/comfy_api_nodes/nodes_veo2.py
index df846d5dd..e25dab2f5 100644
--- a/comfy_api_nodes/nodes_veo2.py
+++ b/comfy_api_nodes/nodes_veo2.py
@@ -1,17 +1,17 @@
import io
import logging
import base64
-import requests
+import aiohttp
import torch
from typing import Optional
from comfy.comfy_types.node_typing import IO, ComfyNodeABC
from comfy_api.input_impl.video_types import VideoFromFile
from comfy_api_nodes.apis import (
- Veo2GenVidRequest,
- Veo2GenVidResponse,
- Veo2GenVidPollRequest,
- Veo2GenVidPollResponse
+ VeoGenVidRequest,
+ VeoGenVidResponse,
+ VeoGenVidPollRequest,
+ VeoGenVidPollResponse
)
from comfy_api_nodes.apis.client import (
ApiEndpoint,
@@ -35,7 +35,7 @@ def convert_image_to_base64(image: torch.Tensor):
return tensor_to_base64_string(scaled_image)
-def get_video_url_from_response(poll_response: Veo2GenVidPollResponse) -> Optional[str]:
+def get_video_url_from_response(poll_response: VeoGenVidPollResponse) -> Optional[str]:
if (
poll_response.response
and hasattr(poll_response.response, "videos")
@@ -130,6 +130,14 @@ class VeoVideoGenerationNode(ComfyNodeABC):
"default": None,
"tooltip": "Optional reference image to guide video generation",
}),
+ "model": (
+ IO.COMBO,
+ {
+ "options": ["veo-2.0-generate-001"],
+ "default": "veo-2.0-generate-001",
+ "tooltip": "Veo 2 model to use for video generation",
+ },
+ ),
},
"hidden": {
"auth_token": "AUTH_TOKEN_COMFY_ORG",
@@ -141,10 +149,10 @@ class VeoVideoGenerationNode(ComfyNodeABC):
RETURN_TYPES = (IO.VIDEO,)
FUNCTION = "generate_video"
CATEGORY = "api node/video/Veo"
- DESCRIPTION = "Generates videos from text prompts using Google's Veo API"
+ DESCRIPTION = "Generates videos from text prompts using Google's Veo 2 API"
API_NODE = True
- def generate_video(
+ async def generate_video(
self,
prompt,
aspect_ratio="16:9",
@@ -154,6 +162,8 @@ class VeoVideoGenerationNode(ComfyNodeABC):
person_generation="ALLOW",
seed=0,
image=None,
+ model="veo-2.0-generate-001",
+ generate_audio=False,
unique_id: Optional[str] = None,
**kwargs,
):
@@ -188,23 +198,26 @@ class VeoVideoGenerationNode(ComfyNodeABC):
parameters["negativePrompt"] = negative_prompt
if seed > 0:
parameters["seed"] = seed
+ # Only add generateAudio for Veo 3 models
+ if "veo-3.0" in model:
+ parameters["generateAudio"] = generate_audio
# Initial request to start video generation
initial_operation = SynchronousOperation(
endpoint=ApiEndpoint(
- path="/proxy/veo/generate",
+ path=f"/proxy/veo/{model}/generate",
method=HttpMethod.POST,
- request_model=Veo2GenVidRequest,
- response_model=Veo2GenVidResponse
+ request_model=VeoGenVidRequest,
+ response_model=VeoGenVidResponse
),
- request=Veo2GenVidRequest(
+ request=VeoGenVidRequest(
instances=instances,
parameters=parameters
),
auth_kwargs=kwargs,
)
- initial_response = initial_operation.execute()
+ initial_response = await initial_operation.execute()
operation_name = initial_response.name
logging.info(f"Veo generation started with operation name: {operation_name}")
@@ -223,16 +236,16 @@ class VeoVideoGenerationNode(ComfyNodeABC):
# Define the polling operation
poll_operation = PollingOperation(
poll_endpoint=ApiEndpoint(
- path="/proxy/veo/poll",
+ path=f"/proxy/veo/{model}/poll",
method=HttpMethod.POST,
- request_model=Veo2GenVidPollRequest,
- response_model=Veo2GenVidPollResponse
+ request_model=VeoGenVidPollRequest,
+ response_model=VeoGenVidPollResponse
),
completed_statuses=["completed"],
failed_statuses=[], # No failed statuses, we'll handle errors after polling
status_extractor=status_extractor,
progress_extractor=progress_extractor,
- request=Veo2GenVidPollRequest(
+ request=VeoGenVidPollRequest(
operationName=operation_name
),
auth_kwargs=kwargs,
@@ -243,7 +256,7 @@ class VeoVideoGenerationNode(ComfyNodeABC):
)
# Execute the polling operation
- poll_response = poll_operation.execute()
+ poll_response = await poll_operation.execute()
# Now check for errors in the final response
# Check for error in poll response
@@ -268,7 +281,6 @@ class VeoVideoGenerationNode(ComfyNodeABC):
raise Exception(error_message)
# Extract video data
- video_data = None
if poll_response.response and hasattr(poll_response.response, 'videos') and poll_response.response.videos and len(poll_response.response.videos) > 0:
video = poll_response.response.videos[0]
@@ -278,9 +290,9 @@ class VeoVideoGenerationNode(ComfyNodeABC):
video_data = base64.b64decode(video.bytesBase64Encoded)
elif hasattr(video, 'gcsUri') and video.gcsUri:
# Download from URL
- video_url = video.gcsUri
- video_response = requests.get(video_url)
- video_data = video_response.content
+ async with aiohttp.ClientSession() as session:
+ async with session.get(video.gcsUri) as video_response:
+ video_data = await video_response.content.read()
else:
raise Exception("Video returned but no data or URL was provided")
else:
@@ -298,11 +310,64 @@ class VeoVideoGenerationNode(ComfyNodeABC):
return (VideoFromFile(video_io),)
-# Register the node
+class Veo3VideoGenerationNode(VeoVideoGenerationNode):
+ """
+ Generates videos from text prompts using Google's Veo 3 API.
+
+ Supported models:
+ - veo-3.0-generate-001
+ - veo-3.0-fast-generate-001
+
+ This node extends the base Veo node with Veo 3 specific features including
+ audio generation and fixed 8-second duration.
+ """
+
+ @classmethod
+ def INPUT_TYPES(s):
+ parent_input = super().INPUT_TYPES()
+
+ # Update model options for Veo 3
+ parent_input["optional"]["model"] = (
+ IO.COMBO,
+ {
+ "options": ["veo-3.0-generate-001", "veo-3.0-fast-generate-001"],
+ "default": "veo-3.0-generate-001",
+ "tooltip": "Veo 3 model to use for video generation",
+ },
+ )
+
+ # Add generateAudio parameter
+ parent_input["optional"]["generate_audio"] = (
+ IO.BOOLEAN,
+ {
+ "default": False,
+ "tooltip": "Generate audio for the video. Supported by all Veo 3 models.",
+ }
+ )
+
+ # Update duration constraints for Veo 3 (only 8 seconds supported)
+ parent_input["optional"]["duration_seconds"] = (
+ IO.INT,
+ {
+ "default": 8,
+ "min": 8,
+ "max": 8,
+ "step": 1,
+ "display": "number",
+ "tooltip": "Duration of the output video in seconds (Veo 3 only supports 8 seconds)",
+ },
+ )
+
+ return parent_input
+
+
+# Register the nodes
NODE_CLASS_MAPPINGS = {
"VeoVideoGenerationNode": VeoVideoGenerationNode,
+ "Veo3VideoGenerationNode": Veo3VideoGenerationNode,
}
NODE_DISPLAY_NAME_MAPPINGS = {
- "VeoVideoGenerationNode": "Google Veo2 Video Generation",
+ "VeoVideoGenerationNode": "Google Veo 2 Video Generation",
+ "Veo3VideoGenerationNode": "Google Veo 3 Video Generation",
}
diff --git a/comfy_api_nodes/nodes_vidu.py b/comfy_api_nodes/nodes_vidu.py
new file mode 100644
index 000000000..2f441948c
--- /dev/null
+++ b/comfy_api_nodes/nodes_vidu.py
@@ -0,0 +1,622 @@
+import logging
+from enum import Enum
+from typing import Any, Callable, Optional, Literal, TypeVar
+from typing_extensions import override
+
+import torch
+from pydantic import BaseModel, Field
+
+from comfy_api.latest import ComfyExtension, io as comfy_io
+from comfy_api_nodes.util.validation_utils import (
+ validate_aspect_ratio_closeness,
+ validate_image_dimensions,
+ validate_image_aspect_ratio_range,
+ get_number_of_images,
+)
+from comfy_api_nodes.apis.client import (
+ ApiEndpoint,
+ HttpMethod,
+ SynchronousOperation,
+ PollingOperation,
+ EmptyRequest,
+)
+from comfy_api_nodes.apinode_utils import download_url_to_video_output, upload_images_to_comfyapi
+
+
+VIDU_TEXT_TO_VIDEO = "/proxy/vidu/text2video"
+VIDU_IMAGE_TO_VIDEO = "/proxy/vidu/img2video"
+VIDU_REFERENCE_VIDEO = "/proxy/vidu/reference2video"
+VIDU_START_END_VIDEO = "/proxy/vidu/start-end2video"
+VIDU_GET_GENERATION_STATUS = "/proxy/vidu/tasks/%s/creations"
+
+R = TypeVar("R")
+
+class VideoModelName(str, Enum):
+ vidu_q1 = 'viduq1'
+
+
+class AspectRatio(str, Enum):
+ r_16_9 = "16:9"
+ r_9_16 = "9:16"
+ r_1_1 = "1:1"
+
+
+class Resolution(str, Enum):
+ r_1080p = "1080p"
+
+
+class MovementAmplitude(str, Enum):
+ auto = "auto"
+ small = "small"
+ medium = "medium"
+ large = "large"
+
+
+class TaskCreationRequest(BaseModel):
+ model: VideoModelName = VideoModelName.vidu_q1
+ prompt: Optional[str] = Field(None, max_length=1500)
+ duration: Optional[Literal[5]] = 5
+ seed: Optional[int] = Field(0, ge=0, le=2147483647)
+ aspect_ratio: Optional[AspectRatio] = AspectRatio.r_16_9
+ resolution: Optional[Resolution] = Resolution.r_1080p
+ movement_amplitude: Optional[MovementAmplitude] = MovementAmplitude.auto
+ images: Optional[list[str]] = Field(None, description="Base64 encoded string or image URL")
+
+
+class TaskStatus(str, Enum):
+ created = "created"
+ queueing = "queueing"
+ processing = "processing"
+ success = "success"
+ failed = "failed"
+
+
+class TaskCreationResponse(BaseModel):
+ task_id: str = Field(...)
+ state: TaskStatus = Field(...)
+ created_at: str = Field(...)
+ code: Optional[int] = Field(None, description="Error code")
+
+
+class TaskResult(BaseModel):
+ id: str = Field(..., description="Creation id")
+ url: str = Field(..., description="The URL of the generated results, valid for one hour")
+ cover_url: str = Field(..., description="The cover URL of the generated results, valid for one hour")
+
+
+class TaskStatusResponse(BaseModel):
+ state: TaskStatus = Field(...)
+ err_code: Optional[str] = Field(None)
+ creations: list[TaskResult] = Field(..., description="Generated results")
+
+
+async def poll_until_finished(
+ auth_kwargs: dict[str, str],
+ api_endpoint: ApiEndpoint[Any, R],
+ result_url_extractor: Optional[Callable[[R], str]] = None,
+ estimated_duration: Optional[int] = None,
+ node_id: Optional[str] = None,
+) -> R:
+ return await PollingOperation(
+ poll_endpoint=api_endpoint,
+ completed_statuses=[TaskStatus.success.value],
+ failed_statuses=[TaskStatus.failed.value],
+ status_extractor=lambda response: response.state.value,
+ auth_kwargs=auth_kwargs,
+ result_url_extractor=result_url_extractor,
+ estimated_duration=estimated_duration,
+ node_id=node_id,
+ poll_interval=16.0,
+ max_poll_attempts=256,
+ ).execute()
+
+
+def get_video_url_from_response(response) -> Optional[str]:
+ if response.creations:
+ return response.creations[0].url
+ return None
+
+
+def get_video_from_response(response) -> TaskResult:
+ if not response.creations:
+ error_msg = f"Vidu request does not contain results. State: {response.state}, Error Code: {response.err_code}"
+ logging.info(error_msg)
+ raise RuntimeError(error_msg)
+ logging.info("Vidu task %s succeeded. Video URL: %s", response.creations[0].id, response.creations[0].url)
+ return response.creations[0]
+
+
+async def execute_task(
+ vidu_endpoint: str,
+ auth_kwargs: Optional[dict[str, str]],
+ payload: TaskCreationRequest,
+ estimated_duration: int,
+ node_id: str,
+) -> R:
+ response = await SynchronousOperation(
+ endpoint=ApiEndpoint(
+ path=vidu_endpoint,
+ method=HttpMethod.POST,
+ request_model=TaskCreationRequest,
+ response_model=TaskCreationResponse,
+ ),
+ request=payload,
+ auth_kwargs=auth_kwargs,
+ ).execute()
+ if response.state == TaskStatus.failed:
+ error_msg = f"Vidu request failed. Code: {response.code}"
+ logging.error(error_msg)
+ raise RuntimeError(error_msg)
+ return await poll_until_finished(
+ auth_kwargs,
+ ApiEndpoint(
+ path=VIDU_GET_GENERATION_STATUS % response.task_id,
+ method=HttpMethod.GET,
+ request_model=EmptyRequest,
+ response_model=TaskStatusResponse,
+ ),
+ result_url_extractor=get_video_url_from_response,
+ estimated_duration=estimated_duration,
+ node_id=node_id,
+ )
+
+
+class ViduTextToVideoNode(comfy_io.ComfyNode):
+
+ @classmethod
+ def define_schema(cls):
+ return comfy_io.Schema(
+ node_id="ViduTextToVideoNode",
+ display_name="Vidu Text To Video Generation",
+ category="api node/video/Vidu",
+ description="Generate video from text prompt",
+ inputs=[
+ comfy_io.Combo.Input(
+ "model",
+ options=[model.value for model in VideoModelName],
+ default=VideoModelName.vidu_q1.value,
+ tooltip="Model name",
+ ),
+ comfy_io.String.Input(
+ "prompt",
+ multiline=True,
+ tooltip="A textual description for video generation",
+ ),
+ comfy_io.Int.Input(
+ "duration",
+ default=5,
+ min=5,
+ max=5,
+ step=1,
+ display_mode=comfy_io.NumberDisplay.number,
+ tooltip="Duration of the output video in seconds",
+ optional=True,
+ ),
+ comfy_io.Int.Input(
+ "seed",
+ default=0,
+ min=0,
+ max=2147483647,
+ step=1,
+ display_mode=comfy_io.NumberDisplay.number,
+ control_after_generate=True,
+ tooltip="Seed for video generation (0 for random)",
+ optional=True,
+ ),
+ comfy_io.Combo.Input(
+ "aspect_ratio",
+ options=[model.value for model in AspectRatio],
+ default=AspectRatio.r_16_9.value,
+ tooltip="The aspect ratio of the output video",
+ optional=True,
+ ),
+ comfy_io.Combo.Input(
+ "resolution",
+ options=[model.value for model in Resolution],
+ default=Resolution.r_1080p.value,
+ tooltip="Supported values may vary by model & duration",
+ optional=True,
+ ),
+ comfy_io.Combo.Input(
+ "movement_amplitude",
+ options=[model.value for model in MovementAmplitude],
+ default=MovementAmplitude.auto.value,
+ tooltip="The movement amplitude of objects in the frame",
+ optional=True,
+ ),
+ ],
+ outputs=[
+ comfy_io.Video.Output(),
+ ],
+ hidden=[
+ comfy_io.Hidden.auth_token_comfy_org,
+ comfy_io.Hidden.api_key_comfy_org,
+ comfy_io.Hidden.unique_id,
+ ],
+ is_api_node=True,
+ )
+
+ @classmethod
+ async def execute(
+ cls,
+ model: str,
+ prompt: str,
+ duration: int,
+ seed: int,
+ aspect_ratio: str,
+ resolution: str,
+ movement_amplitude: str,
+ ) -> comfy_io.NodeOutput:
+ if not prompt:
+ raise ValueError("The prompt field is required and cannot be empty.")
+ payload = TaskCreationRequest(
+ model_name=model,
+ prompt=prompt,
+ duration=duration,
+ seed=seed,
+ aspect_ratio=aspect_ratio,
+ resolution=resolution,
+ movement_amplitude=movement_amplitude,
+ )
+ auth = {
+ "auth_token": cls.hidden.auth_token_comfy_org,
+ "comfy_api_key": cls.hidden.api_key_comfy_org,
+ }
+ results = await execute_task(VIDU_TEXT_TO_VIDEO, auth, payload, 320, cls.hidden.unique_id)
+ return comfy_io.NodeOutput(await download_url_to_video_output(get_video_from_response(results).url))
+
+
+class ViduImageToVideoNode(comfy_io.ComfyNode):
+
+ @classmethod
+ def define_schema(cls):
+ return comfy_io.Schema(
+ node_id="ViduImageToVideoNode",
+ display_name="Vidu Image To Video Generation",
+ category="api node/video/Vidu",
+ description="Generate video from image and optional prompt",
+ inputs=[
+ comfy_io.Combo.Input(
+ "model",
+ options=[model.value for model in VideoModelName],
+ default=VideoModelName.vidu_q1.value,
+ tooltip="Model name",
+ ),
+ comfy_io.Image.Input(
+ "image",
+ tooltip="An image to be used as the start frame of the generated video",
+ ),
+ comfy_io.String.Input(
+ "prompt",
+ multiline=True,
+ default="",
+ tooltip="A textual description for video generation",
+ optional=True,
+ ),
+ comfy_io.Int.Input(
+ "duration",
+ default=5,
+ min=5,
+ max=5,
+ step=1,
+ display_mode=comfy_io.NumberDisplay.number,
+ tooltip="Duration of the output video in seconds",
+ optional=True,
+ ),
+ comfy_io.Int.Input(
+ "seed",
+ default=0,
+ min=0,
+ max=2147483647,
+ step=1,
+ display_mode=comfy_io.NumberDisplay.number,
+ control_after_generate=True,
+ tooltip="Seed for video generation (0 for random)",
+ optional=True,
+ ),
+ comfy_io.Combo.Input(
+ "resolution",
+ options=[model.value for model in Resolution],
+ default=Resolution.r_1080p.value,
+ tooltip="Supported values may vary by model & duration",
+ optional=True,
+ ),
+ comfy_io.Combo.Input(
+ "movement_amplitude",
+ options=[model.value for model in MovementAmplitude],
+ default=MovementAmplitude.auto.value,
+ tooltip="The movement amplitude of objects in the frame",
+ optional=True,
+ ),
+ ],
+ outputs=[
+ comfy_io.Video.Output(),
+ ],
+ hidden=[
+ comfy_io.Hidden.auth_token_comfy_org,
+ comfy_io.Hidden.api_key_comfy_org,
+ comfy_io.Hidden.unique_id,
+ ],
+ is_api_node=True,
+ )
+
+ @classmethod
+ async def execute(
+ cls,
+ model: str,
+ image: torch.Tensor,
+ prompt: str,
+ duration: int,
+ seed: int,
+ resolution: str,
+ movement_amplitude: str,
+ ) -> comfy_io.NodeOutput:
+ if get_number_of_images(image) > 1:
+ raise ValueError("Only one input image is allowed.")
+ validate_image_aspect_ratio_range(image, (1, 4), (4, 1))
+ payload = TaskCreationRequest(
+ model_name=model,
+ prompt=prompt,
+ duration=duration,
+ seed=seed,
+ resolution=resolution,
+ movement_amplitude=movement_amplitude,
+ )
+ auth = {
+ "auth_token": cls.hidden.auth_token_comfy_org,
+ "comfy_api_key": cls.hidden.api_key_comfy_org,
+ }
+ payload.images = await upload_images_to_comfyapi(
+ image,
+ max_images=1,
+ mime_type="image/png",
+ auth_kwargs=auth,
+ )
+ results = await execute_task(VIDU_IMAGE_TO_VIDEO, auth, payload, 120, cls.hidden.unique_id)
+ return comfy_io.NodeOutput(await download_url_to_video_output(get_video_from_response(results).url))
+
+
+class ViduReferenceVideoNode(comfy_io.ComfyNode):
+
+ @classmethod
+ def define_schema(cls):
+ return comfy_io.Schema(
+ node_id="ViduReferenceVideoNode",
+ display_name="Vidu Reference To Video Generation",
+ category="api node/video/Vidu",
+ description="Generate video from multiple images and prompt",
+ inputs=[
+ comfy_io.Combo.Input(
+ "model",
+ options=[model.value for model in VideoModelName],
+ default=VideoModelName.vidu_q1.value,
+ tooltip="Model name",
+ ),
+ comfy_io.Image.Input(
+ "images",
+ tooltip="Images to use as references to generate a video with consistent subjects (max 7 images).",
+ ),
+ comfy_io.String.Input(
+ "prompt",
+ multiline=True,
+ tooltip="A textual description for video generation",
+ ),
+ comfy_io.Int.Input(
+ "duration",
+ default=5,
+ min=5,
+ max=5,
+ step=1,
+ display_mode=comfy_io.NumberDisplay.number,
+ tooltip="Duration of the output video in seconds",
+ optional=True,
+ ),
+ comfy_io.Int.Input(
+ "seed",
+ default=0,
+ min=0,
+ max=2147483647,
+ step=1,
+ display_mode=comfy_io.NumberDisplay.number,
+ control_after_generate=True,
+ tooltip="Seed for video generation (0 for random)",
+ optional=True,
+ ),
+ comfy_io.Combo.Input(
+ "aspect_ratio",
+ options=[model.value for model in AspectRatio],
+ default=AspectRatio.r_16_9.value,
+ tooltip="The aspect ratio of the output video",
+ optional=True,
+ ),
+ comfy_io.Combo.Input(
+ "resolution",
+ options=[model.value for model in Resolution],
+ default=Resolution.r_1080p.value,
+ tooltip="Supported values may vary by model & duration",
+ optional=True,
+ ),
+ comfy_io.Combo.Input(
+ "movement_amplitude",
+ options=[model.value for model in MovementAmplitude],
+ default=MovementAmplitude.auto.value,
+ tooltip="The movement amplitude of objects in the frame",
+ optional=True,
+ ),
+ ],
+ outputs=[
+ comfy_io.Video.Output(),
+ ],
+ hidden=[
+ comfy_io.Hidden.auth_token_comfy_org,
+ comfy_io.Hidden.api_key_comfy_org,
+ comfy_io.Hidden.unique_id,
+ ],
+ is_api_node=True,
+ )
+
+ @classmethod
+ async def execute(
+ cls,
+ model: str,
+ images: torch.Tensor,
+ prompt: str,
+ duration: int,
+ seed: int,
+ aspect_ratio: str,
+ resolution: str,
+ movement_amplitude: str,
+ ) -> comfy_io.NodeOutput:
+ if not prompt:
+ raise ValueError("The prompt field is required and cannot be empty.")
+ a = get_number_of_images(images)
+ if a > 7:
+ raise ValueError("Too many images, maximum allowed is 7.")
+ for image in images:
+ validate_image_aspect_ratio_range(image, (1, 4), (4, 1))
+ validate_image_dimensions(image, min_width=128, min_height=128)
+ payload = TaskCreationRequest(
+ model_name=model,
+ prompt=prompt,
+ duration=duration,
+ seed=seed,
+ aspect_ratio=aspect_ratio,
+ resolution=resolution,
+ movement_amplitude=movement_amplitude,
+ )
+ auth = {
+ "auth_token": cls.hidden.auth_token_comfy_org,
+ "comfy_api_key": cls.hidden.api_key_comfy_org,
+ }
+ payload.images = await upload_images_to_comfyapi(
+ images,
+ max_images=7,
+ mime_type="image/png",
+ auth_kwargs=auth,
+ )
+ results = await execute_task(VIDU_REFERENCE_VIDEO, auth, payload, 120, cls.hidden.unique_id)
+ return comfy_io.NodeOutput(await download_url_to_video_output(get_video_from_response(results).url))
+
+
+class ViduStartEndToVideoNode(comfy_io.ComfyNode):
+
+ @classmethod
+ def define_schema(cls):
+ return comfy_io.Schema(
+ node_id="ViduStartEndToVideoNode",
+ display_name="Vidu Start End To Video Generation",
+ category="api node/video/Vidu",
+ description="Generate a video from start and end frames and a prompt",
+ inputs=[
+ comfy_io.Combo.Input(
+ "model",
+ options=[model.value for model in VideoModelName],
+ default=VideoModelName.vidu_q1.value,
+ tooltip="Model name",
+ ),
+ comfy_io.Image.Input(
+ "first_frame",
+ tooltip="Start frame",
+ ),
+ comfy_io.Image.Input(
+ "end_frame",
+ tooltip="End frame",
+ ),
+ comfy_io.String.Input(
+ "prompt",
+ multiline=True,
+ tooltip="A textual description for video generation",
+ optional=True,
+ ),
+ comfy_io.Int.Input(
+ "duration",
+ default=5,
+ min=5,
+ max=5,
+ step=1,
+ display_mode=comfy_io.NumberDisplay.number,
+ tooltip="Duration of the output video in seconds",
+ optional=True,
+ ),
+ comfy_io.Int.Input(
+ "seed",
+ default=0,
+ min=0,
+ max=2147483647,
+ step=1,
+ display_mode=comfy_io.NumberDisplay.number,
+ control_after_generate=True,
+ tooltip="Seed for video generation (0 for random)",
+ optional=True,
+ ),
+ comfy_io.Combo.Input(
+ "resolution",
+ options=[model.value for model in Resolution],
+ default=Resolution.r_1080p.value,
+ tooltip="Supported values may vary by model & duration",
+ optional=True,
+ ),
+ comfy_io.Combo.Input(
+ "movement_amplitude",
+ options=[model.value for model in MovementAmplitude],
+ default=MovementAmplitude.auto.value,
+ tooltip="The movement amplitude of objects in the frame",
+ optional=True,
+ ),
+ ],
+ outputs=[
+ comfy_io.Video.Output(),
+ ],
+ hidden=[
+ comfy_io.Hidden.auth_token_comfy_org,
+ comfy_io.Hidden.api_key_comfy_org,
+ comfy_io.Hidden.unique_id,
+ ],
+ is_api_node=True,
+ )
+
+ @classmethod
+ async def execute(
+ cls,
+ model: str,
+ first_frame: torch.Tensor,
+ end_frame: torch.Tensor,
+ prompt: str,
+ duration: int,
+ seed: int,
+ resolution: str,
+ movement_amplitude: str,
+ ) -> comfy_io.NodeOutput:
+ validate_aspect_ratio_closeness(first_frame, end_frame, min_rel=0.8, max_rel=1.25, strict=False)
+ payload = TaskCreationRequest(
+ model_name=model,
+ prompt=prompt,
+ duration=duration,
+ seed=seed,
+ resolution=resolution,
+ movement_amplitude=movement_amplitude,
+ )
+ auth = {
+ "auth_token": cls.hidden.auth_token_comfy_org,
+ "comfy_api_key": cls.hidden.api_key_comfy_org,
+ }
+ payload.images = [
+ (await upload_images_to_comfyapi(frame, max_images=1, mime_type="image/png", auth_kwargs=auth))[0]
+ for frame in (first_frame, end_frame)
+ ]
+ results = await execute_task(VIDU_START_END_VIDEO, auth, payload, 96, cls.hidden.unique_id)
+ return comfy_io.NodeOutput(await download_url_to_video_output(get_video_from_response(results).url))
+
+
+class ViduExtension(ComfyExtension):
+ @override
+ async def get_node_list(self) -> list[type[comfy_io.ComfyNode]]:
+ return [
+ ViduTextToVideoNode,
+ ViduImageToVideoNode,
+ ViduReferenceVideoNode,
+ ViduStartEndToVideoNode,
+ ]
+
+async def comfy_entrypoint() -> ViduExtension:
+ return ViduExtension()
diff --git a/comfy_api_nodes/util/validation_utils.py b/comfy_api_nodes/util/validation_utils.py
index 031b9fbd3..606b794bf 100644
--- a/comfy_api_nodes/util/validation_utils.py
+++ b/comfy_api_nodes/util/validation_utils.py
@@ -53,6 +53,53 @@ def validate_image_aspect_ratio(
)
+def validate_image_aspect_ratio_range(
+ image: torch.Tensor,
+ min_ratio: tuple[float, float], # e.g. (1, 4)
+ max_ratio: tuple[float, float], # e.g. (4, 1)
+ *,
+ strict: bool = True, # True -> (min, max); False -> [min, max]
+) -> float:
+ a1, b1 = min_ratio
+ a2, b2 = max_ratio
+ if a1 <= 0 or b1 <= 0 or a2 <= 0 or b2 <= 0:
+ raise ValueError("Ratios must be positive, like (1, 4) or (4, 1).")
+ lo, hi = (a1 / b1), (a2 / b2)
+ if lo > hi:
+ lo, hi = hi, lo
+ a1, b1, a2, b2 = a2, b2, a1, b1 # swap only for error text
+ w, h = get_image_dimensions(image)
+ if w <= 0 or h <= 0:
+ raise ValueError(f"Invalid image dimensions: {w}x{h}")
+ ar = w / h
+ ok = (lo < ar < hi) if strict else (lo <= ar <= hi)
+ if not ok:
+ op = "<" if strict else "≤"
+ raise ValueError(f"Image aspect ratio {ar:.6g} is outside allowed range: {a1}:{b1} {op} ratio {op} {a2}:{b2}")
+ return ar
+
+
+def validate_aspect_ratio_closeness(
+ start_img,
+ end_img,
+ min_rel: float,
+ max_rel: float,
+ *,
+ strict: bool = False, # True => exclusive, False => inclusive
+) -> None:
+ w1, h1 = get_image_dimensions(start_img)
+ w2, h2 = get_image_dimensions(end_img)
+ if min(w1, h1, w2, h2) <= 0:
+ raise ValueError("Invalid image dimensions")
+ ar1 = w1 / h1
+ ar2 = w2 / h2
+ # Normalize so it is symmetric (no need to check both ar1/ar2 and ar2/ar1)
+ closeness = max(ar1, ar2) / min(ar1, ar2)
+ limit = max(max_rel, 1.0 / min_rel) # for 0.8..1.25 this is 1.25
+ if (closeness >= limit) if strict else (closeness > limit):
+ raise ValueError(f"Aspect ratios must be close: start/end={ar1/ar2:.4f}, allowed range {min_rel}–{max_rel}.")
+
+
def validate_video_dimensions(
video: VideoInput,
min_width: Optional[int] = None,
@@ -98,3 +145,9 @@ def validate_video_duration(
raise ValueError(
f"Video duration must be at most {max_duration}s, got {duration}s"
)
+
+
+def get_number_of_images(images):
+ if isinstance(images, torch.Tensor):
+ return images.shape[0] if images.ndim >= 4 else 1
+ return len(images)
diff --git a/comfy_extras/nodes_audio.py b/comfy_extras/nodes_audio.py
index a90b31779..3b23f65d8 100644
--- a/comfy_extras/nodes_audio.py
+++ b/comfy_extras/nodes_audio.py
@@ -346,6 +346,24 @@ class LoadAudio:
return "Invalid audio file: {}".format(audio)
return True
+class RecordAudio:
+ @classmethod
+ def INPUT_TYPES(s):
+ return {"required": {"audio": ("AUDIO_RECORD", {})}}
+
+ CATEGORY = "audio"
+
+ RETURN_TYPES = ("AUDIO", )
+ FUNCTION = "load"
+
+ def load(self, audio):
+ audio_path = folder_paths.get_annotated_filepath(audio)
+
+ waveform, sample_rate = torchaudio.load(audio_path)
+ audio = {"waveform": waveform.unsqueeze(0), "sample_rate": sample_rate}
+ return (audio, )
+
+
NODE_CLASS_MAPPINGS = {
"EmptyLatentAudio": EmptyLatentAudio,
"VAEEncodeAudio": VAEEncodeAudio,
@@ -356,6 +374,7 @@ NODE_CLASS_MAPPINGS = {
"LoadAudio": LoadAudio,
"PreviewAudio": PreviewAudio,
"ConditioningStableAudio": ConditioningStableAudio,
+ "RecordAudio": RecordAudio,
}
NODE_DISPLAY_NAME_MAPPINGS = {
@@ -367,4 +386,5 @@ NODE_DISPLAY_NAME_MAPPINGS = {
"SaveAudio": "Save Audio (FLAC)",
"SaveAudioMP3": "Save Audio (MP3)",
"SaveAudioOpus": "Save Audio (Opus)",
+ "RecordAudio": "Record Audio",
}
diff --git a/comfy_extras/nodes_context_windows.py b/comfy_extras/nodes_context_windows.py
new file mode 100644
index 000000000..1c3d9e697
--- /dev/null
+++ b/comfy_extras/nodes_context_windows.py
@@ -0,0 +1,89 @@
+from __future__ import annotations
+from comfy_api.latest import ComfyExtension, io
+import comfy.context_windows
+import nodes
+
+
+class ContextWindowsManualNode(io.ComfyNode):
+ @classmethod
+ def define_schema(cls) -> io.Schema:
+ return io.Schema(
+ node_id="ContextWindowsManual",
+ display_name="Context Windows (Manual)",
+ category="context",
+ description="Manually set context windows.",
+ inputs=[
+ io.Model.Input("model", tooltip="The model to apply context windows to during sampling."),
+ io.Int.Input("context_length", min=1, default=16, tooltip="The length of the context window."),
+ io.Int.Input("context_overlap", min=0, default=4, tooltip="The overlap of the context window."),
+ io.Combo.Input("context_schedule", options=[
+ comfy.context_windows.ContextSchedules.STATIC_STANDARD,
+ comfy.context_windows.ContextSchedules.UNIFORM_STANDARD,
+ comfy.context_windows.ContextSchedules.UNIFORM_LOOPED,
+ comfy.context_windows.ContextSchedules.BATCHED,
+ ], tooltip="The stride of the context window."),
+ io.Int.Input("context_stride", min=1, default=1, tooltip="The stride of the context window; only applicable to uniform schedules."),
+ io.Boolean.Input("closed_loop", default=False, tooltip="Whether to close the context window loop; only applicable to looped schedules."),
+ io.Combo.Input("fuse_method", options=comfy.context_windows.ContextFuseMethods.LIST_STATIC, default=comfy.context_windows.ContextFuseMethods.PYRAMID, tooltip="The method to use to fuse the context windows."),
+ io.Int.Input("dim", min=0, max=5, default=0, tooltip="The dimension to apply the context windows to."),
+ ],
+ outputs=[
+ io.Model.Output(tooltip="The model with context windows applied during sampling."),
+ ],
+ is_experimental=True,
+ )
+
+ @classmethod
+ def execute(cls, model: io.Model.Type, context_length: int, context_overlap: int, context_schedule: str, context_stride: int, closed_loop: bool, fuse_method: str, dim: int) -> io.Model:
+ model = model.clone()
+ model.model_options["context_handler"] = comfy.context_windows.IndexListContextHandler(
+ context_schedule=comfy.context_windows.get_matching_context_schedule(context_schedule),
+ fuse_method=comfy.context_windows.get_matching_fuse_method(fuse_method),
+ context_length=context_length,
+ context_overlap=context_overlap,
+ context_stride=context_stride,
+ closed_loop=closed_loop,
+ dim=dim)
+ # make memory usage calculation only take into account the context window latents
+ comfy.context_windows.create_prepare_sampling_wrapper(model)
+ return io.NodeOutput(model)
+
+class WanContextWindowsManualNode(ContextWindowsManualNode):
+ @classmethod
+ def define_schema(cls) -> io.Schema:
+ schema = super().define_schema()
+ schema.node_id = "WanContextWindowsManual"
+ schema.display_name = "WAN Context Windows (Manual)"
+ schema.description = "Manually set context windows for WAN-like models (dim=2)."
+ schema.inputs = [
+ io.Model.Input("model", tooltip="The model to apply context windows to during sampling."),
+ io.Int.Input("context_length", min=1, max=nodes.MAX_RESOLUTION, step=4, default=81, tooltip="The length of the context window."),
+ io.Int.Input("context_overlap", min=0, default=30, tooltip="The overlap of the context window."),
+ io.Combo.Input("context_schedule", options=[
+ comfy.context_windows.ContextSchedules.STATIC_STANDARD,
+ comfy.context_windows.ContextSchedules.UNIFORM_STANDARD,
+ comfy.context_windows.ContextSchedules.UNIFORM_LOOPED,
+ comfy.context_windows.ContextSchedules.BATCHED,
+ ], tooltip="The stride of the context window."),
+ io.Int.Input("context_stride", min=1, default=1, tooltip="The stride of the context window; only applicable to uniform schedules."),
+ io.Boolean.Input("closed_loop", default=False, tooltip="Whether to close the context window loop; only applicable to looped schedules."),
+ io.Combo.Input("fuse_method", options=comfy.context_windows.ContextFuseMethods.LIST_STATIC, default=comfy.context_windows.ContextFuseMethods.PYRAMID, tooltip="The method to use to fuse the context windows."),
+ ]
+ return schema
+
+ @classmethod
+ def execute(cls, model: io.Model.Type, context_length: int, context_overlap: int, context_schedule: str, context_stride: int, closed_loop: bool, fuse_method: str) -> io.Model:
+ context_length = max(((context_length - 1) // 4) + 1, 1) # at least length 1
+ context_overlap = max(((context_overlap - 1) // 4) + 1, 0) # at least overlap 0
+ return super().execute(model, context_length, context_overlap, context_schedule, context_stride, closed_loop, fuse_method, dim=2)
+
+
+class ContextWindowsExtension(ComfyExtension):
+ async def get_node_list(self) -> list[type[io.ComfyNode]]:
+ return [
+ ContextWindowsManualNode,
+ WanContextWindowsManualNode,
+ ]
+
+def comfy_entrypoint():
+ return ContextWindowsExtension()
diff --git a/comfy_extras/nodes_flux.py b/comfy_extras/nodes_flux.py
index 8a8a17698..c8db75bb3 100644
--- a/comfy_extras/nodes_flux.py
+++ b/comfy_extras/nodes_flux.py
@@ -100,9 +100,28 @@ class FluxKontextImageScale:
return (image, )
+class FluxKontextMultiReferenceLatentMethod:
+ @classmethod
+ def INPUT_TYPES(s):
+ return {"required": {
+ "conditioning": ("CONDITIONING", ),
+ "reference_latents_method": (("offset", "index"), ),
+ }}
+
+ RETURN_TYPES = ("CONDITIONING",)
+ FUNCTION = "append"
+ EXPERIMENTAL = True
+
+ CATEGORY = "advanced/conditioning/flux"
+
+ def append(self, conditioning, reference_latents_method):
+ c = node_helpers.conditioning_set_values(conditioning, {"reference_latents_method": reference_latents_method})
+ return (c, )
+
NODE_CLASS_MAPPINGS = {
"CLIPTextEncodeFlux": CLIPTextEncodeFlux,
"FluxGuidance": FluxGuidance,
"FluxDisableGuidance": FluxDisableGuidance,
"FluxKontextImageScale": FluxKontextImageScale,
+ "FluxKontextMultiReferenceLatentMethod": FluxKontextMultiReferenceLatentMethod,
}
diff --git a/comfy_extras/nodes_lt.py b/comfy_extras/nodes_lt.py
index b5058667a..f82337a67 100644
--- a/comfy_extras/nodes_lt.py
+++ b/comfy_extras/nodes_lt.py
@@ -166,7 +166,7 @@ class LTXVAddGuide:
negative = self.add_keyframe_index(negative, frame_idx, guiding_latent, scale_factors)
mask = torch.full(
- (noise_mask.shape[0], 1, guiding_latent.shape[2], 1, 1),
+ (noise_mask.shape[0], 1, guiding_latent.shape[2], noise_mask.shape[3], noise_mask.shape[4]),
1.0 - strength,
dtype=noise_mask.dtype,
device=noise_mask.device,
diff --git a/comfy_extras/nodes_model_merging_model_specific.py b/comfy_extras/nodes_model_merging_model_specific.py
index 2c93cd84f..55eb3ccfe 100644
--- a/comfy_extras/nodes_model_merging_model_specific.py
+++ b/comfy_extras/nodes_model_merging_model_specific.py
@@ -314,6 +314,29 @@ class ModelMergeCosmosPredict2_14B(comfy_extras.nodes_model_merging.ModelMergeBl
return {"required": arg_dict}
+class ModelMergeQwenImage(comfy_extras.nodes_model_merging.ModelMergeBlocks):
+ CATEGORY = "advanced/model_merging/model_specific"
+
+ @classmethod
+ def INPUT_TYPES(s):
+ arg_dict = { "model1": ("MODEL",),
+ "model2": ("MODEL",)}
+
+ argument = ("FLOAT", {"default": 1.0, "min": 0.0, "max": 1.0, "step": 0.01})
+
+ arg_dict["pos_embeds."] = argument
+ arg_dict["img_in."] = argument
+ arg_dict["txt_norm."] = argument
+ arg_dict["txt_in."] = argument
+ arg_dict["time_text_embed."] = argument
+
+ for i in range(60):
+ arg_dict["transformer_blocks.{}.".format(i)] = argument
+
+ arg_dict["proj_out."] = argument
+
+ return {"required": arg_dict}
+
NODE_CLASS_MAPPINGS = {
"ModelMergeSD1": ModelMergeSD1,
"ModelMergeSD2": ModelMergeSD1, #SD1 and SD2 have the same blocks
@@ -329,4 +352,5 @@ NODE_CLASS_MAPPINGS = {
"ModelMergeWAN2_1": ModelMergeWAN2_1,
"ModelMergeCosmosPredict2_2B": ModelMergeCosmosPredict2_2B,
"ModelMergeCosmosPredict2_14B": ModelMergeCosmosPredict2_14B,
+ "ModelMergeQwenImage": ModelMergeQwenImage,
}
diff --git a/comfy_extras/nodes_model_patch.py b/comfy_extras/nodes_model_patch.py
new file mode 100644
index 000000000..3eaada9bc
--- /dev/null
+++ b/comfy_extras/nodes_model_patch.py
@@ -0,0 +1,161 @@
+import torch
+import folder_paths
+import comfy.utils
+import comfy.ops
+import comfy.model_management
+import comfy.ldm.common_dit
+import comfy.latent_formats
+
+
+class BlockWiseControlBlock(torch.nn.Module):
+ # [linear, gelu, linear]
+ def __init__(self, dim: int = 3072, device=None, dtype=None, operations=None):
+ super().__init__()
+ self.x_rms = operations.RMSNorm(dim, eps=1e-6)
+ self.y_rms = operations.RMSNorm(dim, eps=1e-6)
+ self.input_proj = operations.Linear(dim, dim)
+ self.act = torch.nn.GELU()
+ self.output_proj = operations.Linear(dim, dim)
+
+ def forward(self, x, y):
+ x, y = self.x_rms(x), self.y_rms(y)
+ x = self.input_proj(x + y)
+ x = self.act(x)
+ x = self.output_proj(x)
+ return x
+
+
+class QwenImageBlockWiseControlNet(torch.nn.Module):
+ def __init__(
+ self,
+ num_layers: int = 60,
+ in_dim: int = 64,
+ additional_in_dim: int = 0,
+ dim: int = 3072,
+ device=None, dtype=None, operations=None
+ ):
+ super().__init__()
+ self.additional_in_dim = additional_in_dim
+ self.img_in = operations.Linear(in_dim + additional_in_dim, dim, device=device, dtype=dtype)
+ self.controlnet_blocks = torch.nn.ModuleList(
+ [
+ BlockWiseControlBlock(dim, device=device, dtype=dtype, operations=operations)
+ for _ in range(num_layers)
+ ]
+ )
+
+ def process_input_latent_image(self, latent_image):
+ latent_image[:, :16] = comfy.latent_formats.Wan21().process_in(latent_image[:, :16])
+ patch_size = 2
+ hidden_states = comfy.ldm.common_dit.pad_to_patch_size(latent_image, (1, patch_size, patch_size))
+ orig_shape = hidden_states.shape
+ hidden_states = hidden_states.view(orig_shape[0], orig_shape[1], orig_shape[-2] // 2, 2, orig_shape[-1] // 2, 2)
+ hidden_states = hidden_states.permute(0, 2, 4, 1, 3, 5)
+ hidden_states = hidden_states.reshape(orig_shape[0], (orig_shape[-2] // 2) * (orig_shape[-1] // 2), orig_shape[1] * 4)
+ return self.img_in(hidden_states)
+
+ def control_block(self, img, controlnet_conditioning, block_id):
+ return self.controlnet_blocks[block_id](img, controlnet_conditioning)
+
+
+class ModelPatchLoader:
+ @classmethod
+ def INPUT_TYPES(s):
+ return {"required": { "name": (folder_paths.get_filename_list("model_patches"), ),
+ }}
+ RETURN_TYPES = ("MODEL_PATCH",)
+ FUNCTION = "load_model_patch"
+ EXPERIMENTAL = True
+
+ CATEGORY = "advanced/loaders"
+
+ def load_model_patch(self, name):
+ model_patch_path = folder_paths.get_full_path_or_raise("model_patches", name)
+ sd = comfy.utils.load_torch_file(model_patch_path, safe_load=True)
+ dtype = comfy.utils.weight_dtype(sd)
+ # TODO: this node will work with more types of model patches
+ additional_in_dim = sd["img_in.weight"].shape[1] - 64
+ model = QwenImageBlockWiseControlNet(additional_in_dim=additional_in_dim, device=comfy.model_management.unet_offload_device(), dtype=dtype, operations=comfy.ops.manual_cast)
+ model.load_state_dict(sd)
+ model = comfy.model_patcher.ModelPatcher(model, load_device=comfy.model_management.get_torch_device(), offload_device=comfy.model_management.unet_offload_device())
+ return (model,)
+
+
+class DiffSynthCnetPatch:
+ def __init__(self, model_patch, vae, image, strength, mask=None):
+ self.model_patch = model_patch
+ self.vae = vae
+ self.image = image
+ self.strength = strength
+ self.mask = mask
+ self.encoded_image = model_patch.model.process_input_latent_image(self.encode_latent_cond(image))
+
+ def encode_latent_cond(self, image):
+ latent_image = self.vae.encode(image)
+ if self.model_patch.model.additional_in_dim > 0:
+ if self.mask is None:
+ mask_ = torch.ones_like(latent_image)[:, :self.model_patch.model.additional_in_dim // 4]
+ else:
+ mask_ = comfy.utils.common_upscale(self.mask.mean(dim=1, keepdim=True), latent_image.shape[-1], latent_image.shape[-2], "bilinear", "none")
+
+ return torch.cat([latent_image, mask_], dim=1)
+ else:
+ return latent_image
+
+ def __call__(self, kwargs):
+ x = kwargs.get("x")
+ img = kwargs.get("img")
+ block_index = kwargs.get("block_index")
+ if self.encoded_image is None or self.encoded_image.shape[1:] != img.shape[1:]:
+ spacial_compression = self.vae.spacial_compression_encode()
+ image_scaled = comfy.utils.common_upscale(self.image.movedim(-1, 1), x.shape[-1] * spacial_compression, x.shape[-2] * spacial_compression, "area", "center")
+ loaded_models = comfy.model_management.loaded_models(only_currently_used=True)
+ self.encoded_image = self.model_patch.model.process_input_latent_image(self.encode_latent_cond(image_scaled.movedim(1, -1)))
+ comfy.model_management.load_models_gpu(loaded_models)
+
+ img = img + (self.model_patch.model.control_block(img, self.encoded_image.to(img.dtype), block_index) * self.strength)
+ kwargs['img'] = img
+ return kwargs
+
+ def to(self, device_or_dtype):
+ if isinstance(device_or_dtype, torch.device):
+ self.encoded_image = self.encoded_image.to(device_or_dtype)
+ return self
+
+ def models(self):
+ return [self.model_patch]
+
+class QwenImageDiffsynthControlnet:
+ @classmethod
+ def INPUT_TYPES(s):
+ return {"required": { "model": ("MODEL",),
+ "model_patch": ("MODEL_PATCH",),
+ "vae": ("VAE",),
+ "image": ("IMAGE",),
+ "strength": ("FLOAT", {"default": 1.0, "min": -10.0, "max": 10.0, "step": 0.01}),
+ },
+ "optional": {"mask": ("MASK",)}}
+ RETURN_TYPES = ("MODEL",)
+ FUNCTION = "diffsynth_controlnet"
+ EXPERIMENTAL = True
+
+ CATEGORY = "advanced/loaders/qwen"
+
+ def diffsynth_controlnet(self, model, model_patch, vae, image, strength, mask=None):
+ model_patched = model.clone()
+ image = image[:, :, :, :3]
+ if mask is not None:
+ if mask.ndim == 3:
+ mask = mask.unsqueeze(1)
+ if mask.ndim == 4:
+ mask = mask.unsqueeze(2)
+ mask = 1.0 - mask
+
+ model_patched.set_model_double_block_patch(DiffSynthCnetPatch(model_patch, vae, image, strength, mask))
+ return (model_patched,)
+
+
+NODE_CLASS_MAPPINGS = {
+ "ModelPatchLoader": ModelPatchLoader,
+ "QwenImageDiffsynthControlnet": QwenImageDiffsynthControlnet,
+}
diff --git a/comfy_extras/nodes_qwen.py b/comfy_extras/nodes_qwen.py
new file mode 100644
index 000000000..fff89556f
--- /dev/null
+++ b/comfy_extras/nodes_qwen.py
@@ -0,0 +1,48 @@
+import node_helpers
+import comfy.utils
+import math
+
+
+class TextEncodeQwenImageEdit:
+ @classmethod
+ def INPUT_TYPES(s):
+ return {"required": {
+ "clip": ("CLIP", ),
+ "prompt": ("STRING", {"multiline": True, "dynamicPrompts": True}),
+ },
+ "optional": {"vae": ("VAE", ),
+ "image": ("IMAGE", ),}}
+
+ RETURN_TYPES = ("CONDITIONING",)
+ FUNCTION = "encode"
+
+ CATEGORY = "advanced/conditioning"
+
+ def encode(self, clip, prompt, vae=None, image=None):
+ ref_latent = None
+ if image is None:
+ images = []
+ else:
+ samples = image.movedim(-1, 1)
+ total = int(1024 * 1024)
+
+ scale_by = math.sqrt(total / (samples.shape[3] * samples.shape[2]))
+ width = round(samples.shape[3] * scale_by)
+ height = round(samples.shape[2] * scale_by)
+
+ s = comfy.utils.common_upscale(samples, width, height, "area", "disabled")
+ image = s.movedim(1, -1)
+ images = [image[:, :, :, :3]]
+ if vae is not None:
+ ref_latent = vae.encode(image[:, :, :, :3])
+
+ tokens = clip.tokenize(prompt, images=images)
+ conditioning = clip.encode_from_tokens_scheduled(tokens)
+ if ref_latent is not None:
+ conditioning = node_helpers.conditioning_set_values(conditioning, {"reference_latents": [ref_latent]}, append=True)
+ return (conditioning, )
+
+
+NODE_CLASS_MAPPINGS = {
+ "TextEncodeQwenImageEdit": TextEncodeQwenImageEdit,
+}
diff --git a/comfy_extras/nodes_wan.py b/comfy_extras/nodes_wan.py
index 0067d054d..0fff02f76 100644
--- a/comfy_extras/nodes_wan.py
+++ b/comfy_extras/nodes_wan.py
@@ -9,29 +9,35 @@ import comfy.clip_vision
import json
import numpy as np
from typing import Tuple
+from typing_extensions import override
+from comfy_api.latest import ComfyExtension, io
-class WanImageToVideo:
+class WanImageToVideo(io.ComfyNode):
@classmethod
- def INPUT_TYPES(s):
- return {"required": {"positive": ("CONDITIONING", ),
- "negative": ("CONDITIONING", ),
- "vae": ("VAE", ),
- "width": ("INT", {"default": 832, "min": 16, "max": nodes.MAX_RESOLUTION, "step": 16}),
- "height": ("INT", {"default": 480, "min": 16, "max": nodes.MAX_RESOLUTION, "step": 16}),
- "length": ("INT", {"default": 81, "min": 1, "max": nodes.MAX_RESOLUTION, "step": 4}),
- "batch_size": ("INT", {"default": 1, "min": 1, "max": 4096}),
- },
- "optional": {"clip_vision_output": ("CLIP_VISION_OUTPUT", ),
- "start_image": ("IMAGE", ),
- }}
+ def define_schema(cls):
+ return io.Schema(
+ node_id="WanImageToVideo",
+ category="conditioning/video_models",
+ inputs=[
+ io.Conditioning.Input("positive"),
+ io.Conditioning.Input("negative"),
+ io.Vae.Input("vae"),
+ io.Int.Input("width", default=832, min=16, max=nodes.MAX_RESOLUTION, step=16),
+ io.Int.Input("height", default=480, min=16, max=nodes.MAX_RESOLUTION, step=16),
+ io.Int.Input("length", default=81, min=1, max=nodes.MAX_RESOLUTION, step=4),
+ io.Int.Input("batch_size", default=1, min=1, max=4096),
+ io.ClipVisionOutput.Input("clip_vision_output", optional=True),
+ io.Image.Input("start_image", optional=True),
+ ],
+ outputs=[
+ io.Conditioning.Output(display_name="positive"),
+ io.Conditioning.Output(display_name="negative"),
+ io.Latent.Output(display_name="latent"),
+ ],
+ )
- RETURN_TYPES = ("CONDITIONING", "CONDITIONING", "LATENT")
- RETURN_NAMES = ("positive", "negative", "latent")
- FUNCTION = "encode"
-
- CATEGORY = "conditioning/video_models"
-
- def encode(self, positive, negative, vae, width, height, length, batch_size, start_image=None, clip_vision_output=None):
+ @classmethod
+ def execute(cls, positive, negative, vae, width, height, length, batch_size, start_image=None, clip_vision_output=None) -> io.NodeOutput:
latent = torch.zeros([batch_size, 16, ((length - 1) // 4) + 1, height // 8, width // 8], device=comfy.model_management.intermediate_device())
if start_image is not None:
start_image = comfy.utils.common_upscale(start_image[:length].movedim(-1, 1), width, height, "bilinear", "center").movedim(1, -1)
@@ -51,32 +57,36 @@ class WanImageToVideo:
out_latent = {}
out_latent["samples"] = latent
- return (positive, negative, out_latent)
+ return io.NodeOutput(positive, negative, out_latent)
-class WanFunControlToVideo:
+class WanFunControlToVideo(io.ComfyNode):
@classmethod
- def INPUT_TYPES(s):
- return {"required": {"positive": ("CONDITIONING", ),
- "negative": ("CONDITIONING", ),
- "vae": ("VAE", ),
- "width": ("INT", {"default": 832, "min": 16, "max": nodes.MAX_RESOLUTION, "step": 16}),
- "height": ("INT", {"default": 480, "min": 16, "max": nodes.MAX_RESOLUTION, "step": 16}),
- "length": ("INT", {"default": 81, "min": 1, "max": nodes.MAX_RESOLUTION, "step": 4}),
- "batch_size": ("INT", {"default": 1, "min": 1, "max": 4096}),
- },
- "optional": {"clip_vision_output": ("CLIP_VISION_OUTPUT", ),
- "start_image": ("IMAGE", ),
- "control_video": ("IMAGE", ),
- }}
+ def define_schema(cls):
+ return io.Schema(
+ node_id="WanFunControlToVideo",
+ category="conditioning/video_models",
+ inputs=[
+ io.Conditioning.Input("positive"),
+ io.Conditioning.Input("negative"),
+ io.Vae.Input("vae"),
+ io.Int.Input("width", default=832, min=16, max=nodes.MAX_RESOLUTION, step=16),
+ io.Int.Input("height", default=480, min=16, max=nodes.MAX_RESOLUTION, step=16),
+ io.Int.Input("length", default=81, min=1, max=nodes.MAX_RESOLUTION, step=4),
+ io.Int.Input("batch_size", default=1, min=1, max=4096),
+ io.ClipVisionOutput.Input("clip_vision_output", optional=True),
+ io.Image.Input("start_image", optional=True),
+ io.Image.Input("control_video", optional=True),
+ ],
+ outputs=[
+ io.Conditioning.Output(display_name="positive"),
+ io.Conditioning.Output(display_name="negative"),
+ io.Latent.Output(display_name="latent"),
+ ],
+ )
- RETURN_TYPES = ("CONDITIONING", "CONDITIONING", "LATENT")
- RETURN_NAMES = ("positive", "negative", "latent")
- FUNCTION = "encode"
-
- CATEGORY = "conditioning/video_models"
-
- def encode(self, positive, negative, vae, width, height, length, batch_size, start_image=None, clip_vision_output=None, control_video=None):
+ @classmethod
+ def execute(cls, positive, negative, vae, width, height, length, batch_size, start_image=None, clip_vision_output=None, control_video=None) -> io.NodeOutput:
latent = torch.zeros([batch_size, 16, ((length - 1) // 4) + 1, height // 8, width // 8], device=comfy.model_management.intermediate_device())
concat_latent = torch.zeros([batch_size, 16, ((length - 1) // 4) + 1, height // 8, width // 8], device=comfy.model_management.intermediate_device())
concat_latent = comfy.latent_formats.Wan21().process_out(concat_latent)
@@ -101,32 +111,96 @@ class WanFunControlToVideo:
out_latent = {}
out_latent["samples"] = latent
- return (positive, negative, out_latent)
+ return io.NodeOutput(positive, negative, out_latent)
-class WanFirstLastFrameToVideo:
+class Wan22FunControlToVideo(io.ComfyNode):
@classmethod
- def INPUT_TYPES(s):
- return {"required": {"positive": ("CONDITIONING", ),
- "negative": ("CONDITIONING", ),
- "vae": ("VAE", ),
- "width": ("INT", {"default": 832, "min": 16, "max": nodes.MAX_RESOLUTION, "step": 16}),
- "height": ("INT", {"default": 480, "min": 16, "max": nodes.MAX_RESOLUTION, "step": 16}),
- "length": ("INT", {"default": 81, "min": 1, "max": nodes.MAX_RESOLUTION, "step": 4}),
- "batch_size": ("INT", {"default": 1, "min": 1, "max": 4096}),
- },
- "optional": {"clip_vision_start_image": ("CLIP_VISION_OUTPUT", ),
- "clip_vision_end_image": ("CLIP_VISION_OUTPUT", ),
- "start_image": ("IMAGE", ),
- "end_image": ("IMAGE", ),
- }}
+ def define_schema(cls):
+ return io.Schema(
+ node_id="Wan22FunControlToVideo",
+ category="conditioning/video_models",
+ inputs=[
+ io.Conditioning.Input("positive"),
+ io.Conditioning.Input("negative"),
+ io.Vae.Input("vae"),
+ io.Int.Input("width", default=832, min=16, max=nodes.MAX_RESOLUTION, step=16),
+ io.Int.Input("height", default=480, min=16, max=nodes.MAX_RESOLUTION, step=16),
+ io.Int.Input("length", default=81, min=1, max=nodes.MAX_RESOLUTION, step=4),
+ io.Int.Input("batch_size", default=1, min=1, max=4096),
+ io.Image.Input("ref_image", optional=True),
+ io.Image.Input("control_video", optional=True),
+ ],
+ outputs=[
+ io.Conditioning.Output(display_name="positive"),
+ io.Conditioning.Output(display_name="negative"),
+ io.Latent.Output(display_name="latent"),
+ ],
+ )
- RETURN_TYPES = ("CONDITIONING", "CONDITIONING", "LATENT")
- RETURN_NAMES = ("positive", "negative", "latent")
- FUNCTION = "encode"
+ @classmethod
+ def execute(cls, positive, negative, vae, width, height, length, batch_size, ref_image=None, start_image=None, control_video=None) -> io.NodeOutput:
+ latent = torch.zeros([batch_size, 16, ((length - 1) // 4) + 1, height // 8, width // 8], device=comfy.model_management.intermediate_device())
+ concat_latent = torch.zeros([batch_size, 16, ((length - 1) // 4) + 1, height // 8, width // 8], device=comfy.model_management.intermediate_device())
+ concat_latent = comfy.latent_formats.Wan21().process_out(concat_latent)
+ concat_latent = concat_latent.repeat(1, 2, 1, 1, 1)
+ mask = torch.ones((1, 1, latent.shape[2] * 4, latent.shape[-2], latent.shape[-1]))
- CATEGORY = "conditioning/video_models"
+ if start_image is not None:
+ start_image = comfy.utils.common_upscale(start_image[:length].movedim(-1, 1), width, height, "bilinear", "center").movedim(1, -1)
+ concat_latent_image = vae.encode(start_image[:, :, :, :3])
+ concat_latent[:,16:,:concat_latent_image.shape[2]] = concat_latent_image[:,:,:concat_latent.shape[2]]
+ mask[:, :, :start_image.shape[0] + 3] = 0.0
- def encode(self, positive, negative, vae, width, height, length, batch_size, start_image=None, end_image=None, clip_vision_start_image=None, clip_vision_end_image=None):
+ ref_latent = None
+ if ref_image is not None:
+ ref_image = comfy.utils.common_upscale(ref_image[:1].movedim(-1, 1), width, height, "bilinear", "center").movedim(1, -1)
+ ref_latent = vae.encode(ref_image[:, :, :, :3])
+
+ if control_video is not None:
+ control_video = comfy.utils.common_upscale(control_video[:length].movedim(-1, 1), width, height, "bilinear", "center").movedim(1, -1)
+ concat_latent_image = vae.encode(control_video[:, :, :, :3])
+ concat_latent[:,:16,:concat_latent_image.shape[2]] = concat_latent_image[:,:,:concat_latent.shape[2]]
+
+ mask = mask.view(1, mask.shape[2] // 4, 4, mask.shape[3], mask.shape[4]).transpose(1, 2)
+ positive = node_helpers.conditioning_set_values(positive, {"concat_latent_image": concat_latent, "concat_mask": mask, "concat_mask_index": 16})
+ negative = node_helpers.conditioning_set_values(negative, {"concat_latent_image": concat_latent, "concat_mask": mask, "concat_mask_index": 16})
+
+ if ref_latent is not None:
+ positive = node_helpers.conditioning_set_values(positive, {"reference_latents": [ref_latent]}, append=True)
+ negative = node_helpers.conditioning_set_values(negative, {"reference_latents": [ref_latent]}, append=True)
+
+ out_latent = {}
+ out_latent["samples"] = latent
+ return io.NodeOutput(positive, negative, out_latent)
+
+class WanFirstLastFrameToVideo(io.ComfyNode):
+ @classmethod
+ def define_schema(cls):
+ return io.Schema(
+ node_id="WanFirstLastFrameToVideo",
+ category="conditioning/video_models",
+ inputs=[
+ io.Conditioning.Input("positive"),
+ io.Conditioning.Input("negative"),
+ io.Vae.Input("vae"),
+ io.Int.Input("width", default=832, min=16, max=nodes.MAX_RESOLUTION, step=16),
+ io.Int.Input("height", default=480, min=16, max=nodes.MAX_RESOLUTION, step=16),
+ io.Int.Input("length", default=81, min=1, max=nodes.MAX_RESOLUTION, step=4),
+ io.Int.Input("batch_size", default=1, min=1, max=4096),
+ io.ClipVisionOutput.Input("clip_vision_start_image", optional=True),
+ io.ClipVisionOutput.Input("clip_vision_end_image", optional=True),
+ io.Image.Input("start_image", optional=True),
+ io.Image.Input("end_image", optional=True),
+ ],
+ outputs=[
+ io.Conditioning.Output(display_name="positive"),
+ io.Conditioning.Output(display_name="negative"),
+ io.Latent.Output(display_name="latent"),
+ ],
+ )
+
+ @classmethod
+ def execute(cls, positive, negative, vae, width, height, length, batch_size, start_image=None, end_image=None, clip_vision_start_image=None, clip_vision_end_image=None) -> io.NodeOutput:
latent = torch.zeros([batch_size, 16, ((length - 1) // 4) + 1, height // 8, width // 8], device=comfy.model_management.intermediate_device())
if start_image is not None:
start_image = comfy.utils.common_upscale(start_image[:length].movedim(-1, 1), width, height, "bilinear", "center").movedim(1, -1)
@@ -167,62 +241,70 @@ class WanFirstLastFrameToVideo:
out_latent = {}
out_latent["samples"] = latent
- return (positive, negative, out_latent)
+ return io.NodeOutput(positive, negative, out_latent)
-class WanFunInpaintToVideo:
+class WanFunInpaintToVideo(io.ComfyNode):
@classmethod
- def INPUT_TYPES(s):
- return {"required": {"positive": ("CONDITIONING", ),
- "negative": ("CONDITIONING", ),
- "vae": ("VAE", ),
- "width": ("INT", {"default": 832, "min": 16, "max": nodes.MAX_RESOLUTION, "step": 16}),
- "height": ("INT", {"default": 480, "min": 16, "max": nodes.MAX_RESOLUTION, "step": 16}),
- "length": ("INT", {"default": 81, "min": 1, "max": nodes.MAX_RESOLUTION, "step": 4}),
- "batch_size": ("INT", {"default": 1, "min": 1, "max": 4096}),
- },
- "optional": {"clip_vision_output": ("CLIP_VISION_OUTPUT", ),
- "start_image": ("IMAGE", ),
- "end_image": ("IMAGE", ),
- }}
+ def define_schema(cls):
+ return io.Schema(
+ node_id="WanFunInpaintToVideo",
+ category="conditioning/video_models",
+ inputs=[
+ io.Conditioning.Input("positive"),
+ io.Conditioning.Input("negative"),
+ io.Vae.Input("vae"),
+ io.Int.Input("width", default=832, min=16, max=nodes.MAX_RESOLUTION, step=16),
+ io.Int.Input("height", default=480, min=16, max=nodes.MAX_RESOLUTION, step=16),
+ io.Int.Input("length", default=81, min=1, max=nodes.MAX_RESOLUTION, step=4),
+ io.Int.Input("batch_size", default=1, min=1, max=4096),
+ io.ClipVisionOutput.Input("clip_vision_output", optional=True),
+ io.Image.Input("start_image", optional=True),
+ io.Image.Input("end_image", optional=True),
+ ],
+ outputs=[
+ io.Conditioning.Output(display_name="positive"),
+ io.Conditioning.Output(display_name="negative"),
+ io.Latent.Output(display_name="latent"),
+ ],
+ )
- RETURN_TYPES = ("CONDITIONING", "CONDITIONING", "LATENT")
- RETURN_NAMES = ("positive", "negative", "latent")
- FUNCTION = "encode"
-
- CATEGORY = "conditioning/video_models"
-
- def encode(self, positive, negative, vae, width, height, length, batch_size, start_image=None, end_image=None, clip_vision_output=None):
+ @classmethod
+ def execute(cls, positive, negative, vae, width, height, length, batch_size, start_image=None, end_image=None, clip_vision_output=None) -> io.NodeOutput:
flfv = WanFirstLastFrameToVideo()
- return flfv.encode(positive, negative, vae, width, height, length, batch_size, start_image=start_image, end_image=end_image, clip_vision_start_image=clip_vision_output)
+ return flfv.execute(positive, negative, vae, width, height, length, batch_size, start_image=start_image, end_image=end_image, clip_vision_start_image=clip_vision_output)
-class WanVaceToVideo:
+class WanVaceToVideo(io.ComfyNode):
@classmethod
- def INPUT_TYPES(s):
- return {"required": {"positive": ("CONDITIONING", ),
- "negative": ("CONDITIONING", ),
- "vae": ("VAE", ),
- "width": ("INT", {"default": 832, "min": 16, "max": nodes.MAX_RESOLUTION, "step": 16}),
- "height": ("INT", {"default": 480, "min": 16, "max": nodes.MAX_RESOLUTION, "step": 16}),
- "length": ("INT", {"default": 81, "min": 1, "max": nodes.MAX_RESOLUTION, "step": 4}),
- "batch_size": ("INT", {"default": 1, "min": 1, "max": 4096}),
- "strength": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 1000.0, "step": 0.01}),
- },
- "optional": {"control_video": ("IMAGE", ),
- "control_masks": ("MASK", ),
- "reference_image": ("IMAGE", ),
- }}
+ def define_schema(cls):
+ return io.Schema(
+ node_id="WanVaceToVideo",
+ category="conditioning/video_models",
+ is_experimental=True,
+ inputs=[
+ io.Conditioning.Input("positive"),
+ io.Conditioning.Input("negative"),
+ io.Vae.Input("vae"),
+ io.Int.Input("width", default=832, min=16, max=nodes.MAX_RESOLUTION, step=16),
+ io.Int.Input("height", default=480, min=16, max=nodes.MAX_RESOLUTION, step=16),
+ io.Int.Input("length", default=81, min=1, max=nodes.MAX_RESOLUTION, step=4),
+ io.Int.Input("batch_size", default=1, min=1, max=4096),
+ io.Float.Input("strength", default=1.0, min=0.0, max=1000.0, step=0.01),
+ io.Image.Input("control_video", optional=True),
+ io.Mask.Input("control_masks", optional=True),
+ io.Image.Input("reference_image", optional=True),
+ ],
+ outputs=[
+ io.Conditioning.Output(display_name="positive"),
+ io.Conditioning.Output(display_name="negative"),
+ io.Latent.Output(display_name="latent"),
+ io.Int.Output(display_name="trim_latent"),
+ ],
+ )
- RETURN_TYPES = ("CONDITIONING", "CONDITIONING", "LATENT", "INT")
- RETURN_NAMES = ("positive", "negative", "latent", "trim_latent")
- FUNCTION = "encode"
-
- CATEGORY = "conditioning/video_models"
-
- EXPERIMENTAL = True
-
- def encode(self, positive, negative, vae, width, height, length, batch_size, strength, control_video=None, control_masks=None, reference_image=None):
+ @classmethod
+ def execute(cls, positive, negative, vae, width, height, length, batch_size, strength, control_video=None, control_masks=None, reference_image=None) -> io.NodeOutput:
latent_length = ((length - 1) // 4) + 1
if control_video is not None:
control_video = comfy.utils.common_upscale(control_video[:length].movedim(-1, 1), width, height, "bilinear", "center").movedim(1, -1)
@@ -279,52 +361,59 @@ class WanVaceToVideo:
latent = torch.zeros([batch_size, 16, latent_length, height // 8, width // 8], device=comfy.model_management.intermediate_device())
out_latent = {}
out_latent["samples"] = latent
- return (positive, negative, out_latent, trim_latent)
+ return io.NodeOutput(positive, negative, out_latent, trim_latent)
-class TrimVideoLatent:
+class TrimVideoLatent(io.ComfyNode):
@classmethod
- def INPUT_TYPES(s):
- return {"required": { "samples": ("LATENT",),
- "trim_amount": ("INT", {"default": 0, "min": 0, "max": 99999}),
- }}
+ def define_schema(cls):
+ return io.Schema(
+ node_id="TrimVideoLatent",
+ category="latent/video",
+ is_experimental=True,
+ inputs=[
+ io.Latent.Input("samples"),
+ io.Int.Input("trim_amount", default=0, min=0, max=99999),
+ ],
+ outputs=[
+ io.Latent.Output(),
+ ],
+ )
- RETURN_TYPES = ("LATENT",)
- FUNCTION = "op"
-
- CATEGORY = "latent/video"
-
- EXPERIMENTAL = True
-
- def op(self, samples, trim_amount):
+ @classmethod
+ def execute(cls, samples, trim_amount) -> io.NodeOutput:
samples_out = samples.copy()
s1 = samples["samples"]
samples_out["samples"] = s1[:, :, trim_amount:]
- return (samples_out,)
+ return io.NodeOutput(samples_out)
-class WanCameraImageToVideo:
+class WanCameraImageToVideo(io.ComfyNode):
@classmethod
- def INPUT_TYPES(s):
- return {"required": {"positive": ("CONDITIONING", ),
- "negative": ("CONDITIONING", ),
- "vae": ("VAE", ),
- "width": ("INT", {"default": 832, "min": 16, "max": nodes.MAX_RESOLUTION, "step": 16}),
- "height": ("INT", {"default": 480, "min": 16, "max": nodes.MAX_RESOLUTION, "step": 16}),
- "length": ("INT", {"default": 81, "min": 1, "max": nodes.MAX_RESOLUTION, "step": 4}),
- "batch_size": ("INT", {"default": 1, "min": 1, "max": 4096}),
- },
- "optional": {"clip_vision_output": ("CLIP_VISION_OUTPUT", ),
- "start_image": ("IMAGE", ),
- "camera_conditions": ("WAN_CAMERA_EMBEDDING", ),
- }}
+ def define_schema(cls):
+ return io.Schema(
+ node_id="WanCameraImageToVideo",
+ category="conditioning/video_models",
+ inputs=[
+ io.Conditioning.Input("positive"),
+ io.Conditioning.Input("negative"),
+ io.Vae.Input("vae"),
+ io.Int.Input("width", default=832, min=16, max=nodes.MAX_RESOLUTION, step=16),
+ io.Int.Input("height", default=480, min=16, max=nodes.MAX_RESOLUTION, step=16),
+ io.Int.Input("length", default=81, min=1, max=nodes.MAX_RESOLUTION, step=4),
+ io.Int.Input("batch_size", default=1, min=1, max=4096),
+ io.ClipVisionOutput.Input("clip_vision_output", optional=True),
+ io.Image.Input("start_image", optional=True),
+ io.WanCameraEmbedding.Input("camera_conditions", optional=True),
+ ],
+ outputs=[
+ io.Conditioning.Output(display_name="positive"),
+ io.Conditioning.Output(display_name="negative"),
+ io.Latent.Output(display_name="latent"),
+ ],
+ )
- RETURN_TYPES = ("CONDITIONING", "CONDITIONING", "LATENT")
- RETURN_NAMES = ("positive", "negative", "latent")
- FUNCTION = "encode"
-
- CATEGORY = "conditioning/video_models"
-
- def encode(self, positive, negative, vae, width, height, length, batch_size, start_image=None, clip_vision_output=None, camera_conditions=None):
+ @classmethod
+ def execute(cls, positive, negative, vae, width, height, length, batch_size, start_image=None, clip_vision_output=None, camera_conditions=None) -> io.NodeOutput:
latent = torch.zeros([batch_size, 16, ((length - 1) // 4) + 1, height // 8, width // 8], device=comfy.model_management.intermediate_device())
concat_latent = torch.zeros([batch_size, 16, ((length - 1) // 4) + 1, height // 8, width // 8], device=comfy.model_management.intermediate_device())
concat_latent = comfy.latent_formats.Wan21().process_out(concat_latent)
@@ -333,9 +422,12 @@ class WanCameraImageToVideo:
start_image = comfy.utils.common_upscale(start_image[:length].movedim(-1, 1), width, height, "bilinear", "center").movedim(1, -1)
concat_latent_image = vae.encode(start_image[:, :, :, :3])
concat_latent[:,:,:concat_latent_image.shape[2]] = concat_latent_image[:,:,:concat_latent.shape[2]]
+ mask = torch.ones((1, 1, latent.shape[2] * 4, latent.shape[-2], latent.shape[-1]))
+ mask[:, :, :start_image.shape[0] + 3] = 0.0
+ mask = mask.view(1, mask.shape[2] // 4, 4, mask.shape[3], mask.shape[4]).transpose(1, 2)
- positive = node_helpers.conditioning_set_values(positive, {"concat_latent_image": concat_latent})
- negative = node_helpers.conditioning_set_values(negative, {"concat_latent_image": concat_latent})
+ positive = node_helpers.conditioning_set_values(positive, {"concat_latent_image": concat_latent, "concat_mask": mask})
+ negative = node_helpers.conditioning_set_values(negative, {"concat_latent_image": concat_latent, "concat_mask": mask})
if camera_conditions is not None:
positive = node_helpers.conditioning_set_values(positive, {'camera_conditions': camera_conditions})
@@ -347,29 +439,34 @@ class WanCameraImageToVideo:
out_latent = {}
out_latent["samples"] = latent
- return (positive, negative, out_latent)
+ return io.NodeOutput(positive, negative, out_latent)
-class WanPhantomSubjectToVideo:
+class WanPhantomSubjectToVideo(io.ComfyNode):
@classmethod
- def INPUT_TYPES(s):
- return {"required": {"positive": ("CONDITIONING", ),
- "negative": ("CONDITIONING", ),
- "vae": ("VAE", ),
- "width": ("INT", {"default": 832, "min": 16, "max": nodes.MAX_RESOLUTION, "step": 16}),
- "height": ("INT", {"default": 480, "min": 16, "max": nodes.MAX_RESOLUTION, "step": 16}),
- "length": ("INT", {"default": 81, "min": 1, "max": nodes.MAX_RESOLUTION, "step": 4}),
- "batch_size": ("INT", {"default": 1, "min": 1, "max": 4096}),
- },
- "optional": {"images": ("IMAGE", ),
- }}
+ def define_schema(cls):
+ return io.Schema(
+ node_id="WanPhantomSubjectToVideo",
+ category="conditioning/video_models",
+ inputs=[
+ io.Conditioning.Input("positive"),
+ io.Conditioning.Input("negative"),
+ io.Vae.Input("vae"),
+ io.Int.Input("width", default=832, min=16, max=nodes.MAX_RESOLUTION, step=16),
+ io.Int.Input("height", default=480, min=16, max=nodes.MAX_RESOLUTION, step=16),
+ io.Int.Input("length", default=81, min=1, max=nodes.MAX_RESOLUTION, step=4),
+ io.Int.Input("batch_size", default=1, min=1, max=4096),
+ io.Image.Input("images", optional=True),
+ ],
+ outputs=[
+ io.Conditioning.Output(display_name="positive"),
+ io.Conditioning.Output(display_name="negative_text"),
+ io.Conditioning.Output(display_name="negative_img_text"),
+ io.Latent.Output(display_name="latent"),
+ ],
+ )
- RETURN_TYPES = ("CONDITIONING", "CONDITIONING", "CONDITIONING", "LATENT")
- RETURN_NAMES = ("positive", "negative_text", "negative_img_text", "latent")
- FUNCTION = "encode"
-
- CATEGORY = "conditioning/video_models"
-
- def encode(self, positive, negative, vae, width, height, length, batch_size, images):
+ @classmethod
+ def execute(cls, positive, negative, vae, width, height, length, batch_size, images) -> io.NodeOutput:
latent = torch.zeros([batch_size, 16, ((length - 1) // 4) + 1, height // 8, width // 8], device=comfy.model_management.intermediate_device())
cond2 = negative
if images is not None:
@@ -385,7 +482,7 @@ class WanPhantomSubjectToVideo:
out_latent = {}
out_latent["samples"] = latent
- return (positive, cond2, negative, out_latent)
+ return io.NodeOutput(positive, cond2, negative, out_latent)
def parse_json_tracks(tracks):
"""Parse JSON track data into a standardized format"""
@@ -598,39 +695,41 @@ def patch_motion(
return out_mask_full, out_feature_full
-class WanTrackToVideo:
+class WanTrackToVideo(io.ComfyNode):
@classmethod
- def INPUT_TYPES(s):
- return {"required": {
- "positive": ("CONDITIONING", ),
- "negative": ("CONDITIONING", ),
- "vae": ("VAE", ),
- "tracks": ("STRING", {"multiline": True, "default": "[]"}),
- "width": ("INT", {"default": 832, "min": 16, "max": nodes.MAX_RESOLUTION, "step": 16}),
- "height": ("INT", {"default": 480, "min": 16, "max": nodes.MAX_RESOLUTION, "step": 16}),
- "length": ("INT", {"default": 81, "min": 1, "max": nodes.MAX_RESOLUTION, "step": 4}),
- "batch_size": ("INT", {"default": 1, "min": 1, "max": 4096}),
- "temperature": ("FLOAT", {"default": 220.0, "min": 1.0, "max": 1000.0, "step": 0.1}),
- "topk": ("INT", {"default": 2, "min": 1, "max": 10}),
- "start_image": ("IMAGE", ),
- },
- "optional": {
- "clip_vision_output": ("CLIP_VISION_OUTPUT", ),
- }}
+ def define_schema(cls):
+ return io.Schema(
+ node_id="WanTrackToVideo",
+ category="conditioning/video_models",
+ inputs=[
+ io.Conditioning.Input("positive"),
+ io.Conditioning.Input("negative"),
+ io.Vae.Input("vae"),
+ io.String.Input("tracks", multiline=True, default="[]"),
+ io.Int.Input("width", default=832, min=16, max=nodes.MAX_RESOLUTION, step=16),
+ io.Int.Input("height", default=480, min=16, max=nodes.MAX_RESOLUTION, step=16),
+ io.Int.Input("length", default=81, min=1, max=nodes.MAX_RESOLUTION, step=4),
+ io.Int.Input("batch_size", default=1, min=1, max=4096),
+ io.Float.Input("temperature", default=220.0, min=1.0, max=1000.0, step=0.1),
+ io.Int.Input("topk", default=2, min=1, max=10),
+ io.Image.Input("start_image"),
+ io.ClipVisionOutput.Input("clip_vision_output", optional=True),
+ ],
+ outputs=[
+ io.Conditioning.Output(display_name="positive"),
+ io.Conditioning.Output(display_name="negative"),
+ io.Latent.Output(display_name="latent"),
+ ],
+ )
- RETURN_TYPES = ("CONDITIONING", "CONDITIONING", "LATENT")
- RETURN_NAMES = ("positive", "negative", "latent")
- FUNCTION = "encode"
-
- CATEGORY = "conditioning/video_models"
-
- def encode(self, positive, negative, vae, tracks, width, height, length, batch_size,
- temperature, topk, start_image=None, clip_vision_output=None):
+ @classmethod
+ def execute(cls, positive, negative, vae, tracks, width, height, length, batch_size,
+ temperature, topk, start_image=None, clip_vision_output=None) -> io.NodeOutput:
tracks_data = parse_json_tracks(tracks)
if not tracks_data:
- return WanImageToVideo().encode(positive, negative, vae, width, height, length, batch_size, start_image=start_image, clip_vision_output=clip_vision_output)
+ return WanImageToVideo().execute(positive, negative, vae, width, height, length, batch_size, start_image=start_image, clip_vision_output=clip_vision_output)
latent = torch.zeros([batch_size, 16, ((length - 1) // 4) + 1, height // 8, width // 8],
device=comfy.model_management.intermediate_device())
@@ -684,34 +783,36 @@ class WanTrackToVideo:
out_latent = {}
out_latent["samples"] = latent
- return (positive, negative, out_latent)
+ return io.NodeOutput(positive, negative, out_latent)
-class Wan22ImageToVideoLatent:
+class Wan22ImageToVideoLatent(io.ComfyNode):
@classmethod
- def INPUT_TYPES(s):
- return {"required": {"vae": ("VAE", ),
- "width": ("INT", {"default": 1280, "min": 32, "max": nodes.MAX_RESOLUTION, "step": 32}),
- "height": ("INT", {"default": 704, "min": 32, "max": nodes.MAX_RESOLUTION, "step": 32}),
- "length": ("INT", {"default": 49, "min": 1, "max": nodes.MAX_RESOLUTION, "step": 4}),
- "batch_size": ("INT", {"default": 1, "min": 1, "max": 4096}),
- },
- "optional": {"start_image": ("IMAGE", ),
- }}
+ def define_schema(cls):
+ return io.Schema(
+ node_id="Wan22ImageToVideoLatent",
+ category="conditioning/inpaint",
+ inputs=[
+ io.Vae.Input("vae"),
+ io.Int.Input("width", default=1280, min=32, max=nodes.MAX_RESOLUTION, step=32),
+ io.Int.Input("height", default=704, min=32, max=nodes.MAX_RESOLUTION, step=32),
+ io.Int.Input("length", default=49, min=1, max=nodes.MAX_RESOLUTION, step=4),
+ io.Int.Input("batch_size", default=1, min=1, max=4096),
+ io.Image.Input("start_image", optional=True),
+ ],
+ outputs=[
+ io.Latent.Output(),
+ ],
+ )
-
- RETURN_TYPES = ("LATENT",)
- FUNCTION = "encode"
-
- CATEGORY = "conditioning/inpaint"
-
- def encode(self, vae, width, height, length, batch_size, start_image=None):
+ @classmethod
+ def execute(cls, vae, width, height, length, batch_size, start_image=None) -> io.NodeOutput:
latent = torch.zeros([1, 48, ((length - 1) // 4) + 1, height // 16, width // 16], device=comfy.model_management.intermediate_device())
if start_image is None:
out_latent = {}
out_latent["samples"] = latent
- return (out_latent,)
+ return io.NodeOutput(out_latent)
mask = torch.ones([latent.shape[0], 1, ((length - 1) // 4) + 1, latent.shape[-2], latent.shape[-1]], device=comfy.model_management.intermediate_device())
@@ -726,18 +827,25 @@ class Wan22ImageToVideoLatent:
latent = latent_format.process_out(latent) * mask + latent * (1.0 - mask)
out_latent["samples"] = latent.repeat((batch_size, ) + (1,) * (latent.ndim - 1))
out_latent["noise_mask"] = mask.repeat((batch_size, ) + (1,) * (mask.ndim - 1))
- return (out_latent,)
+ return io.NodeOutput(out_latent)
-NODE_CLASS_MAPPINGS = {
- "WanTrackToVideo": WanTrackToVideo,
- "WanImageToVideo": WanImageToVideo,
- "WanFunControlToVideo": WanFunControlToVideo,
- "WanFunInpaintToVideo": WanFunInpaintToVideo,
- "WanFirstLastFrameToVideo": WanFirstLastFrameToVideo,
- "WanVaceToVideo": WanVaceToVideo,
- "TrimVideoLatent": TrimVideoLatent,
- "WanCameraImageToVideo": WanCameraImageToVideo,
- "WanPhantomSubjectToVideo": WanPhantomSubjectToVideo,
- "Wan22ImageToVideoLatent": Wan22ImageToVideoLatent,
-}
+class WanExtension(ComfyExtension):
+ @override
+ async def get_node_list(self) -> list[type[io.ComfyNode]]:
+ return [
+ WanTrackToVideo,
+ WanImageToVideo,
+ WanFunControlToVideo,
+ Wan22FunControlToVideo,
+ WanFunInpaintToVideo,
+ WanFirstLastFrameToVideo,
+ WanVaceToVideo,
+ TrimVideoLatent,
+ WanCameraImageToVideo,
+ WanPhantomSubjectToVideo,
+ Wan22ImageToVideoLatent,
+ ]
+
+async def comfy_entrypoint() -> WanExtension:
+ return WanExtension()
diff --git a/comfyui_version.py b/comfyui_version.py
index 7b29e338d..65f06cf37 100644
--- a/comfyui_version.py
+++ b/comfyui_version.py
@@ -1,3 +1,3 @@
# This file is automatically generated by the build process when version is
# updated in pyproject.toml.
-__version__ = "0.3.48"
+__version__ = "0.3.51"
diff --git a/execution.py b/execution.py
index 952f0cc5c..1dc35738b 100644
--- a/execution.py
+++ b/execution.py
@@ -646,8 +646,6 @@ class PromptExecutor:
self.add_message("execution_error", mes, broadcast=False)
def execute(self, prompt, prompt_id, extra_data={}, execute_outputs=[]):
- asyncio_loop = asyncio.new_event_loop()
- asyncio.set_event_loop(asyncio_loop)
asyncio.run(self.execute_async(prompt, prompt_id, extra_data, execute_outputs))
async def execute_async(self, prompt, prompt_id, extra_data={}, execute_outputs=[]):
diff --git a/folder_paths.py b/folder_paths.py
index 9ec952940..b34af39e8 100644
--- a/folder_paths.py
+++ b/folder_paths.py
@@ -46,6 +46,8 @@ folder_names_and_paths["photomaker"] = ([os.path.join(models_dir, "photomaker")]
folder_names_and_paths["classifiers"] = ([os.path.join(models_dir, "classifiers")], {""})
+folder_names_and_paths["model_patches"] = ([os.path.join(models_dir, "model_patches")], supported_pt_extensions)
+
output_directory = os.path.join(base_path, "output")
temp_directory = os.path.join(base_path, "temp")
input_directory = os.path.join(base_path, "input")
diff --git a/models/model_patches/put_model_patches_here b/models/model_patches/put_model_patches_here
new file mode 100644
index 000000000..e69de29bb
diff --git a/nodes.py b/nodes.py
index da4a46366..9681750d3 100644
--- a/nodes.py
+++ b/nodes.py
@@ -925,7 +925,7 @@ class CLIPLoader:
@classmethod
def INPUT_TYPES(s):
return {"required": { "clip_name": (folder_paths.get_filename_list("text_encoders"), ),
- "type": (["stable_diffusion", "stable_cascade", "sd3", "stable_audio", "mochi", "ltxv", "pixart", "cosmos", "lumina2", "wan", "hidream", "chroma", "ace", "omnigen2"], ),
+ "type": (["stable_diffusion", "stable_cascade", "sd3", "stable_audio", "mochi", "ltxv", "pixart", "cosmos", "lumina2", "wan", "hidream", "chroma", "ace", "omnigen2", "qwen_image"], ),
},
"optional": {
"device": (["default", "cpu"], {"advanced": True}),
@@ -1229,12 +1229,12 @@ class RepeatLatentBatch:
s = samples.copy()
s_in = samples["samples"]
- s["samples"] = s_in.repeat((amount, 1,1,1))
+ s["samples"] = s_in.repeat((amount,) + ((1,) * (s_in.ndim - 1)))
if "noise_mask" in samples and samples["noise_mask"].shape[0] > 1:
masks = samples["noise_mask"]
if masks.shape[0] < s_in.shape[0]:
- masks = masks.repeat(math.ceil(s_in.shape[0] / masks.shape[0]), 1, 1, 1)[:s_in.shape[0]]
- s["noise_mask"] = samples["noise_mask"].repeat((amount, 1,1,1))
+ masks = masks.repeat((math.ceil(s_in.shape[0] / masks.shape[0]),) + ((1,) * (masks.ndim - 1)))[:s_in.shape[0]]
+ s["noise_mask"] = samples["noise_mask"].repeat((amount,) + ((1,) * (samples["noise_mask"].ndim - 1)))
if "batch_index" in s:
offset = max(s["batch_index"]) - min(s["batch_index"]) + 1
s["batch_index"] = s["batch_index"] + [x + (i * offset) for i in range(1, amount) for x in s["batch_index"]]
@@ -2320,6 +2320,9 @@ async def init_builtin_extra_nodes():
"nodes_camera_trajectory.py",
"nodes_edit_model.py",
"nodes_tcfg.py",
+ "nodes_context_windows.py",
+ "nodes_qwen.py",
+ "nodes_model_patch.py"
]
import_failed = []
@@ -2349,6 +2352,7 @@ async def init_builtin_api_nodes():
"nodes_moonvalley.py",
"nodes_rodin.py",
"nodes_gemini.py",
+ "nodes_vidu.py",
]
if not await load_custom_node(os.path.join(api_nodes_dir, "canary.py"), module_parent="comfy_api_nodes"):
diff --git a/pyproject.toml b/pyproject.toml
index 256677fad..ecbf04303 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,6 +1,6 @@
[project]
name = "ComfyUI"
-version = "0.3.48"
+version = "0.3.51"
readme = "README.md"
license = { file = "LICENSE" }
requires-python = ">=3.9"
diff --git a/requirements.txt b/requirements.txt
index ffa7dce65..8d928d826 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,6 +1,6 @@
-comfyui-frontend-package==1.23.4
-comfyui-workflow-templates==0.1.47
-comfyui-embedded-docs==0.2.4
+comfyui-frontend-package==1.25.9
+comfyui-workflow-templates==0.1.62
+comfyui-embedded-docs==0.2.6
torch
torchsde
torchvision
@@ -20,11 +20,11 @@ tqdm
psutil
alembic
SQLAlchemy
+av>=14.2.0
#non essential dependencies:
kornia>=0.7.1
spandrel
soundfile
-av>=14.2.0
pydantic~=2.0
pydantic-settings~=2.0
diff --git a/server.py b/server.py
index 0553a0dd7..8f9c88ebf 100644
--- a/server.py
+++ b/server.py
@@ -235,7 +235,7 @@ class PromptServer():
sid,
)
- logging.info(
+ logging.debug(
f"Feature flags negotiated for client {sid}: {client_flags}"
)
first_message = False