From ff452f9fc92ce21a99ed9cbd2c96ec88a8b3648e Mon Sep 17 00:00:00 2001 From: Robin Huang Date: Wed, 9 Apr 2025 11:24:30 -0700 Subject: [PATCH 001/121] Add Ideogram generate node. --- comfy_extras/nodes_api.py | 283 ++++++++++++++++++++++++++++++++++++++ nodes.py | 1 + 2 files changed, 284 insertions(+) create mode 100644 comfy_extras/nodes_api.py diff --git a/comfy_extras/nodes_api.py b/comfy_extras/nodes_api.py new file mode 100644 index 000000000..8ab57bb26 --- /dev/null +++ b/comfy_extras/nodes_api.py @@ -0,0 +1,283 @@ +from inspect import cleandoc +class IdeogramTextToImage: + """ + Generates images synchronously based on a given prompt and optional parameters. + + Images links are available for a limited period of time; if you would like to keep the image, you must download it. + """ + def __init__(self): + pass + + @classmethod + def INPUT_TYPES(s): + """ + Return a dictionary which contains config for all input fields. + Some types (string): "MODEL", "VAE", "CLIP", "CONDITIONING", "LATENT", "IMAGE", "INT", "STRING", "FLOAT". + Input types "INT", "STRING" or "FLOAT" are special values for fields on the node. + The type can be a list for selection. + + Returns: `dict`: + - Key input_fields_group (`string`): Can be either required, hidden or optional. A node class must have property `required` + - Value input_fields (`dict`): Contains input fields config: + * Key field_name (`string`): Name of a entry-point method's argument + * Value field_config (`tuple`): + + First value is a string indicate the type of field or a list for selection. + + Secound value is a config for type "INT", "STRING" or "FLOAT". + """ + return { + "required": { + "prompt": ("STRING", {"multiline": True, + "default": "", "tooltip": "Prompt for the image generation"}), + "model": (["V_2", "V_2_TURBO", "V_1", "V_1_TURBO"], {"default": "V_2"}), + }, + "optional": { + "aspect_ratio": (["ASPECT_1_1", "ASPECT_4_3", "ASPECT_3_4", "ASPECT_16_9", "ASPECT_9_16", + "ASPECT_2_1", "ASPECT_1_2", "ASPECT_3_2", "ASPECT_2_3", "ASPECT_4_5", "ASPECT_5_4"], { + "default": "ASPECT_1_1", + "tooltip": "The aspect ratio for image generation. Cannot be used with resolution" + }), + "resolution": (["1024x1024", "1024x1792", "1792x1024"], { + "default": "1024x1024", + "tooltip": "The resolution for image generation (V2 only). Cannot be used with aspect_ratio" + }), + "magic_prompt_option": (["AUTO", "ON", "OFF"], { + "default": "AUTO", + "tooltip": "Determine if MagicPrompt should be used in generation" + }), + "seed": ("INT", { + "default": 0, + "min": 0, + "max": 2147483647, + "step": 1, + "display": "number" + }), + "style_type": (["NONE", "ANIME", "CINEMATIC", "CREATIVE", "DIGITAL_ART", "PHOTOGRAPHIC"], { + "default": "NONE", + "tooltip": "Style type for generation (V2+ only)" + }), + "negative_prompt": ("STRING", { + "multiline": True, + "default": "", + "tooltip": "Description of what to exclude from the image (V1/V2 only)" + }), + "num_images": ("INT", { + "default": 1, + "min": 1, + "max": 8, + "step": 1, + "display": "number" + }), + "color_palette": ("STRING", { + "multiline": False, + "default": "", + "tooltip": "Color palette preset name or hex colors with weights (V2/V2_TURBO only)" + }), + } + } + + RETURN_TYPES = ("IMAGE",) + #RETURN_NAMES = ("image_output_name",) + DESCRIPTION = cleandoc(__doc__) + FUNCTION = "api_call" + + #OUTPUT_NODE = False + #OUTPUT_TOOLTIPS = ("",) # Tooltips for the output node + + CATEGORY = "Example" + + def api_call(self, prompt, model, aspect_ratio=None, resolution=None, + magic_prompt_option="AUTO", seed=0, style_type="NONE", + negative_prompt="", num_images=1, color_palette=""): + import requests + import torch + from PIL import Image + import io + import numpy as np + import time + + # Build payload with all available parameters + payload = { + "image_request": { + "prompt": prompt, + "model": model, + "num_images": num_images, + "seed": seed, + } + } + + # Make API request + headers = { + "Authorization": "Bearer TBD", # TODO(robin): add authorization key + "Content-Type": "application/json" + } + + response = requests.post( + "http://localhost:8080/proxy/ideogram/generate", + headers=headers, + json=payload + ) + + if response.status_code != 200: + raise Exception(f"API request failed: {response.text}") + + # Parse response + response_data = response.json() + + # Get the image URL from the response + image_url = response_data["data"][0]["url"] + + # Time the image download + download_start = time.time() + img_response = requests.get(image_url) + if img_response.status_code != 200: + raise Exception("Failed to download the image") + download_time = (time.time() - download_start) * 1000 # Convert to milliseconds + print(f"Image download time: {download_time:.2f}ms") + + # Time the conversion process + conversion_start = time.time() + img = Image.open(io.BytesIO(img_response.content)) + img = img.convert("RGB") # Ensure RGB format + + # Convert to numpy array, normalize to float32 between 0 and 1 + img_array = np.array(img).astype(np.float32) / 255.0 + + # Convert to torch tensor and add batch dimension + img_tensor = torch.from_numpy(img_array)[None,] + conversion_time = (time.time() - conversion_start) * 1000 # Convert to milliseconds + print(f"Image conversion time: {conversion_time:.2f}ms") + + return (img_tensor,) + + """ + The node will always be re executed if any of the inputs change but + this method can be used to force the node to execute again even when the inputs don't change. + You can make this node return a number or a string. This value will be compared to the one returned the last time the node was + executed, if it is different the node will be executed again. + This method is used in the core repo for the LoadImage node where they return the image hash as a string, if the image hash + changes between executions the LoadImage node is executed again. + """ + #@classmethod + #def IS_CHANGED(s, image, string_field, int_field, float_field, print_to_screen): + # return "" + + +class RunwayVideoNode: + """ + Generates videos synchronously based on a given image, prompt, and optional parameters using Runway's API. + """ + def __init__(self): + pass + + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "prompt_image": ("IMAGE",), # Will need to handle image URL conversion + "prompt_text": ("STRING", { + "multiline": True, + "default": "", + "tooltip": "Text prompt to guide the video generation" + }), + }, + "optional": { + "seed": ("INT", { + "default": 0, + "min": 0, + "max": 4294967295, + "step": 1, + "display": "number" + }), + "model": (["gen3a_turbo"], { + "default": "gen3a_turbo", + "tooltip": "Model to use for video generation" + }), + "duration": ("FLOAT", { + "default": 5.0, + "min": 1.0, + "max": 10.0, + "step": 0.1, + "display": "number", + "tooltip": "Duration of the generated video in seconds" + }), + "ratio": (["1280:768", "768:1280"], { + "default": "1280:768", + "tooltip": "Aspect ratio of the output video" + }), + "watermark": ("BOOLEAN", { + "default": False, + "tooltip": "Whether to include watermark in the output" + }), + } + } + + RETURN_TYPES = ("VIDEO",) + DESCRIPTION = "Generates videos from images using Runway's API" + FUNCTION = "generate_video" + CATEGORY = "video" + + def generate_video(self, prompt_image, prompt_text, seed=0, model="gen3a_turbo", + duration=5.0, ratio="1280:768", watermark=False): + import requests + import torch + import time + import os + + # Hardcoded API key (temporary solution) + api_key = "key_e861661aa0b307e07e8cc269c1f42cf56fcce876ed6511a507e185ee51f695291da21f4777be1326b4467c34be5a6498b72dc27c9780e483250c692aa410d4c6" # Replace with actual API key + + # Convert torch tensor image to URL (you'll need to implement this part) + # This is a placeholder - you'll need to either save the image temporarily + # or upload it to a service that can host it + image_url = "http://example.com" # Placeholder + + # Build payload + payload = { + "promptImage": image_url, + "promptText": prompt_text, + "seed": seed, + "model": model, + "watermark": watermark, + "duration": duration, + "ratio": ratio + } + + # Make API request + headers = { + "Authorization": f"Bearer {api_key}", + "Content-Type": "application/json", + "X-Runway-Version": "2024-11-06" + } + + # Time the API request + api_start = time.time() + response = requests.post( + "https://api.dev.runwayml.com/v1/image_to_video", + headers=headers, + json=payload + ) + api_time = (time.time() - api_start) * 1000 # Convert to milliseconds + print(f"API request time: {api_time:.2f}ms") + + if response.status_code != 200: + raise Exception(f"API request failed: {response.text}") + + # Parse response + response_data = response.json() + + # Note: You'll need to implement the actual video handling here + # This is a placeholder return + return (None,) + +# A dictionary that contains all nodes you want to export with their names +# NOTE: names should be globally unique +NODE_CLASS_MAPPINGS = { + "IdeogramTextToImage": IdeogramTextToImage, + "RunwayVideoNode": RunwayVideoNode +} + +# A dictionary that contains the friendly/humanly readable titles for the nodes +NODE_DISPLAY_NAME_MAPPINGS = { + "IdeogramTextToImage": "Ideogram Text to Image", + "RunwayVideoNode": "Runway Video Generator" +} diff --git a/nodes.py b/nodes.py index 73a62d930..003034b33 100644 --- a/nodes.py +++ b/nodes.py @@ -2258,6 +2258,7 @@ def init_builtin_extra_nodes(): "nodes_optimalsteps.py", "nodes_hidream.py", "nodes_fresca.py", + "nodes_api.py", ] api_nodes_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), "comfy_api_nodes") From daa25d1b9e5638ebd808ae45aadd589c602f2db4 Mon Sep 17 00:00:00 2001 From: Robin Huang Date: Tue, 15 Apr 2025 13:57:08 -0700 Subject: [PATCH 002/121] Add staging api. --- comfy_extras/nodes_api.py | 101 +++++++++++++++++--------------------- 1 file changed, 46 insertions(+), 55 deletions(-) diff --git a/comfy_extras/nodes_api.py b/comfy_extras/nodes_api.py index 8ab57bb26..bb484bbc4 100644 --- a/comfy_extras/nodes_api.py +++ b/comfy_extras/nodes_api.py @@ -1,5 +1,10 @@ +# Add API base URL at the top of the file +API_BASE = "https://stagingapi.comfy.org" + from inspect import cleandoc -class IdeogramTextToImage: +from comfy.comfy_types.node_typing import ComfyNodeABC, InputTypeDict, IO + +class IdeogramTextToImage(ComfyNodeABC): """ Generates images synchronously based on a given prompt and optional parameters. @@ -9,7 +14,7 @@ class IdeogramTextToImage: pass @classmethod - def INPUT_TYPES(s): + def INPUT_TYPES(cls) -> InputTypeDict: """ Return a dictionary which contains config for all input fields. Some types (string): "MODEL", "VAE", "CLIP", "CONDITIONING", "LATENT", "IMAGE", "INT", "STRING", "FLOAT". @@ -26,58 +31,61 @@ class IdeogramTextToImage: """ return { "required": { - "prompt": ("STRING", {"multiline": True, - "default": "", "tooltip": "Prompt for the image generation"}), - "model": (["V_2", "V_2_TURBO", "V_1", "V_1_TURBO"], {"default": "V_2"}), + "prompt": (IO.STRING, { + "multiline": True, + "default": "", + "tooltip": "Prompt for the image generation", + }), + "model": (IO.COMBO, { "options": ["V_2", "V_2_TURBO", "V_1", "V_1_TURBO"], "default": "V_2", "tooltip": "Model to use for image generation"}), }, "optional": { - "aspect_ratio": (["ASPECT_1_1", "ASPECT_4_3", "ASPECT_3_4", "ASPECT_16_9", "ASPECT_9_16", - "ASPECT_2_1", "ASPECT_1_2", "ASPECT_3_2", "ASPECT_2_3", "ASPECT_4_5", "ASPECT_5_4"], { - "default": "ASPECT_1_1", - "tooltip": "The aspect ratio for image generation. Cannot be used with resolution" + "aspect_ratio": (IO.COMBO, { "options": ["ASPECT_1_1", "ASPECT_4_3", "ASPECT_3_4", "ASPECT_16_9", "ASPECT_9_16", "ASPECT_2_1", "ASPECT_1_2", "ASPECT_3_2", "ASPECT_2_3", "ASPECT_4_5", "ASPECT_5_4"], "default": "ASPECT_1_1", "tooltip": "The aspect ratio for image generation. Cannot be used with resolution" }), - "resolution": (["1024x1024", "1024x1792", "1792x1024"], { + "resolution": (IO.COMBO, { "options": ["1024x1024", "1024x1792", "1792x1024"], "default": "1024x1024", "tooltip": "The resolution for image generation (V2 only). Cannot be used with aspect_ratio" }), - "magic_prompt_option": (["AUTO", "ON", "OFF"], { + "magic_prompt_option": (IO.COMBO, { "options": ["AUTO", "ON", "OFF"], "default": "AUTO", "tooltip": "Determine if MagicPrompt should be used in generation" }), - "seed": ("INT", { + "seed": (IO.INT, { "default": 0, "min": 0, "max": 2147483647, "step": 1, "display": "number" }), - "style_type": (["NONE", "ANIME", "CINEMATIC", "CREATIVE", "DIGITAL_ART", "PHOTOGRAPHIC"], { + "style_type": (IO.COMBO, { "options": ["NONE", "ANIME", "CINEMATIC", "CREATIVE", "DIGITAL_ART", "PHOTOGRAPHIC"], "default": "NONE", "tooltip": "Style type for generation (V2+ only)" }), - "negative_prompt": ("STRING", { + "negative_prompt": (IO.STRING, { "multiline": True, "default": "", "tooltip": "Description of what to exclude from the image (V1/V2 only)" }), - "num_images": ("INT", { + "num_images": (IO.INT, { "default": 1, "min": 1, "max": 8, "step": 1, "display": "number" }), - "color_palette": ("STRING", { + "color_palette": (IO.STRING, { "multiline": False, "default": "", "tooltip": "Color palette preset name or hex colors with weights (V2/V2_TURBO only)" }), + }, + "hidden": { + "auth_token": "AUTH_TOKEN_COMFY_ORG" } } - RETURN_TYPES = ("IMAGE",) + RETURN_TYPES = (IO.IMAGE,) #RETURN_NAMES = ("image_output_name",) - DESCRIPTION = cleandoc(__doc__) + DESCRIPTION = cleandoc(__doc__ or "") # Handle potential None value FUNCTION = "api_call" #OUTPUT_NODE = False @@ -85,15 +93,14 @@ class IdeogramTextToImage: CATEGORY = "Example" - def api_call(self, prompt, model, aspect_ratio=None, resolution=None, - magic_prompt_option="AUTO", seed=0, style_type="NONE", - negative_prompt="", num_images=1, color_palette=""): + def api_call(self, prompt, model, aspect_ratio=None, resolution=None, + magic_prompt_option="AUTO", seed=0, style_type="NONE", + negative_prompt="", num_images=1, color_palette="", auth_token=None): import requests import torch from PIL import Image import io import numpy as np - import time # Build payload with all available parameters payload = { @@ -107,12 +114,12 @@ class IdeogramTextToImage: # Make API request headers = { - "Authorization": "Bearer TBD", # TODO(robin): add authorization key + "Authorization": f"Bearer {auth_token}", "Content-Type": "application/json" } - + response = requests.post( - "http://localhost:8080/proxy/ideogram/generate", + f"{API_BASE}/proxy/ideogram/generate", headers=headers, json=payload ) @@ -122,30 +129,22 @@ class IdeogramTextToImage: # Parse response response_data = response.json() - + # Get the image URL from the response image_url = response_data["data"][0]["url"] - - # Time the image download - download_start = time.time() + img_response = requests.get(image_url) if img_response.status_code != 200: raise Exception("Failed to download the image") - download_time = (time.time() - download_start) * 1000 # Convert to milliseconds - print(f"Image download time: {download_time:.2f}ms") - # Time the conversion process - conversion_start = time.time() img = Image.open(io.BytesIO(img_response.content)) img = img.convert("RGB") # Ensure RGB format - + # Convert to numpy array, normalize to float32 between 0 and 1 img_array = np.array(img).astype(np.float32) / 255.0 - + # Convert to torch tensor and add batch dimension img_tensor = torch.from_numpy(img_array)[None,] - conversion_time = (time.time() - conversion_start) * 1000 # Convert to milliseconds - print(f"Image conversion time: {conversion_time:.2f}ms") return (img_tensor,) @@ -208,7 +207,10 @@ class RunwayVideoNode: "default": False, "tooltip": "Whether to include watermark in the output" }), - } + }, + "hidden": { + "auth_token": "AUTH_TOKEN_COMFY_ORG" + }, } RETURN_TYPES = ("VIDEO",) @@ -216,15 +218,9 @@ class RunwayVideoNode: FUNCTION = "generate_video" CATEGORY = "video" - def generate_video(self, prompt_image, prompt_text, seed=0, model="gen3a_turbo", - duration=5.0, ratio="1280:768", watermark=False): + def generate_video(self, prompt_image, prompt_text, seed=0, model="gen3a_turbo", + duration=5.0, ratio="1280:768", watermark=False, auth_token=None): import requests - import torch - import time - import os - - # Hardcoded API key (temporary solution) - api_key = "key_e861661aa0b307e07e8cc269c1f42cf56fcce876ed6511a507e185ee51f695291da21f4777be1326b4467c34be5a6498b72dc27c9780e483250c692aa410d4c6" # Replace with actual API key # Convert torch tensor image to URL (you'll need to implement this part) # This is a placeholder - you'll need to either save the image temporarily @@ -244,27 +240,22 @@ class RunwayVideoNode: # Make API request headers = { - "Authorization": f"Bearer {api_key}", + "Authorization": f"Bearer {auth_token}", "Content-Type": "application/json", - "X-Runway-Version": "2024-11-06" } - # Time the API request - api_start = time.time() response = requests.post( - "https://api.dev.runwayml.com/v1/image_to_video", + f"{API_BASE}/proxy/runway/image_to_video", headers=headers, json=payload ) - api_time = (time.time() - api_start) * 1000 # Convert to milliseconds - print(f"API request time: {api_time:.2f}ms") if response.status_code != 200: raise Exception(f"API request failed: {response.text}") # Parse response - response_data = response.json() - + # response_data = response.json() + # Note: You'll need to implement the actual video handling here # This is a placeholder return return (None,) From 7922ecdb5a6addf17c57ee4c3eae2660c7714366 Mon Sep 17 00:00:00 2001 From: Robin Huang Date: Wed, 16 Apr 2025 13:38:28 -0700 Subject: [PATCH 003/121] Add API_NODE and common error for missing auth token (#5) --- comfy_extras/nodes_api.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/comfy_extras/nodes_api.py b/comfy_extras/nodes_api.py index bb484bbc4..839e717ea 100644 --- a/comfy_extras/nodes_api.py +++ b/comfy_extras/nodes_api.py @@ -4,6 +4,12 @@ API_BASE = "https://stagingapi.comfy.org" from inspect import cleandoc from comfy.comfy_types.node_typing import ComfyNodeABC, InputTypeDict, IO +def check_auth_token(auth_token): + """Verify that an auth token is present.""" + if auth_token is None: + raise Exception("Please login first to use this node.") + return auth_token + class IdeogramTextToImage(ComfyNodeABC): """ Generates images synchronously based on a given prompt and optional parameters. @@ -84,13 +90,9 @@ class IdeogramTextToImage(ComfyNodeABC): } RETURN_TYPES = (IO.IMAGE,) - #RETURN_NAMES = ("image_output_name",) DESCRIPTION = cleandoc(__doc__ or "") # Handle potential None value FUNCTION = "api_call" - - #OUTPUT_NODE = False - #OUTPUT_TOOLTIPS = ("",) # Tooltips for the output node - + API_NODE = True CATEGORY = "Example" def api_call(self, prompt, model, aspect_ratio=None, resolution=None, @@ -102,6 +104,8 @@ class IdeogramTextToImage(ComfyNodeABC): import io import numpy as np + check_auth_token(auth_token) + # Build payload with all available parameters payload = { "image_request": { @@ -217,11 +221,12 @@ class RunwayVideoNode: DESCRIPTION = "Generates videos from images using Runway's API" FUNCTION = "generate_video" CATEGORY = "video" + API_NODE = True def generate_video(self, prompt_image, prompt_text, seed=0, model="gen3a_turbo", duration=5.0, ratio="1280:768", watermark=False, auth_token=None): import requests - + check_auth_token(auth_token) # Convert torch tensor image to URL (you'll need to implement this part) # This is a placeholder - you'll need to either save the image temporarily # or upload it to a service that can host it From 501001b4dbecbdc82f20412d6faf03d42c76d600 Mon Sep 17 00:00:00 2001 From: Robin Huang Date: Sat, 19 Apr 2025 13:23:26 -0700 Subject: [PATCH 004/121] Add Minimax Video Generation + Async Task queue polling example (#6) --- comfy_api_nodes/apis/client.py | 172 ++++++++++- comfy_api_nodes/apis/stubs.py | 513 +++++++++++++++++++++++++++++++++ comfy_extras/nodes_api.py | 279 ------------------ nodes.py | 4 + uv.lock | 7 + 5 files changed, 695 insertions(+), 280 deletions(-) create mode 100644 comfy_api_nodes/apis/stubs.py delete mode 100644 comfy_extras/nodes_api.py create mode 100644 uv.lock diff --git a/comfy_api_nodes/apis/client.py b/comfy_api_nodes/apis/client.py index 384e559dc..4b585fef2 100644 --- a/comfy_api_nodes/apis/client.py +++ b/comfy_api_nodes/apis/client.py @@ -1,5 +1,6 @@ import logging - +import time +from typing import Callable """ API Client Framework for api.comfy.org. @@ -46,6 +47,49 @@ operation = ApiOperation( ) user_profile = operation.execute(client=api_client) # Returns immediately with the result + +# Example 2: Asynchronous API Operation with Polling +# ------------------------------------------------- +# For an API that starts a task and requires polling for completion: + +# 1. Define the endpoints (initial request and polling) +generate_image_endpoint = ApiEndpoint( + path="/v1/images/generate", + method=HttpMethod.POST, + request_model=ImageGenerationRequest, + response_model=TaskCreatedResponse, + query_params=None +) + +check_task_endpoint = ApiEndpoint( + path="/v1/tasks/{task_id}", + method=HttpMethod.GET, + request_model=EmptyRequest, + response_model=ImageGenerationResult, + query_params=None +) + +# 2. Create the request object +request = ImageGenerationRequest( + prompt="a beautiful sunset over mountains", + width=1024, + height=1024, + num_images=1 +) + +# 3. Create and execute the polling operation +operation = PollingOperation( + initial_endpoint=generate_image_endpoint, + initial_request=request, + poll_endpoint=check_task_endpoint, + task_id_field="task_id", + status_field="status", + completed_statuses=["completed"], + failed_statuses=["failed", "error"] +) + +# This will make the initial request and then poll until completion +result = operation.execute(client=api_client) # Returns the final ImageGenerationResult when done """ from typing import ( @@ -62,8 +106,12 @@ import json import requests from urllib.parse import urljoin +# Import models from your generated stubs + T = TypeVar("T", bound=BaseModel) R = TypeVar("R", bound=BaseModel) +P = TypeVar("P", bound=BaseModel) # For poll response + class EmptyRequest(BaseModel): """Base class for empty request bodies. @@ -335,3 +383,125 @@ class SynchronousOperation(Generic[T, R]): self.response = self.endpoint.response_model.model_validate(resp) logging.debug(f"[DEBUG] Parsed Response: {self.response}") return self.response + + +class TaskStatus(str, Enum): + """Enum for task status values""" + + COMPLETED = "completed" + FAILED = "failed" + PENDING = "pending" + + +class PollingOperation(Generic[T, R]): + """ + Represents an asynchronous API operation that requires polling for completion. + """ + + def __init__( + self, + poll_endpoint: ApiEndpoint[EmptyRequest, R], + completed_statuses: list, + failed_statuses: list, + status_extractor: Callable[[R], str], + request: Optional[T] = None, + api_base: str = "https://stagingapi.comfy.org", + auth_token: Optional[str] = None, + poll_interval: float = 1.0, + ): + self.poll_endpoint = poll_endpoint + self.request = request + self.api_base = api_base + self.auth_token = auth_token + self.poll_interval = poll_interval + + # Polling configuration + self.status_extractor = status_extractor or ( + lambda x: getattr(x, "status", None) + ) + self.completed_statuses = completed_statuses + self.failed_statuses = failed_statuses + + # For storing response data + self.final_response = None + self.error = None + + def execute(self, client: Optional[ApiClient] = None) -> R: + """Execute the polling operation using the provided client. If failed, raise an exception.""" + try: + if client is None: + client = ApiClient( + base_url=self.api_base, + api_key=self.auth_token, + ) + return self._poll_until_complete(client) + except Exception as e: + raise Exception(f"Error during polling: {str(e)}") + + def _check_task_status(self, response: R) -> TaskStatus: + """Check task status using the status extractor function""" + try: + status = self.status_extractor(response) + if status in self.completed_statuses: + return TaskStatus.COMPLETED + elif status in self.failed_statuses: + return TaskStatus.FAILED + return TaskStatus.PENDING + except Exception as e: + logging.debug(f"Error extracting status: {e}") + return TaskStatus.PENDING + + def _poll_until_complete(self, client: ApiClient) -> R: + """Poll until the task is complete""" + poll_count = 0 + while True: + try: + poll_count += 1 + logging.debug(f"[DEBUG] Polling attempt #{poll_count}") + + request_dict = ( + self.request.model_dump(exclude_none=True) + if self.request is not None + else None + ) + + if poll_count == 1: + logging.debug( + f"[DEBUG] Poll Request: {self.poll_endpoint.method.value} {self.poll_endpoint.path}" + ) + logging.debug( + f"[DEBUG] Poll Request Data: {json.dumps(request_dict, indent=2) if request_dict else 'None'}" + ) + + # Query task status + resp = client.request( + method=self.poll_endpoint.method.value, + path=self.poll_endpoint.path, + params=self.poll_endpoint.query_params, + json=request_dict, + ) + + # Parse response + response_obj = self.poll_endpoint.response_model.model_validate(resp) + + # Check if task is complete + status = self._check_task_status(response_obj) + logging.debug(f"[DEBUG] Task Status: {status}") + + if status == TaskStatus.COMPLETED: + logging.debug("[DEBUG] Task completed successfully") + self.final_response = response_obj + return self.final_response + elif status == TaskStatus.FAILED: + logging.debug(f"[DEBUG] Task failed: {json.dumps(resp)}") + raise Exception(f"Task failed: {json.dumps(resp)}") + else: + logging.debug("[DEBUG] Task still pending, continuing to poll...") + + # Wait before polling again + logging.debug(f"[DEBUG] Waiting {self.poll_interval} seconds before next poll") + time.sleep(self.poll_interval) + + except Exception as e: + logging.debug(f"[DEBUG] Polling error: {str(e)}") + raise Exception(f"Error while polling: {str(e)}") diff --git a/comfy_api_nodes/apis/stubs.py b/comfy_api_nodes/apis/stubs.py new file mode 100644 index 000000000..d1da6a5ab --- /dev/null +++ b/comfy_api_nodes/apis/stubs.py @@ -0,0 +1,513 @@ +# generated by datamodel-codegen: +# filename: http://localhost:8080/openapi +# timestamp: 2025-04-18T21:35:21+00:00 + +from __future__ import annotations + +from datetime import datetime +from enum import Enum +from typing import Any, Dict, List, Optional +from uuid import UUID + +from pydantic import BaseModel, Field, conint, constr + + +class ComfyNode(BaseModel): + category: Optional[str] = Field( + None, + description='UI category where the node is listed, used for grouping nodes.', + ) + comfy_node_name: Optional[str] = Field( + None, description='Unique identifier for the node' + ) + deprecated: Optional[bool] = Field( + None, + description='Indicates if the node is deprecated. Deprecated nodes are hidden in the UI.', + ) + description: Optional[str] = Field( + None, description="Brief description of the node's functionality or purpose." + ) + experimental: Optional[bool] = Field( + None, + description='Indicates if the node is experimental, subject to changes or removal.', + ) + function: Optional[str] = Field( + None, description='Name of the entry-point function to execute the node.' + ) + input_types: Optional[str] = Field(None, description='Defines input parameters') + output_is_list: Optional[List[bool]] = Field( + None, description='Boolean values indicating if each output is a list.' + ) + return_names: Optional[str] = Field( + None, description='Names of the outputs for clarity in workflows.' + ) + return_types: Optional[str] = Field( + None, description='Specifies the types of outputs produced by the node.' + ) + + +class ComfyNodeCloudBuildInfo(BaseModel): + build_id: Optional[str] = None + location: Optional[str] = None + project_id: Optional[str] = None + project_number: Optional[str] = None + + +class Customer(BaseModel): + createdAt: Optional[datetime] = Field( + None, description='The date and time the user was created' + ) + email: Optional[str] = Field(None, description='The email address for this user') + id: str = Field(..., description='The firebase UID of the user') + name: Optional[str] = Field(None, description='The name for this user') + updatedAt: Optional[datetime] = Field( + None, description='The date and time the user was last updated' + ) + + +class Error(BaseModel): + details: Optional[List[str]] = Field( + None, + description='Optional detailed information about the error or hints for resolving it.', + ) + message: Optional[str] = Field( + None, description='A clear and concise description of the error.' + ) + + +class ErrorResponse(BaseModel): + error: str + message: str + + +class GitCommitSummary(BaseModel): + author: Optional[str] = Field(None, description='The author of the commit') + branch_name: Optional[str] = Field( + None, description='The branch where the commit was made' + ) + commit_hash: Optional[str] = Field(None, description='The hash of the commit') + commit_name: Optional[str] = Field(None, description='The name of the commit') + status_summary: Optional[Dict[str, str]] = Field( + None, description='A map of operating system to status pairs' + ) + timestamp: Optional[datetime] = Field( + None, description='The timestamp when the commit was made' + ) + + +class ImageRequest(BaseModel): + aspect_ratio: Optional[str] = Field( + None, + description="Optional. The aspect ratio (e.g., 'ASPECT_16_9', 'ASPECT_1_1'). Cannot be used with resolution. Defaults to 'ASPECT_1_1' if unspecified.", + ) + color_palette: Optional[Dict[str, Any]] = Field( + None, description='Optional. Color palette object. Only for V_2, V_2_TURBO.' + ) + magic_prompt_option: Optional[str] = Field( + None, description="Optional. MagicPrompt usage ('AUTO', 'ON', 'OFF')." + ) + model: Optional[str] = Field( + None, + description="Optional. The model used (e.g., 'V_2', 'V_2A_TURBO'). Defaults to 'V_2' if unspecified.", + ) + negative_prompt: Optional[str] = Field( + None, + description='Optional. Description of what to exclude. Only for V_1, V_1_TURBO, V_2, V_2_TURBO.', + ) + num_images: Optional[conint(ge=1, le=8)] = Field( + 1, description='Optional. Number of images to generate (1-8). Defaults to 1.' + ) + prompt: str = Field( + ..., description='Required. The prompt to use to generate the image.' + ) + resolution: Optional[str] = Field( + None, + description="Optional. Resolution (e.g., 'RESOLUTION_1024_1024'). Only for model V_2. Cannot be used with aspect_ratio.", + ) + seed: Optional[conint(ge=0, le=2147483647)] = Field( + None, description='Optional. A number between 0 and 2147483647.' + ) + style_type: Optional[str] = Field( + None, + description="Optional. Style type ('AUTO', 'GENERAL', 'REALISTIC', 'DESIGN', 'RENDER_3D', 'ANIME'). Only for models V_2 and above.", + ) + + +class IdeogramGenerateRequest(BaseModel): + image_request: ImageRequest = Field( + ..., description='The image generation request parameters.' + ) + + +class Datum(BaseModel): + is_image_safe: Optional[bool] = Field( + None, description='Indicates whether the image is considered safe.' + ) + prompt: Optional[str] = Field( + None, description='The prompt used to generate this image.' + ) + resolution: Optional[str] = Field( + None, description="The resolution of the generated image (e.g., '1024x1024')." + ) + seed: Optional[int] = Field( + None, description='The seed value used for this generation.' + ) + style_type: Optional[str] = Field( + None, + description="The style type used for generation (e.g., 'REALISTIC', 'ANIME').", + ) + url: Optional[str] = Field(None, description='URL to the generated image.') + + +class IdeogramGenerateResponse(BaseModel): + created: Optional[datetime] = Field( + None, description='Timestamp when the generation was created.' + ) + data: Optional[List[Datum]] = Field( + None, description='Array of generated image information.' + ) + + +class MachineStats(BaseModel): + cpu_capacity: Optional[str] = Field(None, description='Total CPU on the machine.') + disk_capacity: Optional[str] = Field( + None, description='Total disk capacity on the machine.' + ) + gpu_type: Optional[str] = Field( + None, description='The GPU type. eg. NVIDIA Tesla K80' + ) + initial_cpu: Optional[str] = Field( + None, description='Initial CPU available before the job starts.' + ) + initial_disk: Optional[str] = Field( + None, description='Initial disk available before the job starts.' + ) + initial_ram: Optional[str] = Field( + None, description='Initial RAM available before the job starts.' + ) + machine_name: Optional[str] = Field(None, description='Name of the machine.') + memory_capacity: Optional[str] = Field( + None, description='Total memory on the machine.' + ) + os_version: Optional[str] = Field( + None, description='The operating system version. eg. Ubuntu Linux 20.04' + ) + pip_freeze: Optional[str] = Field(None, description='The pip freeze output') + vram_time_series: Optional[Dict[str, Any]] = Field( + None, description='Time series of VRAM usage.' + ) + + +class MinimaxBaseResponse(BaseModel): + status_code: int = Field( + ..., + description='Status code. 0 indicates success, other values indicate errors.', + ) + status_msg: str = Field( + ..., description='Specific error details or success message.' + ) + + +class File(BaseModel): + bytes: Optional[int] = Field(None, description='File size in bytes') + created_at: Optional[int] = Field( + None, description='Unix timestamp when the file was created, in seconds' + ) + download_url: Optional[str] = Field( + None, description='The URL to download the video' + ) + file_id: Optional[int] = Field(None, description='Unique identifier for the file') + filename: Optional[str] = Field(None, description='The name of the file') + purpose: Optional[str] = Field(None, description='The purpose of using the file') + + +class MinimaxFileRetrieveResponse(BaseModel): + base_resp: MinimaxBaseResponse + file: File + + +class Status(str, Enum): + Queueing = 'Queueing' + Preparing = 'Preparing' + Processing = 'Processing' + Success = 'Success' + Fail = 'Fail' + + +class MinimaxTaskResultResponse(BaseModel): + base_resp: MinimaxBaseResponse + file_id: Optional[str] = Field( + None, + description='After the task status changes to Success, this field returns the file ID corresponding to the generated video.', + ) + status: Status = Field( + ..., + description="Task status: 'Queueing' (in queue), 'Preparing' (task is preparing), 'Processing' (generating), 'Success' (task completed successfully), or 'Fail' (task failed).", + ) + task_id: str = Field(..., description='The task ID being queried.') + + +class Model(str, Enum): + T2V_01_Director = 'T2V-01-Director' + I2V_01_Director = 'I2V-01-Director' + S2V_01 = 'S2V-01' + I2V_01 = 'I2V-01' + I2V_01_live = 'I2V-01-live' + T2V_01 = 'T2V-01' + + +class SubjectReferenceItem(BaseModel): + image: Optional[str] = Field( + None, description='URL or base64 encoding of the subject reference image.' + ) + mask: Optional[str] = Field( + None, + description='URL or base64 encoding of the mask for the subject reference image.', + ) + + +class MinimaxVideoGenerationRequest(BaseModel): + callback_url: Optional[str] = Field( + None, + description='Optional. URL to receive real-time status updates about the video generation task.', + ) + first_frame_image: Optional[str] = Field( + None, + description='URL or base64 encoding of the first frame image. Required when model is I2V-01, I2V-01-Director, or I2V-01-live.', + ) + model: Model = Field( + ..., + description='Required. ID of model. Options: T2V-01-Director, I2V-01-Director, S2V-01, I2V-01, I2V-01-live, T2V-01', + ) + prompt: Optional[constr(max_length=2000)] = Field( + None, + description='Description of the video. Should be less than 2000 characters. Supports camera movement instructions in [brackets].', + ) + prompt_optimizer: Optional[bool] = Field( + True, + description='If true (default), the model will automatically optimize the prompt. Set to false for more precise control.', + ) + subject_reference: Optional[List[SubjectReferenceItem]] = Field( + None, + description='Only available when model is S2V-01. The model will generate a video based on the subject uploaded through this parameter.', + ) + + +class MinimaxVideoGenerationResponse(BaseModel): + base_resp: MinimaxBaseResponse + task_id: str = Field( + ..., description='The task ID for the asynchronous video generation task.' + ) + + +class NodeStatus(str, Enum): + NodeStatusActive = 'NodeStatusActive' + NodeStatusDeleted = 'NodeStatusDeleted' + NodeStatusBanned = 'NodeStatusBanned' + + +class NodeVersionStatus(str, Enum): + NodeVersionStatusActive = 'NodeVersionStatusActive' + NodeVersionStatusDeleted = 'NodeVersionStatusDeleted' + NodeVersionStatusBanned = 'NodeVersionStatusBanned' + NodeVersionStatusPending = 'NodeVersionStatusPending' + NodeVersionStatusFlagged = 'NodeVersionStatusFlagged' + + +class NodeVersionUpdateRequest(BaseModel): + changelog: Optional[str] = Field( + None, description='The changelog describing the version changes.' + ) + deprecated: Optional[bool] = Field( + None, description='Whether the version is deprecated.' + ) + + +class PersonalAccessToken(BaseModel): + createdAt: Optional[datetime] = Field( + None, description='[Output Only]The date and time the token was created.' + ) + description: Optional[str] = Field( + None, + description="Optional. A more detailed description of the token's intended use.", + ) + id: Optional[UUID] = Field(None, description='Unique identifier for the GitCommit') + name: Optional[str] = Field( + None, + description='Required. The name of the token. Can be a simple description.', + ) + token: Optional[str] = Field( + None, + description='[Output Only]. The personal access token. Only returned during creation.', + ) + + +class PublisherStatus(str, Enum): + PublisherStatusActive = 'PublisherStatusActive' + PublisherStatusBanned = 'PublisherStatusBanned' + + +class PublisherUser(BaseModel): + email: Optional[str] = Field(None, description='The email address for this user.') + id: Optional[str] = Field(None, description='The unique id for this user.') + name: Optional[str] = Field(None, description='The name for this user.') + + +class StorageFile(BaseModel): + file_path: Optional[str] = Field(None, description='Path to the file in storage') + id: Optional[UUID] = Field( + None, description='Unique identifier for the storage file' + ) + public_url: Optional[str] = Field(None, description='Public URL') + + +class User(BaseModel): + email: Optional[str] = Field(None, description='The email address for this user.') + id: Optional[str] = Field(None, description='The unique id for this user.') + isAdmin: Optional[bool] = Field( + None, description='Indicates if the user has admin privileges.' + ) + isApproved: Optional[bool] = Field( + None, description='Indicates if the user is approved.' + ) + name: Optional[str] = Field(None, description='The name for this user.') + + +class WorkflowRunStatus(str, Enum): + WorkflowRunStatusStarted = 'WorkflowRunStatusStarted' + WorkflowRunStatusFailed = 'WorkflowRunStatusFailed' + WorkflowRunStatusCompleted = 'WorkflowRunStatusCompleted' + + +class ActionJobResult(BaseModel): + action_job_id: Optional[str] = Field( + None, description='Identifier of the job this result belongs to' + ) + action_run_id: Optional[str] = Field( + None, description='Identifier of the run this result belongs to' + ) + author: Optional[str] = Field(None, description='The author of the commit') + avg_vram: Optional[int] = Field( + None, description='The average VRAM used by the job' + ) + branch_name: Optional[str] = Field( + None, description='Name of the relevant git branch' + ) + comfy_run_flags: Optional[str] = Field( + None, description='The comfy run flags. E.g. `--low-vram`' + ) + commit_hash: Optional[str] = Field(None, description='The hash of the commit') + commit_id: Optional[str] = Field(None, description='The ID of the commit') + commit_message: Optional[str] = Field(None, description='The message of the commit') + commit_time: Optional[int] = Field( + None, description='The Unix timestamp when the commit was made' + ) + cuda_version: Optional[str] = Field(None, description='CUDA version used') + end_time: Optional[int] = Field( + None, description='The end time of the job as a Unix timestamp.' + ) + git_repo: Optional[str] = Field(None, description='The repository name') + id: Optional[UUID] = Field(None, description='Unique identifier for the job result') + job_trigger_user: Optional[str] = Field( + None, description='The user who triggered the job.' + ) + machine_stats: Optional[MachineStats] = None + operating_system: Optional[str] = Field(None, description='Operating system used') + peak_vram: Optional[int] = Field(None, description='The peak VRAM used by the job') + pr_number: Optional[str] = Field(None, description='The pull request number') + python_version: Optional[str] = Field(None, description='PyTorch version used') + pytorch_version: Optional[str] = Field(None, description='PyTorch version used') + start_time: Optional[int] = Field( + None, description='The start time of the job as a Unix timestamp.' + ) + status: Optional[WorkflowRunStatus] = None + storage_file: Optional[StorageFile] = None + workflow_name: Optional[str] = Field(None, description='Name of the workflow') + + +class NodeVersion(BaseModel): + changelog: Optional[str] = Field( + None, description='Summary of changes made in this version' + ) + comfy_node_extract_status: Optional[str] = Field( + None, description='The status of comfy node extraction process.' + ) + createdAt: Optional[datetime] = Field( + None, description='The date and time the version was created.' + ) + dependencies: Optional[List[str]] = Field( + None, description='A list of pip dependencies required by the node.' + ) + deprecated: Optional[bool] = Field( + None, description='Indicates if this version is deprecated.' + ) + downloadUrl: Optional[str] = Field( + None, description='[Output Only] URL to download this version of the node' + ) + id: Optional[str] = None + node_id: Optional[str] = Field( + None, description='The unique identifier of the node.' + ) + status: Optional[NodeVersionStatus] = None + status_reason: Optional[str] = Field( + None, description='The reason for the status change.' + ) + version: Optional[str] = Field( + None, + description='The version identifier, following semantic versioning. Must be unique for the node.', + ) + + +class PublisherMember(BaseModel): + id: Optional[str] = Field( + None, description='The unique identifier for the publisher member.' + ) + role: Optional[str] = Field( + None, description='The role of the user in the publisher.' + ) + user: Optional[PublisherUser] = None + + +class Publisher(BaseModel): + createdAt: Optional[datetime] = Field( + None, description='The date and time the publisher was created.' + ) + description: Optional[str] = None + id: Optional[str] = Field( + None, + description="The unique identifier for the publisher. It's akin to a username. Should be lowercase.", + ) + logo: Optional[str] = Field(None, description="URL to the publisher's logo.") + members: Optional[List[PublisherMember]] = Field( + None, description='A list of members in the publisher.' + ) + name: Optional[str] = None + source_code_repo: Optional[str] = None + status: Optional[PublisherStatus] = None + support: Optional[str] = None + website: Optional[str] = None + + +class Node(BaseModel): + author: Optional[str] = None + category: Optional[str] = Field(None, description='The category of the node.') + description: Optional[str] = None + downloads: Optional[int] = Field( + None, description='The number of downloads of the node.' + ) + icon: Optional[str] = Field(None, description="URL to the node's icon.") + id: Optional[str] = Field(None, description='The unique identifier of the node.') + latest_version: Optional[NodeVersion] = None + license: Optional[str] = Field( + None, description="The path to the LICENSE file in the node's repository." + ) + name: Optional[str] = Field(None, description='The display name of the node.') + publisher: Optional[Publisher] = None + rating: Optional[float] = Field(None, description='The average rating of the node.') + repository: Optional[str] = Field(None, description="URL to the node's repository.") + status: Optional[NodeStatus] = None + status_detail: Optional[str] = Field( + None, description='The status detail of the node.' + ) + tags: Optional[List[str]] = None + translations: Optional[Dict[str, Dict[str, Any]]] = None diff --git a/comfy_extras/nodes_api.py b/comfy_extras/nodes_api.py deleted file mode 100644 index 839e717ea..000000000 --- a/comfy_extras/nodes_api.py +++ /dev/null @@ -1,279 +0,0 @@ -# Add API base URL at the top of the file -API_BASE = "https://stagingapi.comfy.org" - -from inspect import cleandoc -from comfy.comfy_types.node_typing import ComfyNodeABC, InputTypeDict, IO - -def check_auth_token(auth_token): - """Verify that an auth token is present.""" - if auth_token is None: - raise Exception("Please login first to use this node.") - return auth_token - -class IdeogramTextToImage(ComfyNodeABC): - """ - Generates images synchronously based on a given prompt and optional parameters. - - Images links are available for a limited period of time; if you would like to keep the image, you must download it. - """ - def __init__(self): - pass - - @classmethod - def INPUT_TYPES(cls) -> InputTypeDict: - """ - Return a dictionary which contains config for all input fields. - Some types (string): "MODEL", "VAE", "CLIP", "CONDITIONING", "LATENT", "IMAGE", "INT", "STRING", "FLOAT". - Input types "INT", "STRING" or "FLOAT" are special values for fields on the node. - The type can be a list for selection. - - Returns: `dict`: - - Key input_fields_group (`string`): Can be either required, hidden or optional. A node class must have property `required` - - Value input_fields (`dict`): Contains input fields config: - * Key field_name (`string`): Name of a entry-point method's argument - * Value field_config (`tuple`): - + First value is a string indicate the type of field or a list for selection. - + Secound value is a config for type "INT", "STRING" or "FLOAT". - """ - return { - "required": { - "prompt": (IO.STRING, { - "multiline": True, - "default": "", - "tooltip": "Prompt for the image generation", - }), - "model": (IO.COMBO, { "options": ["V_2", "V_2_TURBO", "V_1", "V_1_TURBO"], "default": "V_2", "tooltip": "Model to use for image generation"}), - }, - "optional": { - "aspect_ratio": (IO.COMBO, { "options": ["ASPECT_1_1", "ASPECT_4_3", "ASPECT_3_4", "ASPECT_16_9", "ASPECT_9_16", "ASPECT_2_1", "ASPECT_1_2", "ASPECT_3_2", "ASPECT_2_3", "ASPECT_4_5", "ASPECT_5_4"], "default": "ASPECT_1_1", "tooltip": "The aspect ratio for image generation. Cannot be used with resolution" - }), - "resolution": (IO.COMBO, { "options": ["1024x1024", "1024x1792", "1792x1024"], - "default": "1024x1024", - "tooltip": "The resolution for image generation (V2 only). Cannot be used with aspect_ratio" - }), - "magic_prompt_option": (IO.COMBO, { "options": ["AUTO", "ON", "OFF"], - "default": "AUTO", - "tooltip": "Determine if MagicPrompt should be used in generation" - }), - "seed": (IO.INT, { - "default": 0, - "min": 0, - "max": 2147483647, - "step": 1, - "display": "number" - }), - "style_type": (IO.COMBO, { "options": ["NONE", "ANIME", "CINEMATIC", "CREATIVE", "DIGITAL_ART", "PHOTOGRAPHIC"], - "default": "NONE", - "tooltip": "Style type for generation (V2+ only)" - }), - "negative_prompt": (IO.STRING, { - "multiline": True, - "default": "", - "tooltip": "Description of what to exclude from the image (V1/V2 only)" - }), - "num_images": (IO.INT, { - "default": 1, - "min": 1, - "max": 8, - "step": 1, - "display": "number" - }), - "color_palette": (IO.STRING, { - "multiline": False, - "default": "", - "tooltip": "Color palette preset name or hex colors with weights (V2/V2_TURBO only)" - }), - }, - "hidden": { - "auth_token": "AUTH_TOKEN_COMFY_ORG" - } - } - - RETURN_TYPES = (IO.IMAGE,) - DESCRIPTION = cleandoc(__doc__ or "") # Handle potential None value - FUNCTION = "api_call" - API_NODE = True - CATEGORY = "Example" - - def api_call(self, prompt, model, aspect_ratio=None, resolution=None, - magic_prompt_option="AUTO", seed=0, style_type="NONE", - negative_prompt="", num_images=1, color_palette="", auth_token=None): - import requests - import torch - from PIL import Image - import io - import numpy as np - - check_auth_token(auth_token) - - # Build payload with all available parameters - payload = { - "image_request": { - "prompt": prompt, - "model": model, - "num_images": num_images, - "seed": seed, - } - } - - # Make API request - headers = { - "Authorization": f"Bearer {auth_token}", - "Content-Type": "application/json" - } - - response = requests.post( - f"{API_BASE}/proxy/ideogram/generate", - headers=headers, - json=payload - ) - - if response.status_code != 200: - raise Exception(f"API request failed: {response.text}") - - # Parse response - response_data = response.json() - - # Get the image URL from the response - image_url = response_data["data"][0]["url"] - - img_response = requests.get(image_url) - if img_response.status_code != 200: - raise Exception("Failed to download the image") - - img = Image.open(io.BytesIO(img_response.content)) - img = img.convert("RGB") # Ensure RGB format - - # Convert to numpy array, normalize to float32 between 0 and 1 - img_array = np.array(img).astype(np.float32) / 255.0 - - # Convert to torch tensor and add batch dimension - img_tensor = torch.from_numpy(img_array)[None,] - - return (img_tensor,) - - """ - The node will always be re executed if any of the inputs change but - this method can be used to force the node to execute again even when the inputs don't change. - You can make this node return a number or a string. This value will be compared to the one returned the last time the node was - executed, if it is different the node will be executed again. - This method is used in the core repo for the LoadImage node where they return the image hash as a string, if the image hash - changes between executions the LoadImage node is executed again. - """ - #@classmethod - #def IS_CHANGED(s, image, string_field, int_field, float_field, print_to_screen): - # return "" - - -class RunwayVideoNode: - """ - Generates videos synchronously based on a given image, prompt, and optional parameters using Runway's API. - """ - def __init__(self): - pass - - @classmethod - def INPUT_TYPES(s): - return { - "required": { - "prompt_image": ("IMAGE",), # Will need to handle image URL conversion - "prompt_text": ("STRING", { - "multiline": True, - "default": "", - "tooltip": "Text prompt to guide the video generation" - }), - }, - "optional": { - "seed": ("INT", { - "default": 0, - "min": 0, - "max": 4294967295, - "step": 1, - "display": "number" - }), - "model": (["gen3a_turbo"], { - "default": "gen3a_turbo", - "tooltip": "Model to use for video generation" - }), - "duration": ("FLOAT", { - "default": 5.0, - "min": 1.0, - "max": 10.0, - "step": 0.1, - "display": "number", - "tooltip": "Duration of the generated video in seconds" - }), - "ratio": (["1280:768", "768:1280"], { - "default": "1280:768", - "tooltip": "Aspect ratio of the output video" - }), - "watermark": ("BOOLEAN", { - "default": False, - "tooltip": "Whether to include watermark in the output" - }), - }, - "hidden": { - "auth_token": "AUTH_TOKEN_COMFY_ORG" - }, - } - - RETURN_TYPES = ("VIDEO",) - DESCRIPTION = "Generates videos from images using Runway's API" - FUNCTION = "generate_video" - CATEGORY = "video" - API_NODE = True - - def generate_video(self, prompt_image, prompt_text, seed=0, model="gen3a_turbo", - duration=5.0, ratio="1280:768", watermark=False, auth_token=None): - import requests - check_auth_token(auth_token) - # Convert torch tensor image to URL (you'll need to implement this part) - # This is a placeholder - you'll need to either save the image temporarily - # or upload it to a service that can host it - image_url = "http://example.com" # Placeholder - - # Build payload - payload = { - "promptImage": image_url, - "promptText": prompt_text, - "seed": seed, - "model": model, - "watermark": watermark, - "duration": duration, - "ratio": ratio - } - - # Make API request - headers = { - "Authorization": f"Bearer {auth_token}", - "Content-Type": "application/json", - } - - response = requests.post( - f"{API_BASE}/proxy/runway/image_to_video", - headers=headers, - json=payload - ) - - if response.status_code != 200: - raise Exception(f"API request failed: {response.text}") - - # Parse response - # response_data = response.json() - - # Note: You'll need to implement the actual video handling here - # This is a placeholder return - return (None,) - -# A dictionary that contains all nodes you want to export with their names -# NOTE: names should be globally unique -NODE_CLASS_MAPPINGS = { - "IdeogramTextToImage": IdeogramTextToImage, - "RunwayVideoNode": RunwayVideoNode -} - -# A dictionary that contains the friendly/humanly readable titles for the nodes -NODE_DISPLAY_NAME_MAPPINGS = { - "IdeogramTextToImage": "Ideogram Text to Image", - "RunwayVideoNode": "Runway Video Generator" -} diff --git a/nodes.py b/nodes.py index 003034b33..81428c38d 100644 --- a/nodes.py +++ b/nodes.py @@ -2258,6 +2258,10 @@ def init_builtin_extra_nodes(): "nodes_optimalsteps.py", "nodes_hidream.py", "nodes_fresca.py", + ] + + api_nodes_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), "comfy_api_nodes") + api_nodes_files = [ "nodes_api.py", ] diff --git a/uv.lock b/uv.lock new file mode 100644 index 000000000..df06d6dc7 --- /dev/null +++ b/uv.lock @@ -0,0 +1,7 @@ +version = 1 +requires-python = ">=3.9" + +[[package]] +name = "comfyui" +version = "0.3.26" +source = { virtual = "." } From ded70bb8a14c865c9fe820f7964fe0bbae37d924 Mon Sep 17 00:00:00 2001 From: Christian Byrne Date: Mon, 21 Apr 2025 07:39:52 +0800 Subject: [PATCH 005/121] [Minimax] Show video preview and embed workflow in ouput (#7) --- comfy_api_nodes/nodes_api.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/comfy_api_nodes/nodes_api.py b/comfy_api_nodes/nodes_api.py index 4105ba7e1..927a05fd2 100644 --- a/comfy_api_nodes/nodes_api.py +++ b/comfy_api_nodes/nodes_api.py @@ -278,8 +278,10 @@ class OpenAIGPTImage1(ComfyNodeABC): Uses the proxy at /proxy/openai/images/generations. Returned URLs are short‑lived, so download or cache results if you need to keep them. """ + def __init__(self): - pass + self.output_dir = folder_paths.get_output_directory() + self.type = "output" @classmethod def INPUT_TYPES(cls) -> InputTypeDict: From da72f71bae749cb7998abe81d3689f879b732eba Mon Sep 17 00:00:00 2001 From: Robin Huang Date: Mon, 21 Apr 2025 23:30:39 -0700 Subject: [PATCH 006/121] Remove uv.lock --- uv.lock | 7 ------- 1 file changed, 7 deletions(-) delete mode 100644 uv.lock diff --git a/uv.lock b/uv.lock deleted file mode 100644 index df06d6dc7..000000000 --- a/uv.lock +++ /dev/null @@ -1,7 +0,0 @@ -version = 1 -requires-python = ">=3.9" - -[[package]] -name = "comfyui" -version = "0.3.26" -source = { virtual = "." } From a4aaf8a165d79192bcd1527c9bcd4dbec3217828 Mon Sep 17 00:00:00 2001 From: Robin Huang Date: Mon, 21 Apr 2025 23:37:48 -0700 Subject: [PATCH 007/121] Remove polling operations. --- comfy_api_nodes/apis/client.py | 126 --------------------------------- 1 file changed, 126 deletions(-) diff --git a/comfy_api_nodes/apis/client.py b/comfy_api_nodes/apis/client.py index 4b585fef2..05706710e 100644 --- a/comfy_api_nodes/apis/client.py +++ b/comfy_api_nodes/apis/client.py @@ -106,12 +106,8 @@ import json import requests from urllib.parse import urljoin -# Import models from your generated stubs - T = TypeVar("T", bound=BaseModel) R = TypeVar("R", bound=BaseModel) -P = TypeVar("P", bound=BaseModel) # For poll response - class EmptyRequest(BaseModel): """Base class for empty request bodies. @@ -383,125 +379,3 @@ class SynchronousOperation(Generic[T, R]): self.response = self.endpoint.response_model.model_validate(resp) logging.debug(f"[DEBUG] Parsed Response: {self.response}") return self.response - - -class TaskStatus(str, Enum): - """Enum for task status values""" - - COMPLETED = "completed" - FAILED = "failed" - PENDING = "pending" - - -class PollingOperation(Generic[T, R]): - """ - Represents an asynchronous API operation that requires polling for completion. - """ - - def __init__( - self, - poll_endpoint: ApiEndpoint[EmptyRequest, R], - completed_statuses: list, - failed_statuses: list, - status_extractor: Callable[[R], str], - request: Optional[T] = None, - api_base: str = "https://stagingapi.comfy.org", - auth_token: Optional[str] = None, - poll_interval: float = 1.0, - ): - self.poll_endpoint = poll_endpoint - self.request = request - self.api_base = api_base - self.auth_token = auth_token - self.poll_interval = poll_interval - - # Polling configuration - self.status_extractor = status_extractor or ( - lambda x: getattr(x, "status", None) - ) - self.completed_statuses = completed_statuses - self.failed_statuses = failed_statuses - - # For storing response data - self.final_response = None - self.error = None - - def execute(self, client: Optional[ApiClient] = None) -> R: - """Execute the polling operation using the provided client. If failed, raise an exception.""" - try: - if client is None: - client = ApiClient( - base_url=self.api_base, - api_key=self.auth_token, - ) - return self._poll_until_complete(client) - except Exception as e: - raise Exception(f"Error during polling: {str(e)}") - - def _check_task_status(self, response: R) -> TaskStatus: - """Check task status using the status extractor function""" - try: - status = self.status_extractor(response) - if status in self.completed_statuses: - return TaskStatus.COMPLETED - elif status in self.failed_statuses: - return TaskStatus.FAILED - return TaskStatus.PENDING - except Exception as e: - logging.debug(f"Error extracting status: {e}") - return TaskStatus.PENDING - - def _poll_until_complete(self, client: ApiClient) -> R: - """Poll until the task is complete""" - poll_count = 0 - while True: - try: - poll_count += 1 - logging.debug(f"[DEBUG] Polling attempt #{poll_count}") - - request_dict = ( - self.request.model_dump(exclude_none=True) - if self.request is not None - else None - ) - - if poll_count == 1: - logging.debug( - f"[DEBUG] Poll Request: {self.poll_endpoint.method.value} {self.poll_endpoint.path}" - ) - logging.debug( - f"[DEBUG] Poll Request Data: {json.dumps(request_dict, indent=2) if request_dict else 'None'}" - ) - - # Query task status - resp = client.request( - method=self.poll_endpoint.method.value, - path=self.poll_endpoint.path, - params=self.poll_endpoint.query_params, - json=request_dict, - ) - - # Parse response - response_obj = self.poll_endpoint.response_model.model_validate(resp) - - # Check if task is complete - status = self._check_task_status(response_obj) - logging.debug(f"[DEBUG] Task Status: {status}") - - if status == TaskStatus.COMPLETED: - logging.debug("[DEBUG] Task completed successfully") - self.final_response = response_obj - return self.final_response - elif status == TaskStatus.FAILED: - logging.debug(f"[DEBUG] Task failed: {json.dumps(resp)}") - raise Exception(f"Task failed: {json.dumps(resp)}") - else: - logging.debug("[DEBUG] Task still pending, continuing to poll...") - - # Wait before polling again - logging.debug(f"[DEBUG] Waiting {self.poll_interval} seconds before next poll") - time.sleep(self.poll_interval) - - except Exception as e: - logging.debug(f"[DEBUG] Polling error: {str(e)}") - raise Exception(f"Error while polling: {str(e)}") From 81f55fddf65d53734e38a1f9f896eb13ce5cb160 Mon Sep 17 00:00:00 2001 From: Robin Huang Date: Mon, 21 Apr 2025 23:39:14 -0700 Subject: [PATCH 008/121] Revert "Remove polling operations." This reverts commit 8415404ce8fbc0262b7de54fc700c5c8854a34fc. --- comfy_api_nodes/apis/client.py | 128 +++++++++++++++++++++++++ comfy_api_nodes/nodes_api.py | 170 +++++++++++++++++++++++++++++++++ 2 files changed, 298 insertions(+) diff --git a/comfy_api_nodes/apis/client.py b/comfy_api_nodes/apis/client.py index 05706710e..faee11230 100644 --- a/comfy_api_nodes/apis/client.py +++ b/comfy_api_nodes/apis/client.py @@ -99,15 +99,21 @@ from typing import ( Any, TypeVar, Generic, + Callable, ) from pydantic import BaseModel from enum import Enum +import time import json import requests from urllib.parse import urljoin +# Import models from your generated stubs + T = TypeVar("T", bound=BaseModel) R = TypeVar("R", bound=BaseModel) +P = TypeVar("P", bound=BaseModel) # For poll response + class EmptyRequest(BaseModel): """Base class for empty request bodies. @@ -379,3 +385,125 @@ class SynchronousOperation(Generic[T, R]): self.response = self.endpoint.response_model.model_validate(resp) logging.debug(f"[DEBUG] Parsed Response: {self.response}") return self.response + + +class TaskStatus(str, Enum): + """Enum for task status values""" + + COMPLETED = "completed" + FAILED = "failed" + PENDING = "pending" + + +class PollingOperation(Generic[T, R]): + """ + Represents an asynchronous API operation that requires polling for completion. + """ + + def __init__( + self, + poll_endpoint: ApiEndpoint[EmptyRequest, R], + completed_statuses: list, + failed_statuses: list, + status_extractor: Callable[[R], str], + request: Optional[T] = None, + api_base: str = "https://stagingapi.comfy.org", + auth_token: Optional[str] = None, + poll_interval: float = 1.0, + ): + self.poll_endpoint = poll_endpoint + self.request = request + self.api_base = api_base + self.auth_token = auth_token + self.poll_interval = poll_interval + + # Polling configuration + self.status_extractor = status_extractor or ( + lambda x: getattr(x, "status", None) + ) + self.completed_statuses = completed_statuses + self.failed_statuses = failed_statuses + + # For storing response data + self.final_response = None + self.error = None + + def execute(self, client: Optional[ApiClient] = None) -> R: + """Execute the polling operation using the provided client. If failed, raise an exception.""" + try: + if client is None: + client = ApiClient( + base_url=self.api_base, + api_key=self.auth_token, + ) + return self._poll_until_complete(client) + except Exception as e: + raise Exception(f"Error during polling: {str(e)}") + + def _check_task_status(self, response: R) -> TaskStatus: + """Check task status using the status extractor function""" + try: + status = self.status_extractor(response) + if status in self.completed_statuses: + return TaskStatus.COMPLETED + elif status in self.failed_statuses: + return TaskStatus.FAILED + return TaskStatus.PENDING + except Exception as e: + logging.debug(f"Error extracting status: {e}") + return TaskStatus.PENDING + + def _poll_until_complete(self, client: ApiClient) -> R: + """Poll until the task is complete""" + poll_count = 0 + while True: + try: + poll_count += 1 + logging.debug(f"[DEBUG] Polling attempt #{poll_count}") + + request_dict = ( + self.request.model_dump(exclude_none=True) + if self.request is not None + else None + ) + + if poll_count == 1: + logging.debug( + f"[DEBUG] Poll Request: {self.poll_endpoint.method.value} {self.poll_endpoint.path}" + ) + logging.debug( + f"[DEBUG] Poll Request Data: {json.dumps(request_dict, indent=2) if request_dict else 'None'}" + ) + + # Query task status + resp = client.request( + method=self.poll_endpoint.method.value, + path=self.poll_endpoint.path, + params=self.poll_endpoint.query_params, + json=request_dict, + ) + + # Parse response + response_obj = self.poll_endpoint.response_model.model_validate(resp) + + # Check if task is complete + status = self._check_task_status(response_obj) + logging.debug(f"[DEBUG] Task Status: {status}") + + if status == TaskStatus.COMPLETED: + logging.debug("[DEBUG] Task completed successfully") + self.final_response = response_obj + return self.final_response + elif status == TaskStatus.FAILED: + logging.debug(f"[DEBUG] Task failed: {json.dumps(resp)}") + raise Exception(f"Task failed: {json.dumps(resp)}") + else: + logging.debug("[DEBUG] Task still pending, continuing to poll...") + + # Wait before polling again + logging.debug(f"[DEBUG] Waiting {self.poll_interval} seconds before next poll") + time.sleep(self.poll_interval) + + except Exception as e: + logging.debug(f"[DEBUG] Polling error: {str(e)}") + raise Exception(f"Error while polling: {str(e)}") diff --git a/comfy_api_nodes/nodes_api.py b/comfy_api_nodes/nodes_api.py index 927a05fd2..dbdbcdf60 100644 --- a/comfy_api_nodes/nodes_api.py +++ b/comfy_api_nodes/nodes_api.py @@ -428,6 +428,176 @@ class OpenAIGPTImage1(ComfyNodeABC): return (img_tensor,) +class MinimaxVideoNode: + """ + Generates videos synchronously based on a prompt, and optional parameters using Minimax's API. + """ + + def __init__(self): + self.output_dir = folder_paths.get_output_directory() + self.type = "output" + + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "prompt_text": ( + "STRING", + { + "multiline": True, + "default": "", + "tooltip": "Text prompt to guide the video generation", + }, + ), + "filename_prefix": ("STRING", {"default": "ComfyUI"}), + "model": ( + [ + "T2V-01", + "I2V-01-Director", + "S2V-01", + "I2V-01", + "I2V-01-live", + "T2V-01", + ], + { + "default": "T2V-01", + "tooltip": "Model to use for video generation", + }, + ), + }, + "optional": { + "seed": ( + IO.INT, + { + "default": 0, + "min": 0, + "max": 0xFFFFFFFFFFFFFFFF, + "control_after_generate": True, + "tooltip": "The random seed used for creating the noise.", + }, + ), + }, + "hidden": { + "prompt": "PROMPT", + "extra_pnginfo": "EXTRA_PNGINFO", + "auth_token": "AUTH_TOKEN_COMFY_ORG", + }, + } + + RETURN_TYPES = ("VIDEO",) + DESCRIPTION = "Generates videos from prompts using Minimax's API" + FUNCTION = "generate_video" + CATEGORY = "video" + API_NODE = True + OUTPUT_NODE = True + + def generate_video( + self, + prompt_text, + filename_prefix, + seed=0, + model="T2V-01", + prompt=None, + extra_pnginfo=None, + auth_token=None, + ): + video_generate_operation = SynchronousOperation( + endpoint=ApiEndpoint( + path="/proxy/minimax/video_generation", + method=HttpMethod.POST, + request_model=MinimaxVideoGenerationRequest, + response_model=MinimaxVideoGenerationResponse, + ), + request=MinimaxVideoGenerationRequest( + model=Model(model), + prompt=prompt_text, + callback_url=None, + first_frame_image=None, + subject_reference=None, + prompt_optimizer=None, + ), + auth_token=auth_token, + ) + response = video_generate_operation.execute() + + task_id = response.task_id + + video_generate_operation = PollingOperation( + poll_endpoint=ApiEndpoint( + path="/proxy/minimax/query/video_generation", + method=HttpMethod.GET, + request_model=EmptyRequest, + response_model=MinimaxTaskResultResponse, + query_params={"task_id": task_id}, + ), + completed_statuses=["Success"], + failed_statuses=["Fail"], + status_extractor=lambda x: x.status.value, + auth_token=auth_token, + ) + task_result = video_generate_operation.execute() + + file_id = task_result.file_id + if file_id is None: + raise Exception("Request was not successful. Missing file ID.") + file_retrieve_operation = SynchronousOperation( + endpoint=ApiEndpoint( + path="/proxy/minimax/files/retrieve", + method=HttpMethod.GET, + request_model=EmptyRequest, + response_model=MinimaxFileRetrieveResponse, + query_params={"file_id": int(file_id)}, + ), + request=EmptyRequest(), + auth_token=auth_token, + ) + file_result = file_retrieve_operation.execute() + + file_url = file_result.file.download_url + if file_url is None: + raise Exception(f"No video was found in the response. Full response: {file_result.model_dump()}") + logging.info(f"Generated video URL: {file_url}") + + # Construct the save path + full_output_folder, filename, counter, subfolder, filename_prefix = ( + folder_paths.get_save_image_path(filename_prefix, self.output_dir) + ) + file_basename = f"{filename}_{counter:05}_.mp4" + save_path = os.path.join(full_output_folder, file_basename) + + # Download the video data + video_response = requests.get(file_url) + video_data = video_response.content + + # Save the video data to a file + with open(save_path, "wb") as video_file: + video_file.write(video_data) + + # Add workflow metadata to the video container + if prompt is not None or extra_pnginfo is not None: + try: + container = av.open(save_path, mode="r+") + if prompt is not None: + container.metadata["prompt"] = json.dumps(prompt) + if extra_pnginfo is not None: + for x in extra_pnginfo: + container.metadata[x] = json.dumps(extra_pnginfo[x]) + container.close() + except Exception as e: + logging.warning(f"Failed to add metadata to video: {e}") + + # Create a FileLocator for the frontend to use for the preview + results: list[FileLocator] = [ + { + "filename": file_basename, + "subfolder": subfolder, + "type": self.type, + } + ] + + return {"ui": {"images": results, "animated": (True,)}} + + # A dictionary that contains all nodes you want to export with their names # NOTE: names should be globally unique NODE_CLASS_MAPPINGS = { From a8caf1fbe656f7f2775cc55eb205e25ba8bc503e Mon Sep 17 00:00:00 2001 From: Robin Huang Date: Tue, 22 Apr 2025 00:53:32 -0700 Subject: [PATCH 009/121] Update stubs. --- comfy_api_nodes/apis/stubs.py | 513 ---------------------------------- 1 file changed, 513 deletions(-) delete mode 100644 comfy_api_nodes/apis/stubs.py diff --git a/comfy_api_nodes/apis/stubs.py b/comfy_api_nodes/apis/stubs.py deleted file mode 100644 index d1da6a5ab..000000000 --- a/comfy_api_nodes/apis/stubs.py +++ /dev/null @@ -1,513 +0,0 @@ -# generated by datamodel-codegen: -# filename: http://localhost:8080/openapi -# timestamp: 2025-04-18T21:35:21+00:00 - -from __future__ import annotations - -from datetime import datetime -from enum import Enum -from typing import Any, Dict, List, Optional -from uuid import UUID - -from pydantic import BaseModel, Field, conint, constr - - -class ComfyNode(BaseModel): - category: Optional[str] = Field( - None, - description='UI category where the node is listed, used for grouping nodes.', - ) - comfy_node_name: Optional[str] = Field( - None, description='Unique identifier for the node' - ) - deprecated: Optional[bool] = Field( - None, - description='Indicates if the node is deprecated. Deprecated nodes are hidden in the UI.', - ) - description: Optional[str] = Field( - None, description="Brief description of the node's functionality or purpose." - ) - experimental: Optional[bool] = Field( - None, - description='Indicates if the node is experimental, subject to changes or removal.', - ) - function: Optional[str] = Field( - None, description='Name of the entry-point function to execute the node.' - ) - input_types: Optional[str] = Field(None, description='Defines input parameters') - output_is_list: Optional[List[bool]] = Field( - None, description='Boolean values indicating if each output is a list.' - ) - return_names: Optional[str] = Field( - None, description='Names of the outputs for clarity in workflows.' - ) - return_types: Optional[str] = Field( - None, description='Specifies the types of outputs produced by the node.' - ) - - -class ComfyNodeCloudBuildInfo(BaseModel): - build_id: Optional[str] = None - location: Optional[str] = None - project_id: Optional[str] = None - project_number: Optional[str] = None - - -class Customer(BaseModel): - createdAt: Optional[datetime] = Field( - None, description='The date and time the user was created' - ) - email: Optional[str] = Field(None, description='The email address for this user') - id: str = Field(..., description='The firebase UID of the user') - name: Optional[str] = Field(None, description='The name for this user') - updatedAt: Optional[datetime] = Field( - None, description='The date and time the user was last updated' - ) - - -class Error(BaseModel): - details: Optional[List[str]] = Field( - None, - description='Optional detailed information about the error or hints for resolving it.', - ) - message: Optional[str] = Field( - None, description='A clear and concise description of the error.' - ) - - -class ErrorResponse(BaseModel): - error: str - message: str - - -class GitCommitSummary(BaseModel): - author: Optional[str] = Field(None, description='The author of the commit') - branch_name: Optional[str] = Field( - None, description='The branch where the commit was made' - ) - commit_hash: Optional[str] = Field(None, description='The hash of the commit') - commit_name: Optional[str] = Field(None, description='The name of the commit') - status_summary: Optional[Dict[str, str]] = Field( - None, description='A map of operating system to status pairs' - ) - timestamp: Optional[datetime] = Field( - None, description='The timestamp when the commit was made' - ) - - -class ImageRequest(BaseModel): - aspect_ratio: Optional[str] = Field( - None, - description="Optional. The aspect ratio (e.g., 'ASPECT_16_9', 'ASPECT_1_1'). Cannot be used with resolution. Defaults to 'ASPECT_1_1' if unspecified.", - ) - color_palette: Optional[Dict[str, Any]] = Field( - None, description='Optional. Color palette object. Only for V_2, V_2_TURBO.' - ) - magic_prompt_option: Optional[str] = Field( - None, description="Optional. MagicPrompt usage ('AUTO', 'ON', 'OFF')." - ) - model: Optional[str] = Field( - None, - description="Optional. The model used (e.g., 'V_2', 'V_2A_TURBO'). Defaults to 'V_2' if unspecified.", - ) - negative_prompt: Optional[str] = Field( - None, - description='Optional. Description of what to exclude. Only for V_1, V_1_TURBO, V_2, V_2_TURBO.', - ) - num_images: Optional[conint(ge=1, le=8)] = Field( - 1, description='Optional. Number of images to generate (1-8). Defaults to 1.' - ) - prompt: str = Field( - ..., description='Required. The prompt to use to generate the image.' - ) - resolution: Optional[str] = Field( - None, - description="Optional. Resolution (e.g., 'RESOLUTION_1024_1024'). Only for model V_2. Cannot be used with aspect_ratio.", - ) - seed: Optional[conint(ge=0, le=2147483647)] = Field( - None, description='Optional. A number between 0 and 2147483647.' - ) - style_type: Optional[str] = Field( - None, - description="Optional. Style type ('AUTO', 'GENERAL', 'REALISTIC', 'DESIGN', 'RENDER_3D', 'ANIME'). Only for models V_2 and above.", - ) - - -class IdeogramGenerateRequest(BaseModel): - image_request: ImageRequest = Field( - ..., description='The image generation request parameters.' - ) - - -class Datum(BaseModel): - is_image_safe: Optional[bool] = Field( - None, description='Indicates whether the image is considered safe.' - ) - prompt: Optional[str] = Field( - None, description='The prompt used to generate this image.' - ) - resolution: Optional[str] = Field( - None, description="The resolution of the generated image (e.g., '1024x1024')." - ) - seed: Optional[int] = Field( - None, description='The seed value used for this generation.' - ) - style_type: Optional[str] = Field( - None, - description="The style type used for generation (e.g., 'REALISTIC', 'ANIME').", - ) - url: Optional[str] = Field(None, description='URL to the generated image.') - - -class IdeogramGenerateResponse(BaseModel): - created: Optional[datetime] = Field( - None, description='Timestamp when the generation was created.' - ) - data: Optional[List[Datum]] = Field( - None, description='Array of generated image information.' - ) - - -class MachineStats(BaseModel): - cpu_capacity: Optional[str] = Field(None, description='Total CPU on the machine.') - disk_capacity: Optional[str] = Field( - None, description='Total disk capacity on the machine.' - ) - gpu_type: Optional[str] = Field( - None, description='The GPU type. eg. NVIDIA Tesla K80' - ) - initial_cpu: Optional[str] = Field( - None, description='Initial CPU available before the job starts.' - ) - initial_disk: Optional[str] = Field( - None, description='Initial disk available before the job starts.' - ) - initial_ram: Optional[str] = Field( - None, description='Initial RAM available before the job starts.' - ) - machine_name: Optional[str] = Field(None, description='Name of the machine.') - memory_capacity: Optional[str] = Field( - None, description='Total memory on the machine.' - ) - os_version: Optional[str] = Field( - None, description='The operating system version. eg. Ubuntu Linux 20.04' - ) - pip_freeze: Optional[str] = Field(None, description='The pip freeze output') - vram_time_series: Optional[Dict[str, Any]] = Field( - None, description='Time series of VRAM usage.' - ) - - -class MinimaxBaseResponse(BaseModel): - status_code: int = Field( - ..., - description='Status code. 0 indicates success, other values indicate errors.', - ) - status_msg: str = Field( - ..., description='Specific error details or success message.' - ) - - -class File(BaseModel): - bytes: Optional[int] = Field(None, description='File size in bytes') - created_at: Optional[int] = Field( - None, description='Unix timestamp when the file was created, in seconds' - ) - download_url: Optional[str] = Field( - None, description='The URL to download the video' - ) - file_id: Optional[int] = Field(None, description='Unique identifier for the file') - filename: Optional[str] = Field(None, description='The name of the file') - purpose: Optional[str] = Field(None, description='The purpose of using the file') - - -class MinimaxFileRetrieveResponse(BaseModel): - base_resp: MinimaxBaseResponse - file: File - - -class Status(str, Enum): - Queueing = 'Queueing' - Preparing = 'Preparing' - Processing = 'Processing' - Success = 'Success' - Fail = 'Fail' - - -class MinimaxTaskResultResponse(BaseModel): - base_resp: MinimaxBaseResponse - file_id: Optional[str] = Field( - None, - description='After the task status changes to Success, this field returns the file ID corresponding to the generated video.', - ) - status: Status = Field( - ..., - description="Task status: 'Queueing' (in queue), 'Preparing' (task is preparing), 'Processing' (generating), 'Success' (task completed successfully), or 'Fail' (task failed).", - ) - task_id: str = Field(..., description='The task ID being queried.') - - -class Model(str, Enum): - T2V_01_Director = 'T2V-01-Director' - I2V_01_Director = 'I2V-01-Director' - S2V_01 = 'S2V-01' - I2V_01 = 'I2V-01' - I2V_01_live = 'I2V-01-live' - T2V_01 = 'T2V-01' - - -class SubjectReferenceItem(BaseModel): - image: Optional[str] = Field( - None, description='URL or base64 encoding of the subject reference image.' - ) - mask: Optional[str] = Field( - None, - description='URL or base64 encoding of the mask for the subject reference image.', - ) - - -class MinimaxVideoGenerationRequest(BaseModel): - callback_url: Optional[str] = Field( - None, - description='Optional. URL to receive real-time status updates about the video generation task.', - ) - first_frame_image: Optional[str] = Field( - None, - description='URL or base64 encoding of the first frame image. Required when model is I2V-01, I2V-01-Director, or I2V-01-live.', - ) - model: Model = Field( - ..., - description='Required. ID of model. Options: T2V-01-Director, I2V-01-Director, S2V-01, I2V-01, I2V-01-live, T2V-01', - ) - prompt: Optional[constr(max_length=2000)] = Field( - None, - description='Description of the video. Should be less than 2000 characters. Supports camera movement instructions in [brackets].', - ) - prompt_optimizer: Optional[bool] = Field( - True, - description='If true (default), the model will automatically optimize the prompt. Set to false for more precise control.', - ) - subject_reference: Optional[List[SubjectReferenceItem]] = Field( - None, - description='Only available when model is S2V-01. The model will generate a video based on the subject uploaded through this parameter.', - ) - - -class MinimaxVideoGenerationResponse(BaseModel): - base_resp: MinimaxBaseResponse - task_id: str = Field( - ..., description='The task ID for the asynchronous video generation task.' - ) - - -class NodeStatus(str, Enum): - NodeStatusActive = 'NodeStatusActive' - NodeStatusDeleted = 'NodeStatusDeleted' - NodeStatusBanned = 'NodeStatusBanned' - - -class NodeVersionStatus(str, Enum): - NodeVersionStatusActive = 'NodeVersionStatusActive' - NodeVersionStatusDeleted = 'NodeVersionStatusDeleted' - NodeVersionStatusBanned = 'NodeVersionStatusBanned' - NodeVersionStatusPending = 'NodeVersionStatusPending' - NodeVersionStatusFlagged = 'NodeVersionStatusFlagged' - - -class NodeVersionUpdateRequest(BaseModel): - changelog: Optional[str] = Field( - None, description='The changelog describing the version changes.' - ) - deprecated: Optional[bool] = Field( - None, description='Whether the version is deprecated.' - ) - - -class PersonalAccessToken(BaseModel): - createdAt: Optional[datetime] = Field( - None, description='[Output Only]The date and time the token was created.' - ) - description: Optional[str] = Field( - None, - description="Optional. A more detailed description of the token's intended use.", - ) - id: Optional[UUID] = Field(None, description='Unique identifier for the GitCommit') - name: Optional[str] = Field( - None, - description='Required. The name of the token. Can be a simple description.', - ) - token: Optional[str] = Field( - None, - description='[Output Only]. The personal access token. Only returned during creation.', - ) - - -class PublisherStatus(str, Enum): - PublisherStatusActive = 'PublisherStatusActive' - PublisherStatusBanned = 'PublisherStatusBanned' - - -class PublisherUser(BaseModel): - email: Optional[str] = Field(None, description='The email address for this user.') - id: Optional[str] = Field(None, description='The unique id for this user.') - name: Optional[str] = Field(None, description='The name for this user.') - - -class StorageFile(BaseModel): - file_path: Optional[str] = Field(None, description='Path to the file in storage') - id: Optional[UUID] = Field( - None, description='Unique identifier for the storage file' - ) - public_url: Optional[str] = Field(None, description='Public URL') - - -class User(BaseModel): - email: Optional[str] = Field(None, description='The email address for this user.') - id: Optional[str] = Field(None, description='The unique id for this user.') - isAdmin: Optional[bool] = Field( - None, description='Indicates if the user has admin privileges.' - ) - isApproved: Optional[bool] = Field( - None, description='Indicates if the user is approved.' - ) - name: Optional[str] = Field(None, description='The name for this user.') - - -class WorkflowRunStatus(str, Enum): - WorkflowRunStatusStarted = 'WorkflowRunStatusStarted' - WorkflowRunStatusFailed = 'WorkflowRunStatusFailed' - WorkflowRunStatusCompleted = 'WorkflowRunStatusCompleted' - - -class ActionJobResult(BaseModel): - action_job_id: Optional[str] = Field( - None, description='Identifier of the job this result belongs to' - ) - action_run_id: Optional[str] = Field( - None, description='Identifier of the run this result belongs to' - ) - author: Optional[str] = Field(None, description='The author of the commit') - avg_vram: Optional[int] = Field( - None, description='The average VRAM used by the job' - ) - branch_name: Optional[str] = Field( - None, description='Name of the relevant git branch' - ) - comfy_run_flags: Optional[str] = Field( - None, description='The comfy run flags. E.g. `--low-vram`' - ) - commit_hash: Optional[str] = Field(None, description='The hash of the commit') - commit_id: Optional[str] = Field(None, description='The ID of the commit') - commit_message: Optional[str] = Field(None, description='The message of the commit') - commit_time: Optional[int] = Field( - None, description='The Unix timestamp when the commit was made' - ) - cuda_version: Optional[str] = Field(None, description='CUDA version used') - end_time: Optional[int] = Field( - None, description='The end time of the job as a Unix timestamp.' - ) - git_repo: Optional[str] = Field(None, description='The repository name') - id: Optional[UUID] = Field(None, description='Unique identifier for the job result') - job_trigger_user: Optional[str] = Field( - None, description='The user who triggered the job.' - ) - machine_stats: Optional[MachineStats] = None - operating_system: Optional[str] = Field(None, description='Operating system used') - peak_vram: Optional[int] = Field(None, description='The peak VRAM used by the job') - pr_number: Optional[str] = Field(None, description='The pull request number') - python_version: Optional[str] = Field(None, description='PyTorch version used') - pytorch_version: Optional[str] = Field(None, description='PyTorch version used') - start_time: Optional[int] = Field( - None, description='The start time of the job as a Unix timestamp.' - ) - status: Optional[WorkflowRunStatus] = None - storage_file: Optional[StorageFile] = None - workflow_name: Optional[str] = Field(None, description='Name of the workflow') - - -class NodeVersion(BaseModel): - changelog: Optional[str] = Field( - None, description='Summary of changes made in this version' - ) - comfy_node_extract_status: Optional[str] = Field( - None, description='The status of comfy node extraction process.' - ) - createdAt: Optional[datetime] = Field( - None, description='The date and time the version was created.' - ) - dependencies: Optional[List[str]] = Field( - None, description='A list of pip dependencies required by the node.' - ) - deprecated: Optional[bool] = Field( - None, description='Indicates if this version is deprecated.' - ) - downloadUrl: Optional[str] = Field( - None, description='[Output Only] URL to download this version of the node' - ) - id: Optional[str] = None - node_id: Optional[str] = Field( - None, description='The unique identifier of the node.' - ) - status: Optional[NodeVersionStatus] = None - status_reason: Optional[str] = Field( - None, description='The reason for the status change.' - ) - version: Optional[str] = Field( - None, - description='The version identifier, following semantic versioning. Must be unique for the node.', - ) - - -class PublisherMember(BaseModel): - id: Optional[str] = Field( - None, description='The unique identifier for the publisher member.' - ) - role: Optional[str] = Field( - None, description='The role of the user in the publisher.' - ) - user: Optional[PublisherUser] = None - - -class Publisher(BaseModel): - createdAt: Optional[datetime] = Field( - None, description='The date and time the publisher was created.' - ) - description: Optional[str] = None - id: Optional[str] = Field( - None, - description="The unique identifier for the publisher. It's akin to a username. Should be lowercase.", - ) - logo: Optional[str] = Field(None, description="URL to the publisher's logo.") - members: Optional[List[PublisherMember]] = Field( - None, description='A list of members in the publisher.' - ) - name: Optional[str] = None - source_code_repo: Optional[str] = None - status: Optional[PublisherStatus] = None - support: Optional[str] = None - website: Optional[str] = None - - -class Node(BaseModel): - author: Optional[str] = None - category: Optional[str] = Field(None, description='The category of the node.') - description: Optional[str] = None - downloads: Optional[int] = Field( - None, description='The number of downloads of the node.' - ) - icon: Optional[str] = Field(None, description="URL to the node's icon.") - id: Optional[str] = Field(None, description='The unique identifier of the node.') - latest_version: Optional[NodeVersion] = None - license: Optional[str] = Field( - None, description="The path to the LICENSE file in the node's repository." - ) - name: Optional[str] = Field(None, description='The display name of the node.') - publisher: Optional[Publisher] = None - rating: Optional[float] = Field(None, description='The average rating of the node.') - repository: Optional[str] = Field(None, description="URL to the node's repository.") - status: Optional[NodeStatus] = None - status_detail: Optional[str] = Field( - None, description='The status detail of the node.' - ) - tags: Optional[List[str]] = None - translations: Optional[Dict[str, Dict[str, Any]]] = None From c92bcbccf1028b5ce220cea9b5bd3fc7ccd573d0 Mon Sep 17 00:00:00 2001 From: Robin Huang Date: Thu, 24 Apr 2025 15:40:34 -0700 Subject: [PATCH 010/121] Added Ideogram and Minimax back in. --- .github/workflows/update-api-stubs.yml | 11 +- .gitignore | 3 + comfy_api_nodes/README.md | 30 ++ comfy_api_nodes/apis/PixverseController.py | 4 +- comfy_api_nodes/apis/PixverseDto.py | 4 +- comfy_api_nodes/apis/__init__.py | 371 ++++++++++----------- comfy_api_nodes/apis/client.py | 3 +- comfy_api_nodes/nodes_api.py | 183 +++++++++- comfy_api_nodes/redocly-dev.yaml | 10 + comfy_api_nodes/redocly.yaml | 10 + 10 files changed, 418 insertions(+), 211 deletions(-) create mode 100644 comfy_api_nodes/README.md create mode 100644 comfy_api_nodes/redocly-dev.yaml create mode 100644 comfy_api_nodes/redocly.yaml diff --git a/.github/workflows/update-api-stubs.yml b/.github/workflows/update-api-stubs.yml index 2ae99b673..fe9e4cd6a 100644 --- a/.github/workflows/update-api-stubs.yml +++ b/.github/workflows/update-api-stubs.yml @@ -22,10 +22,19 @@ jobs: run: | python -m pip install --upgrade pip pip install 'datamodel-code-generator[http]' + npm install @redocly/cli + + - name: Download OpenAPI spec + run: | + curl -o openapi.yaml https://api.comfy.org/openapi + + - name: Filter OpenAPI spec with Redocly + run: | + npx @redocly/cli bundle openapi.yaml --output filtered-openapi.yaml --config comfy_api_nodes/redocly.yaml --remove-unused-components - name: Generate API models run: | - datamodel-codegen --use-subclass-enum --url https://api.comfy.org/openapi --output comfy_api_nodes/apis --output-model-type pydantic_v2.BaseModel + datamodel-codegen --use-subclass-enum --input filtered-openapi.yaml --output comfy_api_nodes/apis --output-model-type pydantic_v2.BaseModel - name: Check for changes id: git-check diff --git a/.gitignore b/.gitignore index 61881b8a4..4e8cea71e 100644 --- a/.gitignore +++ b/.gitignore @@ -21,3 +21,6 @@ venv/ *.log web_custom_versions/ .DS_Store +openapi.yaml +filtered-openapi.yaml +uv.lock diff --git a/comfy_api_nodes/README.md b/comfy_api_nodes/README.md new file mode 100644 index 000000000..75b1d696a --- /dev/null +++ b/comfy_api_nodes/README.md @@ -0,0 +1,30 @@ +# ComfyUI API Nodes + +## Introduction + +Below are a collection of nodes that work by calling external APIs. More information available in our [docs](https://docs.comfy.org/tutorials/api-nodes/overview#api-nodes). + +## Development + +API stubs are generated through automatic codegen tools from OpenAPI definitions. Since the Comfy Org OpenAPI definition contains many things from the Comfy Registry as well, we use redocly/cli to filter out only the paths relevant for API nodes. + +### Redocly Instructions + +**Tip** +When developing locally, use the `redocly-dev.yaml` file to generate pydantic models. This lets you use stubs for APIs that are not marked `Released` yet. + +Before your API node PR merges, make sure to add the `Released` tag to the `openapi.yaml` file and test in staging. + +```bash +# Download the OpenAPI file from prod server. +curl -o openapi.yaml https://api.comfy.org/openapi + +# Filter out unneeded API definitions. +npm install -g @redocly/cli +redocly bundle openapi.yaml --output filtered-openapi.yaml --config comfy_api_nodes/redocly-dev.yaml --remove-unused-components + +# Generate the pydantic datamodels for validation. +datamodel-codegen --use-subclass-enum --input filtered-openapi.yaml --output comfy_api_nodes/apis/__init__.py --output-model-type pydantic_v2.BaseModel + +``` + diff --git a/comfy_api_nodes/apis/PixverseController.py b/comfy_api_nodes/apis/PixverseController.py index 29a3ab33b..949fce171 100644 --- a/comfy_api_nodes/apis/PixverseController.py +++ b/comfy_api_nodes/apis/PixverseController.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: -# filename: https://api.comfy.org/openapi -# timestamp: 2025-04-23T15:56:33+00:00 +# filename: filtered-openapi.yaml +# timestamp: 2025-04-24T22:16:48+00:00 from __future__ import annotations diff --git a/comfy_api_nodes/apis/PixverseDto.py b/comfy_api_nodes/apis/PixverseDto.py index 399512214..3a2c95fbc 100644 --- a/comfy_api_nodes/apis/PixverseDto.py +++ b/comfy_api_nodes/apis/PixverseDto.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: -# filename: https://api.comfy.org/openapi -# timestamp: 2025-04-23T15:56:33+00:00 +# filename: filtered-openapi.yaml +# timestamp: 2025-04-24T22:16:48+00:00 from __future__ import annotations diff --git a/comfy_api_nodes/apis/__init__.py b/comfy_api_nodes/apis/__init__.py index e7ea9b332..0e28cc481 100644 --- a/comfy_api_nodes/apis/__init__.py +++ b/comfy_api_nodes/apis/__init__.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: -# filename: https://api.comfy.org/openapi -# timestamp: 2025-04-23T15:56:33+00:00 +# filename: filtered-openapi.yaml +# timestamp: 2025-04-24T22:29:35+00:00 from __future__ import annotations @@ -8,34 +8,14 @@ from datetime import datetime from enum import Enum from typing import Any, Dict, List, Optional -from pydantic import AnyUrl, BaseModel, Field, confloat, conint - -class Customer(BaseModel): - createdAt: Optional[datetime] = Field( - None, description='The date and time the user was created' - ) - email: Optional[str] = Field(None, description='The email address for this user') - id: str = Field(..., description='The firebase UID of the user') - name: Optional[str] = Field(None, description='The name for this user') - updatedAt: Optional[datetime] = Field( - None, description='The date and time the user was last updated' - ) - - -class Error(BaseModel): - details: Optional[List[str]] = Field( - None, - description='Optional detailed information about the error or hints for resolving it.', - ) - message: Optional[str] = Field( - None, description='A clear and concise description of the error.' - ) +from pydantic import BaseModel, Field, conint, constr class ErrorResponse(BaseModel): error: str message: str + class ImageRequest(BaseModel): aspect_ratio: Optional[str] = Field( None, @@ -71,6 +51,12 @@ class ImageRequest(BaseModel): ) +class IdeogramGenerateRequest(BaseModel): + image_request: ImageRequest = Field( + ..., description='The image generation request parameters.' + ) + + class Datum(BaseModel): is_image_safe: Optional[bool] = Field( None, description='Indicates whether the image is considered safe.' @@ -91,6 +77,15 @@ class Datum(BaseModel): url: Optional[str] = Field(None, description='URL to the generated image.') +class IdeogramGenerateResponse(BaseModel): + created: Optional[datetime] = Field( + None, description='Timestamp when the generation was created.' + ) + data: Optional[List[Datum]] = Field( + None, description='Array of generated image information.' + ) + + class Code(Enum): int_1100 = 1100 int_1101 = 1101 @@ -106,135 +101,14 @@ class Code1(Enum): int_1004 = 1004 -class AspectRatio(str, Enum): - field_16_9 = '16:9' - field_9_16 = '9:16' - field_1_1 = '1:1' - - -class Config(BaseModel): - horizontal: Optional[confloat(ge=-10.0, le=10.0)] = None - pan: Optional[confloat(ge=-10.0, le=10.0)] = None - roll: Optional[confloat(ge=-10.0, le=10.0)] = None - tilt: Optional[confloat(ge=-10.0, le=10.0)] = None - vertical: Optional[confloat(ge=-10.0, le=10.0)] = None - zoom: Optional[confloat(ge=-10.0, le=10.0)] = None - - -class Type(str, Enum): - simple = 'simple' - down_back = 'down_back' - forward_up = 'forward_up' - right_turn_forward = 'right_turn_forward' - left_turn_forward = 'left_turn_forward' - - -class CameraControl(BaseModel): - config: Optional[Config] = None - type: Optional[Type] = Field(None, description='Predefined camera movements type') - - -class Duration(str, Enum): - field_5 = 5 - field_10 = 10 - - -class Mode(str, Enum): - std = 'std' - pro = 'pro' - - -class TaskInfo(BaseModel): - external_task_id: Optional[str] = None - - -class Video(BaseModel): - duration: Optional[str] = Field(None, description='Total video duration') - id: Optional[str] = Field(None, description='Generated video ID') - url: Optional[AnyUrl] = Field(None, description='URL for generated video') - - -class TaskResult(BaseModel): - videos: Optional[List[Video]] = None - - -class TaskStatus(str, Enum): - submitted = 'submitted' - processing = 'processing' - succeed = 'succeed' - failed = 'failed' - - -class Data(BaseModel): - created_at: Optional[int] = Field(None, description='Task creation time') - task_id: Optional[str] = Field(None, description='Task ID') - task_info: Optional[TaskInfo] = None - task_result: Optional[TaskResult] = None - task_status: Optional[TaskStatus] = None - updated_at: Optional[int] = Field(None, description='Task update time') - - -class AspectRatio1(str, Enum): - field_16_9 = '16:9' - field_9_16 = '9:16' - field_1_1 = '1:1' - field_4_3 = '4:3' - field_3_4 = '3:4' - field_3_2 = '3:2' - field_2_3 = '2:3' - field_21_9 = '21:9' - - -class ImageReference(str, Enum): - subject = 'subject' - face = 'face' - - -class Image(BaseModel): - index: Optional[int] = Field(None, description='Image Number (0-9)') - url: Optional[AnyUrl] = Field(None, description='URL for generated image') - - -class TaskResult1(BaseModel): - images: Optional[List[Image]] = None - - -class Data1(BaseModel): - created_at: Optional[int] = Field(None, description='Task creation time') - task_id: Optional[str] = Field(None, description='Task ID') - task_result: Optional[TaskResult1] = None - task_status: Optional[TaskStatus] = None - task_status_msg: Optional[str] = Field(None, description='Task status information') - updated_at: Optional[int] = Field(None, description='Task update time') - - -class AspectRatio2(str, Enum): - field_16_9 = '16:9' - field_9_16 = '9:16' - field_1_1 = '1:1' - - -class CameraControl1(BaseModel): - config: Optional[Config] = None - type: Optional[Type] = Field(None, description='Predefined camera movements type') - - -class ModelName2(str, Enum): - kling_v1 = 'kling-v1' - kling_v1_6 = 'kling-v1-6' - - -class TaskResult2(BaseModel): - videos: Optional[List[Video]] = None - - -class Data2(BaseModel): - created_at: Optional[int] = Field(None, description='Task creation time') - task_id: Optional[str] = Field(None, description='Task ID') - task_info: Optional[TaskInfo] = None - task_result: Optional[TaskResult2] = None - task_status: Optional[TaskStatus] = None - updated_at: Optional[int] = Field(None, description='Task update time') +class KlingErrorResponse(BaseModel): + code: int = Field( + ..., description='Error code value as defined in the API documentation' + ) + message: str = Field(..., description='Human-readable error message') + request_id: str = Field( + ..., description='Request ID for tracking and troubleshooting' + ) class Code2(Enum): @@ -244,43 +118,141 @@ class Code2(Enum): int_1203 = 1203 -class ResourcePackType(str, Enum): - decreasing_total = 'decreasing_total' - constant_period = 'constant_period' +class KlingRequestError(KlingErrorResponse): + code: Optional[Code2] = Field( + None, + description='- 1200: Invalid request parameters\n- 1201: Invalid parameters\n- 1202: Invalid request method\n- 1203: Requested resource does not exist\n', + ) + + +class Code3(Enum): + int_5000 = 5000 + int_5001 = 5001 + int_5002 = 5002 + + +class KlingServerError(KlingErrorResponse): + code: Optional[Code3] = Field( + None, + description='- 5000: Internal server error\n- 5001: Service temporarily unavailable\n- 5002: Server internal timeout\n', + ) + + +class Code4(Enum): + int_1300 = 1300 + int_1301 = 1301 + int_1302 = 1302 + int_1303 = 1303 + int_1304 = 1304 + + +class KlingStrategyError(KlingErrorResponse): + code: Optional[Code4] = Field( + None, + description='- 1300: Trigger platform strategy\n- 1301: Trigger content security policy\n- 1302: API request too frequent\n- 1303: Concurrency/QPS exceeds limit\n- 1304: Trigger IP whitelist policy\n', + ) + + +class MinimaxBaseResponse(BaseModel): + status_code: int = Field( + ..., + description='Status code. 0 indicates success, other values indicate errors.', + ) + status_msg: str = Field( + ..., description='Specific error details or success message.' + ) + + +class File(BaseModel): + bytes: Optional[int] = Field(None, description='File size in bytes') + created_at: Optional[int] = Field( + None, description='Unix timestamp when the file was created, in seconds' + ) + download_url: Optional[str] = Field( + None, description='The URL to download the video' + ) + file_id: Optional[int] = Field(None, description='Unique identifier for the file') + filename: Optional[str] = Field(None, description='The name of the file') + purpose: Optional[str] = Field(None, description='The purpose of using the file') + + +class MinimaxFileRetrieveResponse(BaseModel): + base_resp: MinimaxBaseResponse + file: File class Status(str, Enum): - toBeOnline = 'toBeOnline' - online = 'online' - expired = 'expired' - runOut = 'runOut' + Queueing = 'Queueing' + Preparing = 'Preparing' + Processing = 'Processing' + Success = 'Success' + Fail = 'Fail' -class ResourcePackSubscribeInfo(BaseModel): - effective_time: Optional[int] = Field( - None, description='Effective time, Unix timestamp in ms' - ) - invalid_time: Optional[int] = Field( - None, description='Expiration time, Unix timestamp in ms' - ) - purchase_time: Optional[int] = Field( - None, description='Purchase time, Unix timestamp in ms' - ) - remaining_quantity: Optional[float] = Field( - None, description='Remaining quantity (updated with a 12-hour delay)' - ) - resource_pack_id: Optional[str] = Field(None, description='Resource package ID') - resource_pack_name: Optional[str] = Field(None, description='Resource package name') - resource_pack_type: Optional[ResourcePackType] = Field( +class MinimaxTaskResultResponse(BaseModel): + base_resp: MinimaxBaseResponse + file_id: Optional[str] = Field( None, - description='Resource package type (decreasing_total=decreasing total, constant_period=constant periodicity)', + description='After the task status changes to Success, this field returns the file ID corresponding to the generated video.', ) - status: Optional[Status] = Field(None, description='Resource Package Status') - total_quantity: Optional[float] = Field(None, description='Total quantity') + status: Status = Field( + ..., + description="Task status: 'Queueing' (in queue), 'Preparing' (task is preparing), 'Processing' (generating), 'Success' (task completed successfully), or 'Fail' (task failed).", + ) + task_id: str = Field(..., description='The task ID being queried.') -class Background(str, Enum): - transparent = 'transparent' - opaque = 'opaque' + +class Model(str, Enum): + T2V_01_Director = 'T2V-01-Director' + I2V_01_Director = 'I2V-01-Director' + S2V_01 = 'S2V-01' + I2V_01 = 'I2V-01' + I2V_01_live = 'I2V-01-live' + T2V_01 = 'T2V-01' + + +class SubjectReferenceItem(BaseModel): + image: Optional[str] = Field( + None, description='URL or base64 encoding of the subject reference image.' + ) + mask: Optional[str] = Field( + None, + description='URL or base64 encoding of the mask for the subject reference image.', + ) + + +class MinimaxVideoGenerationRequest(BaseModel): + callback_url: Optional[str] = Field( + None, + description='Optional. URL to receive real-time status updates about the video generation task.', + ) + first_frame_image: Optional[str] = Field( + None, + description='URL or base64 encoding of the first frame image. Required when model is I2V-01, I2V-01-Director, or I2V-01-live.', + ) + model: Model = Field( + ..., + description='Required. ID of model. Options: T2V-01-Director, I2V-01-Director, S2V-01, I2V-01, I2V-01-live, T2V-01', + ) + prompt: Optional[constr(max_length=2000)] = Field( + None, + description='Description of the video. Should be less than 2000 characters. Supports camera movement instructions in [brackets].', + ) + prompt_optimizer: Optional[bool] = Field( + True, + description='If true (default), the model will automatically optimize the prompt. Set to false for more precise control.', + ) + subject_reference: Optional[List[SubjectReferenceItem]] = Field( + None, + description='Only available when model is S2V-01. The model will generate a video based on the subject uploaded through this parameter.', + ) + + +class MinimaxVideoGenerationResponse(BaseModel): + base_resp: MinimaxBaseResponse + task_id: str = Field( + ..., description='The task ID for the asynchronous video generation task.' + ) class Moderation(str, Enum): @@ -294,12 +266,6 @@ class OutputFormat(str, Enum): jpeg = 'jpeg' -class Quality(str, Enum): - low = 'low' - medium = 'medium' - high = 'high' - - class OpenAIImageEditRequest(BaseModel): background: Optional[str] = Field( None, description='Background transparency', examples=['opaque'] @@ -337,7 +303,12 @@ class OpenAIImageEditRequest(BaseModel): ) -class Quality1(str, Enum): +class Background(str, Enum): + transparent = 'transparent' + opaque = 'opaque' + + +class Quality(str, Enum): low = 'low' medium = 'medium' high = 'high' @@ -381,7 +352,7 @@ class OpenAIImageGenerationRequest(BaseModel): description='A text description of the desired image', examples=['Draw a rocket in front of a blackhole in deep space'], ) - quality: Optional[Quality1] = Field( + quality: Optional[Quality] = Field( None, description='The quality of the generated image', examples=['high'] ) response_format: Optional[ResponseFormat] = Field( @@ -410,13 +381,17 @@ class Datum1(BaseModel): class OpenAIImageGenerationResponse(BaseModel): data: Optional[List[Datum1]] = None -class User(BaseModel): - email: Optional[str] = Field(None, description='The email address for this user.') - id: Optional[str] = Field(None, description='The unique id for this user.') - isAdmin: Optional[bool] = Field( - None, description='Indicates if the user has admin privileges.' + + +class KlingAccountError(KlingErrorResponse): + code: Optional[Code] = Field( + None, + description='- 1100: Account exception\n- 1101: Account in arrears (postpaid scenario)\n- 1102: Resource pack depleted or expired (prepaid scenario)\n- 1103: Unauthorized access to requested resource\n', ) - isApproved: Optional[bool] = Field( - None, description='Indicates if the user is approved.' + + +class KlingAuthenticationError(KlingErrorResponse): + code: Optional[Code1] = Field( + None, + description='- 1000: Authentication failed\n- 1001: Authorization is empty\n- 1002: Authorization is invalid\n- 1003: Authorization is not yet valid\n- 1004: Authorization has expired\n', ) - name: Optional[str] = Field(None, description='The name for this user.') diff --git a/comfy_api_nodes/apis/client.py b/comfy_api_nodes/apis/client.py index faee11230..c71acf938 100644 --- a/comfy_api_nodes/apis/client.py +++ b/comfy_api_nodes/apis/client.py @@ -99,11 +99,10 @@ from typing import ( Any, TypeVar, Generic, - Callable, + ) from pydantic import BaseModel from enum import Enum -import time import json import requests from urllib.parse import urljoin diff --git a/comfy_api_nodes/nodes_api.py b/comfy_api_nodes/nodes_api.py index dbdbcdf60..0d032d848 100644 --- a/comfy_api_nodes/nodes_api.py +++ b/comfy_api_nodes/nodes_api.py @@ -1,14 +1,23 @@ import io from inspect import cleandoc - +from comfy.comfy_types.node_typing import FileLocator +from typing import Literal from comfy.utils import common_upscale from comfy.comfy_types.node_typing import IO, ComfyNodeABC, InputTypeDict from comfy_api_nodes.apis import ( OpenAIImageGenerationRequest, OpenAIImageEditRequest, - OpenAIImageGenerationResponse + OpenAIImageGenerationResponse, + MinimaxVideoGenerationRequest, + MinimaxVideoGenerationResponse, + MinimaxFileRetrieveResponse, + MinimaxTaskResultResponse, + IdeogramGenerateRequest, + IdeogramGenerateResponse, + ImageRequest, + Model ) -from comfy_api_nodes.apis.client import ApiEndpoint, HttpMethod, SynchronousOperation +from comfy_api_nodes.apis.client import ApiEndpoint, HttpMethod, SynchronousOperation, PollingOperation, EmptyRequest import numpy as np from PIL import Image @@ -16,6 +25,11 @@ import requests import torch import math import base64 +import logging +import json +import av +import os +import folder_paths def downscale_input(image): samples = image.movedim(-1,1) @@ -428,14 +442,168 @@ class OpenAIGPTImage1(ComfyNodeABC): return (img_tensor,) -class MinimaxVideoNode: +class IdeogramTextToImage(ComfyNodeABC): + """ + Generates images synchronously based on a given prompt and optional parameters. + + Images links are available for a limited period of time; if you would like to keep the image, you must download it. + """ + def __init__(self): + pass + + @classmethod + def INPUT_TYPES(cls) -> InputTypeDict: + """ + Return a dictionary which contains config for all input fields. + Some types (string): "MODEL", "VAE", "CLIP", "CONDITIONING", "LATENT", "IMAGE", "INT", "STRING", "FLOAT". + Input types "INT", "STRING" or "FLOAT" are special values for fields on the node. + The type can be a list for selection. + + Returns: `dict`: + - Key input_fields_group (`string`): Can be either required, hidden or optional. A node class must have property `required` + - Value input_fields (`dict`): Contains input fields config: + * Key field_name (`string`): Name of a entry-point method's argument + * Value field_config (`tuple`): + + First value is a string indicate the type of field or a list for selection. + + Secound value is a config for type "INT", "STRING" or "FLOAT". + """ + return { + "required": { + "prompt": (IO.STRING, { + "multiline": True, + "default": "", + "tooltip": "Prompt for the image generation", + }), + "model": (IO.COMBO, { "options": ["V_2", "V_2_TURBO", "V_1", "V_1_TURBO"], "default": "V_2", "tooltip": "Model to use for image generation"}), + }, + "optional": { + "aspect_ratio": (IO.COMBO, { "options": ["ASPECT_1_1", "ASPECT_4_3", "ASPECT_3_4", "ASPECT_16_9", "ASPECT_9_16", "ASPECT_2_1", "ASPECT_1_2", "ASPECT_3_2", "ASPECT_2_3", "ASPECT_4_5", "ASPECT_5_4"], "default": "ASPECT_1_1", "tooltip": "The aspect ratio for image generation. Cannot be used with resolution" + }), + "resolution": (IO.COMBO, { "options": ["1024x1024", "1024x1792", "1792x1024"], + "default": "1024x1024", + "tooltip": "The resolution for image generation (V2 only). Cannot be used with aspect_ratio" + }), + "magic_prompt_option": (IO.COMBO, { "options": ["AUTO", "ON", "OFF"], + "default": "AUTO", + "tooltip": "Determine if MagicPrompt should be used in generation" + }), + "seed": (IO.INT, { + "default": 0, + "min": 0, + "max": 2147483647, + "step": 1, + "display": "number" + }), + "style_type": (IO.COMBO, { "options": ["NONE", "ANIME", "CINEMATIC", "CREATIVE", "DIGITAL_ART", "PHOTOGRAPHIC"], + "default": "NONE", + "tooltip": "Style type for generation (V2+ only)" + }), + "negative_prompt": (IO.STRING, { + "multiline": True, + "default": "", + "tooltip": "Description of what to exclude from the image (V1/V2 only)" + }), + "num_images": (IO.INT, { + "default": 1, + "min": 1, + "max": 8, + "step": 1, + "display": "number" + }), + "color_palette": (IO.STRING, { + "multiline": False, + "default": "", + "tooltip": "Color palette preset name or hex colors with weights (V2/V2_TURBO only)" + }), + }, + "hidden": { + "auth_token": "AUTH_TOKEN_COMFY_ORG" + } + } + + RETURN_TYPES = (IO.IMAGE,) + DESCRIPTION = cleandoc(__doc__ or "") # Handle potential None value + FUNCTION = "api_call" + API_NODE = True + CATEGORY = "Example" + + def api_call(self, prompt, model, aspect_ratio=None, resolution=None, + magic_prompt_option="AUTO", seed=0, style_type="NONE", + negative_prompt="", num_images=1, color_palette="", auth_token=None): + import torch + from PIL import Image + import io + import numpy as np + import requests + + operation = SynchronousOperation( + endpoint=ApiEndpoint( + path="/proxy/ideogram/generate", + method=HttpMethod.POST, + request_model=IdeogramGenerateRequest, + response_model=IdeogramGenerateResponse + ), + request=IdeogramGenerateRequest( + image_request=ImageRequest( + prompt=prompt, + model=model, + num_images=num_images, + seed=seed, + aspect_ratio=aspect_ratio if aspect_ratio != "ASPECT_1_1" else None, + resolution=resolution if resolution != "1024x1024" else None, + magic_prompt_option=magic_prompt_option if magic_prompt_option != "AUTO" else None, + style_type=style_type if style_type != "NONE" else None, + negative_prompt=negative_prompt if negative_prompt else None, + color_palette=None + ) + ), + auth_token=auth_token + ) + + response = operation.execute() + + if not response.data or len(response.data) == 0: + raise Exception("No images were generated in the response") + image_url = response.data[0].url + + if not image_url: + raise Exception("No image URL was generated in the response") + img_response = requests.get(image_url) + if img_response.status_code != 200: + raise Exception("Failed to download the image") + + img = Image.open(io.BytesIO(img_response.content)) + img = img.convert("RGB") # Ensure RGB format + + # Convert to numpy array, normalize to float32 between 0 and 1 + img_array = np.array(img).astype(np.float32) / 255.0 + + # Convert to torch tensor and add batch dimension + img_tensor = torch.from_numpy(img_array)[None,] + + return (img_tensor,) + + """ + The node will always be re executed if any of the inputs change but + this method can be used to force the node to execute again even when the inputs don't change. + You can make this node return a number or a string. This value will be compared to the one returned the last time the node was + executed, if it is different the node will be executed again. + This method is used in the core repo for the LoadImage node where they return the image hash as a string, if the image hash + changes between executions the LoadImage node is executed again. + """ + #@classmethod + #def IS_CHANGED(s, image, string_field, int_field, float_field, print_to_screen): + # return "" + + +class MinimaxTextToVideoNode: """ Generates videos synchronously based on a prompt, and optional parameters using Minimax's API. """ def __init__(self): self.output_dir = folder_paths.get_output_directory() - self.type = "output" + self.type: Literal["output"] = "output" @classmethod def INPUT_TYPES(s): @@ -597,13 +765,14 @@ class MinimaxVideoNode: return {"ui": {"images": results, "animated": (True,)}} - # A dictionary that contains all nodes you want to export with their names # NOTE: names should be globally unique NODE_CLASS_MAPPINGS = { "OpenAIDalle2": OpenAIDalle2, "OpenAIDalle3": OpenAIDalle3, "OpenAIGPTImage1": OpenAIGPTImage1, + "IdeogramTextToImage": IdeogramTextToImage, + "MinimaxTextToVideoNode": MinimaxTextToVideoNode, } # A dictionary that contains the friendly/humanly readable titles for the nodes @@ -611,4 +780,6 @@ NODE_DISPLAY_NAME_MAPPINGS = { "OpenAIDalle2": "OpenAI DALL·E 2", "OpenAIDalle3": "OpenAI DALL·E 3", "OpenAIGPTImage1": "OpenAI GPT Image 1", + "IdeogramTextToImage": "Ideogram Text to Image", + "MinimaxTextToVideoNode": "Minimax Text to Video", } diff --git a/comfy_api_nodes/redocly-dev.yaml b/comfy_api_nodes/redocly-dev.yaml new file mode 100644 index 000000000..d9e3cab70 --- /dev/null +++ b/comfy_api_nodes/redocly-dev.yaml @@ -0,0 +1,10 @@ +# This file is used to filter the Comfy Org OpenAPI spec for schemas related to API Nodes. +# This is used for development purposes to generate stubs for unreleased API endpoints. +apis: + filter: + root: openapi.yaml + decorators: + filter-in: + property: tags + value: ['API Nodes'] + matchStrategy: all diff --git a/comfy_api_nodes/redocly.yaml b/comfy_api_nodes/redocly.yaml new file mode 100644 index 000000000..d102345b1 --- /dev/null +++ b/comfy_api_nodes/redocly.yaml @@ -0,0 +1,10 @@ +# This file is used to filter the Comfy Org OpenAPI spec for schemas related to API Nodes. + +apis: + filter: + root: openapi.yaml + decorators: + filter-in: + property: tags + value: ['API Nodes', 'Released'] + matchStrategy: all From 788bc2f6b424e3218438ce3a2c8608767ad7c4be Mon Sep 17 00:00:00 2001 From: Jedrzej Kosinski Date: Mon, 28 Apr 2025 13:54:40 -0500 Subject: [PATCH 011/121] Added initial BFL Flux 1.1 [pro] Ultra node (#11) --- comfy_api_nodes/apis/__init__.py | 50 +++++++-- comfy_api_nodes/nodes_api.py | 179 ++++++++++++++++++++++++++++++- 2 files changed, 219 insertions(+), 10 deletions(-) diff --git a/comfy_api_nodes/apis/__init__.py b/comfy_api_nodes/apis/__init__.py index 0e28cc481..ee8a70c3a 100644 --- a/comfy_api_nodes/apis/__init__.py +++ b/comfy_api_nodes/apis/__init__.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: filtered-openapi.yaml -# timestamp: 2025-04-24T22:29:35+00:00 +# timestamp: 2025-04-25T03:57:04+00:00 from __future__ import annotations @@ -8,7 +8,47 @@ from datetime import datetime from enum import Enum from typing import Any, Dict, List, Optional -from pydantic import BaseModel, Field, conint, constr +from pydantic import BaseModel, Field, confloat, conint, constr + + +class OutputFormat(str, Enum): + png = 'png' + webp = 'webp' + jpeg = 'jpeg' + + +class BFLFluxProGenerateRequest(BaseModel): + prompt: str = Field(..., description='The text prompt for image generation.') + prompt_upsampling: Optional[bool] = Field( + None, description='Whether to perform upsampling on the prompt. If active, automatically modifies the prompt for more creative generation.' + ) + seed: Optional[int] = Field(None, description='The seed value for reproducibility.') + aspect_ratio: Optional[str] = Field(None, description='Aspect ratio of the image between 21:9 and 9:21.') + safety_tolerance: Optional[conint(ge=0, le=6)] = Field( + 6, description='Tolerance level for input and output moderation. Between 0 and 6, 0 being most strict, 6 being least strict. Defaults to 2.' + ) + output_format: Optional[OutputFormat] = Field( + OutputFormat.png, description="Output format for the generated image. Can be 'jpeg' or 'png'.", examples=['png'] + ) + raw: Optional[bool] = Field(None, description='Generate less processed, more natural-looking images.') + image_prompt: Optional[str] = Field(None, description='Optional image to remix in base64 format') + image_prompt_strength: Optional[confloat(ge=0.0, le=1.0)] = Field( + None, description='Blend between the prompt and the image prompt.' + ) + + +class BFLFluxProGenerateResponse(BaseModel): + id: str = Field(..., description='The unique identifier for the generation task.') + polling_url: str = Field(..., description='URL to poll for the generation result.') + + +class BFLStatus(str, Enum): + task_not_found = "Task not found" + pending = "Pending" + request_moderated = "Request Moderated" + content_moderated = "Content Moderated" + ready = "Ready" + error = "Error" class ErrorResponse(BaseModel): @@ -260,12 +300,6 @@ class Moderation(str, Enum): auto = 'auto' -class OutputFormat(str, Enum): - png = 'png' - webp = 'webp' - jpeg = 'jpeg' - - class OpenAIImageEditRequest(BaseModel): background: Optional[str] = Field( None, description='Background transparency', examples=['opaque'] diff --git a/comfy_api_nodes/nodes_api.py b/comfy_api_nodes/nodes_api.py index 0d032d848..605f84960 100644 --- a/comfy_api_nodes/nodes_api.py +++ b/comfy_api_nodes/nodes_api.py @@ -14,6 +14,9 @@ from comfy_api_nodes.apis import ( MinimaxTaskResultResponse, IdeogramGenerateRequest, IdeogramGenerateResponse, + BFLFluxProGenerateRequest, + BFLFluxProGenerateResponse, + BFLStatus, ImageRequest, Model ) @@ -29,12 +32,13 @@ import logging import json import av import os +import time import folder_paths -def downscale_input(image): +def downscale_input(image, total_pixels=1536*1024): samples = image.movedim(-1,1) #downscaling input images to roughly the same size as the outputs - total = int(1536 * 1024) + total = int(total_pixels) scale_by = math.sqrt(total / (samples.shape[3] * samples.shape[2])) if scale_by >= 1: return image @@ -83,6 +87,25 @@ def validate_and_cast_response(response): return torch.stack(image_tensors, dim=0) +def validate_aspect_ratio(aspect_ratio: str, minimum_ratio: float, maximum_ratio: float, minimum_ratio_str: str, maximum_ratio_str: str): + # get ratio values + numbers = aspect_ratio.split(':') + if len(numbers) != 2: + raise Exception(f"Aspect ratio must be in the format X:Y, such as 16:9, but was {aspect_ratio}.") + try: + numerator = int(numbers[0]) + denominator = int(numbers[1]) + except ValueError: + raise Exception(f"Aspect ratio must contain numbers separated by ':', such as 16:9, but was {aspect_ratio}.") + calculated_ratio = numerator/denominator + # if not close to minimum and maximum, check bounds + if not math.isclose(calculated_ratio, minimum_ratio) or not math.isclose(calculated_ratio, maximum_ratio): + if calculated_ratio < minimum_ratio: + raise Exception(f"Aspect ratio cannot reduce to any less than {minimum_ratio_str} ({minimum_ratio}), but was {aspect_ratio} ({calculated_ratio}).") + elif calculated_ratio > maximum_ratio: + raise Exception(f"Aspect ratio cannot reduce to any greater than {maximum_ratio_str} ({maximum_ratio}), but was {aspect_ratio} ({calculated_ratio}).") + return aspect_ratio + class OpenAIDalle2(ComfyNodeABC): """ Generates images synchronously via OpenAI's DALL·E 2 endpoint. @@ -595,6 +618,156 @@ class IdeogramTextToImage(ComfyNodeABC): #def IS_CHANGED(s, image, string_field, int_field, float_field, print_to_screen): # return "" +class FluxProUltraImageNode(ComfyNodeABC): + """ + Generates images synchronously based on prompt and resolution. + """ + MINIMUM_RATIO = 1/4 + MAXIMUM_RATIO = 4/1 + MINIMUM_RATIO_STR = "1:4" + MAXIMUM_RATIO_STR = "4:1" + + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "prompt": (IO.STRING, { + "multiline": True, + "default": "", + "tooltip": "Prompt for the image generation", + }), + "prompt_upsampling": (IO.BOOLEAN, { + "default": False, + "tooltip": "Whether to perform upsampling on the prompt. If active, automatically modifies the prompt for more creative generation, but results are nondeterministic (same seed will not produce exactly the same result)." + }), + "seed": (IO.INT, { + "default": 0, + "min": 0, + "max": 0xFFFFFFFFFFFFFFFF, + "control_after_generate": True, + "tooltip": "The random seed used for creating the noise.", + }), + "aspect_ratio": (IO.STRING, { + "default": "16:9", + "tooltip": "Aspect ratio of image; must be between 1:4 and 4:1.", + }), + "raw": (IO.BOOLEAN, { + "default": False, + "tooltip": "When True, generate less processed, more natural-looking images." + }), + }, + "optional": { + "image_prompt": (IO.IMAGE, ), + "image_prompt_strength": (IO.FLOAT, { + "default": 0.1, + "min": 0.0, + "max": 1.0, + "step": 0.01, + "tooltip": "Blend between the prompt and the image prompt.", + }), + }, + "hidden": { + "auth_token": "AUTH_TOKEN_COMFY_ORG", + } + } + + @classmethod + def VALIDATE_INPUTS(cls, aspect_ratio: str): + try: + validate_aspect_ratio(aspect_ratio, minimum_ratio=cls.MINIMUM_RATIO, maximum_ratio=cls.MAXIMUM_RATIO, + minimum_ratio_str=cls.MINIMUM_RATIO_STR, maximum_ratio_str=cls.MAXIMUM_RATIO_STR) + except Exception as e: + return str(e) + return True + + RETURN_TYPES = (IO.IMAGE,) + DESCRIPTION = cleandoc(__doc__ or "") # Handle potential None value + FUNCTION = "api_call" + API_NODE = True + CATEGORY = "api node" + + def api_call(self, prompt: str, aspect_ratio: str, prompt_upsampling=False, raw=False, seed=0, image_prompt=None, image_prompt_strength=0.1, auth_token=None, **kwargs): + operation = SynchronousOperation( + endpoint=ApiEndpoint( + path="/proxy/bfl/flux-pro-1.1-ultra/generate", + method=HttpMethod.POST, + request_model=BFLFluxProGenerateRequest, + response_model=BFLFluxProGenerateResponse + ), + request=BFLFluxProGenerateRequest( + prompt=prompt, + prompt_upsampling=prompt_upsampling, + seed=seed, + aspect_ratio=validate_aspect_ratio(aspect_ratio, minimum_ratio=self.MINIMUM_RATIO, maximum_ratio=self.MAXIMUM_RATIO, + minimum_ratio_str=self.MINIMUM_RATIO_STR, maximum_ratio_str=self.MAXIMUM_RATIO_STR), + raw=raw, + image_prompt=image_prompt if image_prompt is None else self._convert_image_to_base64(image_prompt), + image_prompt_strength=None if image_prompt is None else round(image_prompt_strength, 2), + ), + auth_token=auth_token + ) + output_image = self._handle_bfl_synchronous_operation(operation) + return (output_image,) + + def _handle_bfl_synchronous_operation(self, operation: SynchronousOperation, timeout_bfl_calls=360): + response_api: BFLFluxProGenerateResponse = operation.execute() + return self._poll_until_generated(response_api.polling_url, timeout=timeout_bfl_calls) + + def _poll_until_generated(self, polling_url: str, timeout=360): + # used bfl-comfy-nodes to verify code implementation: + # https://github.com/black-forest-labs/bfl-comfy-nodes/tree/main + start_time = time.time() + retries_404 = 0 + max_retries_404 = 5 + retry_404_seconds = 2 + retry_202_seconds = 2 + retry_pending_seconds = 1 + request = requests.Request(method=HttpMethod.GET, url=polling_url) + # NOTE: should True loop be replaced with checking if workflow has been interrupted? + while True: + response = requests.Session().send(request.prepare()) + if response.status_code == 200: + result = response.json() + if result["status"] == BFLStatus.ready: + img_url = result["result"]["sample"] + img_response = requests.get(img_url) + return self._process_bfl_image_response(img_response) + elif result["status"] in [BFLStatus.request_moderated, BFLStatus.content_moderated]: + status = result["status"] + raise Exception(f"BFL API did not return an image due to: {status}.") + elif result["status"] == BFLStatus.error: + raise Exception(f"BFL API encountered an error: {result}.") + elif result["status"] == BFLStatus.pending: + time.sleep(retry_pending_seconds) + continue + elif response.status_code == 404: + if retries_404 < max_retries_404: + retries_404 += 1 + time.sleep(retry_404_seconds) + continue + raise Exception(f"BFL API could not find task after {max_retries_404} tries.") + elif response.status_code == 202: + time.sleep(retry_202_seconds) + elif time.time() - start_time > timeout: + raise Exception(f"BFL API experienced a timeout; could not return request under {timeout} seconds.") + else: + raise Exception(f"BFL API encountered an error: {response.json()}") + + def _process_bfl_image_response(self, response: requests.Response): + image = Image.open(io.BytesIO(response.content)).convert("RGBA") + image_array = np.array(image).astype(np.float32) / 255.0 + return torch.from_numpy(image_array).unsqueeze(0) + + def _convert_image_to_base64(self, image: torch.Tensor): + scaled_image = downscale_input(image, total_pixels=2048*2048) + # remove batch dimension if present + if len(scaled_image.shape) > 3: + scaled_image = scaled_image[0] + image_np = (scaled_image.numpy() * 255).astype(np.uint8) + img = Image.fromarray(image_np) + img_byte_arr = io.BytesIO() + img.save(img_byte_arr, format='PNG') + return base64.b64encode(img_byte_arr.getvalue()).decode() class MinimaxTextToVideoNode: """ @@ -772,6 +945,7 @@ NODE_CLASS_MAPPINGS = { "OpenAIDalle3": OpenAIDalle3, "OpenAIGPTImage1": OpenAIGPTImage1, "IdeogramTextToImage": IdeogramTextToImage, + "FluxProUltraImageNode": FluxProUltraImageNode, "MinimaxTextToVideoNode": MinimaxTextToVideoNode, } @@ -781,5 +955,6 @@ NODE_DISPLAY_NAME_MAPPINGS = { "OpenAIDalle3": "OpenAI DALL·E 3", "OpenAIGPTImage1": "OpenAI GPT Image 1", "IdeogramTextToImage": "Ideogram Text to Image", + "FluxProUltraImageNode": "Flux 1.1 [pro] Ultra Image", "MinimaxTextToVideoNode": "Minimax Text to Video", } From 864d0f739589cb087dbf3c619b6bc3a04432f30f Mon Sep 17 00:00:00 2001 From: Chenlei Hu Date: Mon, 28 Apr 2025 15:07:49 -0400 Subject: [PATCH 012/121] Add --comfy-api-base launch arg (#13) --- comfy/cli_args.py | 8 ++++++++ comfy_api_nodes/apis/client.py | 14 ++++++++------ 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/comfy/cli_args.py b/comfy/cli_args.py index 1b971be3c..74d00e171 100644 --- a/comfy/cli_args.py +++ b/comfy/cli_args.py @@ -191,6 +191,14 @@ parser.add_argument("--user-directory", type=is_valid_directory, default=None, h parser.add_argument("--enable-compress-response-body", action="store_true", help="Enable compressing response body.") +parser.add_argument( + "--comfy-api-base", + type=str, + default="https://api.comfy.org", + choices=["https://api.comfy.org", "https://stagingapi.comfy.org"], + help="Set the base URL for the ComfyUI API.", +) + if comfy.options.args_parsing: args = parser.parse_args() else: diff --git a/comfy_api_nodes/apis/client.py b/comfy_api_nodes/apis/client.py index c71acf938..66ea671ac 100644 --- a/comfy_api_nodes/apis/client.py +++ b/comfy_api_nodes/apis/client.py @@ -1,6 +1,9 @@ import logging import time from typing import Callable + +from comfy.cli_args import args + """ API Client Framework for api.comfy.org. @@ -316,7 +319,7 @@ class SynchronousOperation(Generic[T, R]): endpoint: ApiEndpoint[T, R], request: T, files: Optional[Dict[str, Any]] = None, - api_base: str = "https://api.comfy.org", + api_base: str | None = None, auth_token: Optional[str] = None, timeout: float = 604800.0, verify_ssl: bool = True, @@ -325,18 +328,17 @@ class SynchronousOperation(Generic[T, R]): self.request = request self.response = None self.error = None - self.api_base = api_base + self.api_base: str = api_base or args.comfy_api_base self.auth_token = auth_token self.timeout = timeout self.verify_ssl = verify_ssl self.files = files + def execute(self, client: Optional[ApiClient] = None) -> R: """Execute the API operation using the provided client or create one""" try: # Create client if not provided if client is None: - if self.api_base is None: - raise ValueError("Either client or api_base must be provided") client = ApiClient( base_url=self.api_base, api_key=self.auth_token, @@ -406,13 +408,13 @@ class PollingOperation(Generic[T, R]): failed_statuses: list, status_extractor: Callable[[R], str], request: Optional[T] = None, - api_base: str = "https://stagingapi.comfy.org", + api_base: str | None = None, auth_token: Optional[str] = None, poll_interval: float = 1.0, ): self.poll_endpoint = poll_endpoint self.request = request - self.api_base = api_base + self.api_base: str = api_base or args.comfy_api_base self.auth_token = auth_token self.poll_interval = poll_interval From 019ec67e1dab8aa764a286a9915f4082f2c35162 Mon Sep 17 00:00:00 2001 From: Robin Huang Date: Mon, 28 Apr 2025 13:29:45 -0700 Subject: [PATCH 013/121] Add instructions for staging development. (#14) --- comfy_api_nodes/README.md | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/comfy_api_nodes/README.md b/comfy_api_nodes/README.md index 75b1d696a..86ba379f7 100644 --- a/comfy_api_nodes/README.md +++ b/comfy_api_nodes/README.md @@ -6,6 +6,18 @@ Below are a collection of nodes that work by calling external APIs. More informa ## Development +While developing, you should be testing against the Staging environment. To test against staging: + +**Install ComfyUI_frontend** + +Follow the instructions [here](https://github.com/Comfy-Org/ComfyUI_frontend) to start the frontend server. By default, it will connect to Staging authentication. + +> **Hint:** If you use --front-end-version argument for ComfyUI, it will use production authentication. + +```bash +python run main.py --comfy-api-base https://stagingapi.comfy.org +``` + API stubs are generated through automatic codegen tools from OpenAPI definitions. Since the Comfy Org OpenAPI definition contains many things from the Comfy Registry as well, we use redocly/cli to filter out only the paths relevant for API nodes. ### Redocly Instructions @@ -17,7 +29,7 @@ Before your API node PR merges, make sure to add the `Released` tag to the `open ```bash # Download the OpenAPI file from prod server. -curl -o openapi.yaml https://api.comfy.org/openapi +curl -o openapi.yaml https://stagingapi.comfy.org/openapi # Filter out unneeded API definitions. npm install -g @redocly/cli From 66d42afd23d5d6ec968a6425c8960ed6c509ada4 Mon Sep 17 00:00:00 2001 From: thot-experiment Date: Mon, 28 Apr 2025 15:01:16 -0700 Subject: [PATCH 014/121] remove validation to make it easier to run against LAN copies of the API --- comfy/cli_args.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/comfy/cli_args.py b/comfy/cli_args.py index 74d00e171..155ec53d2 100644 --- a/comfy/cli_args.py +++ b/comfy/cli_args.py @@ -195,8 +195,7 @@ parser.add_argument( "--comfy-api-base", type=str, default="https://api.comfy.org", - choices=["https://api.comfy.org", "https://stagingapi.comfy.org"], - help="Set the base URL for the ComfyUI API.", + help="Set the base URL for the ComfyUI API. (default: https://api.comfy.org)", ) if comfy.options.args_parsing: From 442dc70c07ea82949f056b43a9e2a322e1378313 Mon Sep 17 00:00:00 2001 From: Christian Byrne Date: Mon, 28 Apr 2025 15:17:57 -0700 Subject: [PATCH 015/121] Manually add BFL polling status response schema (#15) --- comfy_api_nodes/apis/BFLPolling.py | 29 +++++++++++++++++++++++++++++ comfy_api_nodes/apis/__init__.py | 9 --------- comfy_api_nodes/nodes_api.py | 2 +- 3 files changed, 30 insertions(+), 10 deletions(-) create mode 100644 comfy_api_nodes/apis/BFLPolling.py diff --git a/comfy_api_nodes/apis/BFLPolling.py b/comfy_api_nodes/apis/BFLPolling.py new file mode 100644 index 000000000..e40bf3344 --- /dev/null +++ b/comfy_api_nodes/apis/BFLPolling.py @@ -0,0 +1,29 @@ +from __future__ import annotations + +from enum import Enum +from typing import Any, Dict, Optional + +from pydantic import BaseModel, Field, confloat + + +class BFLStatus(str, Enum): + task_not_found = "Task not found" + pending = "Pending" + request_moderated = "Request Moderated" + content_moderated = "Content Moderated" + ready = "Ready" + error = "Error" + + +class BFLFluxProStatusResponse(BaseModel): + id: str = Field(..., description="The unique identifier for the generation task.") + status: BFLStatus = Field(..., description="The status of the task.") + result: Optional[Dict[str, Any]] = Field( + None, description="The result of the task (null if not completed)." + ) + progress: confloat(ge=0.0, le=1.0) = Field( + ..., description="The progress of the task (0.0 to 1.0)." + ) + details: Optional[Dict[str, Any]] = Field( + None, description="Additional details about the task (null if not available)." + ) diff --git a/comfy_api_nodes/apis/__init__.py b/comfy_api_nodes/apis/__init__.py index ee8a70c3a..4ca4d5ab9 100644 --- a/comfy_api_nodes/apis/__init__.py +++ b/comfy_api_nodes/apis/__init__.py @@ -42,15 +42,6 @@ class BFLFluxProGenerateResponse(BaseModel): polling_url: str = Field(..., description='URL to poll for the generation result.') -class BFLStatus(str, Enum): - task_not_found = "Task not found" - pending = "Pending" - request_moderated = "Request Moderated" - content_moderated = "Content Moderated" - ready = "Ready" - error = "Error" - - class ErrorResponse(BaseModel): error: str message: str diff --git a/comfy_api_nodes/nodes_api.py b/comfy_api_nodes/nodes_api.py index 605f84960..03100aab5 100644 --- a/comfy_api_nodes/nodes_api.py +++ b/comfy_api_nodes/nodes_api.py @@ -16,10 +16,10 @@ from comfy_api_nodes.apis import ( IdeogramGenerateResponse, BFLFluxProGenerateRequest, BFLFluxProGenerateResponse, - BFLStatus, ImageRequest, Model ) +from comfy_api_nodes.apis.BFLPolling import BFLStatus from comfy_api_nodes.apis.client import ApiEndpoint, HttpMethod, SynchronousOperation, PollingOperation, EmptyRequest import numpy as np From 2018a0d5238bb387124fa9a708cfb1faad915b1d Mon Sep 17 00:00:00 2001 From: Robin Huang Date: Mon, 28 Apr 2025 17:01:25 -0700 Subject: [PATCH 016/121] Add function for uploading files. (#18) --- comfy_api_nodes/apis/client.py | 42 +++++++++++++++++++++++++++++----- 1 file changed, 36 insertions(+), 6 deletions(-) diff --git a/comfy_api_nodes/apis/client.py b/comfy_api_nodes/apis/client.py index 66ea671ac..bec43e648 100644 --- a/comfy_api_nodes/apis/client.py +++ b/comfy_api_nodes/apis/client.py @@ -1,6 +1,7 @@ import logging import time from typing import Callable +import io from comfy.cli_args import args @@ -102,7 +103,6 @@ from typing import ( Any, TypeVar, Generic, - ) from pydantic import BaseModel from enum import Enum @@ -255,7 +255,9 @@ class ApiClient: error_message = f"API Error: {error_json}" except Exception as json_error: # If we can't parse the JSON, fall back to the original error message - logging.debug(f"[DEBUG] Failed to parse error response: {str(json_error)}") + logging.debug( + f"[DEBUG] Failed to parse error response: {str(json_error)}" + ) logging.debug(f"[DEBUG] API Error: {error_message} (Status: {status_code})") if hasattr(e, "response") and e.response.content: @@ -281,6 +283,26 @@ class ApiClient: raise Exception("Unauthorized: Please login first to use this node.") return auth_token + @staticmethod + def upload_file( + upload_url: str, + file: io.BytesIO | str, + ): + """Upload a file to the API. Make sure the file has a filename equal to what the url expects. + + Args: + upload_url: The URL to upload to + file: Either a file path string, BytesIO object, or tuple of (file_path, filename) + mime_type: The mime type of the file + """ + if isinstance(file, io.BytesIO): + file.seek(0) # Ensure we're at the start of the file + data = file.read() + return requests.put(upload_url, data=data) + elif isinstance(file, str): + with open(file, "rb") as f: + data = f.read() + return requests.put(upload_url, data=data) class ApiEndpoint(Generic[T, R]): """Defines an API endpoint with its request and response types""" @@ -347,10 +369,16 @@ class SynchronousOperation(Generic[T, R]): ) # Convert request model to dict, but use None for EmptyRequest - request_dict = None if isinstance(self.request, EmptyRequest) else self.request.model_dump(exclude_none=True) + request_dict = ( + None + if isinstance(self.request, EmptyRequest) + else self.request.model_dump(exclude_none=True) + ) # Debug log for request - logging.debug(f"[DEBUG] API Request: {self.endpoint.method.value} {self.endpoint.path}") + logging.debug( + f"[DEBUG] API Request: {self.endpoint.method.value} {self.endpoint.path}" + ) logging.debug(f"[DEBUG] Request Data: {json.dumps(request_dict, indent=2)}") logging.debug(f"[DEBUG] Query Params: {self.endpoint.query_params}") @@ -473,7 +501,7 @@ class PollingOperation(Generic[T, R]): f"[DEBUG] Poll Request: {self.poll_endpoint.method.value} {self.poll_endpoint.path}" ) logging.debug( - f"[DEBUG] Poll Request Data: {json.dumps(request_dict, indent=2) if request_dict else 'None'}" + f"[DEBUG] Poll Request Data: {json.dumps(request_dict, indent=2) if request_dict else 'None'}" ) # Query task status @@ -502,7 +530,9 @@ class PollingOperation(Generic[T, R]): logging.debug("[DEBUG] Task still pending, continuing to poll...") # Wait before polling again - logging.debug(f"[DEBUG] Waiting {self.poll_interval} seconds before next poll") + logging.debug( + f"[DEBUG] Waiting {self.poll_interval} seconds before next poll" + ) time.sleep(self.poll_interval) except Exception as e: From f8f2f3c5fb6d04f8bb9e0153937e3c5b5d510fce Mon Sep 17 00:00:00 2001 From: Jedrzej Kosinski Date: Mon, 28 Apr 2025 22:54:35 -0500 Subject: [PATCH 017/121] Add Luma nodes (#16) Co-authored-by: Robin Huang --- comfy_api_nodes/apis/client.py | 11 +- comfy_api_nodes/apis/luma_api.py | 128 ++++++++++ comfy_api_nodes/nodes_api.py | 401 ++++++++++++++++++++++++++++++- 3 files changed, 532 insertions(+), 8 deletions(-) create mode 100644 comfy_api_nodes/apis/luma_api.py diff --git a/comfy_api_nodes/apis/client.py b/comfy_api_nodes/apis/client.py index bec43e648..0d822afb5 100644 --- a/comfy_api_nodes/apis/client.py +++ b/comfy_api_nodes/apis/client.py @@ -104,7 +104,7 @@ from typing import ( TypeVar, Generic, ) -from pydantic import BaseModel +from pydantic import BaseModel, Field from enum import Enum import json import requests @@ -124,6 +124,15 @@ class EmptyRequest(BaseModel): pass +class UploadRequest(BaseModel): + filename: str = Field(..., description="Filename to upload") + + +class UploadResponse(BaseModel): + download_url: str = Field(..., description='URL to GET uploaded file') + upload_url: str = Field(..., description='URL to PUT file to upload') + + class HttpMethod(str, Enum): GET = "GET" POST = "POST" diff --git a/comfy_api_nodes/apis/luma_api.py b/comfy_api_nodes/apis/luma_api.py new file mode 100644 index 000000000..e7ab03946 --- /dev/null +++ b/comfy_api_nodes/apis/luma_api.py @@ -0,0 +1,128 @@ +from __future__ import annotations + +from enum import Enum +from typing import Optional, Union + +from pydantic import BaseModel, Field, confloat + + +class LumaImageModel(str, Enum): + photon_1 = "photon-1" + photon_flash_1 = "photon-flash-1" + + +class LumaVideoModel(str, Enum): + ray_2 = "ray-2" + ray_flash_2 = "ray-flash-2" + ray_1_6 = "ray-1-6" + + +class LumaAspectRatio(str, Enum): + ratio_1_1 = "1:1" + ratio_16_9 = "16:9" + ratio_9_16 = "9:16" + ratio_4_3 = "4:3" + ratio_3_4 = "3:4" + ratio_21_9 = "21:9" + ratio_9_21 = "9:21" + + +class LumaVideoOutputResolution(str, Enum): + res_540p = "540p" + res_720p = "720p" + res_1080p = "1080p" + res_4k = "4k" + + +class LumaVideoModelOutputDuration(str, Enum): + dur_5s = "5s" + dur_9s = "9s" + + +class LumaGenerationType(str, Enum): + video = 'video' + image = 'image' + + +class LumaState(str, Enum): + queued = "queued" + dreaming = "dreaming" + completed = "completed" + failed = "failed" + + +class LumaAssets(BaseModel): + video: Optional[str] = Field(None, description='The URL of the video') + image: Optional[str] = Field(None, description='The URL of the image') + progress_video: Optional[str] = Field(None, description='The URL of the progress video') + + +class LumaImageRef(BaseModel): + '''Used for image gen''' + url: str = Field(..., description='The URL of the image reference') + weight: confloat(ge=0.0, le=1.0) = Field(..., description='The weight of the image reference') + + +class LumaImageReference(BaseModel): + '''Used for video gen''' + type: str = Field('image', description='Input type, defaults to image') + url: str = Field(..., description='The URL of the image') + + +class LumaModifyImageRef(BaseModel): + url: str = Field(..., description='The URL of the image reference') + weight: confloat(ge=0.0, le=1.0) = Field(..., description='The weight of the image reference') + + +class LumaCharacterRef(BaseModel): + identity0: LumaImageIdentity = Field(..., description='The image identity object') + + +class LumaImageIdentity(BaseModel): + images: list[str] = Field(..., description='The URLs of the image identity') + + +class LumaGenerationReference(BaseModel): + type: str = Field('generation', description='Input type, defaults to generation') + id: str = Field(..., description='The ID of the generation') + + +class LumaKeyframe(BaseModel): + generation: Optional[LumaGenerationReference] = Field(None, description='Reference to generation') + image: Optional[LumaImageReference] = Field(None, description='Reference to image') + + +class LumaKeyframes(BaseModel): + frame0: Optional[LumaKeyframe] = Field(None, description='') + frame1: Optional[LumaKeyframe] = Field(None, description='') + + +class LumaImageGenerationRequest(BaseModel): + prompt: str = Field(..., description='The prompt of the generation') + model: LumaImageModel = Field(LumaImageModel.photon_1, description='The image model used for the generation') + aspect_ratio: Optional[LumaAspectRatio] = Field(LumaAspectRatio.ratio_16_9, description='The aspect ratio of the generation') + image_ref: Optional[list[LumaImageRef]] = Field(None, description='List of image reference objects') + style_ref: Optional[list[LumaImageRef]] = Field(None, description='List of style reference objects') + character_ref: Optional[LumaCharacterRef] = Field(None, description='The image identity object') + modify_image_ref: Optional[LumaModifyImageRef] = Field(None, description='The modify image reference object') + + +class LumaGenerationRequest(BaseModel): + prompt: str = Field(..., description='The prompt of the generation') + model: LumaVideoModel = Field(LumaVideoModel.ray_2, description='The video model used for the generation') + duration: LumaVideoModelOutputDuration = Field(LumaVideoModelOutputDuration.dur_5s, description='The duration of the generation') + aspect_ratio: Optional[LumaAspectRatio] = Field(None, description='The aspect ratio of the generation') + resolution: Optional[LumaVideoOutputResolution] = Field(None, description='The resolution of the generation') + loop: Optional[bool] = Field(None, description='Whether to loop the video') + keyframes: Optional[LumaKeyframes] = Field(None, description='The keyframes of the generation') + + +class LumaGeneration(BaseModel): + id: str = Field(..., description='The ID of the generation') + generation_type: LumaGenerationType = Field(..., description='Generation type, image or video') + state: LumaState = Field(..., description='The state of the generation') + failure_reason: Optional[str] = Field(None, description='The reason for the state of the generation') + created_at: str = Field(..., description='The date and time when the generation was created') + assets: Optional[LumaAssets] = Field(None, description='The assets of the generation') + model: str = Field(..., description='The model used for the generation') + request: Union[LumaGenerationRequest, LumaImageGenerationRequest] = Field(..., description="The request used for the generation") diff --git a/comfy_api_nodes/nodes_api.py b/comfy_api_nodes/nodes_api.py index 03100aab5..359c2cb7c 100644 --- a/comfy_api_nodes/nodes_api.py +++ b/comfy_api_nodes/nodes_api.py @@ -20,7 +20,21 @@ from comfy_api_nodes.apis import ( Model ) from comfy_api_nodes.apis.BFLPolling import BFLStatus -from comfy_api_nodes.apis.client import ApiEndpoint, HttpMethod, SynchronousOperation, PollingOperation, EmptyRequest +from comfy_api_nodes.apis.luma_api import ( + LumaImageModel, + LumaVideoModel, + LumaVideoOutputResolution, + LumaVideoModelOutputDuration, + LumaAspectRatio, + LumaState, + LumaImageGenerationRequest, + LumaGenerationRequest, + LumaGeneration, + LumaCharacterRef, + LumaModifyImageRef, + LumaImageIdentity, +) +from comfy_api_nodes.apis.client import ApiClient, ApiEndpoint, HttpMethod, SynchronousOperation, PollingOperation, EmptyRequest, UploadRequest, UploadResponse import numpy as np from PIL import Image @@ -33,6 +47,7 @@ import json import av import os import time +import uuid import folder_paths def downscale_input(image, total_pixels=1536*1024): @@ -106,6 +121,76 @@ def validate_aspect_ratio(aspect_ratio: str, minimum_ratio: float, maximum_ratio raise Exception(f"Aspect ratio cannot reduce to any greater than {maximum_ratio_str} ({maximum_ratio}), but was {aspect_ratio} ({calculated_ratio}).") return aspect_ratio +def process_image_response(response: requests.Response): + '''Uses content from a Response object and converts it to a torch.Tensor''' + image = Image.open(io.BytesIO(response.content)).convert("RGBA") + image_array = np.array(image).astype(np.float32) / 255.0 + return torch.from_numpy(image_array).unsqueeze(0) + +def convert_image_to_bytesio(image: torch.Tensor, name: str=None, allow_alpha=True, total_pixels=2048*2048): + img_binary = None + # only care about first image, if it is a batch + if len(image.shape) > 3: + image = image[0] + # TODO: remove alpha if not allowed and present + input_tensor = image.cpu() + input_tensor = downscale_input(input_tensor.unsqueeze(0), total_pixels=total_pixels).squeeze() + image_np = (input_tensor.numpy() * 255).astype(np.uint8) + img = Image.fromarray(image_np) + img_byte_arr = io.BytesIO() + img.save(img_byte_arr, format='PNG') + img_byte_arr.seek(0) + img_binary = img_byte_arr + img_binary.name = f"{name if name else uuid.uuid4()}.png" + return img_binary + +def upload_images_to_comfyapi(image: torch.Tensor, max_images=8, auth_token=None) -> list[str]: + # if batch, try to upload each file if max_images is greater than 0 + idx_image = 0 + download_urls: list[str] = [] + is_batch = len(image.shape) > 3 + batch_length = 1 + if is_batch: + batch_length = image.shape[0] + while True: + curr_image = image + if len(image.shape) > 3: + curr_image = image[idx_image] + # get BytesIO version of image + img_binary = convert_image_to_bytesio(curr_image) + # first, request upload/download urls from comfy API + operation = SynchronousOperation( + endpoint=ApiEndpoint( + path="/customers/storage", + method=HttpMethod.POST, + request_model=UploadRequest, + response_model=UploadResponse + ), + request=UploadRequest( + filename=img_binary.name + ), + auth_token=auth_token + ) + response = operation.execute() + + upload_response = ApiClient.upload_file(response.upload_url, img_binary) + # verify success + try: + upload_response.raise_for_status() + except requests.exceptions.HTTPError as e: + raise Exception(f"Could not upload one or more images: {e}") + # add download_url to list + download_urls.append(response.download_url) + + idx_image += 1 + # stop uploading additional files if done + if is_batch and max_images > 0: + if idx_image >= max_images: + break + if idx_image >= batch_length: + break + return download_urls + class OpenAIDalle2(ComfyNodeABC): """ Generates images synchronously via OpenAI's DALL·E 2 endpoint. @@ -731,7 +816,7 @@ class FluxProUltraImageNode(ComfyNodeABC): if result["status"] == BFLStatus.ready: img_url = result["result"]["sample"] img_response = requests.get(img_url) - return self._process_bfl_image_response(img_response) + return process_image_response(img_response) elif result["status"] in [BFLStatus.request_moderated, BFLStatus.content_moderated]: status = result["status"] raise Exception(f"BFL API did not return an image due to: {status}.") @@ -753,11 +838,6 @@ class FluxProUltraImageNode(ComfyNodeABC): else: raise Exception(f"BFL API encountered an error: {response.json()}") - def _process_bfl_image_response(self, response: requests.Response): - image = Image.open(io.BytesIO(response.content)).convert("RGBA") - image_array = np.array(image).astype(np.float32) / 255.0 - return torch.from_numpy(image_array).unsqueeze(0) - def _convert_image_to_base64(self, image: torch.Tensor): scaled_image = downscale_input(image, total_pixels=2048*2048) # remove batch dimension if present @@ -769,6 +849,307 @@ class FluxProUltraImageNode(ComfyNodeABC): img.save(img_byte_arr, format='PNG') return base64.b64encode(img_byte_arr.getvalue()).decode() +class LumaImageGenerationNode: + """ + Generates images synchronously based on prompt and aspect ratio. + """ + RETURN_TYPES = (IO.IMAGE,) + DESCRIPTION = cleandoc(__doc__ or "") # Handle potential None value + FUNCTION = "api_call" + API_NODE = True + CATEGORY = "api node" + + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "prompt": (IO.STRING, { + "multiline": True, + "default": "", + "tooltip": "Prompt for the image generation", + }), + "model": ([model.value for model in LumaImageModel],), + "aspect_ratio": ([ratio.value for ratio in LumaAspectRatio], { + "default": LumaAspectRatio.ratio_16_9, + }), + "seed": (IO.INT, { + "default": 0, + "min": 0, + "max": 0xFFFFFFFFFFFFFFFF, + "control_after_generate": True, + "tooltip": "Seed to determine if node should re-run; actual results are nondeterministic regardless of seed.", + }), + }, + "optional": { + "character_ref_image": (IO.IMAGE, { + "tooltip": "Character reference images; can be a batch of multiple, only the first 4 images will be considered." + }) + }, + "hidden": { + "auth_token": "AUTH_TOKEN_COMFY_ORG", + } + } + + def api_call(self, prompt: str, model: str, aspect_ratio: str, seed, character_ref_image: torch.Tensor=None, auth_token=None, **kwargs): + # handle character_ref images + character_ref = None + if character_ref_image is not None: + download_urls = upload_images_to_comfyapi(character_ref_image, max_images=4, auth_token=auth_token) + character_ref = LumaCharacterRef(identity0=LumaImageIdentity(images=download_urls)) + + operation = SynchronousOperation( + endpoint=ApiEndpoint( + path="/proxy/luma/generations/image", + method=HttpMethod.POST, + request_model=LumaImageGenerationRequest, + response_model=LumaGeneration + ), + request=LumaImageGenerationRequest( + prompt=prompt, + model=model, + aspect_ratio=aspect_ratio, + character_ref=character_ref + ), + auth_token=auth_token + ) + response_api: LumaGeneration = operation.execute() + + operation = PollingOperation( + poll_endpoint=ApiEndpoint( + path=f"/proxy/luma/generations/{response_api.id}", + method=HttpMethod.GET, + request_model=EmptyRequest, + response_model=LumaGeneration, + ), + completed_statuses=[LumaState.completed], + failed_statuses=[LumaState.failed], + status_extractor=lambda x: x.state, + auth_token=auth_token, + ) + response_poll = operation.execute() + + img_response = requests.get(response_poll.assets.image) + img = process_image_response(img_response) + return (img,) + +class LumaImageModifyNode: + """ + Modifies images synchronously based on prompt and aspect ratio. + """ + RETURN_TYPES = (IO.IMAGE,) + DESCRIPTION = cleandoc(__doc__ or "") # Handle potential None value + FUNCTION = "api_call" + API_NODE = True + CATEGORY = "api node" + + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "image": (IO.IMAGE,), + "prompt": (IO.STRING, { + "multiline": True, + "default": "", + "tooltip": "Prompt for the image generation", + }), + "image_weight": (IO.FLOAT, { + "default": 1.0, + "min": 0.0, + "max": 1.0, + "step": 0.01, + "tooltip": "Weight of the image; the closer to 0.0, the less the image will be modified." + }), + "model": ([model.value for model in LumaImageModel],), + "seed": (IO.INT, { + "default": 0, + "min": 0, + "max": 0xFFFFFFFFFFFFFFFF, + "control_after_generate": True, + "tooltip": "Seed to determine if node should re-run; actual results are nondeterministic regardless of seed.", + }), + }, + "optional": { + }, + "hidden": { + "auth_token": "AUTH_TOKEN_COMFY_ORG", + } + } + + def api_call(self, prompt: str, model: str, image: torch.Tensor, image_weight: float, seed, auth_token=None, **kwargs): + # first, upload image + download_urls = upload_images_to_comfyapi(image, max_images=1, auth_token=auth_token) + image_url = download_urls[0] + # next, make Luma call with download url provided + operation = SynchronousOperation( + endpoint=ApiEndpoint( + path="/proxy/luma/generations/image", + method=HttpMethod.POST, + request_model=LumaImageGenerationRequest, + response_model=LumaGeneration + ), + request=LumaImageGenerationRequest( + prompt=prompt, + model=model, + modify_image_ref=LumaModifyImageRef( + url=image_url, + weight=round(image_weight, 2) + ), + ), + auth_token=auth_token + ) + response_api: LumaGeneration = operation.execute() + + operation = PollingOperation( + poll_endpoint=ApiEndpoint( + path=f"/proxy/luma/generations/{response_api.id}", + method=HttpMethod.GET, + request_model=EmptyRequest, + response_model=LumaGeneration, + ), + completed_statuses=[LumaState.completed], + failed_statuses=[LumaState.failed], + status_extractor=lambda x: x.state, + auth_token=auth_token, + ) + response_poll = operation.execute() + + img_response = requests.get(response_poll.assets.image) + img = process_image_response(img_response) + return (img,) + +class LumaVideoGenerationNode: + """ + Generates videos synchronously based on prompt and output_size. + """ + def __init__(self): + self.output_dir = folder_paths.get_output_directory() + self.type: Literal["output"] = "output" + + RETURN_TYPES = ("IMAGE",) + DESCRIPTION = cleandoc(__doc__ or "") # Handle potential None value + FUNCTION = "api_call" + API_NODE = True + CATEGORY = "api node" + + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "prompt": (IO.STRING, { + "multiline": True, + "default": "", + "tooltip": "Prompt for the video generation", + }), + "model": ([model.value for model in LumaVideoModel],), + "aspect_ratio": ([ratio.value for ratio in LumaAspectRatio], { + "default": LumaAspectRatio.ratio_16_9, + }), + "resolution": ([resolution.value for resolution in LumaVideoOutputResolution], { + "default": LumaVideoOutputResolution.res_540p, + }), + "duration": ([dur.value for dur in LumaVideoModelOutputDuration],), + "seed": (IO.INT, { + "default": 0, + "min": 0, + "max": 0xFFFFFFFFFFFFFFFF, + "control_after_generate": True, + "tooltip": "Seed to determine if node should re-run; actual results are nondeterministic regardless of seed.", + }), + "filename_prefix": ("STRING", {"default": "ComfyUI"}), + }, + "optional": { + }, + "hidden": { + "auth_token": "AUTH_TOKEN_COMFY_ORG", + } + } + + def api_call(self, prompt: str, model: str, aspect_ratio: str, resolution: str, duration: str, seed, filename_prefix: str, + auth_token=None, **kwargs): + extra_pnginfo = None + operation = SynchronousOperation( + endpoint=ApiEndpoint( + path="/proxy/luma/generations", + method=HttpMethod.POST, + request_model=LumaGenerationRequest, + response_model=LumaGeneration + ), + request=LumaGenerationRequest( + prompt=prompt, + model=model, + resolution=resolution, + aspect_ratio=aspect_ratio, + duration=duration, + ), + auth_token=auth_token + ) + response_api: LumaGeneration = operation.execute() + + operation = PollingOperation( + poll_endpoint=ApiEndpoint( + path=f"/proxy/luma/generations/{response_api.id}", + method=HttpMethod.GET, + request_model=EmptyRequest, + response_model=LumaGeneration, + ), + completed_statuses=[LumaState.completed], + failed_statuses=[LumaState.failed], + status_extractor=lambda x: x.state, + auth_token=auth_token, + ) + response_poll = operation.execute() + + vid_response = requests.get(response_poll.assets.video) + self._save_video_locally(vid_response, filename_prefix, extra_pnginfo) + + return (None,) + #return {"ui": {"images": results, "animated": (True,)}} + + def _save_video_locally(self, response: requests.Response, filename_prefix: str, extra_pnginfo): + # Construct the save path + full_output_folder, filename, counter, subfolder, filename_prefix = ( + folder_paths.get_save_image_path(filename_prefix, self.output_dir) + ) + file_basename = f"{filename}_{counter:05}_.mp4" + save_path = os.path.join(full_output_folder, file_basename) + + video_data = response.content + + # Save the video data to a file + with open(save_path, "wb") as video_file: + video_file.write(video_data) + + # Add workflow metadata to the video container + #if prompt is not None or extra_pnginfo is not None: + if extra_pnginfo is not None: + try: + container = av.open(save_path, mode="r+") + # if prompt is not None: + # container.metadata["prompt"] = json.dumps(prompt) + if extra_pnginfo is not None: + for x in extra_pnginfo: + container.metadata[x] = json.dumps(extra_pnginfo[x]) + container.close() + except Exception as e: + logging.warning(f"Failed to add metadata to video: {e}") + + # Create a FileLocator for the frontend to use for the preview + results: list[FileLocator] = [ + { + "filename": file_basename, + "subfolder": subfolder, + "type": self.type, + } + ] + + return results + + def _get_output_type(self, output_size: str): + if output_size in [resolution.value for resolution in LumaVideoOutputResolution]: + return LumaVideoOutputResolution + else: + return LumaAspectRatio + class MinimaxTextToVideoNode: """ Generates videos synchronously based on a prompt, and optional parameters using Minimax's API. @@ -946,6 +1327,9 @@ NODE_CLASS_MAPPINGS = { "OpenAIGPTImage1": OpenAIGPTImage1, "IdeogramTextToImage": IdeogramTextToImage, "FluxProUltraImageNode": FluxProUltraImageNode, + "LumaImageNode": LumaImageGenerationNode, + "LumaImageModifyNode": LumaImageModifyNode, + "LumaVideoNode": LumaVideoGenerationNode, "MinimaxTextToVideoNode": MinimaxTextToVideoNode, } @@ -956,5 +1340,8 @@ NODE_DISPLAY_NAME_MAPPINGS = { "OpenAIGPTImage1": "OpenAI GPT Image 1", "IdeogramTextToImage": "Ideogram Text to Image", "FluxProUltraImageNode": "Flux 1.1 [pro] Ultra Image", + "LumaImageNode": "Luma Generate Image", + "LumaImageModifyNode": "Luma Modify Image", + "LumaVideoNode": "Luma Generate Video", "MinimaxTextToVideoNode": "Minimax Text to Video", } From e9fa0616fd272b6ffa99b82f9eede36ae71f9cc2 Mon Sep 17 00:00:00 2001 From: Christian Byrne Date: Mon, 28 Apr 2025 21:52:45 -0700 Subject: [PATCH 018/121] Refactor util functions (#20) --- comfy_api_nodes/nodes_api.py | 122 +++++++++++++++++++++++++++++++---- 1 file changed, 111 insertions(+), 11 deletions(-) diff --git a/comfy_api_nodes/nodes_api.py b/comfy_api_nodes/nodes_api.py index 359c2cb7c..a935465d8 100644 --- a/comfy_api_nodes/nodes_api.py +++ b/comfy_api_nodes/nodes_api.py @@ -1,7 +1,7 @@ import io from inspect import cleandoc from comfy.comfy_types.node_typing import FileLocator -from typing import Literal +from typing import Literal, Optional from comfy.utils import common_upscale from comfy.comfy_types.node_typing import IO, ComfyNodeABC, InputTypeDict from comfy_api_nodes.apis import ( @@ -49,6 +49,7 @@ import os import time import uuid import folder_paths +from io import BytesIO def downscale_input(image, total_pixels=1536*1024): samples = image.movedim(-1,1) @@ -121,15 +122,54 @@ def validate_aspect_ratio(aspect_ratio: str, minimum_ratio: float, maximum_ratio raise Exception(f"Aspect ratio cannot reduce to any greater than {maximum_ratio_str} ({maximum_ratio}), but was {aspect_ratio} ({calculated_ratio}).") return aspect_ratio -def process_image_response(response: requests.Response): - '''Uses content from a Response object and converts it to a torch.Tensor''' - image = Image.open(io.BytesIO(response.content)).convert("RGBA") + +def mimetype_to_extension(mime_type: str) -> str: + """Converts a MIME type to a file extension.""" + return mime_type.split('/')[-1].lower() + + +def download_url_to_bytesio(url: str, timeout: int = None) -> BytesIO: + """Downloads content from a URL using requests and returns it as BytesIO. + + Args: + url: The URL to download. + timeout: Request timeout in seconds. Defaults to None (no timeout). + + Returns: + BytesIO object containing the downloaded content. + """ + response = requests.get(url, stream=True, timeout=timeout) + response.raise_for_status() # Raises HTTPError for bad responses (4XX or 5XX) + return BytesIO(response.content) + + +def bytesio_to_image_tensor(image_bytesio: BytesIO, mode: str = "RGBA") -> torch.Tensor: + """Converts image data from BytesIO to a torch.Tensor. + + Args: + image_bytesio: BytesIO object containing the image data. + mode: The PIL mode to convert the image to (e.g., "RGB", "RGBA"). + + Returns: + A torch.Tensor representing the image (1, H, W, C). + + Raises: + PIL.UnidentifiedImageError: If the image data cannot be identified. + ValueError: If the specified mode is invalid. + """ + image = Image.open(image_bytesio) + image = image.convert(mode) image_array = np.array(image).astype(np.float32) / 255.0 return torch.from_numpy(image_array).unsqueeze(0) -def convert_image_to_bytesio(image: torch.Tensor, name: str=None, allow_alpha=True, total_pixels=2048*2048): - img_binary = None - # only care about first image, if it is a batch + +def process_image_response(response: requests.Response): + '''Uses content from a Response object and converts it to a torch.Tensor''' + return bytesio_to_image_tensor(BytesIO(response.content)) + + +def _tensor_to_pil(image: torch.Tensor, total_pixels: int = 2048*2048) -> Image.Image: + """Converts a single torch.Tensor image [H, W, C] to a PIL Image, optionally downscaling.""" if len(image.shape) > 3: image = image[0] # TODO: remove alpha if not allowed and present @@ -137,13 +177,73 @@ def convert_image_to_bytesio(image: torch.Tensor, name: str=None, allow_alpha=Tr input_tensor = downscale_input(input_tensor.unsqueeze(0), total_pixels=total_pixels).squeeze() image_np = (input_tensor.numpy() * 255).astype(np.uint8) img = Image.fromarray(image_np) + return img + + +def _pil_to_bytesio(img: Image.Image, mime_type: str = 'image/png') -> BytesIO: + """Converts a PIL Image to a BytesIO object.""" img_byte_arr = io.BytesIO() - img.save(img_byte_arr, format='PNG') + # Derive PIL format from MIME type (e.g., 'image/png' -> 'PNG') + pil_format = mime_type.split('/')[-1].upper() + if pil_format == 'JPG': + pil_format = 'JPEG' + img.save(img_byte_arr, format=pil_format) img_byte_arr.seek(0) - img_binary = img_byte_arr - img_binary.name = f"{name if name else uuid.uuid4()}.png" + return img_byte_arr + + +def tensor_to_bytesio(image: torch.Tensor, name: Optional[str] = None, total_pixels: int = 2048*2048, mime_type: str = 'image/png') -> BytesIO: + """Converts a torch.Tensor image to a named BytesIO object. + + Args: + image: Input torch.Tensor image. + name: Optional filename for the BytesIO object. + total_pixels: Maximum total pixels for potential downscaling. + mime_type: Target image MIME type (e.g., 'image/png', 'image/jpeg', 'image/webp', 'video/mp4'). + + Returns: + Named BytesIO object containing the image data. + """ + pil_image = _tensor_to_pil(image, total_pixels=total_pixels) + img_binary = _pil_to_bytesio(pil_image, mime_type=mime_type) + img_binary.name = f"{name if name else uuid.uuid4()}.{mimetype_to_extension(mime_type)}" return img_binary + +def tensor_to_base64_string(image_tensor: torch.Tensor, total_pixels: int = 2048*2048, mime_type: str = 'image/png') -> str: + """Convert [B, H, W, C] or [H, W, C] tensor to a base64 string. + + Args: + image_tensor: Input torch.Tensor image. + total_pixels: Maximum total pixels for potential downscaling. + mime_type: Target image MIME type (e.g., 'image/png', 'image/jpeg', 'image/webp', 'video/mp4'). + + Returns: + Base64 encoded string of the image. + """ + pil_image = _tensor_to_pil(image_tensor, total_pixels=total_pixels) + img_byte_arr = _pil_to_bytesio(pil_image, mime_type=mime_type) + img_bytes = img_byte_arr.getvalue() + # Encode bytes to base64 string + base64_encoded_string = base64.b64encode(img_bytes).decode("utf-8") + return base64_encoded_string + + +def tensor_to_data_uri(image_tensor: torch.Tensor, total_pixels: int = 2048 * 2048, mime_type: str = 'image/png') -> str: + """Converts a tensor image to a Data URI string. + + Args: + image_tensor: Input torch.Tensor image. + total_pixels: Maximum total pixels for potential downscaling. + mime_type: Target image MIME type (e.g., 'image/png', 'image/jpeg', 'image/webp'). + + Returns: + Data URI string (e.g., 'data:image/png;base64,...'). + """ + base64_string = tensor_to_base64_string(image_tensor, total_pixels, mime_type) + return f"data:{mime_type};base64,{base64_string}" + + def upload_images_to_comfyapi(image: torch.Tensor, max_images=8, auth_token=None) -> list[str]: # if batch, try to upload each file if max_images is greater than 0 idx_image = 0 @@ -157,7 +257,7 @@ def upload_images_to_comfyapi(image: torch.Tensor, max_images=8, auth_token=None if len(image.shape) > 3: curr_image = image[idx_image] # get BytesIO version of image - img_binary = convert_image_to_bytesio(curr_image) + img_binary = tensor_to_bytesio(curr_image) # first, request upload/download urls from comfy API operation = SynchronousOperation( endpoint=ApiEndpoint( From 4f8b235fdab875c75d39b3836dc2fd46b0ff68c1 Mon Sep 17 00:00:00 2001 From: guill Date: Mon, 28 Apr 2025 22:11:32 -0700 Subject: [PATCH 019/121] Add VIDEO type (#21) --- comfy/comfy_types/node_typing.py | 9 +- comfy_api/input/__init__.py | 8 + comfy_api/input/basic_types.py | 20 +++ comfy_api/input/video_types.py | 45 ++++++ comfy_api/input_impl/__init__.py | 7 + comfy_api/input_impl/video_types.py | 224 ++++++++++++++++++++++++++++ comfy_api/util/__init__.py | 8 + comfy_api/util/video_types.py | 51 +++++++ comfy_extras/nodes_video.py | 164 +++++++++++++++++++- folder_paths.py | 4 +- 10 files changed, 532 insertions(+), 8 deletions(-) create mode 100644 comfy_api/input/__init__.py create mode 100644 comfy_api/input/basic_types.py create mode 100644 comfy_api/input/video_types.py create mode 100644 comfy_api/input_impl/__init__.py create mode 100644 comfy_api/input_impl/video_types.py create mode 100644 comfy_api/util/__init__.py create mode 100644 comfy_api/util/video_types.py diff --git a/comfy/comfy_types/node_typing.py b/comfy/comfy_types/node_typing.py index 0bdda032e..9a345586e 100644 --- a/comfy/comfy_types/node_typing.py +++ b/comfy/comfy_types/node_typing.py @@ -48,6 +48,7 @@ class IO(StrEnum): FACE_ANALYSIS = "FACE_ANALYSIS" BBOX = "BBOX" SEGS = "SEGS" + VIDEO = "VIDEO" ANY = "*" """Always matches any type, but at a price. @@ -269,7 +270,7 @@ class ComfyNodeABC(ABC): Comfy Docs: https://docs.comfy.org/custom-nodes/backend/lists#list-processing """ - OUTPUT_IS_LIST: tuple[bool] + OUTPUT_IS_LIST: tuple[bool, ...] """A tuple indicating which node outputs are lists, but will be connected to nodes that expect individual items. Connected nodes that do not implement `INPUT_IS_LIST` will be executed once for every item in the list. @@ -288,7 +289,7 @@ class ComfyNodeABC(ABC): Comfy Docs: https://docs.comfy.org/custom-nodes/backend/lists#list-processing """ - RETURN_TYPES: tuple[IO] + RETURN_TYPES: tuple[IO, ...] """A tuple representing the outputs of this node. Usage:: @@ -297,12 +298,12 @@ class ComfyNodeABC(ABC): Comfy Docs: https://docs.comfy.org/custom-nodes/backend/server_overview#return-types """ - RETURN_NAMES: tuple[str] + RETURN_NAMES: tuple[str, ...] """The output slot names for each item in `RETURN_TYPES`, e.g. ``RETURN_NAMES = ("count", "filter_string")`` Comfy Docs: https://docs.comfy.org/custom-nodes/backend/server_overview#return-names """ - OUTPUT_TOOLTIPS: tuple[str] + OUTPUT_TOOLTIPS: tuple[str, ...] """A tuple of strings to use as tooltips for node outputs, one for each item in `RETURN_TYPES`.""" FUNCTION: str """The name of the function to execute as a literal string, e.g. `FUNCTION = "execute"` diff --git a/comfy_api/input/__init__.py b/comfy_api/input/__init__.py new file mode 100644 index 000000000..66667946f --- /dev/null +++ b/comfy_api/input/__init__.py @@ -0,0 +1,8 @@ +from .basic_types import ImageInput, AudioInput +from .video_types import VideoInput + +__all__ = [ + "ImageInput", + "AudioInput", + "VideoInput", +] diff --git a/comfy_api/input/basic_types.py b/comfy_api/input/basic_types.py new file mode 100644 index 000000000..033fb7e27 --- /dev/null +++ b/comfy_api/input/basic_types.py @@ -0,0 +1,20 @@ +import torch +from typing import TypedDict + +ImageInput = torch.Tensor +""" +An image in format [B, H, W, C] where B is the batch size, C is the number of channels, +""" + +class AudioInput(TypedDict): + """ + TypedDict representing audio input. + """ + + waveform: torch.Tensor + """ + Tensor in the format [B, C, T] where B is the batch size, C is the number of channels, + """ + + sample_rate: int + diff --git a/comfy_api/input/video_types.py b/comfy_api/input/video_types.py new file mode 100644 index 000000000..0676e0e66 --- /dev/null +++ b/comfy_api/input/video_types.py @@ -0,0 +1,45 @@ +from __future__ import annotations +from abc import ABC, abstractmethod +from typing import Optional +from comfy_api.util import VideoContainer, VideoCodec, VideoComponents + +class VideoInput(ABC): + """ + Abstract base class for video input types. + """ + + @abstractmethod + def get_components(self) -> VideoComponents: + """ + Abstract method to get the video components (images, audio, and frame rate). + + Returns: + VideoComponents containing images, audio, and frame rate + """ + pass + + @abstractmethod + def save_to( + self, + path: str, + format: VideoContainer = VideoContainer.AUTO, + codec: VideoCodec = VideoCodec.AUTO, + metadata: Optional[dict] = None + ): + """ + Abstract method to save the video input to a file. + """ + pass + + # Provide a default implementation, but subclasses can provide optimized versions + # if possible. + def get_dimensions(self) -> tuple[int, int]: + """ + Returns the dimensions of the video input. + + Returns: + Tuple of (width, height) + """ + components = self.get_components() + return components.images.shape[2], components.images.shape[1] + diff --git a/comfy_api/input_impl/__init__.py b/comfy_api/input_impl/__init__.py new file mode 100644 index 000000000..02901b8b9 --- /dev/null +++ b/comfy_api/input_impl/__init__.py @@ -0,0 +1,7 @@ +from .video_types import VideoFromFile, VideoFromComponents + +__all__ = [ + # Implementations + "VideoFromFile", + "VideoFromComponents", +] diff --git a/comfy_api/input_impl/video_types.py b/comfy_api/input_impl/video_types.py new file mode 100644 index 000000000..12e5783db --- /dev/null +++ b/comfy_api/input_impl/video_types.py @@ -0,0 +1,224 @@ +from __future__ import annotations +from av.container import InputContainer +from av.subtitles.stream import SubtitleStream +from fractions import Fraction +from typing import Optional +from comfy_api.input import AudioInput +import av +import io +import json +import numpy as np +import torch +from comfy_api.input import VideoInput +from comfy_api.util import VideoContainer, VideoCodec, VideoComponents + +class VideoFromFile(VideoInput): + """ + Class representing video input from a file. + """ + + def __init__(self, file: str | io.BytesIO): + """ + Initialize the VideoFromFile object based off of either a path on disk or a BytesIO object + containing the file contents. + """ + self.__file = file + + def get_dimensions(self) -> tuple[int, int]: + """ + Returns the dimensions of the video input. + + Returns: + Tuple of (width, height) + """ + if isinstance(self.__file, io.BytesIO): + self.__file.seek(0) # Reset the BytesIO object to the beginning + with av.open(self.__file, mode='r') as container: + for stream in container.streams: + if stream.type == 'video': + assert isinstance(stream, av.VideoStream) + return stream.width, stream.height + raise ValueError(f"No video stream found in file '{self.__file}'") + + def get_components_internal(self, container: InputContainer) -> VideoComponents: + # Get video frames + frames = [] + for frame in container.decode(video=0): + img = frame.to_ndarray(format='rgb24') # shape: (H, W, 3) + img = torch.from_numpy(img) / 255.0 # shape: (H, W, 3) + frames.append(img) + + images = torch.stack(frames) if len(frames) > 0 else torch.zeros(0, 3, 0, 0) + + # Get frame rate + video_stream = next(s for s in container.streams if s.type == 'video') + frame_rate = Fraction(video_stream.average_rate) if video_stream and video_stream.average_rate else Fraction(1) + + # Get audio if available + audio = None + try: + container.seek(0) # Reset the container to the beginning + for stream in container.streams: + if stream.type != 'audio': + continue + assert isinstance(stream, av.AudioStream) + audio_frames = [] + for packet in container.demux(stream): + for frame in packet.decode(): + assert isinstance(frame, av.AudioFrame) + audio_frames.append(frame.to_ndarray()) # shape: (channels, samples) + if len(audio_frames) > 0: + audio_data = np.concatenate(audio_frames, axis=1) # shape: (channels, total_samples) + audio_tensor = torch.from_numpy(audio_data).unsqueeze(0) # shape: (1, channels, total_samples) + audio = AudioInput({ + "waveform": audio_tensor, + "sample_rate": int(stream.sample_rate) if stream.sample_rate else 1, + }) + except StopIteration: + pass # No audio stream + + metadata = container.metadata + return VideoComponents(images=images, audio=audio, frame_rate=frame_rate, metadata=metadata) + + def get_components(self) -> VideoComponents: + if isinstance(self.__file, io.BytesIO): + self.__file.seek(0) # Reset the BytesIO object to the beginning + with av.open(self.__file, mode='r') as container: + return self.get_components_internal(container) + raise ValueError(f"No video stream found in file '{self.__file}'") + + def save_to( + self, + path: str, + format: VideoContainer = VideoContainer.AUTO, + codec: VideoCodec = VideoCodec.AUTO, + metadata: Optional[dict] = None + ): + if isinstance(self.__file, io.BytesIO): + self.__file.seek(0) # Reset the BytesIO object to the beginning + with av.open(self.__file, mode='r') as container: + container_format = container.format.name + video_encoding = container.streams.video[0].codec.name if len(container.streams.video) > 0 else None + reuse_streams = True + if format != VideoContainer.AUTO and format not in container_format.split(","): + reuse_streams = False + if codec != VideoCodec.AUTO and codec != video_encoding and video_encoding is not None: + reuse_streams = False + + if not reuse_streams: + components = self.get_components_internal(container) + video = VideoFromComponents(components) + return video.save_to( + path, + format=format, + codec=codec, + metadata=metadata + ) + + streams = container.streams + with av.open(path, mode='w', options={"movflags": "use_metadata_tags"}) as output_container: + # Copy over the original metadata + for key, value in container.metadata.items(): + if metadata is None or key not in metadata: + output_container.metadata[key] = value + + # Add our new metadata + if metadata is not None: + for key, value in metadata.items(): + if isinstance(value, str): + output_container.metadata[key] = value + else: + output_container.metadata[key] = json.dumps(value) + + # Add streams to the new container + stream_map = {} + for stream in streams: + if isinstance(stream, (av.VideoStream, av.AudioStream, SubtitleStream)): + out_stream = output_container.add_stream_from_template(template=stream, opaque=True) + stream_map[stream] = out_stream + + # Write packets to the new container + for packet in container.demux(): + if packet.stream in stream_map and packet.dts is not None: + packet.stream = stream_map[packet.stream] + output_container.mux(packet) + +class VideoFromComponents(VideoInput): + """ + Class representing video input from tensors. + """ + + def __init__(self, components: VideoComponents): + self.__components = components + + def get_components(self) -> VideoComponents: + return VideoComponents( + images=self.__components.images, + audio=self.__components.audio, + frame_rate=self.__components.frame_rate + ) + + def save_to( + self, + path: str, + format: VideoContainer = VideoContainer.AUTO, + codec: VideoCodec = VideoCodec.AUTO, + metadata: Optional[dict] = None + ): + if format != VideoContainer.AUTO and format != VideoContainer.MP4: + raise ValueError("Only MP4 format is supported for now") + if codec != VideoCodec.AUTO and codec != VideoCodec.H264: + raise ValueError("Only H264 codec is supported for now") + with av.open(path, mode='w', options={'movflags': 'use_metadata_tags'}) as output: + # Add metadata before writing any streams + if metadata is not None: + for key, value in metadata.items(): + output.metadata[key] = json.dumps(value) + + frame_rate = Fraction(round(self.__components.frame_rate * 1000), 1000) + # Create a video stream + video_stream = output.add_stream('h264', rate=frame_rate) + video_stream.width = self.__components.images.shape[2] + video_stream.height = self.__components.images.shape[1] + video_stream.pix_fmt = 'yuv420p' + + # Create an audio stream + audio_sample_rate = 1 + audio_stream: Optional[av.AudioStream] = None + if self.__components.audio: + audio_sample_rate = int(self.__components.audio['sample_rate']) + audio_stream = output.add_stream('aac', rate=audio_sample_rate) + audio_stream.sample_rate = audio_sample_rate + audio_stream.format = 'fltp' + + # Encode video + for i, frame in enumerate(self.__components.images): + img = (frame * 255).clamp(0, 255).byte().cpu().numpy() # shape: (H, W, 3) + frame = av.VideoFrame.from_ndarray(img, format='rgb24') + frame = frame.reformat(format='yuv420p') # Convert to YUV420P as required by h264 + packet = video_stream.encode(frame) + output.mux(packet) + + # Flush video + packet = video_stream.encode(None) + output.mux(packet) + + if audio_stream and self.__components.audio: + # Encode audio + samples_per_frame = int(audio_sample_rate / frame_rate) + num_frames = self.__components.audio['waveform'].shape[2] // samples_per_frame + for i in range(num_frames): + start = i * samples_per_frame + end = start + samples_per_frame + # TODO(Feature) - Add support for stereo audio + chunk = self.__components.audio['waveform'][0, 0, start:end].unsqueeze(0).numpy() + audio_frame = av.AudioFrame.from_ndarray(chunk, format='fltp', layout='mono') + audio_frame.sample_rate = audio_sample_rate + audio_frame.pts = i * samples_per_frame + for packet in audio_stream.encode(audio_frame): + output.mux(packet) + + # Flush audio + for packet in audio_stream.encode(None): + output.mux(packet) + diff --git a/comfy_api/util/__init__.py b/comfy_api/util/__init__.py new file mode 100644 index 000000000..9019c46db --- /dev/null +++ b/comfy_api/util/__init__.py @@ -0,0 +1,8 @@ +from .video_types import VideoContainer, VideoCodec, VideoComponents + +__all__ = [ + # Utility Types + "VideoContainer", + "VideoCodec", + "VideoComponents", +] diff --git a/comfy_api/util/video_types.py b/comfy_api/util/video_types.py new file mode 100644 index 000000000..d09663db9 --- /dev/null +++ b/comfy_api/util/video_types.py @@ -0,0 +1,51 @@ +from __future__ import annotations +from dataclasses import dataclass +from enum import Enum +from fractions import Fraction +from typing import Optional +from comfy_api.input import ImageInput, AudioInput + +class VideoCodec(str, Enum): + AUTO = "auto" + H264 = "h264" + + @classmethod + def as_input(cls) -> list[str]: + """ + Returns a list of codec names that can be used as node input. + """ + return [member.value for member in cls] + +class VideoContainer(str, Enum): + AUTO = "auto" + MP4 = "mp4" + + @classmethod + def as_input(cls) -> list[str]: + """ + Returns a list of container names that can be used as node input. + """ + return [member.value for member in cls] + + @classmethod + def get_extension(cls, value) -> str: + """ + Returns the file extension for the container. + """ + if isinstance(value, str): + value = cls(value) + if value == VideoContainer.MP4 or value == VideoContainer.AUTO: + return "mp4" + return "" + +@dataclass +class VideoComponents: + """ + Dataclass representing the components of a video. + """ + + images: ImageInput + frame_rate: Fraction + audio: Optional[AudioInput] = None + metadata: Optional[dict] = None + diff --git a/comfy_extras/nodes_video.py b/comfy_extras/nodes_video.py index a9e244ebe..61f7171b2 100644 --- a/comfy_extras/nodes_video.py +++ b/comfy_extras/nodes_video.py @@ -5,9 +5,13 @@ import av import torch import folder_paths import json +from typing import Optional, Literal from fractions import Fraction -from comfy.comfy_types import FileLocator - +from comfy.comfy_types import IO, FileLocator, ComfyNodeABC +from comfy_api.input import ImageInput, AudioInput, VideoInput +from comfy_api.util import VideoContainer, VideoCodec, VideoComponents +from comfy_api.input_impl import VideoFromFile, VideoFromComponents +from comfy.cli_args import args class SaveWEBM: def __init__(self): @@ -75,7 +79,163 @@ class SaveWEBM: return {"ui": {"images": results, "animated": (True,)}} # TODO: frontend side +class SaveVideo(ComfyNodeABC): + def __init__(self): + self.output_dir = folder_paths.get_output_directory() + self.type: Literal["output"] = "output" + self.prefix_append = "" + + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "video": (IO.VIDEO, {"tooltip": "The video to save."}), + "filename_prefix": ("STRING", {"default": "video/ComfyUI", "tooltip": "The prefix for the file to save. This may include formatting information such as %date:yyyy-MM-dd% or %Empty Latent Image.width% to include values from nodes."}), + "format": (VideoContainer.as_input(), {"default": "auto", "tooltip": "The format to save the video as."}), + "codec": (VideoCodec.as_input(), {"default": "auto", "tooltip": "The codec to use for the video."}), + }, + "hidden": { + "prompt": "PROMPT", + "extra_pnginfo": "EXTRA_PNGINFO" + }, + } + + RETURN_TYPES = () + FUNCTION = "save_video" + + OUTPUT_NODE = True + + CATEGORY = "image/video" + DESCRIPTION = "Saves the input images to your ComfyUI output directory." + + def save_video(self, video: VideoInput, filename_prefix, format, codec, prompt=None, extra_pnginfo=None): + filename_prefix += self.prefix_append + width, height = video.get_dimensions() + full_output_folder, filename, counter, subfolder, filename_prefix = folder_paths.get_save_image_path( + filename_prefix, + self.output_dir, + width, + height + ) + results: list[FileLocator] = list() + saved_metadata = None + if not args.disable_metadata: + metadata = {} + if extra_pnginfo is not None: + metadata.update(extra_pnginfo) + if prompt is not None: + metadata["prompt"] = prompt + if len(metadata) > 0: + saved_metadata = metadata + file = f"{filename}_{counter:05}_.{VideoContainer.get_extension(format)}" + video.save_to( + os.path.join(full_output_folder, file), + format=format, + codec=codec, + metadata=saved_metadata + ) + + results.append({ + "filename": file, + "subfolder": subfolder, + "type": self.type + }) + counter += 1 + + return { "ui": { "images": results, "animated": (True,) } } + +class CreateVideo(ComfyNodeABC): + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "images": (IO.IMAGE, {"tooltip": "The images to create a video from."}), + "fps": ("FLOAT", {"default": 30.0, "min": 1.0, "max": 120.0, "step": 1.0}), + }, + "optional": { + "audio": (IO.AUDIO, {"tooltip": "The audio to add to the video."}), + } + } + + RETURN_TYPES = (IO.VIDEO,) + FUNCTION = "create_video" + + CATEGORY = "image/video" + DESCRIPTION = "Create a video from images." + + def create_video(self, images: ImageInput, fps: float, audio: Optional[AudioInput] = None): + return (VideoFromComponents( + VideoComponents( + images=images, + audio=audio, + frame_rate=Fraction(fps), + ) + ),) + +class GetVideoComponents(ComfyNodeABC): + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "video": (IO.VIDEO, {"tooltip": "The video to extract components from."}), + } + } + RETURN_TYPES = (IO.IMAGE, IO.AUDIO, IO.FLOAT) + RETURN_NAMES = ("images", "audio", "fps") + FUNCTION = "get_components" + + CATEGORY = "image/video" + DESCRIPTION = "Extracts all components from a video: frames, audio, and framerate." + + def get_components(self, video: VideoInput): + components = video.get_components() + + return (components.images, components.audio, float(components.frame_rate)) + +class LoadVideo(ComfyNodeABC): + @classmethod + def INPUT_TYPES(cls): + input_dir = folder_paths.get_input_directory() + files = [f for f in os.listdir(input_dir) if os.path.isfile(os.path.join(input_dir, f))] + files = folder_paths.filter_files_content_types(files, ["video"]) + return {"required": + {"file": (sorted(files), {"video_upload": True})}, + } + + CATEGORY = "image/video" + + RETURN_TYPES = (IO.VIDEO,) + FUNCTION = "load_video" + def load_video(self, file): + video_path = folder_paths.get_annotated_filepath(file) + return (VideoFromFile(video_path),) + + @classmethod + def IS_CHANGED(cls, file): + video_path = folder_paths.get_annotated_filepath(file) + mod_time = os.path.getmtime(video_path) + # Instead of hashing the file, we can just use the modification time to avoid + # rehashing large files. + return mod_time + + @classmethod + def VALIDATE_INPUTS(cls, file): + if not folder_paths.exists_annotated_filepath(file): + return "Invalid video file: {}".format(file) + + return True NODE_CLASS_MAPPINGS = { "SaveWEBM": SaveWEBM, + "SaveVideo": SaveVideo, + "CreateVideo": CreateVideo, + "GetVideoComponents": GetVideoComponents, + "LoadVideo": LoadVideo, +} + +NODE_DISPLAY_NAME_MAPPINGS = { + "SaveVideo": "Save Video", + "CreateVideo": "Create Video", + "GetVideoComponents": "Get Video Components", + "LoadVideo": "Load Video", } diff --git a/folder_paths.py b/folder_paths.py index 9a525e5a1..f0b3fd103 100644 --- a/folder_paths.py +++ b/folder_paths.py @@ -4,7 +4,7 @@ import os import time import mimetypes import logging -from typing import Literal +from typing import Literal, List from collections.abc import Collection from comfy.cli_args import args @@ -141,7 +141,7 @@ def get_directory_by_type(type_name: str) -> str | None: return get_input_directory() return None -def filter_files_content_types(files: list[str], content_types: Literal["image", "video", "audio", "model"]) -> list[str]: +def filter_files_content_types(files: list[str], content_types: List[Literal["image", "video", "audio", "model"]]) -> list[str]: """ Example: files = os.listdir(folder_paths.get_input_directory()) From 1c15da7026d599e0d2b118fc86f986421e339d03 Mon Sep 17 00:00:00 2001 From: Jedrzej Kosinski Date: Tue, 29 Apr 2025 01:43:14 -0500 Subject: [PATCH 020/121] Add rest of Luma node functionality (#19) Co-authored-by: Robin Huang --- comfy_api_nodes/apis/luma_api.py | 52 +++++- comfy_api_nodes/nodes_api.py | 270 ++++++++++++++++++++++++------- 2 files changed, 256 insertions(+), 66 deletions(-) diff --git a/comfy_api_nodes/apis/luma_api.py b/comfy_api_nodes/apis/luma_api.py index e7ab03946..1e9350d87 100644 --- a/comfy_api_nodes/apis/luma_api.py +++ b/comfy_api_nodes/apis/luma_api.py @@ -1,11 +1,52 @@ from __future__ import annotations + +import torch + from enum import Enum from typing import Optional, Union from pydantic import BaseModel, Field, confloat + +class LumaIO: + LUMA_REF = "LUMA_REF" + + +class LumaReference: + def __init__(self, image: torch.Tensor, weight: float): + self.image = image + self.weight = weight + + def create_api_model(self, download_url: str): + return LumaImageRef(url=download_url, weight=self.weight) + +class LumaReferenceChain: + def __init__(self, first_ref: LumaReference=None): + self.refs: list[LumaReference] = [] + if first_ref: + self.refs.append(first_ref) + + def add(self, luma_ref: LumaReference=None): + self.refs.append(luma_ref) + + def create_api_model(self, download_urls: list[str], max_refs=4): + if len(self.refs) == 0: + return None + api_refs: list[LumaImageRef] = [] + for ref, url in zip(self.refs, download_urls): + api_ref = LumaImageRef(url=url, weight=ref.weight) + api_refs.append(api_ref) + return api_refs + + def clone(self): + c = LumaReferenceChain() + for ref in self.refs: + c.add(ref) + return c + + class LumaImageModel(str, Enum): photon_1 = "photon-1" photon_flash_1 = "photon-flash-1" @@ -65,7 +106,7 @@ class LumaImageRef(BaseModel): class LumaImageReference(BaseModel): '''Used for video gen''' - type: str = Field('image', description='Input type, defaults to image') + type: Optional[str] = Field('image', description='Input type, defaults to image') url: str = Field(..., description='The URL of the image') @@ -87,14 +128,9 @@ class LumaGenerationReference(BaseModel): id: str = Field(..., description='The ID of the generation') -class LumaKeyframe(BaseModel): - generation: Optional[LumaGenerationReference] = Field(None, description='Reference to generation') - image: Optional[LumaImageReference] = Field(None, description='Reference to image') - - class LumaKeyframes(BaseModel): - frame0: Optional[LumaKeyframe] = Field(None, description='') - frame1: Optional[LumaKeyframe] = Field(None, description='') + frame0: Optional[Union[LumaImageReference, LumaGenerationReference]] = Field(None, description='') + frame1: Optional[Union[LumaImageReference, LumaGenerationReference]] = Field(None, description='') class LumaImageGenerationRequest(BaseModel): diff --git a/comfy_api_nodes/nodes_api.py b/comfy_api_nodes/nodes_api.py index a935465d8..cfd9b3e40 100644 --- a/comfy_api_nodes/nodes_api.py +++ b/comfy_api_nodes/nodes_api.py @@ -4,6 +4,7 @@ from comfy.comfy_types.node_typing import FileLocator from typing import Literal, Optional from comfy.utils import common_upscale from comfy.comfy_types.node_typing import IO, ComfyNodeABC, InputTypeDict +from comfy_api.input_impl.video_types import VideoFromFile from comfy_api_nodes.apis import ( OpenAIImageGenerationRequest, OpenAIImageEditRequest, @@ -33,6 +34,11 @@ from comfy_api_nodes.apis.luma_api import ( LumaCharacterRef, LumaModifyImageRef, LumaImageIdentity, + LumaReference, + LumaReferenceChain, + LumaImageReference, + LumaKeyframes, + LumaIO, ) from comfy_api_nodes.apis.client import ApiClient, ApiEndpoint, HttpMethod, SynchronousOperation, PollingOperation, EmptyRequest, UploadRequest, UploadResponse @@ -949,6 +955,44 @@ class FluxProUltraImageNode(ComfyNodeABC): img.save(img_byte_arr, format='PNG') return base64.b64encode(img_byte_arr.getvalue()).decode() +class LumaReferenceNode: + """ + Holds an image and weight for use with Luma Generate Image node. + """ + RETURN_TYPES = (LumaIO.LUMA_REF,) + RETURN_NAMES = ("luma_ref",) + DESCRIPTION = cleandoc(__doc__ or "") # Handle potential None value + FUNCTION = "create_luma_reference" + CATEGORY = "api node/Luma" + + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "image": (IO.IMAGE, { + "tooltip": "Image to use as reference.", + }), + "weight": (IO.FLOAT, { + "default": 1.0, + "min": 0.0, + "max": 1.0, + "step": 0.01, + "tooltip": "Weight of image reference.", + }), + }, + "optional": { + "luma_ref": (LumaIO.LUMA_REF,) + } + } + + def create_luma_reference(self, image: torch.Tensor, weight: float, luma_ref: LumaReferenceChain=None): + if luma_ref is not None: + luma_ref = luma_ref.clone() + else: + luma_ref = LumaReferenceChain() + luma_ref.add(LumaReference(image=image, weight=round(weight, 2))) + return (luma_ref, ) + class LumaImageGenerationNode: """ Generates images synchronously based on prompt and aspect ratio. @@ -979,10 +1023,23 @@ class LumaImageGenerationNode: "control_after_generate": True, "tooltip": "Seed to determine if node should re-run; actual results are nondeterministic regardless of seed.", }), + "style_image_weight": (IO.FLOAT, { + "default": 1.0, + "min": 0.0, + "max": 1.0, + "step": 0.01, + "tooltip": "Weight of style image. Ignored if no style_image provided.", + }), }, "optional": { - "character_ref_image": (IO.IMAGE, { - "tooltip": "Character reference images; can be a batch of multiple, only the first 4 images will be considered." + "image_luma_ref": (LumaIO.LUMA_REF, { + "tooltip": "Luma Reference node connection to influence generation with input images; up to 4 images can be considered." + }), + "style_image": (IO.IMAGE, { + "tooltip": "Style reference image; only 1 image will be used." + }), + "character_image": (IO.IMAGE, { + "tooltip": "Character reference images; can be a batch of multiple, up to 4 images can be considered." }) }, "hidden": { @@ -990,11 +1047,21 @@ class LumaImageGenerationNode: } } - def api_call(self, prompt: str, model: str, aspect_ratio: str, seed, character_ref_image: torch.Tensor=None, auth_token=None, **kwargs): + def api_call(self, prompt: str, model: str, aspect_ratio: str, seed, style_image_weight: float, + image_luma_ref: LumaReferenceChain=None, style_image: torch.Tensor=None, character_image: torch.Tensor=None, + auth_token=None, **kwargs): + # handle image_luma_ref + api_image_ref = None + if image_luma_ref is not None: + api_image_ref = self._convert_luma_refs(image_luma_ref, auth_token=auth_token) + # handle style_luma_ref + api_style_ref = None + if style_image is not None: + api_style_ref = self._convert_style_image(style_image, weight=style_image_weight, auth_token=auth_token) # handle character_ref images character_ref = None - if character_ref_image is not None: - download_urls = upload_images_to_comfyapi(character_ref_image, max_images=4, auth_token=auth_token) + if character_image is not None: + download_urls = upload_images_to_comfyapi(character_image, max_images=4, auth_token=auth_token) character_ref = LumaCharacterRef(identity0=LumaImageIdentity(images=download_urls)) operation = SynchronousOperation( @@ -1008,7 +1075,9 @@ class LumaImageGenerationNode: prompt=prompt, model=model, aspect_ratio=aspect_ratio, - character_ref=character_ref + image_ref=api_image_ref, + style_ref=api_style_ref, + character_ref=character_ref, ), auth_token=auth_token ) @@ -1032,6 +1101,21 @@ class LumaImageGenerationNode: img = process_image_response(img_response) return (img,) + def _convert_luma_refs(self, luma_ref: LumaReferenceChain, max_refs: int, auth_token=None): + luma_urls = [] + ref_count = 0 + for ref in luma_ref.refs: + download_urls = upload_images_to_comfyapi(ref.image, max_images=1, auth_token=auth_token) + luma_urls.append(download_urls[0]) + ref_count += 1 + if ref_count >= max_refs: + break + return luma_ref.create_api_model(download_urls=luma_urls, max_refs=max_refs) + + def _convert_style_image(self, style_image: torch.Tensor, weight: float, auth_token=None): + chain = LumaReferenceChain(first_ref=LumaReference(image=style_image, weight=weight)) + return self._convert_luma_refs(chain, max_refs=1, auth_token=auth_token) + class LumaImageModifyNode: """ Modifies images synchronously based on prompt and aspect ratio. @@ -1117,7 +1201,7 @@ class LumaImageModifyNode: img = process_image_response(img_response) return (img,) -class LumaVideoGenerationNode: +class LumaTextToVideoGenerationNode: """ Generates videos synchronously based on prompt and output_size. """ @@ -1125,7 +1209,7 @@ class LumaVideoGenerationNode: self.output_dir = folder_paths.get_output_directory() self.type: Literal["output"] = "output" - RETURN_TYPES = ("IMAGE",) + RETURN_TYPES = (IO.VIDEO,) DESCRIPTION = cleandoc(__doc__ or "") # Handle potential None value FUNCTION = "api_call" API_NODE = True @@ -1148,6 +1232,9 @@ class LumaVideoGenerationNode: "default": LumaVideoOutputResolution.res_540p, }), "duration": ([dur.value for dur in LumaVideoModelOutputDuration],), + "loop": (IO.BOOLEAN, { + "default": False, + }), "seed": (IO.INT, { "default": 0, "min": 0, @@ -1155,7 +1242,6 @@ class LumaVideoGenerationNode: "control_after_generate": True, "tooltip": "Seed to determine if node should re-run; actual results are nondeterministic regardless of seed.", }), - "filename_prefix": ("STRING", {"default": "ComfyUI"}), }, "optional": { }, @@ -1164,9 +1250,8 @@ class LumaVideoGenerationNode: } } - def api_call(self, prompt: str, model: str, aspect_ratio: str, resolution: str, duration: str, seed, filename_prefix: str, + def api_call(self, prompt: str, model: str, aspect_ratio: str, resolution: str, duration: str, loop: bool, seed, auth_token=None, **kwargs): - extra_pnginfo = None operation = SynchronousOperation( endpoint=ApiEndpoint( path="/proxy/luma/generations", @@ -1180,6 +1265,7 @@ class LumaVideoGenerationNode: resolution=resolution, aspect_ratio=aspect_ratio, duration=duration, + loop=loop, ), auth_token=auth_token ) @@ -1200,55 +1286,119 @@ class LumaVideoGenerationNode: response_poll = operation.execute() vid_response = requests.get(response_poll.assets.video) - self._save_video_locally(vid_response, filename_prefix, extra_pnginfo) + return (VideoFromFile(BytesIO(vid_response.content)), ) - return (None,) - #return {"ui": {"images": results, "animated": (True,)}} +class LumaImageToVideoGenerationNode: + """ + Generates videos synchronously based on prompt, input images, and output_size. + """ + def __init__(self): + self.output_dir = folder_paths.get_output_directory() + self.type: Literal["output"] = "output" - def _save_video_locally(self, response: requests.Response, filename_prefix: str, extra_pnginfo): - # Construct the save path - full_output_folder, filename, counter, subfolder, filename_prefix = ( - folder_paths.get_save_image_path(filename_prefix, self.output_dir) - ) - file_basename = f"{filename}_{counter:05}_.mp4" - save_path = os.path.join(full_output_folder, file_basename) + RETURN_TYPES = (IO.VIDEO,) + DESCRIPTION = cleandoc(__doc__ or "") # Handle potential None value + FUNCTION = "api_call" + API_NODE = True + CATEGORY = "api node" - video_data = response.content - - # Save the video data to a file - with open(save_path, "wb") as video_file: - video_file.write(video_data) - - # Add workflow metadata to the video container - #if prompt is not None or extra_pnginfo is not None: - if extra_pnginfo is not None: - try: - container = av.open(save_path, mode="r+") - # if prompt is not None: - # container.metadata["prompt"] = json.dumps(prompt) - if extra_pnginfo is not None: - for x in extra_pnginfo: - container.metadata[x] = json.dumps(extra_pnginfo[x]) - container.close() - except Exception as e: - logging.warning(f"Failed to add metadata to video: {e}") - - # Create a FileLocator for the frontend to use for the preview - results: list[FileLocator] = [ - { - "filename": file_basename, - "subfolder": subfolder, - "type": self.type, + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "prompt": (IO.STRING, { + "multiline": True, + "default": "", + "tooltip": "Prompt for the video generation", + }), + "model": ([model.value for model in LumaVideoModel],), + # "aspect_ratio": ([ratio.value for ratio in LumaAspectRatio], { + # "default": LumaAspectRatio.ratio_16_9, + # }), + "resolution": ([resolution.value for resolution in LumaVideoOutputResolution], { + "default": LumaVideoOutputResolution.res_540p, + }), + "duration": ([dur.value for dur in LumaVideoModelOutputDuration],), + "loop": (IO.BOOLEAN, { + "default": False, + }), + "seed": (IO.INT, { + "default": 0, + "min": 0, + "max": 0xFFFFFFFFFFFFFFFF, + "control_after_generate": True, + "tooltip": "Seed to determine if node should re-run; actual results are nondeterministic regardless of seed.", + }), + }, + "optional": { + "first_image": (IO.IMAGE, { + "tooltip": "First frame of generated video." + }), + "last_image": (IO.IMAGE, { + "tooltip": "Last frame of generated video." + }), + }, + "hidden": { + "auth_token": "AUTH_TOKEN_COMFY_ORG", } - ] + } - return results + def api_call(self, prompt: str, model: str, resolution: str, duration: str, loop: bool, seed, + first_image: torch.Tensor=None, last_image: torch.Tensor=None, + auth_token=None, **kwargs): + if first_image is None and last_image is None: + raise Exception("At least one of first_image and last_image requires an input.") + keyframes = self._convert_to_keyframes(first_image, last_image, auth_token) - def _get_output_type(self, output_size: str): - if output_size in [resolution.value for resolution in LumaVideoOutputResolution]: - return LumaVideoOutputResolution - else: - return LumaAspectRatio + operation = SynchronousOperation( + endpoint=ApiEndpoint( + path="/proxy/luma/generations", + method=HttpMethod.POST, + request_model=LumaGenerationRequest, + response_model=LumaGeneration + ), + request=LumaGenerationRequest( + prompt=prompt, + model=model, + aspect_ratio=LumaAspectRatio.ratio_16_9, + resolution=resolution, + duration=duration, + loop=loop, + keyframes=keyframes + ), + auth_token=auth_token + ) + response_api: LumaGeneration = operation.execute() + + operation = PollingOperation( + poll_endpoint=ApiEndpoint( + path=f"/proxy/luma/generations/{response_api.id}", + method=HttpMethod.GET, + request_model=EmptyRequest, + response_model=LumaGeneration, + ), + completed_statuses=[LumaState.completed], + failed_statuses=[LumaState.failed], + status_extractor=lambda x: x.state, + auth_token=auth_token, + ) + response_poll = operation.execute() + + vid_response = requests.get(response_poll.assets.video) + return (VideoFromFile(BytesIO(vid_response.content)), ) + + def _convert_to_keyframes(self, first_image: torch.Tensor=None, last_image: torch.Tensor=None, auth_token=None): + if first_image is None and last_image is None: + return None + frame0 = None + frame1 = None + if first_image is not None: + download_urls = upload_images_to_comfyapi(first_image, max_images=1, auth_token=auth_token) + frame0 = LumaImageReference(type='image', url=download_urls[0]) + if last_image is not None: + download_urls = upload_images_to_comfyapi(last_image, max_images=1, auth_token=auth_token) + frame1 = LumaImageReference(type='image', url=download_urls[0]) + return LumaKeyframes(frame0=frame0, frame1=frame1) class MinimaxTextToVideoNode: """ @@ -1429,7 +1579,9 @@ NODE_CLASS_MAPPINGS = { "FluxProUltraImageNode": FluxProUltraImageNode, "LumaImageNode": LumaImageGenerationNode, "LumaImageModifyNode": LumaImageModifyNode, - "LumaVideoNode": LumaVideoGenerationNode, + "LumaReferenceNode": LumaReferenceNode, + "LumaVideoNode": LumaTextToVideoGenerationNode, + "LumaImageToVideoNode": LumaImageToVideoGenerationNode, "MinimaxTextToVideoNode": MinimaxTextToVideoNode, } @@ -1440,8 +1592,10 @@ NODE_DISPLAY_NAME_MAPPINGS = { "OpenAIGPTImage1": "OpenAI GPT Image 1", "IdeogramTextToImage": "Ideogram Text to Image", "FluxProUltraImageNode": "Flux 1.1 [pro] Ultra Image", - "LumaImageNode": "Luma Generate Image", - "LumaImageModifyNode": "Luma Modify Image", - "LumaVideoNode": "Luma Generate Video", + "LumaImageNode": "Luma Text to Image", + "LumaImageModifyNode": "Luma Image to Image", + "LumaReferenceNode": "Luma Reference", + "LumaVideoNode": "Luma Text to Video", + "LumaImageToVideoNode": "Luma Image to Video", "MinimaxTextToVideoNode": "Minimax Text to Video", } From 577423b65f5444cf235a77483e76fdf56fea1a23 Mon Sep 17 00:00:00 2001 From: Jedrzej Kosinski Date: Tue, 29 Apr 2025 02:18:13 -0500 Subject: [PATCH 021/121] Fix image_luma_ref not working (#28) Co-authored-by: Robin Huang --- comfy_api_nodes/nodes_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/comfy_api_nodes/nodes_api.py b/comfy_api_nodes/nodes_api.py index cfd9b3e40..45654e61b 100644 --- a/comfy_api_nodes/nodes_api.py +++ b/comfy_api_nodes/nodes_api.py @@ -1053,7 +1053,7 @@ class LumaImageGenerationNode: # handle image_luma_ref api_image_ref = None if image_luma_ref is not None: - api_image_ref = self._convert_luma_refs(image_luma_ref, auth_token=auth_token) + api_image_ref = self._convert_luma_refs(image_luma_ref, max_refs=4, auth_token=auth_token) # handle style_luma_ref api_style_ref = None if style_image is not None: From 9c7e9d883697423fbcd6611a3cf3957086684eee Mon Sep 17 00:00:00 2001 From: Chenlei Hu Date: Tue, 29 Apr 2025 14:31:02 -0400 Subject: [PATCH 022/121] [Bug] Remove duplicated option T2V-01 in MinimaxTextToVideoNode (#31) --- comfy_api_nodes/nodes_api.py | 1 - 1 file changed, 1 deletion(-) diff --git a/comfy_api_nodes/nodes_api.py b/comfy_api_nodes/nodes_api.py index 45654e61b..f6a628328 100644 --- a/comfy_api_nodes/nodes_api.py +++ b/comfy_api_nodes/nodes_api.py @@ -1429,7 +1429,6 @@ class MinimaxTextToVideoNode: "S2V-01", "I2V-01", "I2V-01-live", - "T2V-01", ], { "default": "T2V-01", From acbf26c7d1a37ff1a93ea08a15e5326ebad63808 Mon Sep 17 00:00:00 2001 From: Christian Byrne Date: Tue, 29 Apr 2025 11:53:10 -0700 Subject: [PATCH 023/121] Add utils to map from pydantic model fields to comfy node inputs (#30) --- comfy_api_nodes/README.md | 3 +- comfy_api_nodes/mapper_utils.py | 117 +++++++ .../comfy_api_nodes_test/mapper_utils_test.py | 297 ++++++++++++++++++ 3 files changed, 415 insertions(+), 2 deletions(-) create mode 100644 comfy_api_nodes/mapper_utils.py create mode 100644 tests-unit/comfy_api_nodes_test/mapper_utils_test.py diff --git a/comfy_api_nodes/README.md b/comfy_api_nodes/README.md index 86ba379f7..0ebecae63 100644 --- a/comfy_api_nodes/README.md +++ b/comfy_api_nodes/README.md @@ -36,7 +36,6 @@ npm install -g @redocly/cli redocly bundle openapi.yaml --output filtered-openapi.yaml --config comfy_api_nodes/redocly-dev.yaml --remove-unused-components # Generate the pydantic datamodels for validation. -datamodel-codegen --use-subclass-enum --input filtered-openapi.yaml --output comfy_api_nodes/apis/__init__.py --output-model-type pydantic_v2.BaseModel +datamodel-codegen --use-subclass-enum --field-constraints --input filtered-openapi.yaml --output comfy_api_nodes/apis/__init__.py --output-model-type pydantic_v2.BaseModel ``` - diff --git a/comfy_api_nodes/mapper_utils.py b/comfy_api_nodes/mapper_utils.py new file mode 100644 index 000000000..f8fd5632e --- /dev/null +++ b/comfy_api_nodes/mapper_utils.py @@ -0,0 +1,117 @@ +from enum import Enum + +from pydantic.fields import FieldInfo +from pydantic import BaseModel +from pydantic_core import PydanticUndefined + +from comfy.comfy_types.node_typing import IO, InputTypeOptions + +NodeInput = tuple[IO, InputTypeOptions] + + +def _create_base_config(field_info: FieldInfo) -> InputTypeOptions: + config = {} + if hasattr(field_info, "default") and field_info.default is not PydanticUndefined: + config["default"] = field_info.default + if hasattr(field_info, "description") and field_info.description is not None: + config["tooltip"] = field_info.description + return config + + +def _get_number_constraints_config(field_info: FieldInfo) -> dict: + config = {} + if hasattr(field_info, "metadata"): + metadata = field_info.metadata + for constraint in metadata: + if hasattr(constraint, "ge"): + config["min"] = constraint.ge + if hasattr(constraint, "le"): + config["max"] = constraint.le + if hasattr(constraint, "multiple_of"): + config["step"] = constraint.multiple_of + return config + + +def _model_field_to_image_input(field_info: FieldInfo, **kwargs) -> NodeInput: + return IO.IMAGE, { + **_create_base_config(field_info), + **kwargs, + } + + +def _model_field_to_string_input(field_info: FieldInfo, **kwargs) -> NodeInput: + return IO.STRING, { + **_create_base_config(field_info), + **kwargs, + } + + +def _model_field_to_float_input(field_info: FieldInfo, **kwargs) -> NodeInput: + return IO.FLOAT, { + **_create_base_config(field_info), + **_get_number_constraints_config(field_info), + **kwargs, + } + + +def _model_field_to_int_input(field_info: FieldInfo, **kwargs) -> NodeInput: + return IO.INT, { + **_create_base_config(field_info), + **_get_number_constraints_config(field_info), + **kwargs, + } + + +def _model_field_to_combo_input( + field_info: FieldInfo, enum_type: type[Enum] = None, **kwargs +) -> NodeInput: + combo_config = {} + if enum_type is not None: + combo_config["options"] = [option.value for option in enum_type] + combo_config = { + **combo_config, + **_create_base_config(field_info), + **kwargs, + } + return IO.COMBO, combo_config + + +def model_field_to_node_input( + input_type: IO, base_model: type[BaseModel], field_name: str, **kwargs +) -> NodeInput: + """ + Maps a field from a Pydantic model to a Comfy node input. + + Args: + input_type: The type of the input. + base_model: The Pydantic model to map the field from. + field_name: The name of the field to map. + **kwargs: Additional key/values to include in the input options. + + Note: + For combo inputs, pass an `Enum` to the `enum_type` keyword argument to populate the options automatically. + + Example: + >>> model_field_to_node_input(IO.STRING, MyModel, "my_field", multiline=True) + >>> model_field_to_node_input(IO.COMBO, MyModel, "my_field", enum_type=MyEnum) + >>> model_field_to_node_input(IO.FLOAT, MyModel, "my_field", slider=True) + """ + field_info: FieldInfo = base_model.model_fields[field_name] + result: NodeInput + + match input_type: + case IO.IMAGE: + result = _model_field_to_image_input(field_info, **kwargs) + case IO.STRING: + result = _model_field_to_string_input(field_info, **kwargs) + case IO.FLOAT: + result = _model_field_to_float_input(field_info, **kwargs) + case IO.INT: + result = _model_field_to_int_input(field_info, **kwargs) + case IO.COMBO: + result = _model_field_to_combo_input(field_info, **kwargs) + case _: + message = f"Invalid input type: {input_type}" + raise ValueError(message) + + return result diff --git a/tests-unit/comfy_api_nodes_test/mapper_utils_test.py b/tests-unit/comfy_api_nodes_test/mapper_utils_test.py new file mode 100644 index 000000000..69488f691 --- /dev/null +++ b/tests-unit/comfy_api_nodes_test/mapper_utils_test.py @@ -0,0 +1,297 @@ +from typing import Optional +from enum import Enum + +from pydantic import BaseModel, Field + +from comfy.comfy_types.node_typing import IO +from comfy_api_nodes.mapper_utils import model_field_to_node_input + + +def test_model_field_to_float_input(): + """Tests mapping a float field with constraints.""" + + class ModelWithFloatField(BaseModel): + cfg_scale: Optional[float] = Field( + default=0.5, + description="Flexibility in video generation", + ge=0.0, + le=1.0, + multiple_of=0.001, + ) + + expected_output = ( + IO.FLOAT, + { + "default": 0.5, + "tooltip": "Flexibility in video generation", + "min": 0.0, + "max": 1.0, + "step": 0.001, + }, + ) + + actual_output = model_field_to_node_input( + IO.FLOAT, ModelWithFloatField, "cfg_scale" + ) + + assert actual_output[0] == expected_output[0] + assert actual_output[1] == expected_output[1] + + +def test_model_field_to_float_input_no_constraints(): + """Tests mapping a float field with no constraints.""" + + class ModelWithFloatField(BaseModel): + cfg_scale: Optional[float] = Field(default=0.5) + + expected_output = ( + IO.FLOAT, + { + "default": 0.5, + }, + ) + + actual_output = model_field_to_node_input( + IO.FLOAT, ModelWithFloatField, "cfg_scale" + ) + + assert actual_output[0] == expected_output[0] + assert actual_output[1] == expected_output[1] + + +def test_model_field_to_int_input(): + """Tests mapping an int field with constraints.""" + + class ModelWithIntField(BaseModel): + num_frames: Optional[int] = Field( + default=10, + description="Number of frames to generate", + ge=1, + le=100, + multiple_of=1, + ) + + expected_output = ( + IO.INT, + { + "default": 10, + "tooltip": "Number of frames to generate", + "min": 1, + "max": 100, + "step": 1, + }, + ) + + actual_output = model_field_to_node_input(IO.INT, ModelWithIntField, "num_frames") + + assert actual_output[0] == expected_output[0] + assert actual_output[1] == expected_output[1] + + +def test_model_field_to_string_input(): + """Tests mapping a string field.""" + + class ModelWithStringField(BaseModel): + prompt: Optional[str] = Field( + default="A beautiful sunset over a calm ocean", + description="A prompt for the video generation", + ) + + expected_output = ( + IO.STRING, + { + "default": "A beautiful sunset over a calm ocean", + "tooltip": "A prompt for the video generation", + }, + ) + + actual_output = model_field_to_node_input(IO.STRING, ModelWithStringField, "prompt") + + assert actual_output[0] == expected_output[0] + assert actual_output[1] == expected_output[1] + + +def test_model_field_to_string_input_multiline(): + """Tests mapping a string field.""" + + class ModelWithStringField(BaseModel): + prompt: Optional[str] = Field( + default="A beautiful sunset over a calm ocean", + description="A prompt for the video generation", + ) + + expected_output = ( + IO.STRING, + { + "default": "A beautiful sunset over a calm ocean", + "tooltip": "A prompt for the video generation", + "multiline": True, + }, + ) + + actual_output = model_field_to_node_input( + IO.STRING, ModelWithStringField, "prompt", multiline=True + ) + + assert actual_output[0] == expected_output[0] + assert actual_output[1] == expected_output[1] + + +def test_model_field_to_combo_input(): + """Tests mapping a combo field.""" + + class MockEnum(str, Enum): + option_1 = "option 1" + option_2 = "option 2" + option_3 = "option 3" + + class ModelWithComboField(BaseModel): + model_name: Optional[MockEnum] = Field("option 1", description="Model Name") + + expected_output = ( + IO.COMBO, + { + "options": ["option 1", "option 2", "option 3"], + "default": "option 1", + "tooltip": "Model Name", + }, + ) + + actual_output = model_field_to_node_input( + IO.COMBO, ModelWithComboField, "model_name", enum_type=MockEnum + ) + + assert actual_output[0] == expected_output[0] + assert actual_output[1] == expected_output[1] + + +def test_model_field_to_combo_input_no_options(): + """Tests mapping a combo field with no options.""" + + class ModelWithComboField(BaseModel): + model_name: Optional[str] = Field(description="Model Name") + + expected_output = ( + IO.COMBO, + { + "tooltip": "Model Name", + }, + ) + + actual_output = model_field_to_node_input( + IO.COMBO, ModelWithComboField, "model_name" + ) + + assert actual_output[0] == expected_output[0] + assert actual_output[1] == expected_output[1] + + +def test_model_field_to_image_input(): + """Tests mapping an image field.""" + + class ModelWithImageField(BaseModel): + image: Optional[str] = Field( + default=None, + description="An image for the video generation", + ) + + expected_output = ( + IO.IMAGE, + { + "default": None, + "tooltip": "An image for the video generation", + }, + ) + + actual_output = model_field_to_node_input(IO.IMAGE, ModelWithImageField, "image") + + assert actual_output[0] == expected_output[0] + assert actual_output[1] == expected_output[1] + + +def test_model_field_to_node_input_no_description(): + """Tests mapping a field with no description.""" + + class ModelWithNoDescriptionField(BaseModel): + field: Optional[str] = Field(default="default value") + + expected_output = ( + IO.STRING, + { + "default": "default value", + }, + ) + + actual_output = model_field_to_node_input( + IO.STRING, ModelWithNoDescriptionField, "field" + ) + + assert actual_output[0] == expected_output[0] + assert actual_output[1] == expected_output[1] + + +def test_model_field_to_node_input_no_default(): + """Tests mapping a field with no default.""" + + class ModelWithNoDefaultField(BaseModel): + field: Optional[str] = Field(description="A field with no default") + + expected_output = ( + IO.STRING, + { + "tooltip": "A field with no default", + }, + ) + + actual_output = model_field_to_node_input( + IO.STRING, ModelWithNoDefaultField, "field" + ) + + assert actual_output[0] == expected_output[0] + assert actual_output[1] == expected_output[1] + + +def test_model_field_to_node_input_no_metadata(): + """Tests mapping a field with no metadata or properties defined on the schema.""" + + class ModelWithNoMetadataField(BaseModel): + field: Optional[str] = Field() + + expected_output = ( + IO.STRING, + {}, + ) + + actual_output = model_field_to_node_input( + IO.STRING, ModelWithNoMetadataField, "field" + ) + + assert actual_output[0] == expected_output[0] + assert actual_output[1] == expected_output[1] + + +def test_model_field_to_node_input_default_is_none(): + """ + Tests mapping a field with a default of `None`. + I.e., the default field should be included as the schema explicitly sets it to `None`. + """ + + class ModelWithNoneDefaultField(BaseModel): + field: Optional[str] = Field( + default=None, description="A field with a default of None" + ) + + expected_output = ( + IO.STRING, + { + "default": None, + "tooltip": "A field with a default of None", + }, + ) + + actual_output = model_field_to_node_input( + IO.STRING, ModelWithNoneDefaultField, "field" + ) + + assert actual_output[0] == expected_output[0] + assert actual_output[1] == expected_output[1] From 850f5daff6cd7258d765540fe059f1d526b5fc1d Mon Sep 17 00:00:00 2001 From: thot experiment <94414189+thot-experiment@users.noreply.github.com> Date: Tue, 29 Apr 2025 12:25:41 -0700 Subject: [PATCH 024/121] add veo2, bump av req (#32) --- comfy_api_nodes/apis/PixverseController.py | 4 +- comfy_api_nodes/apis/PixverseDto.py | 4 +- comfy_api_nodes/apis/__init__.py | 1732 +++++++++++++++++++- comfy_api_nodes/nodes_veo2.py | 274 ++++ nodes.py | 6 +- requirements.txt | 2 +- 6 files changed, 1941 insertions(+), 81 deletions(-) create mode 100644 comfy_api_nodes/nodes_veo2.py diff --git a/comfy_api_nodes/apis/PixverseController.py b/comfy_api_nodes/apis/PixverseController.py index 949fce171..6f87c3d2c 100644 --- a/comfy_api_nodes/apis/PixverseController.py +++ b/comfy_api_nodes/apis/PixverseController.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: -# filename: filtered-openapi.yaml -# timestamp: 2025-04-24T22:16:48+00:00 +# filename: https://stagingapi.comfy.org/openapi +# timestamp: 2025-04-29T03:13:19+00:00 from __future__ import annotations diff --git a/comfy_api_nodes/apis/PixverseDto.py b/comfy_api_nodes/apis/PixverseDto.py index 3a2c95fbc..0b08fa2aa 100644 --- a/comfy_api_nodes/apis/PixverseDto.py +++ b/comfy_api_nodes/apis/PixverseDto.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: -# filename: filtered-openapi.yaml -# timestamp: 2025-04-24T22:16:48+00:00 +# filename: https://stagingapi.comfy.org/openapi +# timestamp: 2025-04-29T03:13:19+00:00 from __future__ import annotations diff --git a/comfy_api_nodes/apis/__init__.py b/comfy_api_nodes/apis/__init__.py index 4ca4d5ab9..562a7addb 100644 --- a/comfy_api_nodes/apis/__init__.py +++ b/comfy_api_nodes/apis/__init__.py @@ -1,39 +1,37 @@ # generated by datamodel-codegen: -# filename: filtered-openapi.yaml -# timestamp: 2025-04-25T03:57:04+00:00 +# filename: https://stagingapi.comfy.org/openapi +# timestamp: 2025-04-29T03:13:19+00:00 from __future__ import annotations from datetime import datetime from enum import Enum -from typing import Any, Dict, List, Optional +from typing import Any, Dict, List, Literal, Optional, Union +from uuid import UUID -from pydantic import BaseModel, Field, confloat, conint, constr - - -class OutputFormat(str, Enum): - png = 'png' - webp = 'webp' - jpeg = 'jpeg' +from pydantic import AnyUrl, BaseModel, Field, RootModel, confloat, conint, constr class BFLFluxProGenerateRequest(BaseModel): + guidance_scale: Optional[confloat(ge=1.0, le=20.0)] = Field( + None, description='The guidance scale for generation.' + ) + height: conint(ge=64, le=2048) = Field( + ..., description='The height of the image to generate.' + ) + negative_prompt: Optional[str] = Field( + None, description='The negative prompt for image generation.' + ) + num_images: Optional[conint(ge=1, le=4)] = Field( + None, description='The number of images to generate.' + ) + num_inference_steps: Optional[conint(ge=1, le=100)] = Field( + None, description='The number of inference steps.' + ) prompt: str = Field(..., description='The text prompt for image generation.') - prompt_upsampling: Optional[bool] = Field( - None, description='Whether to perform upsampling on the prompt. If active, automatically modifies the prompt for more creative generation.' - ) seed: Optional[int] = Field(None, description='The seed value for reproducibility.') - aspect_ratio: Optional[str] = Field(None, description='Aspect ratio of the image between 21:9 and 9:21.') - safety_tolerance: Optional[conint(ge=0, le=6)] = Field( - 6, description='Tolerance level for input and output moderation. Between 0 and 6, 0 being most strict, 6 being least strict. Defaults to 2.' - ) - output_format: Optional[OutputFormat] = Field( - OutputFormat.png, description="Output format for the generated image. Can be 'jpeg' or 'png'.", examples=['png'] - ) - raw: Optional[bool] = Field(None, description='Generate less processed, more natural-looking images.') - image_prompt: Optional[str] = Field(None, description='Optional image to remix in base64 format') - image_prompt_strength: Optional[confloat(ge=0.0, le=1.0)] = Field( - None, description='Blend between the prompt and the image prompt.' + width: conint(ge=64, le=2048) = Field( + ..., description='The width of the image to generate.' ) @@ -42,11 +40,106 @@ class BFLFluxProGenerateResponse(BaseModel): polling_url: str = Field(..., description='URL to poll for the generation result.') +class ComfyNode(BaseModel): + category: Optional[str] = Field( + None, + description='UI category where the node is listed, used for grouping nodes.', + ) + comfy_node_name: Optional[str] = Field( + None, description='Unique identifier for the node' + ) + deprecated: Optional[bool] = Field( + None, + description='Indicates if the node is deprecated. Deprecated nodes are hidden in the UI.', + ) + description: Optional[str] = Field( + None, description="Brief description of the node's functionality or purpose." + ) + experimental: Optional[bool] = Field( + None, + description='Indicates if the node is experimental, subject to changes or removal.', + ) + function: Optional[str] = Field( + None, description='Name of the entry-point function to execute the node.' + ) + input_types: Optional[str] = Field(None, description='Defines input parameters') + output_is_list: Optional[List[bool]] = Field( + None, description='Boolean values indicating if each output is a list.' + ) + return_names: Optional[str] = Field( + None, description='Names of the outputs for clarity in workflows.' + ) + return_types: Optional[str] = Field( + None, description='Specifies the types of outputs produced by the node.' + ) + + +class ComfyNodeCloudBuildInfo(BaseModel): + build_id: Optional[str] = None + location: Optional[str] = None + project_id: Optional[str] = None + project_number: Optional[str] = None + + +class Customer(BaseModel): + createdAt: Optional[datetime] = Field( + None, description='The date and time the user was created' + ) + email: Optional[str] = Field(None, description='The email address for this user') + id: str = Field(..., description='The firebase UID of the user') + name: Optional[str] = Field(None, description='The name for this user') + updatedAt: Optional[datetime] = Field( + None, description='The date and time the user was last updated' + ) + + +class CustomerStorageResourceResponse(BaseModel): + download_url: Optional[str] = Field( + None, + description='The signed URL to use for downloading the file from the specified path', + ) + existing_file: Optional[bool] = Field( + None, description='Whether an existing file with the same hash was found' + ) + expires_at: Optional[datetime] = Field( + None, description='When the signed URL will expire' + ) + upload_url: Optional[str] = Field( + None, + description='The signed URL to use for uploading the file to the specified path', + ) + + +class Error(BaseModel): + details: Optional[List[str]] = Field( + None, + description='Optional detailed information about the error or hints for resolving it.', + ) + message: Optional[str] = Field( + None, description='A clear and concise description of the error.' + ) + + class ErrorResponse(BaseModel): error: str message: str +class GitCommitSummary(BaseModel): + author: Optional[str] = Field(None, description='The author of the commit') + branch_name: Optional[str] = Field( + None, description='The branch where the commit was made' + ) + commit_hash: Optional[str] = Field(None, description='The hash of the commit') + commit_name: Optional[str] = Field(None, description='The name of the commit') + status_summary: Optional[Dict[str, str]] = Field( + None, description='A map of operating system to status pairs' + ) + timestamp: Optional[datetime] = Field( + None, description='The timestamp when the commit was made' + ) + + class ImageRequest(BaseModel): aspect_ratio: Optional[str] = Field( None, @@ -117,24 +210,47 @@ class IdeogramGenerateResponse(BaseModel): ) -class Code(Enum): - int_1100 = 1100 - int_1101 = 1101 - int_1102 = 1102 - int_1103 = 1103 +class Duration(str, Enum): + field_5 = '5' + field_10 = '10' -class Code1(Enum): - int_1000 = 1000 - int_1001 = 1001 - int_1002 = 1002 - int_1003 = 1003 - int_1004 = 1004 +class Mode(str, Enum): + std = 'std' + pro = 'pro' + + +class ModelName(str, Enum): + kling_v1 = 'kling-v1' + kling_v1_5 = 'kling-v1-5' + kling_v1_6 = 'kling-v1-6' + + +class KlingDualCharacterEffectInput(BaseModel): + duration: Duration = Field( + ..., + description='Video Length in seconds. Both 5 and 10-second videos are supported.', + ) + images: List[str] = Field( + ..., + description='Reference Image Group. Must contain exactly 2 images. First image will be positioned on left side, second on right side of the composite. Each image follows the same requirements as single image effects.', + max_length=2, + min_length=2, + ) + mode: Optional[Mode] = Field( + 'std', + description='Video generation mode. std (Standard Mode) is cost-effective, pro (Professional Mode) generates videos with longer duration and higher quality.', + ) + model_name: Optional[ModelName] = Field( + 'kling-v1', + description='Model Name. Can be kling-v1, kling-v1-5, or kling-v1-6.', + ) class KlingErrorResponse(BaseModel): code: int = Field( - ..., description='Error code value as defined in the API documentation' + ..., + description='- 1000: Authentication failed\n- 1001: Authorization is empty\n- 1002: Authorization is invalid\n- 1003: Authorization is not yet valid\n- 1004: Authorization has expired\n- 1100: Account exception\n- 1101: Account in arrears (postpaid scenario)\n- 1102: Resource pack depleted or expired (prepaid scenario)\n- 1103: Unauthorized access to requested resource\n- 1200: Invalid request parameters\n- 1201: Invalid parameters\n- 1202: Invalid request method\n- 1203: Requested resource does not exist\n- 1300: Trigger platform strategy\n- 1301: Trigger content security policy\n- 1302: API request too frequent\n- 1303: Concurrency/QPS exceeds limit\n- 1304: Trigger IP whitelist policy\n- 5000: Internal server error\n- 5001: Service temporarily unavailable\n- 5002: Server internal timeout\n', ) message: str = Field(..., description='Human-readable error message') request_id: str = Field( @@ -142,45 +258,779 @@ class KlingErrorResponse(BaseModel): ) -class Code2(Enum): - int_1200 = 1200 - int_1201 = 1201 - int_1202 = 1202 - int_1203 = 1203 +class AspectRatio(str, Enum): + field_16_9 = '16:9' + field_9_16 = '9:16' + field_1_1 = '1:1' -class KlingRequestError(KlingErrorResponse): - code: Optional[Code2] = Field( +class Config(BaseModel): + horizontal: Optional[confloat(ge=-10.0, le=10.0)] = Field( None, - description='- 1200: Invalid request parameters\n- 1201: Invalid parameters\n- 1202: Invalid request method\n- 1203: Requested resource does not exist\n', + description="Controls camera's movement along horizontal axis (x-axis). Negative indicates left, positive indicates right.", + ) + pan: Optional[confloat(ge=-10.0, le=10.0)] = Field( + None, + description="Controls camera's rotation in vertical plane (x-axis). Negative indicates downward rotation, positive indicates upward rotation.", + ) + roll: Optional[confloat(ge=-10.0, le=10.0)] = Field( + None, + description="Controls camera's rolling amount (z-axis). Negative indicates counterclockwise, positive indicates clockwise.", + ) + tilt: Optional[confloat(ge=-10.0, le=10.0)] = Field( + None, + description="Controls camera's rotation in horizontal plane (y-axis). Negative indicates left rotation, positive indicates right rotation.", + ) + vertical: Optional[confloat(ge=-10.0, le=10.0)] = Field( + None, + description="Controls camera's movement along vertical axis (y-axis). Negative indicates downward, positive indicates upward.", + ) + zoom: Optional[confloat(ge=-10.0, le=10.0)] = Field( + None, + description="Controls change in camera's focal length. Negative indicates narrower field of view, positive indicates wider field of view.", ) -class Code3(Enum): - int_5000 = 5000 - int_5001 = 5001 - int_5002 = 5002 +class Type(str, Enum): + simple = 'simple' + down_back = 'down_back' + forward_up = 'forward_up' + right_turn_forward = 'right_turn_forward' + left_turn_forward = 'left_turn_forward' -class KlingServerError(KlingErrorResponse): - code: Optional[Code3] = Field( +class CameraControl(BaseModel): + config: Optional[Config] = None + type: Optional[Type] = Field( None, - description='- 5000: Internal server error\n- 5001: Service temporarily unavailable\n- 5002: Server internal timeout\n', + description='Predefined camera movements type. simple: Customizable camera movement. down_back: Camera descends and moves backward. forward_up: Camera moves forward and tilts up. right_turn_forward: Rotate right and move forward. left_turn_forward: Rotate left and move forward.', ) -class Code4(Enum): - int_1300 = 1300 - int_1301 = 1301 - int_1302 = 1302 - int_1303 = 1303 - int_1304 = 1304 - - -class KlingStrategyError(KlingErrorResponse): - code: Optional[Code4] = Field( +class Trajectory(BaseModel): + x: Optional[int] = Field( None, - description='- 1300: Trigger platform strategy\n- 1301: Trigger content security policy\n- 1302: API request too frequent\n- 1303: Concurrency/QPS exceeds limit\n- 1304: Trigger IP whitelist policy\n', + description='The horizontal coordinate of trajectory point. Based on bottom-left corner of image as origin (0,0).', + ) + y: Optional[int] = Field( + None, + description='The vertical coordinate of trajectory point. Based on bottom-left corner of image as origin (0,0).', + ) + + +class DynamicMask(BaseModel): + mask: Optional[AnyUrl] = Field( + None, + description='Dynamic Brush Application Area (Mask image created by users using the motion brush). The aspect ratio must match the input image.', + ) + trajectories: Optional[List[Trajectory]] = None + + +class KlingImage2VideoRequest(BaseModel): + aspect_ratio: Optional[AspectRatio] = '16:9' + callback_url: Optional[AnyUrl] = Field( + None, + description='The callback notification address. Server will notify when the task status changes.', + ) + camera_control: Optional[CameraControl] = None + cfg_scale: Optional[confloat(ge=0.0, le=1.0)] = Field( + 0.5, + description="Flexibility in video generation. The higher the value, the lower the model's degree of flexibility, and the stronger the relevance to the user's prompt.", + ) + duration: Optional[Duration] = Field(5, description='Video length in seconds') + dynamic_masks: Optional[List[DynamicMask]] = Field( + None, + description='Dynamic Brush Configuration List (up to 6 groups). For 5-second videos, trajectory length must not exceed 77 coordinates.', + ) + external_task_id: Optional[str] = Field( + None, + description='Customized Task ID. Must be unique within a single user account.', + ) + image: Optional[AnyUrl] = Field( + None, + description='Reference Image - URL or Base64 encoded string, cannot exceed 10MB, resolution not less than 300*300px, aspect ratio between 1:2.5 ~ 2.5:1. Base64 should not include data:image prefix.', + ) + image_tail: Optional[AnyUrl] = Field( + None, + description='Reference Image - End frame control. URL or Base64 encoded string, cannot exceed 10MB, resolution not less than 300*300px. Base64 should not include data:image prefix.', + ) + mode: Optional[Mode] = Field( + 'std', + description='Video generation mode. std: Standard Mode, which is cost-effective. pro: Professional Mode, generates videos with longer duration but higher quality output.', + ) + model_name: Optional[ModelName] = Field('kling-v1', description='Model Name') + negative_prompt: Optional[constr(max_length=2500)] = Field( + None, description='Negative text prompt' + ) + prompt: Optional[constr(max_length=2500)] = Field( + None, description='Positive text prompt' + ) + static_mask: Optional[AnyUrl] = Field( + None, + description='Static Brush Application Area (Mask image created by users using the motion brush). The aspect ratio must match the input image.', + ) + + +class TaskInfo(BaseModel): + external_task_id: Optional[str] = None + + +class Video(BaseModel): + duration: Optional[str] = Field(None, description='Total video duration') + id: Optional[str] = Field(None, description='Generated video ID') + url: Optional[AnyUrl] = Field(None, description='URL for generated video') + + +class TaskResult(BaseModel): + videos: Optional[List[Video]] = None + + +class TaskStatus(str, Enum): + submitted = 'submitted' + processing = 'processing' + succeed = 'succeed' + failed = 'failed' + + +class Data(BaseModel): + created_at: Optional[int] = Field(None, description='Task creation time') + task_id: Optional[str] = Field(None, description='Task ID') + task_info: Optional[TaskInfo] = None + task_result: Optional[TaskResult] = None + task_status: Optional[TaskStatus] = None + updated_at: Optional[int] = Field(None, description='Task update time') + + +class KlingImage2VideoResponse(BaseModel): + code: Optional[int] = Field(None, description='Error code') + data: Optional[Data] = None + message: Optional[str] = Field(None, description='Error message') + request_id: Optional[str] = Field(None, description='Request ID') + + +class AspectRatio1(str, Enum): + field_16_9 = '16:9' + field_9_16 = '9:16' + field_1_1 = '1:1' + field_4_3 = '4:3' + field_3_4 = '3:4' + field_3_2 = '3:2' + field_2_3 = '2:3' + field_21_9 = '21:9' + + +class ImageReference(str, Enum): + subject = 'subject' + face = 'face' + + +class ModelName2(str, Enum): + kling_v1 = 'kling-v1' + kling_v1_5 = 'kling-v1-5' + + +class KlingImageGenerationsRequest(BaseModel): + aspect_ratio: Optional[AspectRatio1] = Field( + '16:9', description='Aspect ratio of the generated images' + ) + callback_url: Optional[AnyUrl] = Field( + None, description='The callback notification address' + ) + human_fidelity: Optional[confloat(ge=0.0, le=1.0)] = Field( + 0.45, description='Subject reference similarity' + ) + image: Optional[str] = Field( + None, description='Reference Image - Base64 encoded string or image URL' + ) + image_fidelity: Optional[confloat(ge=0.0, le=1.0)] = Field( + 0.5, description='Reference intensity for user-uploaded images' + ) + image_reference: Optional[ImageReference] = Field( + None, description='Image reference type' + ) + model_name: Optional[ModelName2] = Field('kling-v1', description='Model Name') + n: Optional[conint(ge=1, le=9)] = Field(1, description='Number of generated images') + negative_prompt: Optional[constr(max_length=200)] = Field( + None, description='Negative text prompt' + ) + prompt: constr(max_length=500) = Field(..., description='Positive text prompt') + + +class Image(BaseModel): + index: Optional[int] = Field(None, description='Image Number (0-9)') + url: Optional[AnyUrl] = Field(None, description='URL for generated image') + + +class TaskResult1(BaseModel): + images: Optional[List[Image]] = None + + +class Data1(BaseModel): + created_at: Optional[int] = Field(None, description='Task creation time') + task_id: Optional[str] = Field(None, description='Task ID') + task_result: Optional[TaskResult1] = None + task_status: Optional[TaskStatus] = None + task_status_msg: Optional[str] = Field(None, description='Task status information') + updated_at: Optional[int] = Field(None, description='Task update time') + + +class KlingImageGenerationsResponse(BaseModel): + code: Optional[int] = Field(None, description='Error code') + data: Optional[Data1] = None + message: Optional[str] = Field(None, description='Error message') + request_id: Optional[str] = Field(None, description='Request ID') + + +class AudioType(str, Enum): + file = 'file' + url = 'url' + + +class Mode2(str, Enum): + text2video = 'text2video' + audio2video = 'audio2video' + + +class VoiceLanguage(str, Enum): + zh = 'zh' + en = 'en' + + +class Input(BaseModel): + audio_file: Optional[str] = Field( + None, + description='Local Path of Audio File. Supported formats: .mp3/.wav/.m4a/.aac, maximum file size of 5MB. Base64 code.', + ) + audio_type: Optional[AudioType] = Field( + None, + description='Method of Transmitting Audio Files for Lip-Sync. Required when mode is audio2video.', + ) + audio_url: Optional[AnyUrl] = Field( + None, + description='Audio File Download URL. Supported formats: .mp3/.wav/.m4a/.aac, maximum file size of 5MB.', + ) + mode: Mode2 = Field( + ..., + description='Video Generation Mode. text2video: Text-to-video generation mode; audio2video: Audio-to-video generation mode', + ) + text: Optional[str] = Field( + None, + description='Text Content for Lip-Sync Video Generation. Required when mode is text2video. Maximum length is 120 characters.', + ) + video_id: Optional[str] = Field( + None, + description='The ID of the video generated by Kling AI. Only supports 5-second and 10-second videos generated within the last 30 days.', + ) + video_url: Optional[AnyUrl] = Field( + None, + description='Get link for uploaded video. Video files support .mp4/.mov, file size does not exceed 100MB, video length between 2-10s.', + ) + voice_id: Optional[str] = Field( + None, + description='Voice ID. Required when mode is text2video. The system offers a variety of voice options to choose from.', + ) + voice_language: Optional[VoiceLanguage] = Field( + 'zh', description='The voice language corresponds to the Voice ID.' + ) + voice_speed: Optional[confloat(ge=0.8, le=2.0)] = Field( + 1, + description='Speech Rate. Valid range: 0.8~2.0, accurate to one decimal place.', + ) + + +class KlingLipSyncRequest(BaseModel): + callback_url: Optional[AnyUrl] = Field( + None, + description='The callback notification address. Server will notify when the task status changes.', + ) + input: Input + + +class TaskResult2(BaseModel): + videos: Optional[List[Video]] = None + + +class Data2(BaseModel): + created_at: Optional[int] = Field(None, description='Task creation time') + task_id: Optional[str] = Field(None, description='Task ID') + task_info: Optional[TaskInfo] = None + task_result: Optional[TaskResult2] = None + task_status: Optional[TaskStatus] = None + updated_at: Optional[int] = Field(None, description='Task update time') + + +class KlingLipSyncResponse(BaseModel): + code: Optional[int] = Field(None, description='Error code') + data: Optional[Data2] = None + message: Optional[str] = Field(None, description='Error message') + request_id: Optional[str] = Field(None, description='Request ID') + + +class ResourcePackType(str, Enum): + decreasing_total = 'decreasing_total' + constant_period = 'constant_period' + + +class Status(str, Enum): + toBeOnline = 'toBeOnline' + online = 'online' + expired = 'expired' + runOut = 'runOut' + + +class ResourcePackSubscribeInfo(BaseModel): + effective_time: Optional[int] = Field( + None, description='Effective time, Unix timestamp in ms' + ) + invalid_time: Optional[int] = Field( + None, description='Expiration time, Unix timestamp in ms' + ) + purchase_time: Optional[int] = Field( + None, description='Purchase time, Unix timestamp in ms' + ) + remaining_quantity: Optional[float] = Field( + None, description='Remaining quantity (updated with a 12-hour delay)' + ) + resource_pack_id: Optional[str] = Field(None, description='Resource package ID') + resource_pack_name: Optional[str] = Field(None, description='Resource package name') + resource_pack_type: Optional[ResourcePackType] = Field( + None, + description='Resource package type (decreasing_total=decreasing total, constant_period=constant periodicity)', + ) + status: Optional[Status] = Field(None, description='Resource Package Status') + total_quantity: Optional[float] = Field(None, description='Total quantity') + + +class Data3(BaseModel): + code: Optional[int] = Field(None, description='Error code; 0 indicates success') + msg: Optional[str] = Field(None, description='Error information') + resource_pack_subscribe_infos: Optional[List[ResourcePackSubscribeInfo]] = Field( + None, description='Resource package list' + ) + + +class KlingResourcePackageResponse(BaseModel): + code: Optional[int] = Field(None, description='Error code; 0 indicates success') + data: Optional[Data3] = None + message: Optional[str] = Field(None, description='Error information') + request_id: Optional[str] = Field( + None, + description='Request ID, generated by the system, used to track requests and troubleshoot problems', + ) + + +class Duration2(str, Enum): + field_5 = '5' + + +class ModelName3(str, Enum): + kling_v1_6 = 'kling-v1-6' + + +class KlingSingleImageEffectInput(BaseModel): + duration: Duration2 = Field( + ..., description='Video Length in seconds. Only 5-second videos are supported.' + ) + image: str = Field( + ..., + description='Reference Image. URL or Base64 encoded string (without data:image prefix). File size cannot exceed 10MB, resolution not less than 300*300px, aspect ratio between 1:2.5 ~ 2.5:1.', + ) + model_name: ModelName3 = Field( + ..., + description='Model Name. Only kling-v1-6 is supported for single image effects.', + ) + + +class AspectRatio2(str, Enum): + field_16_9 = '16:9' + field_9_16 = '9:16' + field_1_1 = '1:1' + + +class Config1(BaseModel): + horizontal: Optional[confloat(ge=-10.0, le=10.0)] = None + pan: Optional[confloat(ge=-10.0, le=10.0)] = None + roll: Optional[confloat(ge=-10.0, le=10.0)] = None + tilt: Optional[confloat(ge=-10.0, le=10.0)] = None + vertical: Optional[confloat(ge=-10.0, le=10.0)] = None + zoom: Optional[confloat(ge=-10.0, le=10.0)] = None + + +class CameraControl1(BaseModel): + config: Optional[Config1] = None + type: Optional[Type] = Field(None, description='Predefined camera movements type') + + +class Duration3(str, Enum): + field_5 = 5 + field_10 = 10 + + +class Mode3(str, Enum): + std = 'std' + pro = 'pro' + + +class ModelName4(str, Enum): + kling_v1 = 'kling-v1' + kling_v1_6 = 'kling-v1-6' + + +class KlingText2VideoRequest(BaseModel): + aspect_ratio: Optional[AspectRatio2] = '16:9' + callback_url: Optional[AnyUrl] = Field( + None, description='The callback notification address' + ) + camera_control: Optional[CameraControl1] = None + cfg_scale: Optional[confloat(ge=0.0, le=1.0)] = Field( + 0.5, description='Flexibility in video generation' + ) + duration: Optional[Duration3] = 5 + external_task_id: Optional[str] = Field(None, description='Customized Task ID') + mode: Optional[Mode3] = Field('std', description='Video generation mode') + model_name: Optional[ModelName4] = Field('kling-v1', description='Model Name') + negative_prompt: Optional[constr(max_length=2500)] = Field( + None, description='Negative text prompt' + ) + prompt: Optional[constr(max_length=2500)] = Field( + None, description='Positive text prompt' + ) + + +class TaskResult3(BaseModel): + videos: Optional[List[Video]] = None + + +class Data4(BaseModel): + created_at: Optional[int] = Field(None, description='Task creation time') + task_id: Optional[str] = Field(None, description='Task ID') + task_info: Optional[TaskInfo] = None + task_result: Optional[TaskResult3] = None + task_status: Optional[TaskStatus] = None + updated_at: Optional[int] = Field(None, description='Task update time') + + +class KlingText2VideoResponse(BaseModel): + code: Optional[int] = Field(None, description='Error code') + data: Optional[Data4] = None + message: Optional[str] = Field(None, description='Error message') + request_id: Optional[str] = Field(None, description='Request ID') + + +class KlingVideoEffectsInput( + RootModel[Union[KlingSingleImageEffectInput, KlingDualCharacterEffectInput]] +): + root: Union[KlingSingleImageEffectInput, KlingDualCharacterEffectInput] + + +class EffectScene(str, Enum): + bloombloom = 'bloombloom' + dizzydizzy = 'dizzydizzy' + fuzzyfuzzy = 'fuzzyfuzzy' + squish = 'squish' + expansion = 'expansion' + hug = 'hug' + kiss = 'kiss' + heart_gesture = 'heart_gesture' + + +class KlingVideoEffectsRequest(BaseModel): + callback_url: Optional[AnyUrl] = Field( + None, + description='The callback notification address for the result of this task.', + ) + effect_scene: EffectScene = Field( + ..., + description='Scene Name. Single Image Effects (bloombloom, dizzydizzy, fuzzyfuzzy, squish, expansion) or Dual-character Effects (hug, kiss, heart_gesture).', + ) + external_task_id: Optional[str] = Field( + None, + description='Customized Task ID. Must be unique within a single user account.', + ) + input: Optional[KlingVideoEffectsInput] = None + + +class TaskResult4(BaseModel): + videos: Optional[List[Video]] = None + + +class Data5(BaseModel): + created_at: Optional[int] = Field(None, description='Task creation time') + task_id: Optional[str] = Field(None, description='Task ID') + task_info: Optional[TaskInfo] = None + task_result: Optional[TaskResult4] = None + task_status: Optional[TaskStatus] = None + updated_at: Optional[int] = Field(None, description='Task update time') + + +class KlingVideoEffectsResponse(BaseModel): + code: Optional[int] = Field(None, description='Error code') + data: Optional[Data5] = None + message: Optional[str] = Field(None, description='Error message') + request_id: Optional[str] = Field(None, description='Request ID') + + +class KlingVideoExtendRequest(BaseModel): + callback_url: Optional[AnyUrl] = Field( + None, + description='The callback notification address. Server will notify when the task status changes.', + ) + cfg_scale: Optional[confloat(ge=0.0, le=1.0)] = Field( + 0.5, + description="Flexibility in video generation. The higher the value, the lower the model's flexibility and the stronger the relevance to the user's prompt.", + ) + negative_prompt: Optional[constr(max_length=2500)] = Field( + None, + description='Negative text prompt for elements to avoid in the extended video', + ) + prompt: Optional[constr(max_length=2500)] = Field( + None, description='Positive text prompt for guiding the video extension' + ) + video_id: Optional[str] = Field( + None, + description='The ID of the video to be extended. Supports videos generated by text-to-video, image-to-video, and previous video extension operations. Cannot exceed 3 minutes total duration after extension.', + ) + + +class TaskResult5(BaseModel): + videos: Optional[List[Video]] = None + + +class Data6(BaseModel): + created_at: Optional[int] = Field(None, description='Task creation time') + task_id: Optional[str] = Field(None, description='Task ID') + task_info: Optional[TaskInfo] = None + task_result: Optional[TaskResult5] = None + task_status: Optional[TaskStatus] = None + updated_at: Optional[int] = Field(None, description='Task update time') + + +class KlingVideoExtendResponse(BaseModel): + code: Optional[int] = Field(None, description='Error code') + data: Optional[Data6] = None + message: Optional[str] = Field(None, description='Error message') + request_id: Optional[str] = Field(None, description='Request ID') + + +class ModelName5(str, Enum): + kolors_virtual_try_on_v1 = 'kolors-virtual-try-on-v1' + kolors_virtual_try_on_v1_5 = 'kolors-virtual-try-on-v1-5' + + +class KlingVirtualTryOnRequest(BaseModel): + callback_url: Optional[AnyUrl] = Field( + None, description='The callback notification address' + ) + cloth_image: Optional[str] = Field( + None, + description='Reference clothing image - Base64 encoded string or image URL', + ) + human_image: str = Field( + ..., description='Reference human image - Base64 encoded string or image URL' + ) + model_name: Optional[ModelName5] = Field( + 'kolors-virtual-try-on-v1', description='Model Name' + ) + + +class Image1(BaseModel): + index: Optional[int] = Field(None, description='Image Number') + url: Optional[AnyUrl] = Field(None, description='URL for generated image') + + +class TaskResult6(BaseModel): + images: Optional[List[Image1]] = None + + +class Data7(BaseModel): + created_at: Optional[int] = Field(None, description='Task creation time') + task_id: Optional[str] = Field(None, description='Task ID') + task_result: Optional[TaskResult6] = None + task_status: Optional[TaskStatus] = None + task_status_msg: Optional[str] = Field(None, description='Task status information') + updated_at: Optional[int] = Field(None, description='Task update time') + + +class KlingVirtualTryOnResponse(BaseModel): + code: Optional[int] = Field(None, description='Error code') + data: Optional[Data7] = None + message: Optional[str] = Field(None, description='Error message') + request_id: Optional[str] = Field(None, description='Request ID') + + +class LumaAspectRatio(str, Enum): + field_1_1 = '1:1' + field_16_9 = '16:9' + field_9_16 = '9:16' + field_4_3 = '4:3' + field_3_4 = '3:4' + field_21_9 = '21:9' + field_9_21 = '9:21' + + +class LumaAssets(BaseModel): + image: Optional[AnyUrl] = Field(None, description='The URL of the image') + progress_video: Optional[AnyUrl] = Field( + None, description='The URL of the progress video' + ) + video: Optional[AnyUrl] = Field(None, description='The URL of the video') + + +class GenerationType(str, Enum): + add_audio = 'add_audio' + + +class LumaAudioGenerationRequest(BaseModel): + callback_url: Optional[AnyUrl] = Field( + None, description='The callback URL for the audio' + ) + generation_type: Optional[GenerationType] = 'add_audio' + negative_prompt: Optional[str] = Field( + None, description='The negative prompt of the audio' + ) + prompt: Optional[str] = Field(None, description='The prompt of the audio') + + +class LumaError(BaseModel): + detail: Optional[str] = Field(None, description='The error message') + + +class Type2(str, Enum): + generation = 'generation' + + +class LumaGenerationReference(BaseModel): + id: UUID = Field(..., description='The ID of the generation') + type: Literal['generation'] + + +class GenerationType1(str, Enum): + video = 'video' + + +class LumaGenerationType(str, Enum): + video = 'video' + image = 'image' + + +class GenerationType2(str, Enum): + image = 'image' + + +class LumaImageIdentity(BaseModel): + images: Optional[List[AnyUrl]] = Field( + None, description='The URLs of the image identity' + ) + + +class LumaImageModel(str, Enum): + photon_1 = 'photon-1' + photon_flash_1 = 'photon-flash-1' + + +class LumaImageRef(BaseModel): + url: Optional[AnyUrl] = Field(None, description='The URL of the image reference') + weight: Optional[float] = Field( + None, description='The weight of the image reference' + ) + + +class Type3(str, Enum): + image = 'image' + + +class LumaImageReference(BaseModel): + type: Literal['image'] + url: AnyUrl = Field(..., description='The URL of the image') + + +class LumaKeyframe(RootModel[Union[LumaGenerationReference, LumaImageReference]]): + root: Union[LumaGenerationReference, LumaImageReference] = Field( + ..., + description='A keyframe can be either a Generation reference, an Image, or a Video', + discriminator='type', + ) + + +class LumaKeyframes(BaseModel): + frame0: Optional[LumaKeyframe] = None + frame1: Optional[LumaKeyframe] = None + + +class LumaModifyImageRef(BaseModel): + url: Optional[AnyUrl] = Field(None, description='The URL of the image reference') + weight: Optional[float] = Field( + None, description='The weight of the modify image reference' + ) + + +class LumaState(str, Enum): + queued = 'queued' + dreaming = 'dreaming' + completed = 'completed' + failed = 'failed' + + +class GenerationType3(str, Enum): + upscale_video = 'upscale_video' + + +class LumaVideoModel(str, Enum): + ray_2 = 'ray-2' + ray_2_flash = 'ray-2-flash' + + +class LumaVideoModelOutputDuration1(str, Enum): + field_5s = '5s' + field_9s = '9s' + + +class LumaVideoModelOutputDuration( + RootModel[Union[LumaVideoModelOutputDuration1, str]] +): + root: Union[LumaVideoModelOutputDuration1, str] + + +class LumaVideoModelOutputResolution1(str, Enum): + field_540p = '540p' + field_720p = '720p' + field_1080p = '1080p' + field_4k = '4k' + + +class LumaVideoModelOutputResolution( + RootModel[Union[LumaVideoModelOutputResolution1, str]] +): + root: Union[LumaVideoModelOutputResolution1, str] + + +class MachineStats(BaseModel): + cpu_capacity: Optional[str] = Field(None, description='Total CPU on the machine.') + disk_capacity: Optional[str] = Field( + None, description='Total disk capacity on the machine.' + ) + gpu_type: Optional[str] = Field( + None, description='The GPU type. eg. NVIDIA Tesla K80' + ) + initial_cpu: Optional[str] = Field( + None, description='Initial CPU available before the job starts.' + ) + initial_disk: Optional[str] = Field( + None, description='Initial disk available before the job starts.' + ) + initial_ram: Optional[str] = Field( + None, description='Initial RAM available before the job starts.' + ) + machine_name: Optional[str] = Field(None, description='Name of the machine.') + memory_capacity: Optional[str] = Field( + None, description='Total memory on the machine.' + ) + os_version: Optional[str] = Field( + None, description='The operating system version. eg. Ubuntu Linux 20.04' + ) + pip_freeze: Optional[str] = Field(None, description='The pip freeze output') + vram_time_series: Optional[Dict[str, Any]] = Field( + None, description='Time series of VRAM usage.' ) @@ -212,7 +1062,7 @@ class MinimaxFileRetrieveResponse(BaseModel): file: File -class Status(str, Enum): +class Status1(str, Enum): Queueing = 'Queueing' Preparing = 'Preparing' Processing = 'Processing' @@ -226,7 +1076,7 @@ class MinimaxTaskResultResponse(BaseModel): None, description='After the task status changes to Success, this field returns the file ID corresponding to the generated video.', ) - status: Status = Field( + status: Status1 = Field( ..., description="Task status: 'Queueing' (in queue), 'Preparing' (task is preparing), 'Processing' (generating), 'Success' (task completed successfully), or 'Fail' (task failed).", ) @@ -286,11 +1136,40 @@ class MinimaxVideoGenerationResponse(BaseModel): ) +class NodeStatus(str, Enum): + NodeStatusActive = 'NodeStatusActive' + NodeStatusDeleted = 'NodeStatusDeleted' + NodeStatusBanned = 'NodeStatusBanned' + + +class NodeVersionStatus(str, Enum): + NodeVersionStatusActive = 'NodeVersionStatusActive' + NodeVersionStatusDeleted = 'NodeVersionStatusDeleted' + NodeVersionStatusBanned = 'NodeVersionStatusBanned' + NodeVersionStatusPending = 'NodeVersionStatusPending' + NodeVersionStatusFlagged = 'NodeVersionStatusFlagged' + + +class NodeVersionUpdateRequest(BaseModel): + changelog: Optional[str] = Field( + None, description='The changelog describing the version changes.' + ) + deprecated: Optional[bool] = Field( + None, description='Whether the version is deprecated.' + ) + + class Moderation(str, Enum): low = 'low' auto = 'auto' +class OutputFormat(str, Enum): + png = 'png' + webp = 'webp' + jpeg = 'jpeg' + + class OpenAIImageEditRequest(BaseModel): background: Optional[str] = Field( None, description='Background transparency', examples=['opaque'] @@ -404,19 +1283,730 @@ class Datum1(BaseModel): url: Optional[str] = Field(None, description='URL of the image') +class InputTokensDetails(BaseModel): + image_tokens: Optional[int] = None + text_tokens: Optional[int] = None + + +class Usage(BaseModel): + input_tokens: Optional[int] = None + input_tokens_details: Optional[InputTokensDetails] = None + output_tokens: Optional[int] = None + total_tokens: Optional[int] = None + + class OpenAIImageGenerationResponse(BaseModel): data: Optional[List[Datum1]] = None + usage: Optional[Usage] = None -class KlingAccountError(KlingErrorResponse): - code: Optional[Code] = Field( +class PersonalAccessToken(BaseModel): + createdAt: Optional[datetime] = Field( + None, description='[Output Only]The date and time the token was created.' + ) + description: Optional[str] = Field( None, - description='- 1100: Account exception\n- 1101: Account in arrears (postpaid scenario)\n- 1102: Resource pack depleted or expired (prepaid scenario)\n- 1103: Unauthorized access to requested resource\n', + description="Optional. A more detailed description of the token's intended use.", + ) + id: Optional[UUID] = Field(None, description='Unique identifier for the GitCommit') + name: Optional[str] = Field( + None, + description='Required. The name of the token. Can be a simple description.', + ) + token: Optional[str] = Field( + None, + description='[Output Only]. The personal access token. Only returned during creation.', ) -class KlingAuthenticationError(KlingErrorResponse): - code: Optional[Code1] = Field( - None, - description='- 1000: Authentication failed\n- 1001: Authorization is empty\n- 1002: Authorization is invalid\n- 1003: Authorization is not yet valid\n- 1004: Authorization has expired\n', +class PublisherStatus(str, Enum): + PublisherStatusActive = 'PublisherStatusActive' + PublisherStatusBanned = 'PublisherStatusBanned' + + +class PublisherUser(BaseModel): + email: Optional[str] = Field(None, description='The email address for this user.') + id: Optional[str] = Field(None, description='The unique id for this user.') + name: Optional[str] = Field(None, description='The name for this user.') + + +class RecraftImageGenerationRequest(BaseModel): + model: str = Field( + ..., description='The model to use for generation (e.g., "recraftv3")' ) + n: conint(ge=1, le=4) = Field(..., description='The number of images to generate') + prompt: str = Field( + ..., description='The text prompt describing the image to generate' + ) + size: str = Field( + ..., description='The size of the generated image (e.g., "1024x1024")' + ) + style: Optional[str] = Field( + None, + description='The style to apply to the generated image (e.g., "digital_illustration")', + ) + + +class Datum2(BaseModel): + image_id: Optional[str] = Field( + None, description='Unique identifier for the generated image' + ) + url: Optional[str] = Field(None, description='URL to access the generated image') + + +class RecraftImageGenerationResponse(BaseModel): + created: int = Field( + ..., description='Unix timestamp when the generation was created' + ) + credits: int = Field(..., description='Number of credits used for the generation') + data: List[Datum2] = Field(..., description='Array of generated image information') + + +class RunwayAspectRatioEnum(str, Enum): + field_1280_720 = '1280:720' + field_720_1280 = '720:1280' + field_1104_832 = '1104:832' + field_832_1104 = '832:1104' + field_960_960 = '960:960' + field_1584_672 = '1584:672' + field_1280_768 = '1280:768' + field_768_1280 = '768:1280' + + +class RunwayDurationEnum(int, Enum): + integer_5 = 5 + integer_10 = 10 + + +class RunwayImageToVideoResponse(BaseModel): + id: Optional[str] = Field(None, description='Task ID') + + +class RunwayModelEnum(str, Enum): + gen4_turbo = 'gen4_turbo' + gen3a_turbo = 'gen3a_turbo' + + +class Position(str, Enum): + first = 'first' + last = 'last' + + +class RunwayPromptImageDetailedObject(BaseModel): + position: Position = Field( + ..., + description="The position of the image in the output video. 'last' is currently supported for gen3a_turbo only.", + ) + uri: AnyUrl = Field( + ..., description='A HTTPS URL or data URI containing an encoded image.' + ) + + +class RunwayPromptImageObject( + RootModel[Union[AnyUrl, List[RunwayPromptImageDetailedObject]]] +): + root: Union[AnyUrl, List[RunwayPromptImageDetailedObject]] = Field( + ..., + description='Image(s) to use for the video generation. Can be a single URI or an array of image objects with positions.', + ) + + +class RunwayTaskStatusEnum(str, Enum): + SUCCEEDED = 'SUCCEEDED' + RUNNING = 'RUNNING' + FAILED = 'FAILED' + PENDING = 'PENDING' + CANCELLED = 'CANCELLED' + THROTTLED = 'THROTTLED' + + +class RunwayTaskStatusResponse(BaseModel): + createdAt: Optional[datetime] = Field(None, description='Task creation timestamp') + id: Optional[str] = Field(None, description='Task ID') + output: Optional[List[str]] = Field(None, description='Array of output video URLs') + status: Optional[RunwayTaskStatusEnum] = None + + +class StorageFile(BaseModel): + file_path: Optional[str] = Field(None, description='Path to the file in storage') + id: Optional[UUID] = Field( + None, description='Unique identifier for the storage file' + ) + public_url: Optional[str] = Field(None, description='Public URL') + + +class StripeAddress(BaseModel): + city: Optional[str] = None + country: Optional[str] = None + line1: Optional[str] = None + line2: Optional[str] = None + postal_code: Optional[str] = None + state: Optional[str] = None + + +class StripeAmountDetails(BaseModel): + tip: Optional[Dict[str, Any]] = None + + +class StripeBillingDetails(BaseModel): + address: Optional[StripeAddress] = None + email: Optional[str] = None + name: Optional[str] = None + phone: Optional[str] = None + tax_id: Optional[Any] = None + + +class Checks(BaseModel): + address_line1_check: Optional[Any] = None + address_postal_code_check: Optional[Any] = None + cvc_check: Optional[str] = None + + +class ExtendedAuthorization(BaseModel): + status: Optional[str] = None + + +class IncrementalAuthorization(BaseModel): + status: Optional[str] = None + + +class Multicapture(BaseModel): + status: Optional[str] = None + + +class NetworkToken(BaseModel): + used: Optional[bool] = None + + +class Overcapture(BaseModel): + maximum_amount_capturable: Optional[int] = None + status: Optional[str] = None + + +class StripeCardDetails(BaseModel): + amount_authorized: Optional[int] = None + authorization_code: Optional[Any] = None + brand: Optional[str] = None + checks: Optional[Checks] = None + country: Optional[str] = None + exp_month: Optional[int] = None + exp_year: Optional[int] = None + extended_authorization: Optional[ExtendedAuthorization] = None + fingerprint: Optional[str] = None + funding: Optional[str] = None + incremental_authorization: Optional[IncrementalAuthorization] = None + installments: Optional[Any] = None + last4: Optional[str] = None + mandate: Optional[Any] = None + multicapture: Optional[Multicapture] = None + network: Optional[str] = None + network_token: Optional[NetworkToken] = None + network_transaction_id: Optional[str] = None + overcapture: Optional[Overcapture] = None + regulated_status: Optional[str] = None + three_d_secure: Optional[Any] = None + wallet: Optional[Any] = None + + +class Object(str, Enum): + charge = 'charge' + + +class Object1(str, Enum): + event = 'event' + + +class Type4(str, Enum): + payment_intent_succeeded = 'payment_intent.succeeded' + + +class StripeOutcome(BaseModel): + advice_code: Optional[Any] = None + network_advice_code: Optional[Any] = None + network_decline_code: Optional[Any] = None + network_status: Optional[str] = None + reason: Optional[Any] = None + risk_level: Optional[str] = None + risk_score: Optional[int] = None + seller_message: Optional[str] = None + type: Optional[str] = None + + +class Object2(str, Enum): + payment_intent = 'payment_intent' + + +class StripePaymentMethodDetails(BaseModel): + card: Optional[StripeCardDetails] = None + type: Optional[str] = None + + +class Card(BaseModel): + installments: Optional[Any] = None + mandate_options: Optional[Any] = None + network: Optional[Any] = None + request_three_d_secure: Optional[str] = None + + +class StripePaymentMethodOptions(BaseModel): + card: Optional[Card] = None + + +class StripeRefundList(BaseModel): + data: Optional[List[Dict[str, Any]]] = None + has_more: Optional[bool] = None + object: Optional[str] = None + total_count: Optional[int] = None + url: Optional[str] = None + + +class StripeRequestInfo(BaseModel): + id: Optional[str] = None + idempotency_key: Optional[str] = None + + +class StripeShipping(BaseModel): + address: Optional[StripeAddress] = None + carrier: Optional[str] = None + name: Optional[str] = None + phone: Optional[str] = None + tracking_number: Optional[str] = None + + +class User(BaseModel): + email: Optional[str] = Field(None, description='The email address for this user.') + id: Optional[str] = Field(None, description='The unique id for this user.') + isAdmin: Optional[bool] = Field( + None, description='Indicates if the user has admin privileges.' + ) + isApproved: Optional[bool] = Field( + None, description='Indicates if the user is approved.' + ) + name: Optional[str] = Field(None, description='The name for this user.') + + +class Veo2GenVidPollRequest(BaseModel): + operationName: str = Field( + ..., + description='Full operation name (from predict response)', + examples=[ + 'projects/PROJECT_ID/locations/us-central1/publishers/google/models/MODEL_ID/operations/OPERATION_ID' + ], + ) + + +class Video5(BaseModel): + bytesBase64Encoded: Optional[str] = Field( + None, description='Base64-encoded video content' + ) + gcsUri: Optional[str] = Field(None, description='Cloud Storage URI of the video') + mimeType: Optional[str] = Field(None, description='Video MIME type') + + +class Response(BaseModel): + field_type: Optional[str] = Field( + None, + alias='@type', + examples=[ + 'type.googleapis.com/cloud.ai.large_models.vision.GenerateVideoResponse' + ], + ) + raiMediaFilteredCount: Optional[int] = Field( + None, description='Count of media filtered by responsible AI policies' + ) + videos: Optional[List[Video5]] = None + + +class Veo2GenVidPollResponse(BaseModel): + done: Optional[bool] = None + name: Optional[str] = None + response: Optional[Response] = Field( + None, description='The actual prediction response if done is true' + ) + + +class Image2(BaseModel): + bytesBase64Encoded: str + gcsUri: Optional[str] = None + mimeType: Optional[str] = None + + +class Image3(BaseModel): + bytesBase64Encoded: Optional[str] = None + gcsUri: str + mimeType: Optional[str] = None + + +class Instance(BaseModel): + image: Optional[Union[Image2, Image3]] = Field( + None, description='Optional image to guide video generation' + ) + prompt: str = Field(..., description='Text description of the video') + + +class PersonGeneration(str, Enum): + ALLOW = 'ALLOW' + BLOCK = 'BLOCK' + + +class Parameters(BaseModel): + aspectRatio: Optional[str] = Field(None, examples=['16:9']) + durationSeconds: Optional[int] = None + enhancePrompt: Optional[bool] = None + negativePrompt: Optional[str] = None + personGeneration: Optional[PersonGeneration] = None + sampleCount: Optional[int] = None + seed: Optional[int] = None + storageUri: Optional[str] = Field( + None, description='Optional Cloud Storage URI to upload the video' + ) + + +class Veo2GenVidRequest(BaseModel): + instances: Optional[List[Instance]] = None + parameters: Optional[Parameters] = None + + +class Veo2GenVidResponse(BaseModel): + name: str = Field( + ..., + description='Operation resource name', + examples=[ + 'projects/PROJECT_ID/locations/us-central1/publishers/google/models/MODEL_ID/operations/a1b07c8e-7b5a-4aba-bb34-3e1ccb8afcc8' + ], + ) + + +class WorkflowRunStatus(str, Enum): + WorkflowRunStatusStarted = 'WorkflowRunStatusStarted' + WorkflowRunStatusFailed = 'WorkflowRunStatusFailed' + WorkflowRunStatusCompleted = 'WorkflowRunStatusCompleted' + + +class ActionJobResult(BaseModel): + action_job_id: Optional[str] = Field( + None, description='Identifier of the job this result belongs to' + ) + action_run_id: Optional[str] = Field( + None, description='Identifier of the run this result belongs to' + ) + author: Optional[str] = Field(None, description='The author of the commit') + avg_vram: Optional[int] = Field( + None, description='The average VRAM used by the job' + ) + branch_name: Optional[str] = Field( + None, description='Name of the relevant git branch' + ) + comfy_run_flags: Optional[str] = Field( + None, description='The comfy run flags. E.g. `--low-vram`' + ) + commit_hash: Optional[str] = Field(None, description='The hash of the commit') + commit_id: Optional[str] = Field(None, description='The ID of the commit') + commit_message: Optional[str] = Field(None, description='The message of the commit') + commit_time: Optional[int] = Field( + None, description='The Unix timestamp when the commit was made' + ) + cuda_version: Optional[str] = Field(None, description='CUDA version used') + end_time: Optional[int] = Field( + None, description='The end time of the job as a Unix timestamp.' + ) + git_repo: Optional[str] = Field(None, description='The repository name') + id: Optional[UUID] = Field(None, description='Unique identifier for the job result') + job_trigger_user: Optional[str] = Field( + None, description='The user who triggered the job.' + ) + machine_stats: Optional[MachineStats] = None + operating_system: Optional[str] = Field(None, description='Operating system used') + peak_vram: Optional[int] = Field(None, description='The peak VRAM used by the job') + pr_number: Optional[str] = Field(None, description='The pull request number') + python_version: Optional[str] = Field(None, description='PyTorch version used') + pytorch_version: Optional[str] = Field(None, description='PyTorch version used') + start_time: Optional[int] = Field( + None, description='The start time of the job as a Unix timestamp.' + ) + status: Optional[WorkflowRunStatus] = None + storage_file: Optional[StorageFile] = None + workflow_name: Optional[str] = Field(None, description='Name of the workflow') + + +class LumaGenerationRequest(BaseModel): + aspect_ratio: LumaAspectRatio + callback_url: Optional[AnyUrl] = Field( + None, + description='The callback URL of the generation, a POST request with Generation object will be sent to the callback URL when the generation is dreaming, completed, or failed', + ) + duration: LumaVideoModelOutputDuration + generation_type: Optional[GenerationType1] = 'video' + keyframes: Optional[LumaKeyframes] = None + loop: Optional[bool] = Field(None, description='Whether to loop the video') + model: LumaVideoModel + prompt: str = Field(..., description='The prompt of the generation') + resolution: LumaVideoModelOutputResolution + + +class CharacterRef(BaseModel): + identity0: Optional[LumaImageIdentity] = None + + +class LumaImageGenerationRequest(BaseModel): + aspect_ratio: Optional[LumaAspectRatio] = '16:9' + callback_url: Optional[AnyUrl] = Field( + None, description='The callback URL for the generation' + ) + character_ref: Optional[CharacterRef] = None + generation_type: Optional[GenerationType2] = 'image' + image_ref: Optional[List[LumaImageRef]] = None + model: Optional[LumaImageModel] = 'photon-1' + modify_image_ref: Optional[LumaModifyImageRef] = None + prompt: Optional[str] = Field(None, description='The prompt of the generation') + style_ref: Optional[List[LumaImageRef]] = None + + +class LumaUpscaleVideoGenerationRequest(BaseModel): + callback_url: Optional[AnyUrl] = Field( + None, description='The callback URL for the upscale' + ) + generation_type: Optional[GenerationType3] = 'upscale_video' + resolution: Optional[LumaVideoModelOutputResolution] = None + + +class NodeVersion(BaseModel): + changelog: Optional[str] = Field( + None, description='Summary of changes made in this version' + ) + comfy_node_extract_status: Optional[str] = Field( + None, description='The status of comfy node extraction process.' + ) + createdAt: Optional[datetime] = Field( + None, description='The date and time the version was created.' + ) + dependencies: Optional[List[str]] = Field( + None, description='A list of pip dependencies required by the node.' + ) + deprecated: Optional[bool] = Field( + None, description='Indicates if this version is deprecated.' + ) + downloadUrl: Optional[str] = Field( + None, description='[Output Only] URL to download this version of the node' + ) + id: Optional[str] = None + node_id: Optional[str] = Field( + None, description='The unique identifier of the node.' + ) + status: Optional[NodeVersionStatus] = None + status_reason: Optional[str] = Field( + None, description='The reason for the status change.' + ) + version: Optional[str] = Field( + None, + description='The version identifier, following semantic versioning. Must be unique for the node.', + ) + + +class PublisherMember(BaseModel): + id: Optional[str] = Field( + None, description='The unique identifier for the publisher member.' + ) + role: Optional[str] = Field( + None, description='The role of the user in the publisher.' + ) + user: Optional[PublisherUser] = None + + +class RunwayImageToVideoRequest(BaseModel): + duration: RunwayDurationEnum + model: RunwayModelEnum + promptImage: RunwayPromptImageObject + promptText: Optional[constr(max_length=1000)] = Field( + None, description='Text prompt for the generation' + ) + ratio: RunwayAspectRatioEnum + seed: conint(ge=0, le=4294967295) = Field( + ..., description='Random seed for generation' + ) + + +class StripeCharge(BaseModel): + amount: Optional[int] = None + amount_captured: Optional[int] = None + amount_refunded: Optional[int] = None + application: Optional[str] = None + application_fee: Optional[str] = None + application_fee_amount: Optional[int] = None + balance_transaction: Optional[str] = None + billing_details: Optional[StripeBillingDetails] = None + calculated_statement_descriptor: Optional[str] = None + captured: Optional[bool] = None + created: Optional[int] = None + currency: Optional[str] = None + customer: Optional[str] = None + description: Optional[str] = None + destination: Optional[Any] = None + dispute: Optional[Any] = None + disputed: Optional[bool] = None + failure_balance_transaction: Optional[Any] = None + failure_code: Optional[Any] = None + failure_message: Optional[Any] = None + fraud_details: Optional[Dict[str, Any]] = None + id: Optional[str] = None + invoice: Optional[Any] = None + livemode: Optional[bool] = None + metadata: Optional[Dict[str, Any]] = None + object: Optional[Object] = None + on_behalf_of: Optional[Any] = None + order: Optional[Any] = None + outcome: Optional[StripeOutcome] = None + paid: Optional[bool] = None + payment_intent: Optional[str] = None + payment_method: Optional[str] = None + payment_method_details: Optional[StripePaymentMethodDetails] = None + radar_options: Optional[Dict[str, Any]] = None + receipt_email: Optional[str] = None + receipt_number: Optional[str] = None + receipt_url: Optional[str] = None + refunded: Optional[bool] = None + refunds: Optional[StripeRefundList] = None + review: Optional[Any] = None + shipping: Optional[StripeShipping] = None + source: Optional[Any] = None + source_transfer: Optional[Any] = None + statement_descriptor: Optional[Any] = None + statement_descriptor_suffix: Optional[Any] = None + status: Optional[str] = None + transfer_data: Optional[Any] = None + transfer_group: Optional[Any] = None + + +class StripeChargeList(BaseModel): + data: Optional[List[StripeCharge]] = None + has_more: Optional[bool] = None + object: Optional[str] = None + total_count: Optional[int] = None + url: Optional[str] = None + + +class StripePaymentIntent(BaseModel): + amount: Optional[int] = None + amount_capturable: Optional[int] = None + amount_details: Optional[StripeAmountDetails] = None + amount_received: Optional[int] = None + application: Optional[str] = None + application_fee_amount: Optional[int] = None + automatic_payment_methods: Optional[Any] = None + canceled_at: Optional[int] = None + cancellation_reason: Optional[str] = None + capture_method: Optional[str] = None + charges: Optional[StripeChargeList] = None + client_secret: Optional[str] = None + confirmation_method: Optional[str] = None + created: Optional[int] = None + currency: Optional[str] = None + customer: Optional[str] = None + description: Optional[str] = None + id: Optional[str] = None + invoice: Optional[str] = None + last_payment_error: Optional[Any] = None + latest_charge: Optional[str] = None + livemode: Optional[bool] = None + metadata: Optional[Dict[str, Any]] = None + next_action: Optional[Any] = None + object: Optional[Object2] = None + on_behalf_of: Optional[Any] = None + payment_method: Optional[str] = None + payment_method_configuration_details: Optional[Any] = None + payment_method_options: Optional[StripePaymentMethodOptions] = None + payment_method_types: Optional[List[str]] = None + processing: Optional[Any] = None + receipt_email: Optional[str] = None + review: Optional[Any] = None + setup_future_usage: Optional[Any] = None + shipping: Optional[StripeShipping] = None + source: Optional[Any] = None + statement_descriptor: Optional[Any] = None + statement_descriptor_suffix: Optional[Any] = None + status: Optional[str] = None + transfer_data: Optional[Any] = None + transfer_group: Optional[Any] = None + + +class LumaGeneration(BaseModel): + assets: Optional[LumaAssets] = None + created_at: Optional[datetime] = Field( + None, description='The date and time when the generation was created' + ) + failure_reason: Optional[str] = Field( + None, description='The reason for the state of the generation' + ) + generation_type: Optional[LumaGenerationType] = None + id: Optional[UUID] = Field(None, description='The ID of the generation') + model: Optional[str] = Field(None, description='The model used for the generation') + request: Optional[ + Union[ + LumaGenerationRequest, + LumaImageGenerationRequest, + LumaUpscaleVideoGenerationRequest, + LumaAudioGenerationRequest, + ] + ] = Field(None, description='The request of the generation') + state: Optional[LumaState] = None + + +class Publisher(BaseModel): + createdAt: Optional[datetime] = Field( + None, description='The date and time the publisher was created.' + ) + description: Optional[str] = None + id: Optional[str] = Field( + None, + description="The unique identifier for the publisher. It's akin to a username. Should be lowercase.", + ) + logo: Optional[str] = Field(None, description="URL to the publisher's logo.") + members: Optional[List[PublisherMember]] = Field( + None, description='A list of members in the publisher.' + ) + name: Optional[str] = None + source_code_repo: Optional[str] = None + status: Optional[PublisherStatus] = None + support: Optional[str] = None + website: Optional[str] = None + + +class Data8(BaseModel): + object: Optional[StripePaymentIntent] = None + + +class StripeEvent(BaseModel): + api_version: Optional[str] = None + created: Optional[int] = None + data: Data8 + id: str + livemode: Optional[bool] = None + object: Object1 + pending_webhooks: Optional[int] = None + request: Optional[StripeRequestInfo] = None + type: Type4 + + +class Node(BaseModel): + author: Optional[str] = None + category: Optional[str] = Field(None, description='The category of the node.') + description: Optional[str] = None + downloads: Optional[int] = Field( + None, description='The number of downloads of the node.' + ) + icon: Optional[str] = Field(None, description="URL to the node's icon.") + id: Optional[str] = Field(None, description='The unique identifier of the node.') + latest_version: Optional[NodeVersion] = None + license: Optional[str] = Field( + None, description="The path to the LICENSE file in the node's repository." + ) + name: Optional[str] = Field(None, description='The display name of the node.') + publisher: Optional[Publisher] = None + rating: Optional[float] = Field(None, description='The average rating of the node.') + repository: Optional[str] = Field(None, description="URL to the node's repository.") + status: Optional[NodeStatus] = None + status_detail: Optional[str] = Field( + None, description='The status detail of the node.' + ) + tags: Optional[List[str]] = None + translations: Optional[Dict[str, Dict[str, Any]]] = None diff --git a/comfy_api_nodes/nodes_veo2.py b/comfy_api_nodes/nodes_veo2.py new file mode 100644 index 000000000..c6defe00a --- /dev/null +++ b/comfy_api_nodes/nodes_veo2.py @@ -0,0 +1,274 @@ +import io +import logging +import base64 +import requests +import math +import torch +import numpy as np +from PIL import Image + +from comfy.comfy_types.node_typing import IO, ComfyNodeABC +from comfy.utils import common_upscale +from comfy_api.input_impl.video_types import VideoFromFile +from comfy_api_nodes.apis import ( + Veo2GenVidRequest, + Veo2GenVidResponse, + Veo2GenVidPollRequest, + Veo2GenVidPollResponse +) +from comfy_api_nodes.apis.client import ( + ApiEndpoint, + HttpMethod, + SynchronousOperation, +) + +def downscale_input(image, total_pixels=1536*1024): + samples = image.movedim(-1,1) + # Downscaling input images to roughly the same size as the outputs + total = int(total_pixels) + scale_by = math.sqrt(total / (samples.shape[3] * samples.shape[2])) + if scale_by >= 1: + return image + width = round(samples.shape[3] * scale_by) + height = round(samples.shape[2] * scale_by) + + s = common_upscale(samples, width, height, "lanczos", "disabled") + s = s.movedim(1,-1) + return s + +class VeoVideoGenerationNode(ComfyNodeABC): + """ + Generates videos from text prompts using Google's Veo API. + + This node can create videos from text descriptions and optional image inputs, + with control over parameters like aspect ratio, duration, and more. + """ + + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "prompt": ( + IO.STRING, + { + "multiline": True, + "default": "", + "tooltip": "Text description of the video", + }, + ), + "aspect_ratio": ( + IO.COMBO, + { + "options": ["16:9", "9:16"], + "default": "16:9", + "tooltip": "Aspect ratio of the output video", + }, + ), + }, + "optional": { + "negative_prompt": ( + IO.STRING, + { + "multiline": True, + "default": "", + "tooltip": "Negative text prompt to guide what to avoid in the video", + }, + ), + "duration_seconds": ( + IO.INT, + { + "default": 5, + "min": 5, + "max": 8, + "step": 1, + "display": "number", + "tooltip": "Duration of the output video in seconds", + }, + ), + "enhance_prompt": ( + IO.BOOLEAN, + { + "default": True, + "tooltip": "Whether to enhance the prompt with AI assistance", + } + ), + "person_generation": ( + IO.COMBO, + { + "options": ["ALLOW", "BLOCK"], + "default": "ALLOW", + "tooltip": "Whether to allow generating people in the video", + }, + ), + "seed": ( + IO.INT, + { + "default": 0, + "min": 0, + "max": 0xFFFFFFFF, + "step": 1, + "display": "number", + "tooltip": "Seed for video generation (0 for random)", + }, + ), + "image": (IO.IMAGE, { + "default": None, + "tooltip": "Optional reference image to guide video generation", + }), + }, + "hidden": { + "auth_token": "AUTH_TOKEN_COMFY_ORG", + }, + } + + RETURN_TYPES = (IO.VIDEO,) + FUNCTION = "generate_video" + CATEGORY = "api node/video" + DESCRIPTION = "Generates videos from text prompts using Google's Veo API" + API_NODE = True + + def _convert_image_to_base64(self, image: torch.Tensor): + if image is None: + return None + + scaled_image = downscale_input(image, total_pixels=2048*2048) + + # Remove batch dimension if present + if len(scaled_image.shape) > 3: + scaled_image = scaled_image[0] + + # Convert to numpy array and then to PIL Image + image_np = (scaled_image.numpy() * 255).astype(np.uint8) + img = Image.fromarray(image_np) + + # Convert to base64 + buffer = io.BytesIO() + img.save(buffer, format="PNG") + return base64.b64encode(buffer.getvalue()).decode('utf-8') + + def generate_video( + self, + prompt, + aspect_ratio="16:9", + negative_prompt="", + duration_seconds=5, + enhance_prompt=True, + person_generation="ALLOW", + seed=0, + image=None, + auth_token=None, + ): + # Prepare the instances for the request + instances = [] + + instance = { + "prompt": prompt + } + + # Add image if provided + if image is not None: + image_base64 = self._convert_image_to_base64(image) + if image_base64: + instance["image"] = { + "bytesBase64Encoded": image_base64, + "mimeType": "image/png" + } + + instances.append(instance) + + # Create parameters dictionary + parameters = { + "aspectRatio": aspect_ratio, + "personGeneration": person_generation, + "durationSeconds": duration_seconds, + "enhancePrompt": enhance_prompt, + } + + # Add optional parameters if provided + if negative_prompt: + parameters["negativePrompt"] = negative_prompt + if seed > 0: + parameters["seed"] = seed + + # Initial request to start video generation + initial_operation = SynchronousOperation( + endpoint=ApiEndpoint( + path="/proxy/veo/generate", + method=HttpMethod.POST, + request_model=Veo2GenVidRequest, + response_model=Veo2GenVidResponse + ), + request=Veo2GenVidRequest( + instances=instances, + parameters=parameters + ), + auth_token=auth_token + ) + + initial_response = initial_operation.execute() + operation_name = initial_response.name + + logging.info(f"Veo generation started with operation name: {operation_name}") + + # Poll until operation is complete + video_data = None + while True: + poll_operation = SynchronousOperation( + endpoint=ApiEndpoint( + path="/proxy/veo/poll", + method=HttpMethod.POST, + request_model=Veo2GenVidPollRequest, + response_model=Veo2GenVidPollResponse + ), + request=Veo2GenVidPollRequest( + operationName=operation_name + ), + auth_token=auth_token + ) + + poll_response = poll_operation.execute() + + if poll_response.done: + if poll_response.response and poll_response.response.videos and len(poll_response.response.videos) > 0: + video = poll_response.response.videos[0] + + # Check if video is provided as base64 or URL + if hasattr(video, 'bytesBase64Encoded') and video.bytesBase64Encoded: + # Decode base64 string to bytes + video_data = base64.b64decode(video.bytesBase64Encoded) + break + elif hasattr(video, 'gcsUri') and video.gcsUri: + # Download from URL + video_url = video.gcsUri + video_response = requests.get(video_url) + video_data = video_response.content + break + else: + raise Exception("Video returned but no data or URL was provided") + else: + raise Exception("Video generation completed but no video was returned") + + # Wait before polling again + import time + time.sleep(5) + + if not video_data: + raise Exception("No video data was returned") + + logging.info("Video generation completed successfully") + + # Convert video data to BytesIO object + video_io = io.BytesIO(video_data) + + # Return VideoFromFile object + return (VideoFromFile(video_io),) + + +# Register the node +NODE_CLASS_MAPPINGS = { + "VeoVideoGenerationNode": VeoVideoGenerationNode, +} + +NODE_DISPLAY_NAME_MAPPINGS = { + "VeoVideoGenerationNode": "Google Veo2 Video Generation", +} diff --git a/nodes.py b/nodes.py index 81428c38d..c7b6daf66 100644 --- a/nodes.py +++ b/nodes.py @@ -2263,11 +2263,7 @@ def init_builtin_extra_nodes(): api_nodes_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), "comfy_api_nodes") api_nodes_files = [ "nodes_api.py", - ] - - api_nodes_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), "comfy_api_nodes") - api_nodes_files = [ - "nodes_api.py", + "nodes_veo2.py" ] import_failed = [] diff --git a/requirements.txt b/requirements.txt index 10cc177af..f64a05947 100644 --- a/requirements.txt +++ b/requirements.txt @@ -22,5 +22,5 @@ psutil kornia>=0.7.1 spandrel soundfile -av>=14.1.0 +av>=14.2.0 pydantic~=2.0 From e92f6e1c727d8680b48200f5007169adf7e6eac8 Mon Sep 17 00:00:00 2001 From: Jedrzej Kosinski Date: Tue, 29 Apr 2025 14:33:00 -0500 Subject: [PATCH 025/121] Add Recraft nodes (#29) --- comfy_api_nodes/apis/recraft_api.py | 180 ++++++++++++++++++++++++++++ comfy_api_nodes/nodes_api.py | 152 +++++++++++++++++++++++ 2 files changed, 332 insertions(+) create mode 100644 comfy_api_nodes/apis/recraft_api.py diff --git a/comfy_api_nodes/apis/recraft_api.py b/comfy_api_nodes/apis/recraft_api.py new file mode 100644 index 000000000..7defea873 --- /dev/null +++ b/comfy_api_nodes/apis/recraft_api.py @@ -0,0 +1,180 @@ +from __future__ import annotations + + + +from enum import Enum +from typing import Optional + +from pydantic import BaseModel, Field, conint + + +class RecraftStyle: + def __init__(self, style: str, substyle: str=None): + self.style = style + self.substyle = substyle + + +class RecraftIO: + STYLEV3 = "RECRAFT_V3_STYLE" + + +class RecraftStyleV3(str, Enum): + #any = 'any' NOTE: this does not work for some reason... why? + realistic_image = 'realistic_image' + digital_illustration = 'digital_illustration' + vector_illustration = 'vector_illustration' + logo_raster = 'logo_raster' + + +def get_v3_substyles(style_v3: str, include_none=True) -> list[str]: + substyles: list[str] = [] + if include_none: + substyles.append("None") + return substyles + dict_recraft_substyles_v3.get(style_v3, []) + + +dict_recraft_substyles_v3 = { + RecraftStyleV3.realistic_image: [ + "b_and_w", + "enterprise", + "evening_light", + "faded_nostalgia", + "forest_life", + "hard_flash", + "hdr", + "motion_blur", + "mystic_naturalism", + "natural_light", + "natural_tones", + "organic_calm", + "real_life_glow", + "retro_realism", + "retro_snapshot", + "studio_portrait", + "urban_drama", + "village_realism", + "warm_folk" + ], + RecraftStyleV3.digital_illustration: [ + "2d_art_poster", + "2d_art_poster_2", + "antiquarian", + "bold_fantasy", + "child_book", + "child_books", + "cover", + "crosshatch", + "digital_engraving", + "engraving_color", + "expressionism", + "freehand_details", + "grain", + "grain_20", + "graphic_intensity", + "hand_drawn", + "hand_drawn_outline", + "handmade_3d", + "hard_comics", + "infantile_sketch", + "long_shadow", + "modern_folk", + "multicolor", + "neon_calm", + "noir", + "nostalgic_pastel", + "outline_details", + "pastel_gradient", + "pastel_sketch", + "pixel_art", + "plastic", + "pop_art", + "pop_renaissance", + "seamless", + "street_art", + "tablet_sketch", + "urban_glow", + "urban_sketching", + "vanilla_dreams", + "young_adult_book", + "young_adult_book_2" + ], + RecraftStyleV3.vector_illustration: [ + "bold_stroke", + "chemistry", + "colored_stencil", + "contour_pop_art", + "cosmics", + "cutout", + "depressive", + "editorial", + "emotional_flat", + "engraving", + "infographical", + "line_art", + "line_circuit", + "linocut", + "marker_outline", + "mosaic", + "naivector", + "roundish_flat", + "seamless", + "segmented_colors", + "sharp_contrast", + "thin", + "vector_photo", + "vivid_shapes" + ], + RecraftStyleV3.logo_raster: [ + "emblem_graffiti", + "emblem_pop_art", + "emblem_punk", + "emblem_stamp", + "emblem_vintage" + ], +} + + +class RecraftModel(str, Enum): + recraftv3 = 'recraftv3' + recraftv2 = 'recraftv2' + + +class RecraftImageSize(str, Enum): + res_1024x1024 = '1024x1024' + res_1365x1024 = '1365x1024' + res_1024x1365 = '1024x1365' + res_1536x1024 = '1536x1024' + res_1024x1536 = '1024x1536' + res_1820x1024 = '1820x1024' + res_1024x1820 = '1024x1820' + res_1024x2048 = '1024x2048' + res_2048x1024 = '2048x1024' + res_1434x1024 = '1434x1024' + res_1024x1434 = '1024x1434' + res_1024x1280 = '1024x1280' + res_1280x1024 = '1280x1024' + res_1024x1707 = '1024x1707' + res_1707x1024 = '1707x1024' + + +class RecraftImageGenerationRequest(BaseModel): + prompt: str = Field(..., description='The text prompt describing the image to generate') + size: RecraftImageSize = Field(..., description='The size of the generated image (e.g., "1024x1024")') + n: conint(ge=1, le=6) = Field(..., description='The number of images to generate') + negative_prompts: Optional[str] = Field(None, description='A text description of undesired elements on an image') + model: Optional[RecraftModel] = Field(RecraftModel.recraftv3, description='The model to use for generation (e.g., "recraftv3")') + style: Optional[str] = Field(None, description='The style to apply to the generated image (e.g., "digital_illustration")') + substyle: Optional[str] = Field(None, description='The substyle to apply to the generated image, depending on the style input') + # text_layout + # controls + + +class RecraftReturnedObject(BaseModel): + image_id: str = Field(..., description='Unique identifier for the generated image') + url: str = Field(..., description='URL to access the generated image') + + +class RecraftImageGenerationResponse(BaseModel): + created: int = Field(..., description='Unix timestamp when the generation was created') + credits: int = Field(..., description='Number of credits used for the generation') + data: list[RecraftReturnedObject] = Field(..., description=' Array of generated image information') diff --git a/comfy_api_nodes/nodes_api.py b/comfy_api_nodes/nodes_api.py index f6a628328..f9f6e3a0c 100644 --- a/comfy_api_nodes/nodes_api.py +++ b/comfy_api_nodes/nodes_api.py @@ -40,6 +40,16 @@ from comfy_api_nodes.apis.luma_api import ( LumaKeyframes, LumaIO, ) +from comfy_api_nodes.apis.recraft_api import ( + RecraftImageGenerationRequest, + RecraftImageGenerationResponse, + RecraftImageSize, + RecraftModel, + RecraftStyle, + RecraftStyleV3, + RecraftIO, + get_v3_substyles, +) from comfy_api_nodes.apis.client import ApiClient, ApiEndpoint, HttpMethod, SynchronousOperation, PollingOperation, EmptyRequest, UploadRequest, UploadResponse import numpy as np @@ -1400,6 +1410,138 @@ class LumaImageToVideoGenerationNode: frame1 = LumaImageReference(type='image', url=download_urls[0]) return LumaKeyframes(frame0=frame0, frame1=frame1) + +class RecraftStyleV3RealisticImageNode: + """ + Select realistic_image style and optional substyle. + """ + + RETURN_TYPES = (RecraftIO.STYLEV3,) + RETURN_NAMES = ("recraft_style",) + FUNCTION = "create_style" + CATEGORY = "api node/Recraft" + + RECRAFT_STYLE = RecraftStyleV3.realistic_image + + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "substyle": (get_v3_substyles(s.RECRAFT_STYLE),), + } + } + + def create_style(self, substyle: str): + if substyle == "None": + substyle = None + return (RecraftStyle(self.RECRAFT_STYLE, substyle),) + +class RecraftStyleV3DigitalIllustrationNode(RecraftStyleV3RealisticImageNode): + """ + Select digital_illustration style and optional substyle. + """ + RECRAFT_STYLE = RecraftStyleV3.digital_illustration + +class RecraftStyleV3VectorIllustrationNode(RecraftStyleV3RealisticImageNode): + """ + Select vector_illustration style and optional substyle. + """ + RECRAFT_STYLE = RecraftStyleV3.vector_illustration + +class RecraftStyleV3LogoRasterNode(RecraftStyleV3RealisticImageNode): + """ + Select vector_illustration style and optional substyle. + """ + RECRAFT_STYLE = RecraftStyleV3.logo_raster + +class RecraftTextToImageNode: + """ + Generates images synchronously based on prompt and resolution. + """ + + RETURN_TYPES = (IO.IMAGE,) + DESCRIPTION = cleandoc(__doc__ or "") # Handle potential None value + FUNCTION = "api_call" + API_NODE = True + CATEGORY = "api node" + + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "prompt": (IO.STRING, { + "multiline": True, + "default": "", + "tooltip": "Prompt for the image generation.", + }), + "size": ([res.value for res in RecraftImageSize], { + "default": RecraftImageSize.res_1024x1024, + "tooltip": "The size of the generated image." + }), + "n": (IO.INT, { + "default": 1, + "min": 1, + "max": 6, + "tooltip": "The number of images to generate." + }), + "seed": (IO.INT, { + "default": 0, + "min": 0, + "max": 0xFFFFFFFFFFFFFFFF, + "control_after_generate": True, + "tooltip": "Seed to determine if node should re-run; actual results are nondeterministic regardless of seed.", + }), + }, + "optional": { + "recraft_style": (RecraftIO.STYLEV3,), + "negative_prompt": (IO.STRING, { + "default": "", + "forceInput": True, + "tooltip": "An optional text description of undesired elements on an image." + }), + }, + "hidden": { + "auth_token": "AUTH_TOKEN_COMFY_ORG", + } + } + + def api_call(self, prompt: str, size: str, n: int, seed, recraft_style: RecraftStyle=None, negative_prompt: str=None, auth_token=None, **kwargs): + default_style = RecraftStyle(RecraftStyleV3.digital_illustration) + if recraft_style is None: + recraft_style = default_style + + if not negative_prompt: + negative_prompt = None + + operation = SynchronousOperation( + endpoint=ApiEndpoint( + path="/proxy/recraft/image_generation", + method=HttpMethod.POST, + request_model=RecraftImageGenerationRequest, + response_model=RecraftImageGenerationResponse + ), + request=RecraftImageGenerationRequest( + prompt=prompt, + negative_prompts=negative_prompt, + model=RecraftModel.recraftv3, + size=size, + n=n, + style=recraft_style.style, + substyle=recraft_style.substyle + ), + auth_token=auth_token + ) + response: RecraftImageGenerationResponse = operation.execute() + images = [] + for data in response.data: + image = bytesio_to_image_tensor(download_url_to_bytesio(data.url, timeout=1024)) + if len(image.shape) < 4: + image = image.unsqueeze(0) + images.append(image) + output_image = torch.cat(images, dim=0) + + return (output_image, ) + class MinimaxTextToVideoNode: """ Generates videos synchronously based on a prompt, and optional parameters using Minimax's API. @@ -1581,6 +1723,11 @@ NODE_CLASS_MAPPINGS = { "LumaReferenceNode": LumaReferenceNode, "LumaVideoNode": LumaTextToVideoGenerationNode, "LumaImageToVideoNode": LumaImageToVideoGenerationNode, + "RecraftTextToImageNode": RecraftTextToImageNode, + #"RecraftStyleV3RealisticImage": RecraftStyleV3RealisticImageNode, + "RecraftStyleV3DigitalIllustration": RecraftStyleV3DigitalIllustrationNode, + #"RecraftStyleV3VectorIllustration": RecraftStyleV3VectorIllustrationNode, + #"RecraftStyleV3LogoRaster": RecraftStyleV3LogoRasterNode, "MinimaxTextToVideoNode": MinimaxTextToVideoNode, } @@ -1596,5 +1743,10 @@ NODE_DISPLAY_NAME_MAPPINGS = { "LumaReferenceNode": "Luma Reference", "LumaVideoNode": "Luma Text to Video", "LumaImageToVideoNode": "Luma Image to Video", + "RecraftTextToImageNode": "Recraft Text to Image", + "RecraftStyleV3RealisticImage": "Recraft Style - Realistic Image", + "RecraftStyleV3DigitalIllustration": "Recraft Style - Digital Illustration", + "RecraftStyleV3VectorIllustration": "Recraft Style - Vector Illustration", + "RecraftStyleV3LogoRaster": "Recraft Style - Logo Raster", "MinimaxTextToVideoNode": "Minimax Text to Video", } From 4a67cdb23a9be93ce674d4d562dd3eded543894e Mon Sep 17 00:00:00 2001 From: Christian Byrne Date: Tue, 29 Apr 2025 12:44:23 -0700 Subject: [PATCH 026/121] Add Kling Nodes (#12) --- comfy_api_nodes/apis/__init__.py | 2286 ++++++++---------------------- comfy_api_nodes/apis/client.py | 1 - comfy_api_nodes/nodes_kling.py | 479 +++++++ nodes.py | 3 +- 4 files changed, 1070 insertions(+), 1699 deletions(-) create mode 100644 comfy_api_nodes/nodes_kling.py diff --git a/comfy_api_nodes/apis/__init__.py b/comfy_api_nodes/apis/__init__.py index 562a7addb..7c41e936d 100644 --- a/comfy_api_nodes/apis/__init__.py +++ b/comfy_api_nodes/apis/__init__.py @@ -1,123 +1,14 @@ # generated by datamodel-codegen: -# filename: https://stagingapi.comfy.org/openapi -# timestamp: 2025-04-29T03:13:19+00:00 +# filename: filtered-openapi.yaml +# timestamp: 2025-04-29T19:38:18+00:00 from __future__ import annotations from datetime import datetime from enum import Enum -from typing import Any, Dict, List, Literal, Optional, Union -from uuid import UUID +from typing import Any, Dict, List, Optional, Union -from pydantic import AnyUrl, BaseModel, Field, RootModel, confloat, conint, constr - - -class BFLFluxProGenerateRequest(BaseModel): - guidance_scale: Optional[confloat(ge=1.0, le=20.0)] = Field( - None, description='The guidance scale for generation.' - ) - height: conint(ge=64, le=2048) = Field( - ..., description='The height of the image to generate.' - ) - negative_prompt: Optional[str] = Field( - None, description='The negative prompt for image generation.' - ) - num_images: Optional[conint(ge=1, le=4)] = Field( - None, description='The number of images to generate.' - ) - num_inference_steps: Optional[conint(ge=1, le=100)] = Field( - None, description='The number of inference steps.' - ) - prompt: str = Field(..., description='The text prompt for image generation.') - seed: Optional[int] = Field(None, description='The seed value for reproducibility.') - width: conint(ge=64, le=2048) = Field( - ..., description='The width of the image to generate.' - ) - - -class BFLFluxProGenerateResponse(BaseModel): - id: str = Field(..., description='The unique identifier for the generation task.') - polling_url: str = Field(..., description='URL to poll for the generation result.') - - -class ComfyNode(BaseModel): - category: Optional[str] = Field( - None, - description='UI category where the node is listed, used for grouping nodes.', - ) - comfy_node_name: Optional[str] = Field( - None, description='Unique identifier for the node' - ) - deprecated: Optional[bool] = Field( - None, - description='Indicates if the node is deprecated. Deprecated nodes are hidden in the UI.', - ) - description: Optional[str] = Field( - None, description="Brief description of the node's functionality or purpose." - ) - experimental: Optional[bool] = Field( - None, - description='Indicates if the node is experimental, subject to changes or removal.', - ) - function: Optional[str] = Field( - None, description='Name of the entry-point function to execute the node.' - ) - input_types: Optional[str] = Field(None, description='Defines input parameters') - output_is_list: Optional[List[bool]] = Field( - None, description='Boolean values indicating if each output is a list.' - ) - return_names: Optional[str] = Field( - None, description='Names of the outputs for clarity in workflows.' - ) - return_types: Optional[str] = Field( - None, description='Specifies the types of outputs produced by the node.' - ) - - -class ComfyNodeCloudBuildInfo(BaseModel): - build_id: Optional[str] = None - location: Optional[str] = None - project_id: Optional[str] = None - project_number: Optional[str] = None - - -class Customer(BaseModel): - createdAt: Optional[datetime] = Field( - None, description='The date and time the user was created' - ) - email: Optional[str] = Field(None, description='The email address for this user') - id: str = Field(..., description='The firebase UID of the user') - name: Optional[str] = Field(None, description='The name for this user') - updatedAt: Optional[datetime] = Field( - None, description='The date and time the user was last updated' - ) - - -class CustomerStorageResourceResponse(BaseModel): - download_url: Optional[str] = Field( - None, - description='The signed URL to use for downloading the file from the specified path', - ) - existing_file: Optional[bool] = Field( - None, description='Whether an existing file with the same hash was found' - ) - expires_at: Optional[datetime] = Field( - None, description='When the signed URL will expire' - ) - upload_url: Optional[str] = Field( - None, - description='The signed URL to use for uploading the file to the specified path', - ) - - -class Error(BaseModel): - details: Optional[List[str]] = Field( - None, - description='Optional detailed information about the error or hints for resolving it.', - ) - message: Optional[str] = Field( - None, description='A clear and concise description of the error.' - ) +from pydantic import AnyUrl, BaseModel, Field, RootModel class ErrorResponse(BaseModel): @@ -125,53 +16,44 @@ class ErrorResponse(BaseModel): message: str -class GitCommitSummary(BaseModel): - author: Optional[str] = Field(None, description='The author of the commit') - branch_name: Optional[str] = Field( - None, description='The branch where the commit was made' - ) - commit_hash: Optional[str] = Field(None, description='The hash of the commit') - commit_name: Optional[str] = Field(None, description='The name of the commit') - status_summary: Optional[Dict[str, str]] = Field( - None, description='A map of operating system to status pairs' - ) - timestamp: Optional[datetime] = Field( - None, description='The timestamp when the commit was made' - ) - - class ImageRequest(BaseModel): + prompt: str = Field( + ..., description='Required. The prompt to use to generate the image.' + ) aspect_ratio: Optional[str] = Field( None, description="Optional. The aspect ratio (e.g., 'ASPECT_16_9', 'ASPECT_1_1'). Cannot be used with resolution. Defaults to 'ASPECT_1_1' if unspecified.", ) - color_palette: Optional[Dict[str, Any]] = Field( - None, description='Optional. Color palette object. Only for V_2, V_2_TURBO.' - ) + model: str = Field(..., description="The model used (e.g., 'V_2', 'V_2A_TURBO')") magic_prompt_option: Optional[str] = Field( None, description="Optional. MagicPrompt usage ('AUTO', 'ON', 'OFF')." ) - model: str = Field(..., description="The model used (e.g., 'V_2', 'V_2A_TURBO')") + seed: Optional[int] = Field( + None, + description='Optional. A number between 0 and 2147483647.', + ge=0, + le=2147483647, + ) + style_type: Optional[str] = Field( + None, + description="Optional. Style type ('AUTO', 'GENERAL', 'REALISTIC', 'DESIGN', 'RENDER_3D', 'ANIME'). Only for models V_2 and above.", + ) negative_prompt: Optional[str] = Field( None, description='Optional. Description of what to exclude. Only for V_1, V_1_TURBO, V_2, V_2_TURBO.', ) - num_images: Optional[conint(ge=1, le=8)] = Field( - 1, description='Optional. Number of images to generate (1-8). Defaults to 1.' - ) - prompt: str = Field( - ..., description='Required. The prompt to use to generate the image.' + num_images: Optional[int] = Field( + 1, + description='Optional. Number of images to generate (1-8). Defaults to 1.', + ge=1, + le=8, ) resolution: Optional[str] = Field( None, description="Optional. Resolution (e.g., 'RESOLUTION_1024_1024'). Only for model V_2. Cannot be used with aspect_ratio.", ) - seed: Optional[conint(ge=0, le=2147483647)] = Field( - None, description='Optional. A number between 0 and 2147483647.' - ) - style_type: Optional[str] = Field( - None, - description="Optional. Style type ('AUTO', 'GENERAL', 'REALISTIC', 'DESIGN', 'RENDER_3D', 'ANIME'). Only for models V_2 and above.", + color_palette: Optional[Dict[str, Any]] = Field( + None, description='Optional. Color palette object. Only for V_2, V_2_TURBO.' ) @@ -182,23 +64,23 @@ class IdeogramGenerateRequest(BaseModel): class Datum(BaseModel): - is_image_safe: Optional[bool] = Field( - None, description='Indicates whether the image is considered safe.' - ) prompt: Optional[str] = Field( None, description='The prompt used to generate this image.' ) resolution: Optional[str] = Field( None, description="The resolution of the generated image (e.g., '1024x1024')." ) + is_image_safe: Optional[bool] = Field( + None, description='Indicates whether the image is considered safe.' + ) seed: Optional[int] = Field( None, description='The seed value used for this generation.' ) + url: Optional[str] = Field(None, description='URL to the generated image.') style_type: Optional[str] = Field( None, description="The style type used for generation (e.g., 'REALISTIC', 'ANIME').", ) - url: Optional[str] = Field(None, description='URL to the generated image.') class IdeogramGenerateResponse(BaseModel): @@ -210,9 +92,9 @@ class IdeogramGenerateResponse(BaseModel): ) -class Duration(str, Enum): - field_5 = '5' - field_10 = '10' +class ModelName(str, Enum): + kling_v1 = 'kling-v1' + kling_v1_6 = 'kling-v1-6' class Mode(str, Enum): @@ -220,77 +102,6 @@ class Mode(str, Enum): pro = 'pro' -class ModelName(str, Enum): - kling_v1 = 'kling-v1' - kling_v1_5 = 'kling-v1-5' - kling_v1_6 = 'kling-v1-6' - - -class KlingDualCharacterEffectInput(BaseModel): - duration: Duration = Field( - ..., - description='Video Length in seconds. Both 5 and 10-second videos are supported.', - ) - images: List[str] = Field( - ..., - description='Reference Image Group. Must contain exactly 2 images. First image will be positioned on left side, second on right side of the composite. Each image follows the same requirements as single image effects.', - max_length=2, - min_length=2, - ) - mode: Optional[Mode] = Field( - 'std', - description='Video generation mode. std (Standard Mode) is cost-effective, pro (Professional Mode) generates videos with longer duration and higher quality.', - ) - model_name: Optional[ModelName] = Field( - 'kling-v1', - description='Model Name. Can be kling-v1, kling-v1-5, or kling-v1-6.', - ) - - -class KlingErrorResponse(BaseModel): - code: int = Field( - ..., - description='- 1000: Authentication failed\n- 1001: Authorization is empty\n- 1002: Authorization is invalid\n- 1003: Authorization is not yet valid\n- 1004: Authorization has expired\n- 1100: Account exception\n- 1101: Account in arrears (postpaid scenario)\n- 1102: Resource pack depleted or expired (prepaid scenario)\n- 1103: Unauthorized access to requested resource\n- 1200: Invalid request parameters\n- 1201: Invalid parameters\n- 1202: Invalid request method\n- 1203: Requested resource does not exist\n- 1300: Trigger platform strategy\n- 1301: Trigger content security policy\n- 1302: API request too frequent\n- 1303: Concurrency/QPS exceeds limit\n- 1304: Trigger IP whitelist policy\n- 5000: Internal server error\n- 5001: Service temporarily unavailable\n- 5002: Server internal timeout\n', - ) - message: str = Field(..., description='Human-readable error message') - request_id: str = Field( - ..., description='Request ID for tracking and troubleshooting' - ) - - -class AspectRatio(str, Enum): - field_16_9 = '16:9' - field_9_16 = '9:16' - field_1_1 = '1:1' - - -class Config(BaseModel): - horizontal: Optional[confloat(ge=-10.0, le=10.0)] = Field( - None, - description="Controls camera's movement along horizontal axis (x-axis). Negative indicates left, positive indicates right.", - ) - pan: Optional[confloat(ge=-10.0, le=10.0)] = Field( - None, - description="Controls camera's rotation in vertical plane (x-axis). Negative indicates downward rotation, positive indicates upward rotation.", - ) - roll: Optional[confloat(ge=-10.0, le=10.0)] = Field( - None, - description="Controls camera's rolling amount (z-axis). Negative indicates counterclockwise, positive indicates clockwise.", - ) - tilt: Optional[confloat(ge=-10.0, le=10.0)] = Field( - None, - description="Controls camera's rotation in horizontal plane (y-axis). Negative indicates left rotation, positive indicates right rotation.", - ) - vertical: Optional[confloat(ge=-10.0, le=10.0)] = Field( - None, - description="Controls camera's movement along vertical axis (y-axis). Negative indicates downward, positive indicates upward.", - ) - zoom: Optional[confloat(ge=-10.0, le=10.0)] = Field( - None, - description="Controls change in camera's focal length. Negative indicates narrower field of view, positive indicates wider field of view.", - ) - - class Type(str, Enum): simple = 'simple' down_back = 'down_back' @@ -299,12 +110,93 @@ class Type(str, Enum): left_turn_forward = 'left_turn_forward' +class Config(BaseModel): + horizontal: Optional[float] = Field(None, ge=-10.0, le=10.0) + vertical: Optional[float] = Field(None, ge=-10.0, le=10.0) + pan: Optional[float] = Field(None, ge=-10.0, le=10.0) + tilt: Optional[float] = Field(None, ge=-10.0, le=10.0) + roll: Optional[float] = Field(None, ge=-10.0, le=10.0) + zoom: Optional[float] = Field(None, ge=-10.0, le=10.0) + + class CameraControl(BaseModel): + type: Optional[Type] = Field(None, description='Predefined camera movements type') config: Optional[Config] = None - type: Optional[Type] = Field( - None, - description='Predefined camera movements type. simple: Customizable camera movement. down_back: Camera descends and moves backward. forward_up: Camera moves forward and tilts up. right_turn_forward: Rotate right and move forward. left_turn_forward: Rotate left and move forward.', + + +class AspectRatio(str, Enum): + field_16_9 = '16:9' + field_9_16 = '9:16' + field_1_1 = '1:1' + + +class Duration(str, Enum): + field_5 = '5' + field_10 = '10' + + +class KlingText2VideoRequest(BaseModel): + model_name: Optional[ModelName] = Field('kling-v1', description='Model Name') + prompt: Optional[str] = Field( + None, description='Positive text prompt', max_length=2500 ) + negative_prompt: Optional[str] = Field( + None, description='Negative text prompt', max_length=2500 + ) + cfg_scale: Optional[float] = Field( + 0.5, description='Flexibility in video generation', ge=0.0, le=1.0 + ) + mode: Optional[Mode] = Field('std', description='Video generation mode') + camera_control: Optional[CameraControl] = None + aspect_ratio: Optional[AspectRatio] = '16:9' + duration: Optional[Duration] = '5' + callback_url: Optional[AnyUrl] = Field( + None, description='The callback notification address' + ) + external_task_id: Optional[str] = Field(None, description='Customized Task ID') + + +class TaskStatus(str, Enum): + submitted = 'submitted' + processing = 'processing' + succeed = 'succeed' + failed = 'failed' + + +class TaskInfo(BaseModel): + external_task_id: Optional[str] = None + + +class Video(BaseModel): + id: Optional[str] = Field(None, description='Generated video ID') + url: Optional[AnyUrl] = Field(None, description='URL for generated video') + duration: Optional[str] = Field(None, description='Total video duration') + + +class TaskResult(BaseModel): + videos: Optional[List[Video]] = None + + +class Data(BaseModel): + task_id: Optional[str] = Field(None, description='Task ID') + task_status: Optional[TaskStatus] = None + task_info: Optional[TaskInfo] = None + created_at: Optional[int] = Field(None, description='Task creation time') + updated_at: Optional[int] = Field(None, description='Task update time') + task_result: Optional[TaskResult] = None + + +class KlingText2VideoResponse(BaseModel): + code: Optional[int] = Field(None, description='Error code') + message: Optional[str] = Field(None, description='Error message') + request_id: Optional[str] = Field(None, description='Request ID') + data: Optional[Data] = None + + +class ModelName1(str, Enum): + kling_v1 = 'kling-v1' + kling_v1_5 = 'kling-v1-5' + kling_v1_6 = 'kling-v1-6' class Trajectory(BaseModel): @@ -326,761 +218,118 @@ class DynamicMask(BaseModel): trajectories: Optional[List[Trajectory]] = None +class Config1(BaseModel): + horizontal: Optional[float] = Field( + None, + description="Controls camera's movement along horizontal axis (x-axis). Negative indicates left, positive indicates right.", + ge=-10.0, + le=10.0, + ) + vertical: Optional[float] = Field( + None, + description="Controls camera's movement along vertical axis (y-axis). Negative indicates downward, positive indicates upward.", + ge=-10.0, + le=10.0, + ) + pan: Optional[float] = Field( + None, + description="Controls camera's rotation in vertical plane (x-axis). Negative indicates downward rotation, positive indicates upward rotation.", + ge=-10.0, + le=10.0, + ) + tilt: Optional[float] = Field( + None, + description="Controls camera's rotation in horizontal plane (y-axis). Negative indicates left rotation, positive indicates right rotation.", + ge=-10.0, + le=10.0, + ) + roll: Optional[float] = Field( + None, + description="Controls camera's rolling amount (z-axis). Negative indicates counterclockwise, positive indicates clockwise.", + ge=-10.0, + le=10.0, + ) + zoom: Optional[float] = Field( + None, + description="Controls change in camera's focal length. Negative indicates narrower field of view, positive indicates wider field of view.", + ge=-10.0, + le=10.0, + ) + + +class CameraControl1(BaseModel): + type: Optional[Type] = Field( + None, + description='Predefined camera movements type. simple: Customizable camera movement. down_back: Camera descends and moves backward. forward_up: Camera moves forward and tilts up. right_turn_forward: Rotate right and move forward. left_turn_forward: Rotate left and move forward.', + ) + config: Optional[Config1] = None + + class KlingImage2VideoRequest(BaseModel): - aspect_ratio: Optional[AspectRatio] = '16:9' - callback_url: Optional[AnyUrl] = Field( - None, - description='The callback notification address. Server will notify when the task status changes.', - ) - camera_control: Optional[CameraControl] = None - cfg_scale: Optional[confloat(ge=0.0, le=1.0)] = Field( - 0.5, - description="Flexibility in video generation. The higher the value, the lower the model's degree of flexibility, and the stronger the relevance to the user's prompt.", - ) - duration: Optional[Duration] = Field(5, description='Video length in seconds') - dynamic_masks: Optional[List[DynamicMask]] = Field( - None, - description='Dynamic Brush Configuration List (up to 6 groups). For 5-second videos, trajectory length must not exceed 77 coordinates.', - ) - external_task_id: Optional[str] = Field( - None, - description='Customized Task ID. Must be unique within a single user account.', - ) - image: Optional[AnyUrl] = Field( + model_name: Optional[ModelName1] = Field('kling-v1', description='Model Name') + image: Optional[str] = Field( None, description='Reference Image - URL or Base64 encoded string, cannot exceed 10MB, resolution not less than 300*300px, aspect ratio between 1:2.5 ~ 2.5:1. Base64 should not include data:image prefix.', ) - image_tail: Optional[AnyUrl] = Field( + image_tail: Optional[str] = Field( None, description='Reference Image - End frame control. URL or Base64 encoded string, cannot exceed 10MB, resolution not less than 300*300px. Base64 should not include data:image prefix.', ) + prompt: Optional[str] = Field( + None, description='Positive text prompt', max_length=2500 + ) + negative_prompt: Optional[str] = Field( + None, description='Negative text prompt', max_length=2500 + ) + cfg_scale: Optional[float] = Field( + 0.5, + description="Flexibility in video generation. The higher the value, the lower the model's degree of flexibility, and the stronger the relevance to the user's prompt.", + ge=0.0, + le=1.0, + ) mode: Optional[Mode] = Field( 'std', description='Video generation mode. std: Standard Mode, which is cost-effective. pro: Professional Mode, generates videos with longer duration but higher quality output.', ) - model_name: Optional[ModelName] = Field('kling-v1', description='Model Name') - negative_prompt: Optional[constr(max_length=2500)] = Field( - None, description='Negative text prompt' - ) - prompt: Optional[constr(max_length=2500)] = Field( - None, description='Positive text prompt' - ) static_mask: Optional[AnyUrl] = Field( None, description='Static Brush Application Area (Mask image created by users using the motion brush). The aspect ratio must match the input image.', ) - - -class TaskInfo(BaseModel): - external_task_id: Optional[str] = None - - -class Video(BaseModel): - duration: Optional[str] = Field(None, description='Total video duration') - id: Optional[str] = Field(None, description='Generated video ID') - url: Optional[AnyUrl] = Field(None, description='URL for generated video') - - -class TaskResult(BaseModel): - videos: Optional[List[Video]] = None - - -class TaskStatus(str, Enum): - submitted = 'submitted' - processing = 'processing' - succeed = 'succeed' - failed = 'failed' - - -class Data(BaseModel): - created_at: Optional[int] = Field(None, description='Task creation time') - task_id: Optional[str] = Field(None, description='Task ID') - task_info: Optional[TaskInfo] = None - task_result: Optional[TaskResult] = None - task_status: Optional[TaskStatus] = None - updated_at: Optional[int] = Field(None, description='Task update time') - - -class KlingImage2VideoResponse(BaseModel): - code: Optional[int] = Field(None, description='Error code') - data: Optional[Data] = None - message: Optional[str] = Field(None, description='Error message') - request_id: Optional[str] = Field(None, description='Request ID') - - -class AspectRatio1(str, Enum): - field_16_9 = '16:9' - field_9_16 = '9:16' - field_1_1 = '1:1' - field_4_3 = '4:3' - field_3_4 = '3:4' - field_3_2 = '3:2' - field_2_3 = '2:3' - field_21_9 = '21:9' - - -class ImageReference(str, Enum): - subject = 'subject' - face = 'face' - - -class ModelName2(str, Enum): - kling_v1 = 'kling-v1' - kling_v1_5 = 'kling-v1-5' - - -class KlingImageGenerationsRequest(BaseModel): - aspect_ratio: Optional[AspectRatio1] = Field( - '16:9', description='Aspect ratio of the generated images' - ) - callback_url: Optional[AnyUrl] = Field( - None, description='The callback notification address' - ) - human_fidelity: Optional[confloat(ge=0.0, le=1.0)] = Field( - 0.45, description='Subject reference similarity' - ) - image: Optional[str] = Field( - None, description='Reference Image - Base64 encoded string or image URL' - ) - image_fidelity: Optional[confloat(ge=0.0, le=1.0)] = Field( - 0.5, description='Reference intensity for user-uploaded images' - ) - image_reference: Optional[ImageReference] = Field( - None, description='Image reference type' - ) - model_name: Optional[ModelName2] = Field('kling-v1', description='Model Name') - n: Optional[conint(ge=1, le=9)] = Field(1, description='Number of generated images') - negative_prompt: Optional[constr(max_length=200)] = Field( - None, description='Negative text prompt' - ) - prompt: constr(max_length=500) = Field(..., description='Positive text prompt') - - -class Image(BaseModel): - index: Optional[int] = Field(None, description='Image Number (0-9)') - url: Optional[AnyUrl] = Field(None, description='URL for generated image') - - -class TaskResult1(BaseModel): - images: Optional[List[Image]] = None - - -class Data1(BaseModel): - created_at: Optional[int] = Field(None, description='Task creation time') - task_id: Optional[str] = Field(None, description='Task ID') - task_result: Optional[TaskResult1] = None - task_status: Optional[TaskStatus] = None - task_status_msg: Optional[str] = Field(None, description='Task status information') - updated_at: Optional[int] = Field(None, description='Task update time') - - -class KlingImageGenerationsResponse(BaseModel): - code: Optional[int] = Field(None, description='Error code') - data: Optional[Data1] = None - message: Optional[str] = Field(None, description='Error message') - request_id: Optional[str] = Field(None, description='Request ID') - - -class AudioType(str, Enum): - file = 'file' - url = 'url' - - -class Mode2(str, Enum): - text2video = 'text2video' - audio2video = 'audio2video' - - -class VoiceLanguage(str, Enum): - zh = 'zh' - en = 'en' - - -class Input(BaseModel): - audio_file: Optional[str] = Field( + dynamic_masks: Optional[List[DynamicMask]] = Field( None, - description='Local Path of Audio File. Supported formats: .mp3/.wav/.m4a/.aac, maximum file size of 5MB. Base64 code.', + description='Dynamic Brush Configuration List (up to 6 groups). For 5-second videos, trajectory length must not exceed 77 coordinates.', ) - audio_type: Optional[AudioType] = Field( - None, - description='Method of Transmitting Audio Files for Lip-Sync. Required when mode is audio2video.', - ) - audio_url: Optional[AnyUrl] = Field( - None, - description='Audio File Download URL. Supported formats: .mp3/.wav/.m4a/.aac, maximum file size of 5MB.', - ) - mode: Mode2 = Field( - ..., - description='Video Generation Mode. text2video: Text-to-video generation mode; audio2video: Audio-to-video generation mode', - ) - text: Optional[str] = Field( - None, - description='Text Content for Lip-Sync Video Generation. Required when mode is text2video. Maximum length is 120 characters.', - ) - video_id: Optional[str] = Field( - None, - description='The ID of the video generated by Kling AI. Only supports 5-second and 10-second videos generated within the last 30 days.', - ) - video_url: Optional[AnyUrl] = Field( - None, - description='Get link for uploaded video. Video files support .mp4/.mov, file size does not exceed 100MB, video length between 2-10s.', - ) - voice_id: Optional[str] = Field( - None, - description='Voice ID. Required when mode is text2video. The system offers a variety of voice options to choose from.', - ) - voice_language: Optional[VoiceLanguage] = Field( - 'zh', description='The voice language corresponds to the Voice ID.' - ) - voice_speed: Optional[confloat(ge=0.8, le=2.0)] = Field( - 1, - description='Speech Rate. Valid range: 0.8~2.0, accurate to one decimal place.', - ) - - -class KlingLipSyncRequest(BaseModel): + camera_control: Optional[CameraControl1] = None + aspect_ratio: Optional[AspectRatio] = '16:9' + duration: Optional[Duration] = Field('5', description='Video length in seconds') callback_url: Optional[AnyUrl] = Field( None, description='The callback notification address. Server will notify when the task status changes.', ) - input: Input - - -class TaskResult2(BaseModel): - videos: Optional[List[Video]] = None - - -class Data2(BaseModel): - created_at: Optional[int] = Field(None, description='Task creation time') - task_id: Optional[str] = Field(None, description='Task ID') - task_info: Optional[TaskInfo] = None - task_result: Optional[TaskResult2] = None - task_status: Optional[TaskStatus] = None - updated_at: Optional[int] = Field(None, description='Task update time') - - -class KlingLipSyncResponse(BaseModel): - code: Optional[int] = Field(None, description='Error code') - data: Optional[Data2] = None - message: Optional[str] = Field(None, description='Error message') - request_id: Optional[str] = Field(None, description='Request ID') - - -class ResourcePackType(str, Enum): - decreasing_total = 'decreasing_total' - constant_period = 'constant_period' - - -class Status(str, Enum): - toBeOnline = 'toBeOnline' - online = 'online' - expired = 'expired' - runOut = 'runOut' - - -class ResourcePackSubscribeInfo(BaseModel): - effective_time: Optional[int] = Field( - None, description='Effective time, Unix timestamp in ms' - ) - invalid_time: Optional[int] = Field( - None, description='Expiration time, Unix timestamp in ms' - ) - purchase_time: Optional[int] = Field( - None, description='Purchase time, Unix timestamp in ms' - ) - remaining_quantity: Optional[float] = Field( - None, description='Remaining quantity (updated with a 12-hour delay)' - ) - resource_pack_id: Optional[str] = Field(None, description='Resource package ID') - resource_pack_name: Optional[str] = Field(None, description='Resource package name') - resource_pack_type: Optional[ResourcePackType] = Field( - None, - description='Resource package type (decreasing_total=decreasing total, constant_period=constant periodicity)', - ) - status: Optional[Status] = Field(None, description='Resource Package Status') - total_quantity: Optional[float] = Field(None, description='Total quantity') - - -class Data3(BaseModel): - code: Optional[int] = Field(None, description='Error code; 0 indicates success') - msg: Optional[str] = Field(None, description='Error information') - resource_pack_subscribe_infos: Optional[List[ResourcePackSubscribeInfo]] = Field( - None, description='Resource package list' - ) - - -class KlingResourcePackageResponse(BaseModel): - code: Optional[int] = Field(None, description='Error code; 0 indicates success') - data: Optional[Data3] = None - message: Optional[str] = Field(None, description='Error information') - request_id: Optional[str] = Field( - None, - description='Request ID, generated by the system, used to track requests and troubleshoot problems', - ) - - -class Duration2(str, Enum): - field_5 = '5' - - -class ModelName3(str, Enum): - kling_v1_6 = 'kling-v1-6' - - -class KlingSingleImageEffectInput(BaseModel): - duration: Duration2 = Field( - ..., description='Video Length in seconds. Only 5-second videos are supported.' - ) - image: str = Field( - ..., - description='Reference Image. URL or Base64 encoded string (without data:image prefix). File size cannot exceed 10MB, resolution not less than 300*300px, aspect ratio between 1:2.5 ~ 2.5:1.', - ) - model_name: ModelName3 = Field( - ..., - description='Model Name. Only kling-v1-6 is supported for single image effects.', - ) - - -class AspectRatio2(str, Enum): - field_16_9 = '16:9' - field_9_16 = '9:16' - field_1_1 = '1:1' - - -class Config1(BaseModel): - horizontal: Optional[confloat(ge=-10.0, le=10.0)] = None - pan: Optional[confloat(ge=-10.0, le=10.0)] = None - roll: Optional[confloat(ge=-10.0, le=10.0)] = None - tilt: Optional[confloat(ge=-10.0, le=10.0)] = None - vertical: Optional[confloat(ge=-10.0, le=10.0)] = None - zoom: Optional[confloat(ge=-10.0, le=10.0)] = None - - -class CameraControl1(BaseModel): - config: Optional[Config1] = None - type: Optional[Type] = Field(None, description='Predefined camera movements type') - - -class Duration3(str, Enum): - field_5 = 5 - field_10 = 10 - - -class Mode3(str, Enum): - std = 'std' - pro = 'pro' - - -class ModelName4(str, Enum): - kling_v1 = 'kling-v1' - kling_v1_6 = 'kling-v1-6' - - -class KlingText2VideoRequest(BaseModel): - aspect_ratio: Optional[AspectRatio2] = '16:9' - callback_url: Optional[AnyUrl] = Field( - None, description='The callback notification address' - ) - camera_control: Optional[CameraControl1] = None - cfg_scale: Optional[confloat(ge=0.0, le=1.0)] = Field( - 0.5, description='Flexibility in video generation' - ) - duration: Optional[Duration3] = 5 - external_task_id: Optional[str] = Field(None, description='Customized Task ID') - mode: Optional[Mode3] = Field('std', description='Video generation mode') - model_name: Optional[ModelName4] = Field('kling-v1', description='Model Name') - negative_prompt: Optional[constr(max_length=2500)] = Field( - None, description='Negative text prompt' - ) - prompt: Optional[constr(max_length=2500)] = Field( - None, description='Positive text prompt' - ) - - -class TaskResult3(BaseModel): - videos: Optional[List[Video]] = None - - -class Data4(BaseModel): - created_at: Optional[int] = Field(None, description='Task creation time') - task_id: Optional[str] = Field(None, description='Task ID') - task_info: Optional[TaskInfo] = None - task_result: Optional[TaskResult3] = None - task_status: Optional[TaskStatus] = None - updated_at: Optional[int] = Field(None, description='Task update time') - - -class KlingText2VideoResponse(BaseModel): - code: Optional[int] = Field(None, description='Error code') - data: Optional[Data4] = None - message: Optional[str] = Field(None, description='Error message') - request_id: Optional[str] = Field(None, description='Request ID') - - -class KlingVideoEffectsInput( - RootModel[Union[KlingSingleImageEffectInput, KlingDualCharacterEffectInput]] -): - root: Union[KlingSingleImageEffectInput, KlingDualCharacterEffectInput] - - -class EffectScene(str, Enum): - bloombloom = 'bloombloom' - dizzydizzy = 'dizzydizzy' - fuzzyfuzzy = 'fuzzyfuzzy' - squish = 'squish' - expansion = 'expansion' - hug = 'hug' - kiss = 'kiss' - heart_gesture = 'heart_gesture' - - -class KlingVideoEffectsRequest(BaseModel): - callback_url: Optional[AnyUrl] = Field( - None, - description='The callback notification address for the result of this task.', - ) - effect_scene: EffectScene = Field( - ..., - description='Scene Name. Single Image Effects (bloombloom, dizzydizzy, fuzzyfuzzy, squish, expansion) or Dual-character Effects (hug, kiss, heart_gesture).', - ) external_task_id: Optional[str] = Field( None, description='Customized Task ID. Must be unique within a single user account.', ) - input: Optional[KlingVideoEffectsInput] = None -class TaskResult4(BaseModel): +class TaskResult1(BaseModel): videos: Optional[List[Video]] = None -class Data5(BaseModel): - created_at: Optional[int] = Field(None, description='Task creation time') +class Data1(BaseModel): task_id: Optional[str] = Field(None, description='Task ID') + task_status: Optional[TaskStatus] = None task_info: Optional[TaskInfo] = None - task_result: Optional[TaskResult4] = None - task_status: Optional[TaskStatus] = None - updated_at: Optional[int] = Field(None, description='Task update time') - - -class KlingVideoEffectsResponse(BaseModel): - code: Optional[int] = Field(None, description='Error code') - data: Optional[Data5] = None - message: Optional[str] = Field(None, description='Error message') - request_id: Optional[str] = Field(None, description='Request ID') - - -class KlingVideoExtendRequest(BaseModel): - callback_url: Optional[AnyUrl] = Field( - None, - description='The callback notification address. Server will notify when the task status changes.', - ) - cfg_scale: Optional[confloat(ge=0.0, le=1.0)] = Field( - 0.5, - description="Flexibility in video generation. The higher the value, the lower the model's flexibility and the stronger the relevance to the user's prompt.", - ) - negative_prompt: Optional[constr(max_length=2500)] = Field( - None, - description='Negative text prompt for elements to avoid in the extended video', - ) - prompt: Optional[constr(max_length=2500)] = Field( - None, description='Positive text prompt for guiding the video extension' - ) - video_id: Optional[str] = Field( - None, - description='The ID of the video to be extended. Supports videos generated by text-to-video, image-to-video, and previous video extension operations. Cannot exceed 3 minutes total duration after extension.', - ) - - -class TaskResult5(BaseModel): - videos: Optional[List[Video]] = None - - -class Data6(BaseModel): created_at: Optional[int] = Field(None, description='Task creation time') - task_id: Optional[str] = Field(None, description='Task ID') - task_info: Optional[TaskInfo] = None - task_result: Optional[TaskResult5] = None - task_status: Optional[TaskStatus] = None updated_at: Optional[int] = Field(None, description='Task update time') + task_result: Optional[TaskResult1] = None -class KlingVideoExtendResponse(BaseModel): +class KlingImage2VideoResponse(BaseModel): code: Optional[int] = Field(None, description='Error code') - data: Optional[Data6] = None message: Optional[str] = Field(None, description='Error message') request_id: Optional[str] = Field(None, description='Request ID') - - -class ModelName5(str, Enum): - kolors_virtual_try_on_v1 = 'kolors-virtual-try-on-v1' - kolors_virtual_try_on_v1_5 = 'kolors-virtual-try-on-v1-5' - - -class KlingVirtualTryOnRequest(BaseModel): - callback_url: Optional[AnyUrl] = Field( - None, description='The callback notification address' - ) - cloth_image: Optional[str] = Field( - None, - description='Reference clothing image - Base64 encoded string or image URL', - ) - human_image: str = Field( - ..., description='Reference human image - Base64 encoded string or image URL' - ) - model_name: Optional[ModelName5] = Field( - 'kolors-virtual-try-on-v1', description='Model Name' - ) - - -class Image1(BaseModel): - index: Optional[int] = Field(None, description='Image Number') - url: Optional[AnyUrl] = Field(None, description='URL for generated image') - - -class TaskResult6(BaseModel): - images: Optional[List[Image1]] = None - - -class Data7(BaseModel): - created_at: Optional[int] = Field(None, description='Task creation time') - task_id: Optional[str] = Field(None, description='Task ID') - task_result: Optional[TaskResult6] = None - task_status: Optional[TaskStatus] = None - task_status_msg: Optional[str] = Field(None, description='Task status information') - updated_at: Optional[int] = Field(None, description='Task update time') - - -class KlingVirtualTryOnResponse(BaseModel): - code: Optional[int] = Field(None, description='Error code') - data: Optional[Data7] = None - message: Optional[str] = Field(None, description='Error message') - request_id: Optional[str] = Field(None, description='Request ID') - - -class LumaAspectRatio(str, Enum): - field_1_1 = '1:1' - field_16_9 = '16:9' - field_9_16 = '9:16' - field_4_3 = '4:3' - field_3_4 = '3:4' - field_21_9 = '21:9' - field_9_21 = '9:21' - - -class LumaAssets(BaseModel): - image: Optional[AnyUrl] = Field(None, description='The URL of the image') - progress_video: Optional[AnyUrl] = Field( - None, description='The URL of the progress video' - ) - video: Optional[AnyUrl] = Field(None, description='The URL of the video') - - -class GenerationType(str, Enum): - add_audio = 'add_audio' - - -class LumaAudioGenerationRequest(BaseModel): - callback_url: Optional[AnyUrl] = Field( - None, description='The callback URL for the audio' - ) - generation_type: Optional[GenerationType] = 'add_audio' - negative_prompt: Optional[str] = Field( - None, description='The negative prompt of the audio' - ) - prompt: Optional[str] = Field(None, description='The prompt of the audio') - - -class LumaError(BaseModel): - detail: Optional[str] = Field(None, description='The error message') - - -class Type2(str, Enum): - generation = 'generation' - - -class LumaGenerationReference(BaseModel): - id: UUID = Field(..., description='The ID of the generation') - type: Literal['generation'] - - -class GenerationType1(str, Enum): - video = 'video' - - -class LumaGenerationType(str, Enum): - video = 'video' - image = 'image' - - -class GenerationType2(str, Enum): - image = 'image' - - -class LumaImageIdentity(BaseModel): - images: Optional[List[AnyUrl]] = Field( - None, description='The URLs of the image identity' - ) - - -class LumaImageModel(str, Enum): - photon_1 = 'photon-1' - photon_flash_1 = 'photon-flash-1' - - -class LumaImageRef(BaseModel): - url: Optional[AnyUrl] = Field(None, description='The URL of the image reference') - weight: Optional[float] = Field( - None, description='The weight of the image reference' - ) - - -class Type3(str, Enum): - image = 'image' - - -class LumaImageReference(BaseModel): - type: Literal['image'] - url: AnyUrl = Field(..., description='The URL of the image') - - -class LumaKeyframe(RootModel[Union[LumaGenerationReference, LumaImageReference]]): - root: Union[LumaGenerationReference, LumaImageReference] = Field( - ..., - description='A keyframe can be either a Generation reference, an Image, or a Video', - discriminator='type', - ) - - -class LumaKeyframes(BaseModel): - frame0: Optional[LumaKeyframe] = None - frame1: Optional[LumaKeyframe] = None - - -class LumaModifyImageRef(BaseModel): - url: Optional[AnyUrl] = Field(None, description='The URL of the image reference') - weight: Optional[float] = Field( - None, description='The weight of the modify image reference' - ) - - -class LumaState(str, Enum): - queued = 'queued' - dreaming = 'dreaming' - completed = 'completed' - failed = 'failed' - - -class GenerationType3(str, Enum): - upscale_video = 'upscale_video' - - -class LumaVideoModel(str, Enum): - ray_2 = 'ray-2' - ray_2_flash = 'ray-2-flash' - - -class LumaVideoModelOutputDuration1(str, Enum): - field_5s = '5s' - field_9s = '9s' - - -class LumaVideoModelOutputDuration( - RootModel[Union[LumaVideoModelOutputDuration1, str]] -): - root: Union[LumaVideoModelOutputDuration1, str] - - -class LumaVideoModelOutputResolution1(str, Enum): - field_540p = '540p' - field_720p = '720p' - field_1080p = '1080p' - field_4k = '4k' - - -class LumaVideoModelOutputResolution( - RootModel[Union[LumaVideoModelOutputResolution1, str]] -): - root: Union[LumaVideoModelOutputResolution1, str] - - -class MachineStats(BaseModel): - cpu_capacity: Optional[str] = Field(None, description='Total CPU on the machine.') - disk_capacity: Optional[str] = Field( - None, description='Total disk capacity on the machine.' - ) - gpu_type: Optional[str] = Field( - None, description='The GPU type. eg. NVIDIA Tesla K80' - ) - initial_cpu: Optional[str] = Field( - None, description='Initial CPU available before the job starts.' - ) - initial_disk: Optional[str] = Field( - None, description='Initial disk available before the job starts.' - ) - initial_ram: Optional[str] = Field( - None, description='Initial RAM available before the job starts.' - ) - machine_name: Optional[str] = Field(None, description='Name of the machine.') - memory_capacity: Optional[str] = Field( - None, description='Total memory on the machine.' - ) - os_version: Optional[str] = Field( - None, description='The operating system version. eg. Ubuntu Linux 20.04' - ) - pip_freeze: Optional[str] = Field(None, description='The pip freeze output') - vram_time_series: Optional[Dict[str, Any]] = Field( - None, description='Time series of VRAM usage.' - ) - - -class MinimaxBaseResponse(BaseModel): - status_code: int = Field( - ..., - description='Status code. 0 indicates success, other values indicate errors.', - ) - status_msg: str = Field( - ..., description='Specific error details or success message.' - ) - - -class File(BaseModel): - bytes: Optional[int] = Field(None, description='File size in bytes') - created_at: Optional[int] = Field( - None, description='Unix timestamp when the file was created, in seconds' - ) - download_url: Optional[str] = Field( - None, description='The URL to download the video' - ) - file_id: Optional[int] = Field(None, description='Unique identifier for the file') - filename: Optional[str] = Field(None, description='The name of the file') - purpose: Optional[str] = Field(None, description='The purpose of using the file') - - -class MinimaxFileRetrieveResponse(BaseModel): - base_resp: MinimaxBaseResponse - file: File - - -class Status1(str, Enum): - Queueing = 'Queueing' - Preparing = 'Preparing' - Processing = 'Processing' - Success = 'Success' - Fail = 'Fail' - - -class MinimaxTaskResultResponse(BaseModel): - base_resp: MinimaxBaseResponse - file_id: Optional[str] = Field( - None, - description='After the task status changes to Success, this field returns the file ID corresponding to the generated video.', - ) - status: Status1 = Field( - ..., - description="Task status: 'Queueing' (in queue), 'Preparing' (task is preparing), 'Processing' (generating), 'Success' (task completed successfully), or 'Fail' (task failed).", - ) - task_id: str = Field(..., description='The task ID being queried.') + data: Optional[Data1] = None class Model(str, Enum): @@ -1103,251 +352,135 @@ class SubjectReferenceItem(BaseModel): class MinimaxVideoGenerationRequest(BaseModel): - callback_url: Optional[str] = Field( - None, - description='Optional. URL to receive real-time status updates about the video generation task.', - ) - first_frame_image: Optional[str] = Field( - None, - description='URL or base64 encoding of the first frame image. Required when model is I2V-01, I2V-01-Director, or I2V-01-live.', - ) model: Model = Field( ..., description='Required. ID of model. Options: T2V-01-Director, I2V-01-Director, S2V-01, I2V-01, I2V-01-live, T2V-01', ) - prompt: Optional[constr(max_length=2000)] = Field( + prompt: Optional[str] = Field( None, description='Description of the video. Should be less than 2000 characters. Supports camera movement instructions in [brackets].', + max_length=2000, ) prompt_optimizer: Optional[bool] = Field( True, description='If true (default), the model will automatically optimize the prompt. Set to false for more precise control.', ) + first_frame_image: Optional[str] = Field( + None, + description='URL or base64 encoding of the first frame image. Required when model is I2V-01, I2V-01-Director, or I2V-01-live.', + ) subject_reference: Optional[List[SubjectReferenceItem]] = Field( None, description='Only available when model is S2V-01. The model will generate a video based on the subject uploaded through this parameter.', ) + callback_url: Optional[str] = Field( + None, + description='Optional. URL to receive real-time status updates about the video generation task.', + ) + + +class MinimaxBaseResponse(BaseModel): + status_code: int = Field( + ..., + description='Status code. 0 indicates success, other values indicate errors.', + ) + status_msg: str = Field( + ..., description='Specific error details or success message.' + ) class MinimaxVideoGenerationResponse(BaseModel): - base_resp: MinimaxBaseResponse task_id: str = Field( ..., description='The task ID for the asynchronous video generation task.' ) + base_resp: MinimaxBaseResponse -class NodeStatus(str, Enum): - NodeStatusActive = 'NodeStatusActive' - NodeStatusDeleted = 'NodeStatusDeleted' - NodeStatusBanned = 'NodeStatusBanned' - - -class NodeVersionStatus(str, Enum): - NodeVersionStatusActive = 'NodeVersionStatusActive' - NodeVersionStatusDeleted = 'NodeVersionStatusDeleted' - NodeVersionStatusBanned = 'NodeVersionStatusBanned' - NodeVersionStatusPending = 'NodeVersionStatusPending' - NodeVersionStatusFlagged = 'NodeVersionStatusFlagged' - - -class NodeVersionUpdateRequest(BaseModel): - changelog: Optional[str] = Field( - None, description='The changelog describing the version changes.' +class File(BaseModel): + file_id: Optional[int] = Field(None, description='Unique identifier for the file') + bytes: Optional[int] = Field(None, description='File size in bytes') + created_at: Optional[int] = Field( + None, description='Unix timestamp when the file was created, in seconds' ) - deprecated: Optional[bool] = Field( - None, description='Whether the version is deprecated.' + filename: Optional[str] = Field(None, description='The name of the file') + purpose: Optional[str] = Field(None, description='The purpose of using the file') + download_url: Optional[str] = Field( + None, description='The URL to download the video' ) -class Moderation(str, Enum): - low = 'low' - auto = 'auto' +class MinimaxFileRetrieveResponse(BaseModel): + file: File + base_resp: MinimaxBaseResponse -class OutputFormat(str, Enum): - png = 'png' - webp = 'webp' - jpeg = 'jpeg' +class Status(str, Enum): + Queueing = 'Queueing' + Preparing = 'Preparing' + Processing = 'Processing' + Success = 'Success' + Fail = 'Fail' -class OpenAIImageEditRequest(BaseModel): - background: Optional[str] = Field( - None, description='Background transparency', examples=['opaque'] - ) - model: str = Field( - ..., description='The model to use for image editing', examples=['gpt-image-1'] - ) - moderation: Optional[Moderation] = Field( - None, description='Content moderation setting', examples=['auto'] - ) - n: Optional[int] = Field( - None, description='The number of images to generate', examples=[1] - ) - output_compression: Optional[int] = Field( - None, description='Compression level for JPEG or WebP (0-100)', examples=[100] - ) - output_format: Optional[OutputFormat] = Field( - None, description='Format of the output image', examples=['png'] - ) - prompt: str = Field( +class MinimaxTaskResultResponse(BaseModel): + task_id: str = Field(..., description='The task ID being queried.') + status: Status = Field( ..., - description='A text description of the desired edit', - examples=['Give the rocketship rainbow coloring'], + description="Task status: 'Queueing' (in queue), 'Preparing' (task is preparing), 'Processing' (generating), 'Success' (task completed successfully), or 'Fail' (task failed).", ) - quality: Optional[str] = Field( - None, description='The quality of the edited image', examples=['low'] - ) - size: Optional[str] = Field( - None, description='Size of the output image', examples=['1024x1024'] - ) - user: Optional[str] = Field( + file_id: Optional[str] = Field( None, - description='A unique identifier for end-user monitoring', - examples=['user-1234'], + description='After the task status changes to Success, this field returns the file ID corresponding to the generated video.', + ) + base_resp: MinimaxBaseResponse + + +class BFLFluxProGenerateRequest(BaseModel): + prompt: str = Field(..., description='The text prompt for image generation.') + negative_prompt: Optional[str] = Field( + None, description='The negative prompt for image generation.' + ) + width: int = Field( + ..., description='The width of the image to generate.', ge=64, le=2048 + ) + height: int = Field( + ..., description='The height of the image to generate.', ge=64, le=2048 + ) + num_inference_steps: Optional[int] = Field( + None, description='The number of inference steps.', ge=1, le=100 + ) + guidance_scale: Optional[float] = Field( + None, description='The guidance scale for generation.', ge=1.0, le=20.0 + ) + seed: Optional[int] = Field(None, description='The seed value for reproducibility.') + num_images: Optional[int] = Field( + None, description='The number of images to generate.', ge=1, le=4 ) -class Background(str, Enum): - transparent = 'transparent' - opaque = 'opaque' - - -class Quality(str, Enum): - low = 'low' - medium = 'medium' - high = 'high' - standard = 'standard' - hd = 'hd' - - -class ResponseFormat(str, Enum): - url = 'url' - b64_json = 'b64_json' - - -class Style(str, Enum): - vivid = 'vivid' - natural = 'natural' - - -class OpenAIImageGenerationRequest(BaseModel): - background: Optional[Background] = Field( - None, description='Background transparency', examples=['opaque'] - ) - model: Optional[str] = Field( - None, description='The model to use for image generation', examples=['dall-e-3'] - ) - moderation: Optional[Moderation] = Field( - None, description='Content moderation setting', examples=['auto'] - ) - n: Optional[int] = Field( - None, - description='The number of images to generate (1-10). Only 1 supported for dall-e-3.', - examples=[1], - ) - output_compression: Optional[int] = Field( - None, description='Compression level for JPEG or WebP (0-100)', examples=[100] - ) - output_format: Optional[OutputFormat] = Field( - None, description='Format of the output image', examples=['png'] - ) - prompt: str = Field( - ..., - description='A text description of the desired image', - examples=['Draw a rocket in front of a blackhole in deep space'], - ) - quality: Optional[Quality] = Field( - None, description='The quality of the generated image', examples=['high'] - ) - response_format: Optional[ResponseFormat] = Field( - None, description='Response format of image data', examples=['b64_json'] - ) - size: Optional[str] = Field( - None, - description='Size of the image (e.g., 1024x1024, 1536x1024, auto)', - examples=['1024x1536'], - ) - style: Optional[Style] = Field( - None, description='Style of the image (only for dall-e-3)', examples=['vivid'] - ) - user: Optional[str] = Field( - None, - description='A unique identifier for end-user monitoring', - examples=['user-1234'], - ) - - -class Datum1(BaseModel): - b64_json: Optional[str] = Field(None, description='Base64 encoded image data') - revised_prompt: Optional[str] = Field(None, description='Revised prompt') - url: Optional[str] = Field(None, description='URL of the image') - - -class InputTokensDetails(BaseModel): - image_tokens: Optional[int] = None - text_tokens: Optional[int] = None - - -class Usage(BaseModel): - input_tokens: Optional[int] = None - input_tokens_details: Optional[InputTokensDetails] = None - output_tokens: Optional[int] = None - total_tokens: Optional[int] = None - - -class OpenAIImageGenerationResponse(BaseModel): - data: Optional[List[Datum1]] = None - usage: Optional[Usage] = None - - -class PersonalAccessToken(BaseModel): - createdAt: Optional[datetime] = Field( - None, description='[Output Only]The date and time the token was created.' - ) - description: Optional[str] = Field( - None, - description="Optional. A more detailed description of the token's intended use.", - ) - id: Optional[UUID] = Field(None, description='Unique identifier for the GitCommit') - name: Optional[str] = Field( - None, - description='Required. The name of the token. Can be a simple description.', - ) - token: Optional[str] = Field( - None, - description='[Output Only]. The personal access token. Only returned during creation.', - ) - - -class PublisherStatus(str, Enum): - PublisherStatusActive = 'PublisherStatusActive' - PublisherStatusBanned = 'PublisherStatusBanned' - - -class PublisherUser(BaseModel): - email: Optional[str] = Field(None, description='The email address for this user.') - id: Optional[str] = Field(None, description='The unique id for this user.') - name: Optional[str] = Field(None, description='The name for this user.') +class BFLFluxProGenerateResponse(BaseModel): + id: str = Field(..., description='The unique identifier for the generation task.') + polling_url: str = Field(..., description='URL to poll for the generation result.') class RecraftImageGenerationRequest(BaseModel): - model: str = Field( - ..., description='The model to use for generation (e.g., "recraftv3")' - ) - n: conint(ge=1, le=4) = Field(..., description='The number of images to generate') prompt: str = Field( ..., description='The text prompt describing the image to generate' ) - size: str = Field( - ..., description='The size of the generated image (e.g., "1024x1024")' + model: str = Field( + ..., description='The model to use for generation (e.g., "recraftv3")' ) style: Optional[str] = Field( None, description='The style to apply to the generated image (e.g., "digital_illustration")', ) + size: str = Field( + ..., description='The size of the generated image (e.g., "1024x1024")' + ) + n: int = Field(..., description='The number of images to generate', ge=1, le=4) -class Datum2(BaseModel): +class Datum1(BaseModel): image_id: Optional[str] = Field( None, description='Unique identifier for the generated image' ) @@ -1359,289 +492,37 @@ class RecraftImageGenerationResponse(BaseModel): ..., description='Unix timestamp when the generation was created' ) credits: int = Field(..., description='Number of credits used for the generation') - data: List[Datum2] = Field(..., description='Array of generated image information') + data: List[Datum1] = Field(..., description='Array of generated image information') -class RunwayAspectRatioEnum(str, Enum): - field_1280_720 = '1280:720' - field_720_1280 = '720:1280' - field_1104_832 = '1104:832' - field_832_1104 = '832:1104' - field_960_960 = '960:960' - field_1584_672 = '1584:672' - field_1280_768 = '1280:768' - field_768_1280 = '768:1280' - - -class RunwayDurationEnum(int, Enum): - integer_5 = 5 - integer_10 = 10 - - -class RunwayImageToVideoResponse(BaseModel): - id: Optional[str] = Field(None, description='Task ID') - - -class RunwayModelEnum(str, Enum): - gen4_turbo = 'gen4_turbo' - gen3a_turbo = 'gen3a_turbo' - - -class Position(str, Enum): - first = 'first' - last = 'last' - - -class RunwayPromptImageDetailedObject(BaseModel): - position: Position = Field( +class KlingErrorResponse(BaseModel): + code: int = Field( ..., - description="The position of the image in the output video. 'last' is currently supported for gen3a_turbo only.", + description='- 1000: Authentication failed\n- 1001: Authorization is empty\n- 1002: Authorization is invalid\n- 1003: Authorization is not yet valid\n- 1004: Authorization has expired\n- 1100: Account exception\n- 1101: Account in arrears (postpaid scenario)\n- 1102: Resource pack depleted or expired (prepaid scenario)\n- 1103: Unauthorized access to requested resource\n- 1200: Invalid request parameters\n- 1201: Invalid parameters\n- 1202: Invalid request method\n- 1203: Requested resource does not exist\n- 1300: Trigger platform strategy\n- 1301: Trigger content security policy\n- 1302: API request too frequent\n- 1303: Concurrency/QPS exceeds limit\n- 1304: Trigger IP whitelist policy\n- 5000: Internal server error\n- 5001: Service temporarily unavailable\n- 5002: Server internal timeout\n', ) - uri: AnyUrl = Field( - ..., description='A HTTPS URL or data URI containing an encoded image.' + message: str = Field(..., description='Human-readable error message') + request_id: str = Field( + ..., description='Request ID for tracking and troubleshooting' ) -class RunwayPromptImageObject( - RootModel[Union[AnyUrl, List[RunwayPromptImageDetailedObject]]] -): - root: Union[AnyUrl, List[RunwayPromptImageDetailedObject]] = Field( - ..., - description='Image(s) to use for the video generation. Can be a single URI or an array of image objects with positions.', - ) - - -class RunwayTaskStatusEnum(str, Enum): - SUCCEEDED = 'SUCCEEDED' - RUNNING = 'RUNNING' - FAILED = 'FAILED' - PENDING = 'PENDING' - CANCELLED = 'CANCELLED' - THROTTLED = 'THROTTLED' - - -class RunwayTaskStatusResponse(BaseModel): - createdAt: Optional[datetime] = Field(None, description='Task creation timestamp') - id: Optional[str] = Field(None, description='Task ID') - output: Optional[List[str]] = Field(None, description='Array of output video URLs') - status: Optional[RunwayTaskStatusEnum] = None - - -class StorageFile(BaseModel): - file_path: Optional[str] = Field(None, description='Path to the file in storage') - id: Optional[UUID] = Field( - None, description='Unique identifier for the storage file' - ) - public_url: Optional[str] = Field(None, description='Public URL') - - -class StripeAddress(BaseModel): - city: Optional[str] = None - country: Optional[str] = None - line1: Optional[str] = None - line2: Optional[str] = None - postal_code: Optional[str] = None - state: Optional[str] = None - - -class StripeAmountDetails(BaseModel): - tip: Optional[Dict[str, Any]] = None - - -class StripeBillingDetails(BaseModel): - address: Optional[StripeAddress] = None - email: Optional[str] = None - name: Optional[str] = None - phone: Optional[str] = None - tax_id: Optional[Any] = None - - -class Checks(BaseModel): - address_line1_check: Optional[Any] = None - address_postal_code_check: Optional[Any] = None - cvc_check: Optional[str] = None - - -class ExtendedAuthorization(BaseModel): - status: Optional[str] = None - - -class IncrementalAuthorization(BaseModel): - status: Optional[str] = None - - -class Multicapture(BaseModel): - status: Optional[str] = None - - -class NetworkToken(BaseModel): - used: Optional[bool] = None - - -class Overcapture(BaseModel): - maximum_amount_capturable: Optional[int] = None - status: Optional[str] = None - - -class StripeCardDetails(BaseModel): - amount_authorized: Optional[int] = None - authorization_code: Optional[Any] = None - brand: Optional[str] = None - checks: Optional[Checks] = None - country: Optional[str] = None - exp_month: Optional[int] = None - exp_year: Optional[int] = None - extended_authorization: Optional[ExtendedAuthorization] = None - fingerprint: Optional[str] = None - funding: Optional[str] = None - incremental_authorization: Optional[IncrementalAuthorization] = None - installments: Optional[Any] = None - last4: Optional[str] = None - mandate: Optional[Any] = None - multicapture: Optional[Multicapture] = None - network: Optional[str] = None - network_token: Optional[NetworkToken] = None - network_transaction_id: Optional[str] = None - overcapture: Optional[Overcapture] = None - regulated_status: Optional[str] = None - three_d_secure: Optional[Any] = None - wallet: Optional[Any] = None - - -class Object(str, Enum): - charge = 'charge' - - -class Object1(str, Enum): - event = 'event' - - -class Type4(str, Enum): - payment_intent_succeeded = 'payment_intent.succeeded' - - -class StripeOutcome(BaseModel): - advice_code: Optional[Any] = None - network_advice_code: Optional[Any] = None - network_decline_code: Optional[Any] = None - network_status: Optional[str] = None - reason: Optional[Any] = None - risk_level: Optional[str] = None - risk_score: Optional[int] = None - seller_message: Optional[str] = None - type: Optional[str] = None - - -class Object2(str, Enum): - payment_intent = 'payment_intent' - - -class StripePaymentMethodDetails(BaseModel): - card: Optional[StripeCardDetails] = None - type: Optional[str] = None - - -class Card(BaseModel): - installments: Optional[Any] = None - mandate_options: Optional[Any] = None - network: Optional[Any] = None - request_three_d_secure: Optional[str] = None - - -class StripePaymentMethodOptions(BaseModel): - card: Optional[Card] = None - - -class StripeRefundList(BaseModel): - data: Optional[List[Dict[str, Any]]] = None - has_more: Optional[bool] = None - object: Optional[str] = None - total_count: Optional[int] = None - url: Optional[str] = None - - -class StripeRequestInfo(BaseModel): - id: Optional[str] = None - idempotency_key: Optional[str] = None - - -class StripeShipping(BaseModel): - address: Optional[StripeAddress] = None - carrier: Optional[str] = None - name: Optional[str] = None - phone: Optional[str] = None - tracking_number: Optional[str] = None - - -class User(BaseModel): - email: Optional[str] = Field(None, description='The email address for this user.') - id: Optional[str] = Field(None, description='The unique id for this user.') - isAdmin: Optional[bool] = Field( - None, description='Indicates if the user has admin privileges.' - ) - isApproved: Optional[bool] = Field( - None, description='Indicates if the user is approved.' - ) - name: Optional[str] = Field(None, description='The name for this user.') - - -class Veo2GenVidPollRequest(BaseModel): - operationName: str = Field( - ..., - description='Full operation name (from predict response)', - examples=[ - 'projects/PROJECT_ID/locations/us-central1/publishers/google/models/MODEL_ID/operations/OPERATION_ID' - ], - ) - - -class Video5(BaseModel): - bytesBase64Encoded: Optional[str] = Field( - None, description='Base64-encoded video content' - ) - gcsUri: Optional[str] = Field(None, description='Cloud Storage URI of the video') - mimeType: Optional[str] = Field(None, description='Video MIME type') - - -class Response(BaseModel): - field_type: Optional[str] = Field( - None, - alias='@type', - examples=[ - 'type.googleapis.com/cloud.ai.large_models.vision.GenerateVideoResponse' - ], - ) - raiMediaFilteredCount: Optional[int] = Field( - None, description='Count of media filtered by responsible AI policies' - ) - videos: Optional[List[Video5]] = None - - -class Veo2GenVidPollResponse(BaseModel): - done: Optional[bool] = None - name: Optional[str] = None - response: Optional[Response] = Field( - None, description='The actual prediction response if done is true' - ) - - -class Image2(BaseModel): +class Image(BaseModel): bytesBase64Encoded: str gcsUri: Optional[str] = None mimeType: Optional[str] = None -class Image3(BaseModel): +class Image1(BaseModel): bytesBase64Encoded: Optional[str] = None gcsUri: str mimeType: Optional[str] = None class Instance(BaseModel): - image: Optional[Union[Image2, Image3]] = Field( + prompt: str = Field(..., description='Text description of the video') + image: Optional[Union[Image, Image1]] = Field( None, description='Optional image to guide video generation' ) - prompt: str = Field(..., description='Text description of the video') class PersonGeneration(str, Enum): @@ -1651,8 +532,6 @@ class PersonGeneration(str, Enum): class Parameters(BaseModel): aspectRatio: Optional[str] = Field(None, examples=['16:9']) - durationSeconds: Optional[int] = None - enhancePrompt: Optional[bool] = None negativePrompt: Optional[str] = None personGeneration: Optional[PersonGeneration] = None sampleCount: Optional[int] = None @@ -1660,6 +539,8 @@ class Parameters(BaseModel): storageUri: Optional[str] = Field( None, description='Optional Cloud Storage URI to upload the video' ) + durationSeconds: Optional[int] = None + enhancePrompt: Optional[bool] = None class Veo2GenVidRequest(BaseModel): @@ -1677,336 +558,347 @@ class Veo2GenVidResponse(BaseModel): ) -class WorkflowRunStatus(str, Enum): - WorkflowRunStatusStarted = 'WorkflowRunStatusStarted' - WorkflowRunStatusFailed = 'WorkflowRunStatusFailed' - WorkflowRunStatusCompleted = 'WorkflowRunStatusCompleted' +class Veo2GenVidPollRequest(BaseModel): + operationName: str = Field( + ..., + description='Full operation name (from predict response)', + examples=[ + 'projects/PROJECT_ID/locations/us-central1/publishers/google/models/MODEL_ID/operations/OPERATION_ID' + ], + ) -class ActionJobResult(BaseModel): - action_job_id: Optional[str] = Field( - None, description='Identifier of the job this result belongs to' +class Video2(BaseModel): + gcsUri: Optional[str] = Field(None, description='Cloud Storage URI of the video') + bytesBase64Encoded: Optional[str] = Field( + None, description='Base64-encoded video content' ) - action_run_id: Optional[str] = Field( - None, description='Identifier of the run this result belongs to' - ) - author: Optional[str] = Field(None, description='The author of the commit') - avg_vram: Optional[int] = Field( - None, description='The average VRAM used by the job' - ) - branch_name: Optional[str] = Field( - None, description='Name of the relevant git branch' - ) - comfy_run_flags: Optional[str] = Field( - None, description='The comfy run flags. E.g. `--low-vram`' - ) - commit_hash: Optional[str] = Field(None, description='The hash of the commit') - commit_id: Optional[str] = Field(None, description='The ID of the commit') - commit_message: Optional[str] = Field(None, description='The message of the commit') - commit_time: Optional[int] = Field( - None, description='The Unix timestamp when the commit was made' - ) - cuda_version: Optional[str] = Field(None, description='CUDA version used') - end_time: Optional[int] = Field( - None, description='The end time of the job as a Unix timestamp.' - ) - git_repo: Optional[str] = Field(None, description='The repository name') - id: Optional[UUID] = Field(None, description='Unique identifier for the job result') - job_trigger_user: Optional[str] = Field( - None, description='The user who triggered the job.' - ) - machine_stats: Optional[MachineStats] = None - operating_system: Optional[str] = Field(None, description='Operating system used') - peak_vram: Optional[int] = Field(None, description='The peak VRAM used by the job') - pr_number: Optional[str] = Field(None, description='The pull request number') - python_version: Optional[str] = Field(None, description='PyTorch version used') - pytorch_version: Optional[str] = Field(None, description='PyTorch version used') - start_time: Optional[int] = Field( - None, description='The start time of the job as a Unix timestamp.' - ) - status: Optional[WorkflowRunStatus] = None - storage_file: Optional[StorageFile] = None - workflow_name: Optional[str] = Field(None, description='Name of the workflow') + mimeType: Optional[str] = Field(None, description='Video MIME type') -class LumaGenerationRequest(BaseModel): - aspect_ratio: LumaAspectRatio - callback_url: Optional[AnyUrl] = Field( +class Response(BaseModel): + field_type: Optional[str] = Field( None, - description='The callback URL of the generation, a POST request with Generation object will be sent to the callback URL when the generation is dreaming, completed, or failed', + alias='@type', + examples=[ + 'type.googleapis.com/cloud.ai.large_models.vision.GenerateVideoResponse' + ], ) - duration: LumaVideoModelOutputDuration - generation_type: Optional[GenerationType1] = 'video' - keyframes: Optional[LumaKeyframes] = None - loop: Optional[bool] = Field(None, description='Whether to loop the video') - model: LumaVideoModel - prompt: str = Field(..., description='The prompt of the generation') - resolution: LumaVideoModelOutputResolution + raiMediaFilteredCount: Optional[int] = Field( + None, description='Count of media filtered by responsible AI policies' + ) + videos: Optional[List[Video2]] = None -class CharacterRef(BaseModel): - identity0: Optional[LumaImageIdentity] = None +class Veo2GenVidPollResponse(BaseModel): + name: Optional[str] = None + done: Optional[bool] = None + response: Optional[Response] = Field( + None, description='The actual prediction response if done is true' + ) -class LumaImageGenerationRequest(BaseModel): - aspect_ratio: Optional[LumaAspectRatio] = '16:9' - callback_url: Optional[AnyUrl] = Field( - None, description='The callback URL for the generation' - ) - character_ref: Optional[CharacterRef] = None - generation_type: Optional[GenerationType2] = 'image' - image_ref: Optional[List[LumaImageRef]] = None - model: Optional[LumaImageModel] = 'photon-1' - modify_image_ref: Optional[LumaModifyImageRef] = None - prompt: Optional[str] = Field(None, description='The prompt of the generation') - style_ref: Optional[List[LumaImageRef]] = None +class RunwayImageToVideoResponse(BaseModel): + id: Optional[str] = Field(None, description='Task ID') -class LumaUpscaleVideoGenerationRequest(BaseModel): - callback_url: Optional[AnyUrl] = Field( - None, description='The callback URL for the upscale' - ) - generation_type: Optional[GenerationType3] = 'upscale_video' - resolution: Optional[LumaVideoModelOutputResolution] = None +class RunwayTaskStatusEnum(str, Enum): + SUCCEEDED = 'SUCCEEDED' + RUNNING = 'RUNNING' + FAILED = 'FAILED' + PENDING = 'PENDING' + CANCELLED = 'CANCELLED' + THROTTLED = 'THROTTLED' -class NodeVersion(BaseModel): - changelog: Optional[str] = Field( - None, description='Summary of changes made in this version' +class RunwayModelEnum(str, Enum): + gen4_turbo = 'gen4_turbo' + gen3a_turbo = 'gen3a_turbo' + + +class Position(str, Enum): + first = 'first' + last = 'last' + + +class RunwayPromptImageDetailedObject(BaseModel): + uri: AnyUrl = Field( + ..., description='A HTTPS URL or data URI containing an encoded image.' ) - comfy_node_extract_status: Optional[str] = Field( - None, description='The status of comfy node extraction process.' + position: Position = Field( + ..., + description="The position of the image in the output video. 'last' is currently supported for gen3a_turbo only.", ) - createdAt: Optional[datetime] = Field( - None, description='The date and time the version was created.' + + +class RunwayDurationEnum(int, Enum): + integer_5 = 5 + integer_10 = 10 + + +class RunwayAspectRatioEnum(str, Enum): + field_1280_720 = '1280:720' + field_720_1280 = '720:1280' + field_1104_832 = '1104:832' + field_832_1104 = '832:1104' + field_960_960 = '960:960' + field_1584_672 = '1584:672' + field_1280_768 = '1280:768' + field_768_1280 = '768:1280' + + +class RunwayPromptImageObject( + RootModel[Union[AnyUrl, List[RunwayPromptImageDetailedObject]]] +): + root: Union[AnyUrl, List[RunwayPromptImageDetailedObject]] = Field( + ..., + description='Image(s) to use for the video generation. Can be a single URI or an array of image objects with positions.', ) - dependencies: Optional[List[str]] = Field( - None, description='A list of pip dependencies required by the node.' + + +class Datum2(BaseModel): + b64_json: Optional[str] = Field(None, description='Base64 encoded image data') + url: Optional[str] = Field(None, description='URL of the image') + revised_prompt: Optional[str] = Field(None, description='Revised prompt') + + +class InputTokensDetails(BaseModel): + text_tokens: Optional[int] = None + image_tokens: Optional[int] = None + + +class Usage(BaseModel): + input_tokens: Optional[int] = None + input_tokens_details: Optional[InputTokensDetails] = None + output_tokens: Optional[int] = None + total_tokens: Optional[int] = None + + +class OpenAIImageGenerationResponse(BaseModel): + data: Optional[List[Datum2]] = None + usage: Optional[Usage] = None + + +class Quality(str, Enum): + low = 'low' + medium = 'medium' + high = 'high' + standard = 'standard' + hd = 'hd' + + +class OutputFormat(str, Enum): + png = 'png' + webp = 'webp' + jpeg = 'jpeg' + + +class Moderation(str, Enum): + low = 'low' + auto = 'auto' + + +class Background(str, Enum): + transparent = 'transparent' + opaque = 'opaque' + + +class ResponseFormat(str, Enum): + url = 'url' + b64_json = 'b64_json' + + +class Style(str, Enum): + vivid = 'vivid' + natural = 'natural' + + +class OpenAIImageGenerationRequest(BaseModel): + model: Optional[str] = Field( + None, description='The model to use for image generation', examples=['dall-e-3'] ) - deprecated: Optional[bool] = Field( - None, description='Indicates if this version is deprecated.' + prompt: str = Field( + ..., + description='A text description of the desired image', + examples=['Draw a rocket in front of a blackhole in deep space'], ) - downloadUrl: Optional[str] = Field( - None, description='[Output Only] URL to download this version of the node' - ) - id: Optional[str] = None - node_id: Optional[str] = Field( - None, description='The unique identifier of the node.' - ) - status: Optional[NodeVersionStatus] = None - status_reason: Optional[str] = Field( - None, description='The reason for the status change.' - ) - version: Optional[str] = Field( + n: Optional[int] = Field( None, - description='The version identifier, following semantic versioning. Must be unique for the node.', + description='The number of images to generate (1-10). Only 1 supported for dall-e-3.', + examples=[1], + ) + quality: Optional[Quality] = Field( + None, description='The quality of the generated image', examples=['high'] + ) + size: Optional[str] = Field( + None, + description='Size of the image (e.g., 1024x1024, 1536x1024, auto)', + examples=['1024x1536'], + ) + output_format: Optional[OutputFormat] = Field( + None, description='Format of the output image', examples=['png'] + ) + output_compression: Optional[int] = Field( + None, description='Compression level for JPEG or WebP (0-100)', examples=[100] + ) + moderation: Optional[Moderation] = Field( + None, description='Content moderation setting', examples=['auto'] + ) + background: Optional[Background] = Field( + None, description='Background transparency', examples=['opaque'] + ) + response_format: Optional[ResponseFormat] = Field( + None, description='Response format of image data', examples=['b64_json'] + ) + style: Optional[Style] = Field( + None, description='Style of the image (only for dall-e-3)', examples=['vivid'] + ) + user: Optional[str] = Field( + None, + description='A unique identifier for end-user monitoring', + examples=['user-1234'], ) -class PublisherMember(BaseModel): - id: Optional[str] = Field( - None, description='The unique identifier for the publisher member.' +class OpenAIImageEditRequest(BaseModel): + model: str = Field( + ..., description='The model to use for image editing', examples=['gpt-image-1'] ) - role: Optional[str] = Field( - None, description='The role of the user in the publisher.' + prompt: str = Field( + ..., + description='A text description of the desired edit', + examples=['Give the rocketship rainbow coloring'], ) - user: Optional[PublisherUser] = None + n: Optional[int] = Field( + None, description='The number of images to generate', examples=[1] + ) + quality: Optional[str] = Field( + None, description='The quality of the edited image', examples=['low'] + ) + size: Optional[str] = Field( + None, description='Size of the output image', examples=['1024x1024'] + ) + output_format: Optional[OutputFormat] = Field( + None, description='Format of the output image', examples=['png'] + ) + output_compression: Optional[int] = Field( + None, description='Compression level for JPEG or WebP (0-100)', examples=[100] + ) + moderation: Optional[Moderation] = Field( + None, description='Content moderation setting', examples=['auto'] + ) + background: Optional[str] = Field( + None, description='Background transparency', examples=['opaque'] + ) + user: Optional[str] = Field( + None, + description='A unique identifier for end-user monitoring', + examples=['user-1234'], + ) + + +class AspectRatio2(RootModel[float]): + root: float = Field( + ..., + description='Aspect ratio (width / height)', + ge=0.4, + le=2.5, + title='Aspectratio', + ) + + +class PikaBodyGenerate22T2vGenerate22T2vPost(BaseModel): + promptText: str = Field(..., title='Prompttext') + negativePrompt: Optional[str] = Field(None, title='Negativeprompt') + seed: Optional[int] = Field(None, title='Seed') + resolution: Optional[str] = Field('1080p', title='Resolution') + duration: Optional[int] = Field(5, title='Duration') + aspectRatio: Optional[AspectRatio2] = Field( + None, description='Aspect ratio (width / height)', title='Aspectratio' + ) + + +class PikaGenerateResponse(BaseModel): + video_id: str = Field(..., title='Video Id') + + +class PikaBodyGenerate22I2vGenerate22I2vPost(BaseModel): + image: bytes = Field(..., title='Image') + promptText: Optional[str] = Field(None, title='Prompttext') + negativePrompt: Optional[str] = Field(None, title='Negativeprompt') + seed: Optional[int] = Field(None, title='Seed') + resolution: Optional[str] = Field('1080p', title='Resolution') + duration: Optional[int] = Field(5, title='Duration') + + +class IngredientsMode(str, Enum): + creative = 'creative' + precise = 'precise' + + +class PikaBodyGenerate22C2vGenerate22PikascenesPost(BaseModel): + images: List[bytes] = Field( + ..., description='Array of images to process', title='Images' + ) + ingredientsMode: IngredientsMode = Field(..., title='Ingredientsmode') + promptText: Optional[str] = Field(None, title='Prompttext') + negativePrompt: Optional[str] = Field(None, title='Negativeprompt') + seed: Optional[int] = Field(None, title='Seed') + resolution: Optional[str] = Field('1080p', title='Resolution') + duration: Optional[int] = Field(5, title='Duration') + aspectRatio: Optional[AspectRatio2] = Field( + None, description='Aspect ratio (width / height)', title='Aspectratio' + ) + + +class PikaBodyGenerate22KeyframeGenerate22PikaframesPost(BaseModel): + keyFrames: List[bytes] = Field( + ..., description='Array of keyframe images', title='Keyframes' + ) + promptText: str = Field(..., title='Prompttext') + negativePrompt: Optional[str] = Field(None, title='Negativeprompt') + seed: Optional[int] = Field(None, title='Seed') + resolution: Optional[str] = Field('1080p', title='Resolution') + duration: Optional[int] = Field(5, title='Duration') + + +class PikaVideoResponse(BaseModel): + id: str = Field(..., title='Id') + status: str = Field(..., title='Status') + url: str = Field(..., title='Url') + progress: int = Field(..., title='Progress') + + +class PikaValidationError(BaseModel): + loc: List[Union[str, int]] = Field(..., title='Location') + msg: str = Field(..., title='Message') + type: str = Field(..., title='Error Type') class RunwayImageToVideoRequest(BaseModel): - duration: RunwayDurationEnum - model: RunwayModelEnum promptImage: RunwayPromptImageObject - promptText: Optional[constr(max_length=1000)] = Field( - None, description='Text prompt for the generation' + seed: int = Field( + ..., description='Random seed for generation', ge=0, le=4294967295 ) - ratio: RunwayAspectRatioEnum - seed: conint(ge=0, le=4294967295) = Field( - ..., description='Random seed for generation' + model: RunwayModelEnum = Field(..., description='Model to use for generation') + promptText: Optional[str] = Field( + None, description='Text prompt for the generation', max_length=1000 + ) + duration: RunwayDurationEnum = Field( + ..., description='The number of seconds of duration for the output video.' + ) + ratio: RunwayAspectRatioEnum = Field( + ..., + description='The resolution (aspect ratio) of the output video. Allowable values depend on the selected model. 1280:768 and 768:1280 are only supported for gen3a_turbo.', ) -class StripeCharge(BaseModel): - amount: Optional[int] = None - amount_captured: Optional[int] = None - amount_refunded: Optional[int] = None - application: Optional[str] = None - application_fee: Optional[str] = None - application_fee_amount: Optional[int] = None - balance_transaction: Optional[str] = None - billing_details: Optional[StripeBillingDetails] = None - calculated_statement_descriptor: Optional[str] = None - captured: Optional[bool] = None - created: Optional[int] = None - currency: Optional[str] = None - customer: Optional[str] = None - description: Optional[str] = None - destination: Optional[Any] = None - dispute: Optional[Any] = None - disputed: Optional[bool] = None - failure_balance_transaction: Optional[Any] = None - failure_code: Optional[Any] = None - failure_message: Optional[Any] = None - fraud_details: Optional[Dict[str, Any]] = None - id: Optional[str] = None - invoice: Optional[Any] = None - livemode: Optional[bool] = None - metadata: Optional[Dict[str, Any]] = None - object: Optional[Object] = None - on_behalf_of: Optional[Any] = None - order: Optional[Any] = None - outcome: Optional[StripeOutcome] = None - paid: Optional[bool] = None - payment_intent: Optional[str] = None - payment_method: Optional[str] = None - payment_method_details: Optional[StripePaymentMethodDetails] = None - radar_options: Optional[Dict[str, Any]] = None - receipt_email: Optional[str] = None - receipt_number: Optional[str] = None - receipt_url: Optional[str] = None - refunded: Optional[bool] = None - refunds: Optional[StripeRefundList] = None - review: Optional[Any] = None - shipping: Optional[StripeShipping] = None - source: Optional[Any] = None - source_transfer: Optional[Any] = None - statement_descriptor: Optional[Any] = None - statement_descriptor_suffix: Optional[Any] = None - status: Optional[str] = None - transfer_data: Optional[Any] = None - transfer_group: Optional[Any] = None +class RunwayTaskStatusResponse(BaseModel): + id: Optional[str] = Field(None, description='Task ID') + status: Optional[RunwayTaskStatusEnum] = Field(None, description='Task status') + createdAt: Optional[datetime] = Field(None, description='Task creation timestamp') + output: Optional[List[str]] = Field(None, description='Array of output video URLs') -class StripeChargeList(BaseModel): - data: Optional[List[StripeCharge]] = None - has_more: Optional[bool] = None - object: Optional[str] = None - total_count: Optional[int] = None - url: Optional[str] = None - - -class StripePaymentIntent(BaseModel): - amount: Optional[int] = None - amount_capturable: Optional[int] = None - amount_details: Optional[StripeAmountDetails] = None - amount_received: Optional[int] = None - application: Optional[str] = None - application_fee_amount: Optional[int] = None - automatic_payment_methods: Optional[Any] = None - canceled_at: Optional[int] = None - cancellation_reason: Optional[str] = None - capture_method: Optional[str] = None - charges: Optional[StripeChargeList] = None - client_secret: Optional[str] = None - confirmation_method: Optional[str] = None - created: Optional[int] = None - currency: Optional[str] = None - customer: Optional[str] = None - description: Optional[str] = None - id: Optional[str] = None - invoice: Optional[str] = None - last_payment_error: Optional[Any] = None - latest_charge: Optional[str] = None - livemode: Optional[bool] = None - metadata: Optional[Dict[str, Any]] = None - next_action: Optional[Any] = None - object: Optional[Object2] = None - on_behalf_of: Optional[Any] = None - payment_method: Optional[str] = None - payment_method_configuration_details: Optional[Any] = None - payment_method_options: Optional[StripePaymentMethodOptions] = None - payment_method_types: Optional[List[str]] = None - processing: Optional[Any] = None - receipt_email: Optional[str] = None - review: Optional[Any] = None - setup_future_usage: Optional[Any] = None - shipping: Optional[StripeShipping] = None - source: Optional[Any] = None - statement_descriptor: Optional[Any] = None - statement_descriptor_suffix: Optional[Any] = None - status: Optional[str] = None - transfer_data: Optional[Any] = None - transfer_group: Optional[Any] = None - - -class LumaGeneration(BaseModel): - assets: Optional[LumaAssets] = None - created_at: Optional[datetime] = Field( - None, description='The date and time when the generation was created' - ) - failure_reason: Optional[str] = Field( - None, description='The reason for the state of the generation' - ) - generation_type: Optional[LumaGenerationType] = None - id: Optional[UUID] = Field(None, description='The ID of the generation') - model: Optional[str] = Field(None, description='The model used for the generation') - request: Optional[ - Union[ - LumaGenerationRequest, - LumaImageGenerationRequest, - LumaUpscaleVideoGenerationRequest, - LumaAudioGenerationRequest, - ] - ] = Field(None, description='The request of the generation') - state: Optional[LumaState] = None - - -class Publisher(BaseModel): - createdAt: Optional[datetime] = Field( - None, description='The date and time the publisher was created.' - ) - description: Optional[str] = None - id: Optional[str] = Field( - None, - description="The unique identifier for the publisher. It's akin to a username. Should be lowercase.", - ) - logo: Optional[str] = Field(None, description="URL to the publisher's logo.") - members: Optional[List[PublisherMember]] = Field( - None, description='A list of members in the publisher.' - ) - name: Optional[str] = None - source_code_repo: Optional[str] = None - status: Optional[PublisherStatus] = None - support: Optional[str] = None - website: Optional[str] = None - - -class Data8(BaseModel): - object: Optional[StripePaymentIntent] = None - - -class StripeEvent(BaseModel): - api_version: Optional[str] = None - created: Optional[int] = None - data: Data8 - id: str - livemode: Optional[bool] = None - object: Object1 - pending_webhooks: Optional[int] = None - request: Optional[StripeRequestInfo] = None - type: Type4 - - -class Node(BaseModel): - author: Optional[str] = None - category: Optional[str] = Field(None, description='The category of the node.') - description: Optional[str] = None - downloads: Optional[int] = Field( - None, description='The number of downloads of the node.' - ) - icon: Optional[str] = Field(None, description="URL to the node's icon.") - id: Optional[str] = Field(None, description='The unique identifier of the node.') - latest_version: Optional[NodeVersion] = None - license: Optional[str] = Field( - None, description="The path to the LICENSE file in the node's repository." - ) - name: Optional[str] = Field(None, description='The display name of the node.') - publisher: Optional[Publisher] = None - rating: Optional[float] = Field(None, description='The average rating of the node.') - repository: Optional[str] = Field(None, description="URL to the node's repository.") - status: Optional[NodeStatus] = None - status_detail: Optional[str] = Field( - None, description='The status detail of the node.' - ) - tags: Optional[List[str]] = None - translations: Optional[Dict[str, Dict[str, Any]]] = None +class PikaHTTPValidationError(BaseModel): + detail: Optional[List[PikaValidationError]] = Field(None, title='Detail') diff --git a/comfy_api_nodes/apis/client.py b/comfy_api_nodes/apis/client.py index 0d822afb5..e9477737c 100644 --- a/comfy_api_nodes/apis/client.py +++ b/comfy_api_nodes/apis/client.py @@ -523,7 +523,6 @@ class PollingOperation(Generic[T, R]): # Parse response response_obj = self.poll_endpoint.response_model.model_validate(resp) - # Check if task is complete status = self._check_task_status(response_obj) logging.debug(f"[DEBUG] Task Status: {status}") diff --git a/comfy_api_nodes/nodes_kling.py b/comfy_api_nodes/nodes_kling.py new file mode 100644 index 000000000..258ae23b0 --- /dev/null +++ b/comfy_api_nodes/nodes_kling.py @@ -0,0 +1,479 @@ +from inspect import cleandoc +from typing import Union, Optional +import math +import logging +import torch + +from comfy_api_nodes.apis import ( + KlingText2VideoRequest, + KlingText2VideoResponse, + TaskStatus, + CameraControl, + Config as CameraConfig, + Type as CameraType, + Duration, + Mode, + AspectRatio, + ModelName, + KlingImage2VideoRequest, + KlingImage2VideoResponse, +) +from comfy_api_nodes.apis.client import ( + ApiEndpoint, + HttpMethod, + SynchronousOperation, + PollingOperation, + EmptyRequest, +) +from comfy_api_nodes.nodes_api import ( + tensor_to_base64_string, + download_url_to_bytesio, +) +from comfy.comfy_types.node_typing import IO, InputTypeOptions, ComfyNodeABC +from comfy_api.input_impl import VideoFromFile +from comfy_api_nodes.mapper_utils import model_field_to_node_input + +KLING_API_VERSION = "v1" +PATH_TEXT_TO_VIDEO = f"/proxy/kling/{KLING_API_VERSION}/videos/text2video" +PATH_IMAGE_TO_VIDEO = f"/proxy/kling/{KLING_API_VERSION}/videos/image2video" +PATH_VIDEO_EXTEND = f"/proxy/kling/{KLING_API_VERSION}/videos/video-extend" +PATH_LIP_SYNC = f"/proxy/kling/{KLING_API_VERSION}/videos/lip-sync" +PATH_VIDEO_EFFECTS = f"/proxy/kling/{KLING_API_VERSION}/videos/effects" +PATH_CHARACTER_IMAGE = f"/proxy/kling/{KLING_API_VERSION}/images/generations" +PATH_VIRTUAL_TRY_ON = f"/proxy/kling/{KLING_API_VERSION}/images/kolors-virtual-try-on" + + +class KlingApiError(Exception): + """Base exception for Kling API errors.""" + + pass + + +def is_valid_camera_control_configs(configs: list[float]) -> bool: + """Verifies that at least one camera control configuration is non-zero.""" + return any(not math.isclose(value, 0.0) for value in configs) + + +def is_valid_prompt(prompt: str) -> bool: + """Verifies that the prompt is not empty.""" + return bool(prompt) + + +def is_valid_initial_response(response: KlingText2VideoResponse) -> bool: + """Verifies that the initial response contains a task ID.""" + return bool(response.data.task_id) + + +def is_valid_video_response(response: KlingText2VideoResponse) -> bool: + """Verifies that the response contains a task result with at least one video.""" + return ( + response.data.task_result + and response.data.task_result.videos + and len(response.data.task_result.videos) > 0 + ) + + +def is_camera_control_supported(model_name: str, duration: str, mode: str) -> bool: + """`camera_control` is only supported in `pro` mode with `5s` duration and `kling-v1-5`""" + return model_name == "kling-v1-5" and duration == "5" and mode == "pro" + + +def get_camera_control_input_config( + tooltip: str, default: float = 0.0 +) -> tuple[IO, InputTypeOptions]: + """Returns common InputTypeOptions for Kling camera control configurations.""" + input_config = { + "default": default, + "min": -10.0, + "max": 10.0, + "step": 0.25, + "display": "slider", + "tooltip": tooltip, + } + return IO.FLOAT, input_config + + +def _get_camera_control_inputs() -> dict[str, tuple[IO, InputTypeOptions]]: + """Returns a dictionary of camera control inputs common to Kling video generation nodes.""" + return { + "camera_control_type": ( + IO.COMBO, + { + "options": [ + camera_control_type.value for camera_control_type in CameraType + ], + "default": "simple", + "tooltip": "Predefined camera movements type. simple: Customizable camera movement. down_back: Camera descends and moves backward. forward_up: Camera moves forward and tilts up. right_turn_forward: Rotate right and move forward. left_turn_forward: Rotate left and move forward.", + }, + ), + "camera_control_horizontal": get_camera_control_input_config( + "Controls camera's movement along horizontal axis (x-axis). Negative indicates left, positive indicates right" + ), + "camera_control_vertical": get_camera_control_input_config( + "Controls camera's movement along vertical axis (y-axis). Negative indicates downward, positive indicates upward." + ), + "camera_control_pan": get_camera_control_input_config( + "Controls camera's rotation in vertical plane (x-axis). Negative indicates downward rotation, positive indicates upward rotation.", + default=0.5, + ), + "camera_control_roll": get_camera_control_input_config( + "Controls camera's rotation in horizontal plane (y-axis). Negative indicates left rotation, positive indicates right rotation.", + ), + "camera_control_tilt": get_camera_control_input_config( + "Controls camera's rolling amount (z-axis). Negative indicates counterclockwise, positive indicates clockwise.", + ), + "camera_control_zoom": get_camera_control_input_config( + "Controls change in camera's focal length. Negative indicates narrower field of view, positive indicates wider field of view.", + ), + } + + +def download_url_to_video_output(video_url: str) -> tuple[VideoFromFile]: + """Downloads a video from a URL and returns a VIDEO output.""" + video_io = download_url_to_bytesio(video_url) + if video_io is None: + error_msg = f"Failed to download video from {video_url}" + logging.error(error_msg) + raise KlingApiError(error_msg) + return (VideoFromFile(video_io),) + + +class KlingNodeABC(ComfyNodeABC): + """Base class for Kling nodes.""" + + @classmethod + def VALIDATE_INPUTS( + cls, + prompt, + negative_prompt, + camera_control_horizontal, + camera_control_vertical, + camera_control_pan, + camera_control_roll, + camera_control_tilt, + camera_control_zoom, + ) -> Union[str, bool]: + if not is_valid_prompt(prompt): + return "Prompt is required" + if len(prompt) >= 2500: + return "Prompt must be less than 2500 characters" + if negative_prompt and len(negative_prompt) >= 2500: + return "Negative prompt must be less than 2500 characters" + if not is_valid_camera_control_configs( + [ + camera_control_horizontal, + camera_control_vertical, + camera_control_pan, + camera_control_roll, + camera_control_tilt, + camera_control_zoom, + ] + ): + return "Invalid camera control configs" + return True + + DESCRIPTION = cleandoc(__doc__ or "") + FUNCTION = "api_call" + CATEGORY = "api node/video/kling" + API_NODE = True + + +class KlingTextToVideoNode(KlingNodeABC): + """ + Kling Text to Video Node. + """ + + @staticmethod + def poll_for_task_status(task_id: str, auth_token: str) -> KlingText2VideoResponse: + """Polls the Kling API endpoint until the task reaches a terminal state.""" + polling_operation = PollingOperation( + poll_endpoint=ApiEndpoint( + path=f"{PATH_TEXT_TO_VIDEO}/{task_id}", + method=HttpMethod.GET, + request_model=EmptyRequest, + response_model=KlingText2VideoResponse, + ), + completed_statuses=[ + TaskStatus.succeed.value, + ], + failed_statuses=[TaskStatus.failed.value], + status_extractor=lambda response: ( + response.data.task_status.value + if response.data and response.data.task_status + else None + ), + auth_token=auth_token, + ) + return polling_operation.execute() + + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "prompt": model_field_to_node_input( + IO.STRING, KlingText2VideoRequest, "prompt", multiline=True + ), + "negative_prompt": model_field_to_node_input( + IO.STRING, KlingText2VideoRequest, "negative_prompt", multiline=True + ), + "cfg_scale": model_field_to_node_input( + IO.FLOAT, KlingText2VideoRequest, "cfg_scale" + ), + "mode": model_field_to_node_input( + IO.COMBO, KlingText2VideoRequest, "mode", enum_type=Mode + ), + "duration": model_field_to_node_input( + IO.COMBO, KlingText2VideoRequest, "duration", enum_type=Duration + ), + "aspect_ratio": model_field_to_node_input( + IO.COMBO, + KlingText2VideoRequest, + "aspect_ratio", + enum_type=AspectRatio, + ), + **_get_camera_control_inputs(), + }, + "hidden": {"auth_token": "AUTH_TOKEN_COMFY_ORG"}, + } + + RETURN_TYPES = ("VIDEO",) + + def api_call( + self, + prompt: str, + negative_prompt: str, + duration: int, + mode: str, + cfg_scale: float, + aspect_ratio: str, + camera_control_type: str, + camera_control_horizontal: float, + camera_control_vertical: float, + camera_control_pan: float, + camera_control_roll: float, + camera_control_tilt: float, + camera_control_zoom: float, + auth_token: Optional[str] = None, + ) -> tuple[VideoFromFile]: + camera_control = None + if is_camera_control_supported("kling-v1-6", duration, mode): + camera_control = CameraControl( + type=CameraType(camera_control_type), + config=CameraConfig( + horizontal=camera_control_horizontal, + vertical=camera_control_vertical, + pan=camera_control_pan, + roll=camera_control_roll, + tilt=camera_control_tilt, + zoom=camera_control_zoom, + ).model_dump(exclude_none=True), + ) + + initial_operation = SynchronousOperation( + endpoint=ApiEndpoint( + path=PATH_TEXT_TO_VIDEO, + method=HttpMethod.POST, + request_model=KlingText2VideoRequest, + response_model=KlingText2VideoResponse, + ), + request=KlingText2VideoRequest( + prompt=prompt if prompt else None, + negative_prompt=negative_prompt if negative_prompt else None, + duration=Duration(duration), + mode=Mode(mode), + cfg_scale=cfg_scale, + aspect_ratio=AspectRatio(aspect_ratio), + camera_control=camera_control, + ), + auth_token=auth_token, + ) + + initial_response = initial_operation.execute() + if not is_valid_initial_response(initial_response): + error_msg = f"Kling initial request failed. Code: {initial_response.code}, Message: {initial_response.message}, Data: {initial_response.data}" + logging.error(error_msg) + raise KlingApiError(error_msg) + + task_id = initial_response.data.task_id + logging.debug("Kling task submitted. Task ID: %s", task_id) + + final_response = self.poll_for_task_status(task_id, auth_token) + if not is_valid_video_response(final_response): + error_msg = ( + f"Kling task {task_id} succeeded but no video data found in response." + ) + logging.error(error_msg) + raise KlingApiError(error_msg) + + video_url = str(final_response.data.task_result.videos[0].url) + logging.debug("Kling task %s succeeded. Video URL: %s", task_id, video_url) + + return download_url_to_video_output(video_url) + + +class KlingImage2VideoNode(KlingNodeABC): + """ + Kling Image to Video Node. + """ + + @staticmethod + def poll_for_task_status(task_id: str, auth_token: str) -> KlingImage2VideoResponse: + """Polls the Kling API endpoint until the task reaches a terminal state.""" + polling_operation = PollingOperation( + poll_endpoint=ApiEndpoint( + path=f"{PATH_IMAGE_TO_VIDEO}/{task_id}", + method=HttpMethod.GET, + request_model=KlingImage2VideoRequest, + response_model=KlingImage2VideoResponse, + ), + completed_statuses=[TaskStatus.succeed.value], + failed_statuses=[TaskStatus.failed.value], + status_extractor=lambda response: ( + response.data.task_status.value + if response.data and response.data.task_status + else None + ), + auth_token=auth_token, + ) + return polling_operation.execute() + + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "model_name": model_field_to_node_input( + IO.COMBO, KlingImage2VideoRequest, "model_name", enum_type=ModelName + ), + "start_frame": model_field_to_node_input( + IO.IMAGE, KlingImage2VideoRequest, "image" + ), + "prompt": model_field_to_node_input( + IO.STRING, KlingImage2VideoRequest, "prompt", multiline=True + ), + "negative_prompt": model_field_to_node_input( + IO.STRING, + KlingImage2VideoRequest, + "negative_prompt", + multiline=True, + ), + "cfg_scale": model_field_to_node_input( + IO.FLOAT, KlingImage2VideoRequest, "cfg_scale" + ), + "mode": model_field_to_node_input( + IO.COMBO, KlingImage2VideoRequest, "mode", enum_type=Mode + ), + "aspect_ratio": model_field_to_node_input( + IO.COMBO, + KlingImage2VideoRequest, + "aspect_ratio", + enum_type=AspectRatio, + ), + "duration": model_field_to_node_input( + IO.COMBO, KlingImage2VideoRequest, "duration", enum_type=Duration + ), + **_get_camera_control_inputs(), + }, + "optional": { + "end_frame": model_field_to_node_input( + IO.IMAGE, KlingImage2VideoRequest, "image_tail" + ), + }, + "hidden": {"auth_token": "AUTH_TOKEN_COMFY_ORG"}, + } + + RETURN_TYPES = ("VIDEO",) + + def api_call( + self, + model_name: str, + start_frame: torch.Tensor, + prompt: str, + negative_prompt: str, + cfg_scale: float, + mode: str, + aspect_ratio: str, + duration: str, + camera_control_type: str, + camera_control_horizontal: float, + camera_control_vertical: float, + camera_control_pan: float, + camera_control_roll: float, + camera_control_tilt: float, + camera_control_zoom: float, + end_frame: Optional[torch.Tensor] = None, + auth_token: Optional[str] = None, + ) -> tuple[VideoFromFile]: + camera_control = None + if is_camera_control_supported(model_name, duration, mode): + config = None + if camera_control_type != "right_turn_forward": + config = CameraConfig( + horizontal=camera_control_horizontal, + vertical=camera_control_vertical, + pan=camera_control_pan, + roll=camera_control_roll, + tilt=camera_control_tilt, + zoom=camera_control_zoom, + ) + camera_control = CameraControl( + type=CameraType(camera_control_type), + config=config if config else None, + ) + + initial_operation = SynchronousOperation( + endpoint=ApiEndpoint( + path=PATH_IMAGE_TO_VIDEO, + method=HttpMethod.POST, + request_model=KlingImage2VideoRequest, + response_model=KlingImage2VideoResponse, + ), + request=KlingImage2VideoRequest( + model_name=ModelName(model_name), + image=tensor_to_base64_string(start_frame), + image_tail=( + tensor_to_base64_string(end_frame) + if end_frame is not None + else None + ), + prompt=prompt, + negative_prompt=negative_prompt if negative_prompt else None, + cfg_scale=cfg_scale, + mode=Mode(mode), + aspect_ratio=AspectRatio(aspect_ratio), + duration=Duration(duration), + camera_control=camera_control, + ), + auth_token=auth_token, + ) + initial_response = initial_operation.execute() + if not is_valid_initial_response(initial_response): + error_msg = f"Kling initial request failed. Code: {initial_response.code}, Message: {initial_response.message}, Data: {initial_response.data}" + logging.error(error_msg) + raise KlingApiError(error_msg) + + task_id = initial_response.data.task_id + logging.debug("Kling task submitted. Task ID: %s", task_id) + + final_response = KlingImage2VideoNode.poll_for_task_status(task_id, auth_token) + if not is_valid_video_response(final_response): + error_msg = ( + f"Kling task {task_id} succeeded but no video data found in response." + ) + logging.error(error_msg) + raise KlingApiError(error_msg) + + video_url = str(final_response.data.task_result.videos[0].url) + logging.info("Attempting to download video from URL: %s", video_url) + + return download_url_to_video_output(video_url) + + +NODE_CLASS_MAPPINGS = { + "KlingTextToVideoNode": KlingTextToVideoNode, + "KlingImage2VideoNode": KlingImage2VideoNode, +} + +NODE_DISPLAY_NAME_MAPPINGS = { + "KlingTextToVideoNode": "Kling Text to Video", + "KlingImage2VideoNode": "Kling Image to Video", +} diff --git a/nodes.py b/nodes.py index c7b6daf66..9d9ba38d1 100644 --- a/nodes.py +++ b/nodes.py @@ -2263,7 +2263,8 @@ def init_builtin_extra_nodes(): api_nodes_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), "comfy_api_nodes") api_nodes_files = [ "nodes_api.py", - "nodes_veo2.py" + "nodes_veo2.py", + "nodes_kling.py" ] import_failed = [] From 557328a9ee7db8225178faf8be2c5c2e247ff6c3 Mon Sep 17 00:00:00 2001 From: Jedrzej Kosinski Date: Tue, 29 Apr 2025 15:50:09 -0500 Subject: [PATCH 027/121] Add Camera Concepts (luma_concepts) to Luma Video nodes (#33) Co-authored-by: Robin Huang --- comfy_api_nodes/apis/luma_api.py | 89 ++++++++++++++++++++++++++++++++ comfy_api_nodes/nodes_api.py | 66 +++++++++++++++++++---- 2 files changed, 144 insertions(+), 11 deletions(-) diff --git a/comfy_api_nodes/apis/luma_api.py b/comfy_api_nodes/apis/luma_api.py index 1e9350d87..037b2415c 100644 --- a/comfy_api_nodes/apis/luma_api.py +++ b/comfy_api_nodes/apis/luma_api.py @@ -12,6 +12,7 @@ from pydantic import BaseModel, Field, confloat class LumaIO: LUMA_REF = "LUMA_REF" + LUMA_CONCEPTS = "LUMA_CONCEPTS" class LumaReference: @@ -47,6 +48,89 @@ class LumaReferenceChain: return c +class LumaConcept: + def __init__(self, key: str): + self.key = key + + +class LumaConceptChain: + def __init__(self, str_list: list[str] = None): + self.concepts: list[LumaConcept] = [] + if str_list is not None: + for c in str_list: + if c != "None": + self.add(LumaConcept(key=c)) + + def add(self, concept: LumaConcept): + self.concepts.append(concept) + + def create_api_model(self): + if len(self.concepts) == 0: + return None + api_concepts: list[LumaConceptObject] = [] + for concept in self.concepts: + if concept.key == "None": + continue + api_concepts.append(LumaConceptObject(key=concept.key)) + if len(api_concepts) == 0: + return None + return api_concepts + + def clone(self): + c = LumaConceptChain() + for concept in self.concepts: + c.add(concept) + return c + + def clone_and_merge(self, other: LumaConceptChain): + c = self.clone() + for concept in other.concepts: + c.add(concept) + return c + + +def get_luma_concepts(include_none=False): + concepts = [] + if include_none: + concepts.append("None") + return concepts + [ + "truck_left", + "pan_right", + "pedestal_down", + "low_angle", + "pedestal_up", + "selfie", + "pan_left", + "roll_right", + "zoom_in", + "over_the_shoulder", + "orbit_right", + "orbit_left", + "static", + "tiny_planet", + "high_angle", + "bolt_cam", + "dolly_zoom", + "overhead", + "zoom_out", + "handheld", + "roll_left", + "pov", + "aerial_drone", + "push_in", + "crane_down", + "truck_right", + "tilt_down", + "elevator_doors", + "tilt_up", + "ground_level", + "pull_out", + "aerial", + "crane_up", + "eye_level" + ] + + class LumaImageModel(str, Enum): photon_1 = "photon-1" photon_flash_1 = "photon-flash-1" @@ -133,6 +217,10 @@ class LumaKeyframes(BaseModel): frame1: Optional[Union[LumaImageReference, LumaGenerationReference]] = Field(None, description='') +class LumaConceptObject(BaseModel): + key: str = Field(..., description='Camera Concept name') + + class LumaImageGenerationRequest(BaseModel): prompt: str = Field(..., description='The prompt of the generation') model: LumaImageModel = Field(LumaImageModel.photon_1, description='The image model used for the generation') @@ -151,6 +239,7 @@ class LumaGenerationRequest(BaseModel): resolution: Optional[LumaVideoOutputResolution] = Field(None, description='The resolution of the generation') loop: Optional[bool] = Field(None, description='Whether to loop the video') keyframes: Optional[LumaKeyframes] = Field(None, description='The keyframes of the generation') + concepts: Optional[list[LumaConceptObject]] = Field(None, description='Camera Concepts to apply to generation') class LumaGeneration(BaseModel): diff --git a/comfy_api_nodes/nodes_api.py b/comfy_api_nodes/nodes_api.py index f9f6e3a0c..66b464ac5 100644 --- a/comfy_api_nodes/nodes_api.py +++ b/comfy_api_nodes/nodes_api.py @@ -38,7 +38,9 @@ from comfy_api_nodes.apis.luma_api import ( LumaReferenceChain, LumaImageReference, LumaKeyframes, + LumaConceptChain, LumaIO, + get_luma_concepts, ) from comfy_api_nodes.apis.recraft_api import ( RecraftImageGenerationRequest, @@ -965,7 +967,7 @@ class FluxProUltraImageNode(ComfyNodeABC): img.save(img_byte_arr, format='PNG') return base64.b64encode(img_byte_arr.getvalue()).decode() -class LumaReferenceNode: +class LumaReferenceNode(ComfyNodeABC): """ Holds an image and weight for use with Luma Generate Image node. """ @@ -1003,7 +1005,39 @@ class LumaReferenceNode: luma_ref.add(LumaReference(image=image, weight=round(weight, 2))) return (luma_ref, ) -class LumaImageGenerationNode: +class LumaConceptsNode(ComfyNodeABC): + """ + Holds one or more Camera Concepts for use with Luma Text to Video and Luma Image to Video nodes. + """ + RETURN_TYPES = (LumaIO.LUMA_CONCEPTS,) + RETURN_NAMES = ("luma_concepts",) + DESCRIPTION = cleandoc(__doc__ or "") # Handle potential None value + FUNCTION = "create_concepts" + CATEGORY = "api node/Luma" + + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "concept1": (get_luma_concepts(include_none=True), ), + "concept2": (get_luma_concepts(include_none=True), ), + "concept3": (get_luma_concepts(include_none=True), ), + "concept4": (get_luma_concepts(include_none=True), ), + }, + "optional": { + "luma_concepts": (LumaIO.LUMA_CONCEPTS, { + "tooltip": "Optional Camera Concepts to add to the ones chosen here." + }), + } + } + + def create_concepts(self, concept1: str, concept2: str, concept3: str, concept4: str, luma_concepts: LumaConceptChain=None): + chain = LumaConceptChain(str_list=[concept1, concept2, concept3, concept4]) + if luma_concepts is not None: + chain = luma_concepts.clone_and_merge(chain) + return (chain,) + +class LumaImageGenerationNode(ComfyNodeABC): """ Generates images synchronously based on prompt and aspect ratio. """ @@ -1126,7 +1160,7 @@ class LumaImageGenerationNode: chain = LumaReferenceChain(first_ref=LumaReference(image=style_image, weight=weight)) return self._convert_luma_refs(chain, max_refs=1, auth_token=auth_token) -class LumaImageModifyNode: +class LumaImageModifyNode(ComfyNodeABC): """ Modifies images synchronously based on prompt and aspect ratio. """ @@ -1211,7 +1245,7 @@ class LumaImageModifyNode: img = process_image_response(img_response) return (img,) -class LumaTextToVideoGenerationNode: +class LumaTextToVideoGenerationNode(ComfyNodeABC): """ Generates videos synchronously based on prompt and output_size. """ @@ -1254,6 +1288,9 @@ class LumaTextToVideoGenerationNode: }), }, "optional": { + "luma_concepts": (LumaIO.LUMA_CONCEPTS, { + "tooltip": "Optional Camera Concepts to dictate camera motion via the Luma Concepts node." + }), }, "hidden": { "auth_token": "AUTH_TOKEN_COMFY_ORG", @@ -1261,7 +1298,7 @@ class LumaTextToVideoGenerationNode: } def api_call(self, prompt: str, model: str, aspect_ratio: str, resolution: str, duration: str, loop: bool, seed, - auth_token=None, **kwargs): + luma_concepts: LumaConceptChain=None, auth_token=None, **kwargs): operation = SynchronousOperation( endpoint=ApiEndpoint( path="/proxy/luma/generations", @@ -1276,6 +1313,7 @@ class LumaTextToVideoGenerationNode: aspect_ratio=aspect_ratio, duration=duration, loop=loop, + concepts=luma_concepts.create_api_model() if luma_concepts else None ), auth_token=auth_token ) @@ -1298,7 +1336,7 @@ class LumaTextToVideoGenerationNode: vid_response = requests.get(response_poll.assets.video) return (VideoFromFile(BytesIO(vid_response.content)), ) -class LumaImageToVideoGenerationNode: +class LumaImageToVideoGenerationNode(ComfyNodeABC): """ Generates videos synchronously based on prompt, input images, and output_size. """ @@ -1347,6 +1385,9 @@ class LumaImageToVideoGenerationNode: "last_image": (IO.IMAGE, { "tooltip": "Last frame of generated video." }), + "luma_concepts": (LumaIO.LUMA_CONCEPTS, { + "tooltip": "Optional Camera Concepts to dictate camera motion via the Luma Concepts node." + }), }, "hidden": { "auth_token": "AUTH_TOKEN_COMFY_ORG", @@ -1354,7 +1395,7 @@ class LumaImageToVideoGenerationNode: } def api_call(self, prompt: str, model: str, resolution: str, duration: str, loop: bool, seed, - first_image: torch.Tensor=None, last_image: torch.Tensor=None, + first_image: torch.Tensor=None, last_image: torch.Tensor=None, luma_concepts: LumaConceptChain=None, auth_token=None, **kwargs): if first_image is None and last_image is None: raise Exception("At least one of first_image and last_image requires an input.") @@ -1370,11 +1411,12 @@ class LumaImageToVideoGenerationNode: request=LumaGenerationRequest( prompt=prompt, model=model, - aspect_ratio=LumaAspectRatio.ratio_16_9, + aspect_ratio=LumaAspectRatio.ratio_16_9, # ignored, but still needed by the API for some reason resolution=resolution, duration=duration, loop=loop, - keyframes=keyframes + keyframes=keyframes, + concepts=luma_concepts.create_api_model() if luma_concepts else None ), auth_token=auth_token ) @@ -1720,9 +1762,10 @@ NODE_CLASS_MAPPINGS = { "FluxProUltraImageNode": FluxProUltraImageNode, "LumaImageNode": LumaImageGenerationNode, "LumaImageModifyNode": LumaImageModifyNode, - "LumaReferenceNode": LumaReferenceNode, "LumaVideoNode": LumaTextToVideoGenerationNode, "LumaImageToVideoNode": LumaImageToVideoGenerationNode, + "LumaReferenceNode": LumaReferenceNode, + "LumaConceptsNode": LumaConceptsNode, "RecraftTextToImageNode": RecraftTextToImageNode, #"RecraftStyleV3RealisticImage": RecraftStyleV3RealisticImageNode, "RecraftStyleV3DigitalIllustration": RecraftStyleV3DigitalIllustrationNode, @@ -1740,9 +1783,10 @@ NODE_DISPLAY_NAME_MAPPINGS = { "FluxProUltraImageNode": "Flux 1.1 [pro] Ultra Image", "LumaImageNode": "Luma Text to Image", "LumaImageModifyNode": "Luma Image to Image", - "LumaReferenceNode": "Luma Reference", "LumaVideoNode": "Luma Text to Video", "LumaImageToVideoNode": "Luma Image to Video", + "LumaReferenceNode": "Luma Reference", + "LumaConceptsNode": "Luma Concepts", "RecraftTextToImageNode": "Recraft Text to Image", "RecraftStyleV3RealisticImage": "Recraft Style - Realistic Image", "RecraftStyleV3DigitalIllustration": "Recraft Style - Digital Illustration", From 520d5f996bd7e0f2bd443084341ce82243630b33 Mon Sep 17 00:00:00 2001 From: Christian Byrne Date: Tue, 29 Apr 2025 14:02:21 -0700 Subject: [PATCH 028/121] Add Runway nodes (#17) --- comfy_api_nodes/apis/client.py | 11 ++ comfy_api_nodes/nodes_runway.py | 265 ++++++++++++++++++++++++++++++++ nodes.py | 3 +- 3 files changed, 278 insertions(+), 1 deletion(-) create mode 100644 comfy_api_nodes/nodes_runway.py diff --git a/comfy_api_nodes/apis/client.py b/comfy_api_nodes/apis/client.py index e9477737c..412fe0e44 100644 --- a/comfy_api_nodes/apis/client.py +++ b/comfy_api_nodes/apis/client.py @@ -4,6 +4,7 @@ from typing import Callable import io from comfy.cli_args import args +from comfy import utils """ API Client Framework for api.comfy.org. @@ -444,6 +445,7 @@ class PollingOperation(Generic[T, R]): completed_statuses: list, failed_statuses: list, status_extractor: Callable[[R], str], + progress_extractor: Callable[[R], float] = None, request: Optional[T] = None, api_base: str | None = None, auth_token: Optional[str] = None, @@ -459,6 +461,7 @@ class PollingOperation(Generic[T, R]): self.status_extractor = status_extractor or ( lambda x: getattr(x, "status", None) ) + self.progress_extractor = progress_extractor self.completed_statuses = completed_statuses self.failed_statuses = failed_statuses @@ -494,6 +497,10 @@ class PollingOperation(Generic[T, R]): def _poll_until_complete(self, client: ApiClient) -> R: """Poll until the task is complete""" poll_count = 0 + progress = 0 + if self.progress_extractor: + progress = utils.ProgressBar(100) + while True: try: poll_count += 1 @@ -527,6 +534,10 @@ class PollingOperation(Generic[T, R]): status = self._check_task_status(response_obj) logging.debug(f"[DEBUG] Task Status: {status}") + # If progress extractor is provided, extract progress + if self.progress_extractor: + progress.update(self.progress_extractor(response_obj)) + if status == TaskStatus.COMPLETED: logging.debug("[DEBUG] Task completed successfully") self.final_response = response_obj diff --git a/comfy_api_nodes/nodes_runway.py b/comfy_api_nodes/nodes_runway.py new file mode 100644 index 000000000..faa364314 --- /dev/null +++ b/comfy_api_nodes/nodes_runway.py @@ -0,0 +1,265 @@ +from inspect import cleandoc +from typing import Union, Optional +import logging + +import torch +from comfy_api_nodes.apis import ( + RunwayImageToVideoRequest, + RunwayImageToVideoResponse, + RunwayTaskStatusResponse as TaskStatusResponse, + RunwayTaskStatusEnum as TaskStatus, + RunwayModelEnum as Model, + RunwayDurationEnum as Duration, + RunwayAspectRatioEnum as AspectRatio, + RunwayPromptImageObject, + RunwayPromptImageDetailedObject, +) +from comfy_api_nodes.apis.client import ( + ApiEndpoint, + HttpMethod, + SynchronousOperation, + PollingOperation, + EmptyRequest, +) +from comfy_api_nodes.nodes_api import ( + download_url_to_bytesio, + upload_images_to_comfyapi, +) +from comfy.comfy_types.node_typing import IO, ComfyNodeABC +from comfy_api.input_impl import VideoFromFile +from comfy_api_nodes.mapper_utils import model_field_to_node_input + +PATH_IMAGE_TO_VIDEO = "/proxy/runway/image-to-video" + + +class RunwayApiError(Exception): + """Base exception for Runway API errors.""" + + pass + + +class RunwayImageToVideoNode(ComfyNodeABC): + """ + Runway Image to Video Node. + """ + + @staticmethod + def is_ratio_supported(model: str, ratio: str) -> bool: + """ + Checks if the chosen aspect ratio is supported by the chosen model. + """ + if model != "gen3a_turbo" and ratio in [ + "1280:768", + "768:1280", + ]: + return False + return True + + @staticmethod + def is_end_frame_supported(model: str) -> bool: + """ + Checks if the chosen model supports the end frame input. + """ + return model == "gen3a_turbo" + + @staticmethod + def is_valid_prompt(prompt: str) -> bool: + return bool(prompt) + + @staticmethod + def is_valid_initial_response(response: RunwayImageToVideoResponse) -> bool: + return bool(response.id) + + @staticmethod + def is_valid_image(image: torch.Tensor) -> bool: + """https://docs.dev.runwayml.com/assets/inputs/#common-error-reasons""" + return image.shape[2] < 8000 and image.shape[1] < 8000 + + @staticmethod + def is_valid_video_response(response: RunwayImageToVideoResponse) -> bool: + return response.output and len(response.output) > 0 + + @staticmethod + def poll_for_task_status(task_id: str, auth_token: str) -> TaskStatusResponse: + """ + Polls the Runway API endpoint until the task reaches a terminal state. + """ + polling_operation = PollingOperation( + poll_endpoint=ApiEndpoint( + path=f"{PATH_IMAGE_TO_VIDEO}/{task_id}", + method=HttpMethod.GET, + request_model=EmptyRequest, + response_model=TaskStatusResponse, + ), + completed_statuses=[ + TaskStatus.SUCCEEDED.value, + ], + failed_statuses=[ + TaskStatus.FAILED.value, + TaskStatus.CANCELLED.value, + ], + progress_extractor=lambda response: (response.progress * 100), + status_extractor=lambda response: (response.status.value), + auth_token=auth_token, + ) + return polling_operation.execute() + + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "model": model_field_to_node_input( + IO.COMBO, RunwayImageToVideoRequest, "model", enum_type=Model + ), + "prompt": model_field_to_node_input( + IO.STRING, RunwayImageToVideoRequest, "promptText", multiline=True + ), + "duration": model_field_to_node_input( + IO.COMBO, RunwayImageToVideoRequest, "duration", enum_type=Duration + ), + "ratio": model_field_to_node_input( + IO.COMBO, RunwayImageToVideoRequest, "ratio", enum_type=AspectRatio + ), + "seed": model_field_to_node_input( + IO.INT, RunwayImageToVideoRequest, "seed" + ), + }, + "optional": { + "start_frame": ( + IO.IMAGE, + {"tooltip": "Start frame to be used for the video"}, + ), + "end_frame": ( + IO.IMAGE, + { + "tooltip": "End frame to be used for the video. Supported for gen3a_turbo only." + }, + ), + }, + "hidden": {"auth_token": "AUTH_TOKEN_COMFY_ORG"}, + } + + RETURN_TYPES = ("VIDEO",) + FUNCTION = "api_call" + CATEGORY = "api node/video/runway" + DESCRIPTION = cleandoc(__doc__ or "") + API_NODE = True + + @classmethod + def VALIDATE_INPUTS( + cls, + model: str, + ratio: str, + ) -> Union[str, bool]: + if not RunwayImageToVideoNode.is_ratio_supported(model, ratio): + return "Invalid aspect ratio for the chosen model. 1280:768 and 768:1280 are only supported for gen3a_turbo." + return True + + def api_call( + self, + model: str, + prompt: str, + duration: str, + ratio: str, + seed: int, + start_frame: Optional[torch.Tensor] = None, + end_frame: Optional[torch.Tensor] = None, + auth_token: Optional[str] = None, + ) -> tuple[VideoFromFile]: + # Validate manually because optional inputs are not passed to VALIDATE_INPUTS. + if start_frame is None and end_frame is None: + message = "Start frame and end frame cannot both be empty." + raise RunwayApiError(message) + if end_frame is not None and not RunwayImageToVideoNode.is_end_frame_supported( + model + ): + message = "End frame is only supported for gen3a_turbo model." + raise RunwayApiError(message) + + prompt_images_tensors: list[torch.Tensor] = [] + if start_frame is not None: + if not RunwayImageToVideoNode.is_valid_image(start_frame): + message = "Start frame is not a valid image." + raise RunwayApiError(message) + prompt_images_tensors.append(start_frame) + + if end_frame != None: + if not RunwayImageToVideoNode.is_valid_image(end_frame): + message = "End frame is not a valid image." + raise RunwayApiError(message) + prompt_images_tensors.append(end_frame) + + # stack tensors + prompt_images_tensor = torch.cat(prompt_images_tensors, dim=0) + + download_urls = upload_images_to_comfyapi( + prompt_images_tensor, max_images=2, auth_token=auth_token + ) + + # Create a list of detailed image objects + prompt_image_details: list[RunwayPromptImageDetailedObject] = [ + RunwayPromptImageDetailedObject(uri=str(download_urls[0]), position="first") + ] + if len(download_urls) > 1: + prompt_image_details.append( + RunwayPromptImageDetailedObject(uri=str(download_urls[1]), position="last") + ) + + # Wrap the list in the main object if details exist + prompt_image_object: Optional[RunwayPromptImageObject] = None + if prompt_image_details: + prompt_image_object = RunwayPromptImageObject( + root=prompt_image_details + ) + + initial_operation = SynchronousOperation( + endpoint=ApiEndpoint( + path=PATH_IMAGE_TO_VIDEO, + method=HttpMethod.POST, + request_model=RunwayImageToVideoRequest, + response_model=RunwayImageToVideoResponse, + ), + request=RunwayImageToVideoRequest( + promptText=prompt, + seed=seed, + model=Model(model), + duration=Duration(duration), + ratio=AspectRatio(ratio), + promptImage=prompt_image_object, + ), + auth_token=auth_token, + ) + + initial_response = initial_operation.execute() + if not RunwayImageToVideoNode.is_valid_initial_response(initial_response): + error_message = "Invalid initial response from Runway API." + logging.error(error_message) + raise RunwayApiError(error_message) + + task_id = initial_response.id + logging.debug("Runway task submitted. Task ID: %s", task_id) + + final_response = self.poll_for_task_status(task_id, auth_token) + if not RunwayImageToVideoNode.is_valid_video_response(final_response): + error_message = "Runway task succeeded but no video data found in response." + logging.error(error_message) + raise RunwayApiError(error_message) + + video_url = final_response.output[0] + logging.debug("Attempting to download video from URL: %s", video_url) + + video_io = download_url_to_bytesio(video_url) + if video_io is None: + error_msg = f"Failed to download video from {video_url}" + logging.error(error_msg) + raise RunwayApiError(error_msg) + return (VideoFromFile(video_io),) + + +NODE_CLASS_MAPPINGS = { + "RunwayImageToVideoNode": RunwayImageToVideoNode, +} + +NODE_DISPLAY_NAME_MAPPINGS = { + "RunwayImageToVideoNode": "Runway Image to Video", +} diff --git a/nodes.py b/nodes.py index 9d9ba38d1..240f4c248 100644 --- a/nodes.py +++ b/nodes.py @@ -2264,7 +2264,8 @@ def init_builtin_extra_nodes(): api_nodes_files = [ "nodes_api.py", "nodes_veo2.py", - "nodes_kling.py" + "nodes_kling.py", + "nodes_runway.py", ] import_failed = [] From ce7d007872c831f50dbd9d3deec8b5eca98d022e Mon Sep 17 00:00:00 2001 From: Christian Byrne Date: Tue, 29 Apr 2025 14:40:42 -0700 Subject: [PATCH 029/121] Convert Minimax node to use VIDEO output type (#34) --- comfy_api_nodes/nodes_api.py | 1279 +++++++++++++++++++++------------- 1 file changed, 800 insertions(+), 479 deletions(-) diff --git a/comfy_api_nodes/nodes_api.py b/comfy_api_nodes/nodes_api.py index 66b464ac5..725995b3e 100644 --- a/comfy_api_nodes/nodes_api.py +++ b/comfy_api_nodes/nodes_api.py @@ -1,6 +1,5 @@ import io from inspect import cleandoc -from comfy.comfy_types.node_typing import FileLocator from typing import Literal, Optional from comfy.utils import common_upscale from comfy.comfy_types.node_typing import IO, ComfyNodeABC, InputTypeDict @@ -61,9 +60,6 @@ import torch import math import base64 import logging -import json -import av -import os import time import uuid import folder_paths @@ -282,16 +278,16 @@ def upload_images_to_comfyapi(image: torch.Tensor, max_images=8, auth_token=None path="/customers/storage", method=HttpMethod.POST, request_model=UploadRequest, - response_model=UploadResponse + response_model=UploadResponse, ), - request=UploadRequest( - filename=img_binary.name - ), - auth_token=auth_token + request=UploadRequest(filename=img_binary.name), + auth_token=auth_token, ) response = operation.execute() - upload_response = ApiClient.upload_file(response.upload_url, img_binary) + upload_response = ApiClient.upload_file( + response.upload_url, img_binary + ) # verify success try: upload_response.raise_for_status() @@ -309,6 +305,7 @@ def upload_images_to_comfyapi(image: torch.Tensor, max_images=8, auth_token=None break return download_urls + class OpenAIDalle2(ComfyNodeABC): """ Generates images synchronously via OpenAI's DALL·E 2 endpoint. @@ -316,6 +313,7 @@ class OpenAIDalle2(ComfyNodeABC): Uses the proxy at /proxy/openai/images/generations. Returned URLs are short‑lived, so download or cache results if you need to keep them. """ + def __init__(self): pass @@ -323,46 +321,62 @@ class OpenAIDalle2(ComfyNodeABC): def INPUT_TYPES(cls) -> InputTypeDict: return { "required": { - "prompt": (IO.STRING, { - "multiline": True, - "default": "", - "tooltip": "Text prompt for DALL·E", - }), + "prompt": ( + IO.STRING, + { + "multiline": True, + "default": "", + "tooltip": "Text prompt for DALL·E", + }, + ), }, "optional": { - "seed": (IO.INT, { - "default": 0, - "min": 0, - "max": 2**31-1, - "step": 1, - "display": "number", - "tooltip": "not implemented yet in backend", - }), - "size": (IO.COMBO, { - "options": ["256x256", "512x512", "1024x1024"], - "default": "1024x1024", - "tooltip": "Image size", - }), - "n": (IO.INT, { - "default": 1, - "min": 1, - "max": 8, - "step": 1, - "display": "number", - "tooltip": "How many images to generate", - }), - "image": (IO.IMAGE, { - "default": None, - "tooltip": "Optional reference image for image editing.", - }), - "mask": (IO.MASK, { - "default": None, - "tooltip": "Optional mask for inpainting (white areas will be replaced)", - }), + "seed": ( + IO.INT, + { + "default": 0, + "min": 0, + "max": 2**31 - 1, + "step": 1, + "display": "number", + "tooltip": "not implemented yet in backend", + }, + ), + "size": ( + IO.COMBO, + { + "options": ["256x256", "512x512", "1024x1024"], + "default": "1024x1024", + "tooltip": "Image size", + }, + ), + "n": ( + IO.INT, + { + "default": 1, + "min": 1, + "max": 8, + "step": 1, + "display": "number", + "tooltip": "How many images to generate", + }, + ), + "image": ( + IO.IMAGE, + { + "default": None, + "tooltip": "Optional reference image for image editing.", + }, + ), + "mask": ( + IO.MASK, + { + "default": None, + "tooltip": "Optional mask for inpainting (white areas will be replaced)", + }, + ), }, - "hidden": { - "auth_token": "AUTH_TOKEN_COMFY_ORG" - } + "hidden": {"auth_token": "AUTH_TOKEN_COMFY_ORG"}, } RETURN_TYPES = (IO.IMAGE,) @@ -371,7 +385,16 @@ class OpenAIDalle2(ComfyNodeABC): DESCRIPTION = cleandoc(__doc__ or "") API_NODE = True - def api_call(self, prompt, seed=0, image=None, mask=None, n=1, size="1024x1024", auth_token=None): + def api_call( + self, + prompt, + seed=0, + image=None, + mask=None, + n=1, + size="1024x1024", + auth_token=None, + ): model = "dall-e-2" path = "/proxy/openai/images/generations" request_class = OpenAIImageGenerationRequest @@ -388,16 +411,16 @@ class OpenAIDalle2(ComfyNodeABC): if mask.shape[1:] != image.shape[1:-1]: raise Exception("Mask and Image must be the same size") - rgba_tensor[:,:,3] = (1-mask.squeeze().cpu()) + rgba_tensor[:, :, 3] = 1 - mask.squeeze().cpu() rgba_tensor = downscale_input(rgba_tensor.unsqueeze(0)).squeeze() image_np = (rgba_tensor.numpy() * 255).astype(np.uint8) img = Image.fromarray(image_np) img_byte_arr = io.BytesIO() - img.save(img_byte_arr, format='PNG') + img.save(img_byte_arr, format="PNG") img_byte_arr.seek(0) - img_binary = img_byte_arr#.getvalue() + img_binary = img_byte_arr # .getvalue() img_binary.name = "image.png" elif image is not None or mask is not None: raise Exception("Dall-E 2 image editing requires an image AND a mask") @@ -408,7 +431,7 @@ class OpenAIDalle2(ComfyNodeABC): path=path, method=HttpMethod.POST, request_model=request_class, - response_model=OpenAIImageGenerationResponse + response_model=OpenAIImageGenerationResponse, ), request=request_class( model=model, @@ -417,10 +440,14 @@ class OpenAIDalle2(ComfyNodeABC): size=size, seed=seed, ), - files={ - "image": img_binary, - } if img_binary else None, - auth_token=auth_token + files=( + { + "image": img_binary, + } + if img_binary + else None + ), + auth_token=auth_token, ) response = operation.execute() @@ -428,6 +455,7 @@ class OpenAIDalle2(ComfyNodeABC): img_tensor = validate_and_cast_response(response) return (img_tensor,) + class OpenAIDalle3(ComfyNodeABC): """ Generates images synchronously via OpenAI's DALL·E 3 endpoint. @@ -435,6 +463,7 @@ class OpenAIDalle3(ComfyNodeABC): Uses the proxy at /proxy/openai/images/generations. Returned URLs are short‑lived, so download or cache results if you need to keep them. """ + def __init__(self): pass @@ -442,40 +471,53 @@ class OpenAIDalle3(ComfyNodeABC): def INPUT_TYPES(cls) -> InputTypeDict: return { "required": { - "prompt": (IO.STRING, { - "multiline": True, - "default": "", - "tooltip": "Text prompt for DALL·E", - }), + "prompt": ( + IO.STRING, + { + "multiline": True, + "default": "", + "tooltip": "Text prompt for DALL·E", + }, + ), }, "optional": { - "seed": (IO.INT, { - "default": 0, - "min": 0, - "max": 2**31-1, - "step": 1, - "display": "number", - "tooltip": "not implemented yet in backend", - }), - "quality" : (IO.COMBO, { - "options": ["standard","hd"], - "default": "standard", - "tooltip": "Image quality", - }), - "style": (IO.COMBO, { - "options": ["natural","vivid"], - "default": "natural", - "tooltip": "Vivid causes the model to lean towards generating hyper-real and dramatic images. Natural causes the model to produce more natural, less hyper-real looking images.", - }), - "size": (IO.COMBO, { - "options": ["1024x1024", "1024x1792", "1792x1024"], - "default": "1024x1024", - "tooltip": "Image size", - }), + "seed": ( + IO.INT, + { + "default": 0, + "min": 0, + "max": 2**31 - 1, + "step": 1, + "display": "number", + "tooltip": "not implemented yet in backend", + }, + ), + "quality": ( + IO.COMBO, + { + "options": ["standard", "hd"], + "default": "standard", + "tooltip": "Image quality", + }, + ), + "style": ( + IO.COMBO, + { + "options": ["natural", "vivid"], + "default": "natural", + "tooltip": "Vivid causes the model to lean towards generating hyper-real and dramatic images. Natural causes the model to produce more natural, less hyper-real looking images.", + }, + ), + "size": ( + IO.COMBO, + { + "options": ["1024x1024", "1024x1792", "1792x1024"], + "default": "1024x1024", + "tooltip": "Image size", + }, + ), }, - "hidden": { - "auth_token": "AUTH_TOKEN_COMFY_ORG" - } + "hidden": {"auth_token": "AUTH_TOKEN_COMFY_ORG"}, } RETURN_TYPES = (IO.IMAGE,) @@ -484,7 +526,15 @@ class OpenAIDalle3(ComfyNodeABC): DESCRIPTION = cleandoc(__doc__ or "") API_NODE = True - def api_call(self, prompt, seed=0, style="natural", quality="standard", size="1024x1024", auth_token=None): + def api_call( + self, + prompt, + seed=0, + style="natural", + quality="standard", + size="1024x1024", + auth_token=None, + ): model = "dall-e-3" # build the operation @@ -493,7 +543,7 @@ class OpenAIDalle3(ComfyNodeABC): path="/proxy/openai/images/generations", method=HttpMethod.POST, request_model=OpenAIImageGenerationRequest, - response_model=OpenAIImageGenerationResponse + response_model=OpenAIImageGenerationResponse, ), request=OpenAIImageGenerationRequest( model=model, @@ -503,7 +553,7 @@ class OpenAIDalle3(ComfyNodeABC): style=style, seed=seed, ), - auth_token=auth_token + auth_token=auth_token, ) response = operation.execute() @@ -511,6 +561,7 @@ class OpenAIDalle3(ComfyNodeABC): img_tensor = validate_and_cast_response(response) return (img_tensor,) + class OpenAIGPTImage1(ComfyNodeABC): """ Generates images synchronously via OpenAI's GPT Image 1 endpoint. @@ -527,56 +578,78 @@ class OpenAIGPTImage1(ComfyNodeABC): def INPUT_TYPES(cls) -> InputTypeDict: return { "required": { - "prompt": (IO.STRING, { - "multiline": True, - "default": "", - "tooltip": "Text prompt for GPT Image 1", - }), + "prompt": ( + IO.STRING, + { + "multiline": True, + "default": "", + "tooltip": "Text prompt for GPT Image 1", + }, + ), }, "optional": { - "seed": (IO.INT, { - "default": 0, - "min": 0, - "max": 2**31-1, - "step": 1, - "display": "number", - "tooltip": "not implemented yet in backend", - }), - "quality": (IO.COMBO, { - "options": ["low","medium","high"], - "default": "low", - "tooltip": "Image quality, affects cost and generation time.", - }), - "background": (IO.COMBO, { - "options": ["opaque","transparent"], - "default": "opaque", - "tooltip": "Return image with or without background", - }), - "size": (IO.COMBO, { - "options": ["auto", "1024x1024", "1024x1536", "1536x1024"], - "default": "auto", - "tooltip": "Image size", - }), - "n": (IO.INT, { - "default": 1, - "min": 1, - "max": 8, - "step": 1, - "display": "number", - "tooltip": "How many images to generate", - }), - "image": (IO.IMAGE, { - "default": None, - "tooltip": "Optional reference image for image editing.", - }), - "mask": (IO.MASK, { - "default": None, - "tooltip": "Optional mask for inpainting (white areas will be replaced)", - }), + "seed": ( + IO.INT, + { + "default": 0, + "min": 0, + "max": 2**31 - 1, + "step": 1, + "display": "number", + "tooltip": "not implemented yet in backend", + }, + ), + "quality": ( + IO.COMBO, + { + "options": ["low", "medium", "high"], + "default": "low", + "tooltip": "Image quality, affects cost and generation time.", + }, + ), + "background": ( + IO.COMBO, + { + "options": ["opaque", "transparent"], + "default": "opaque", + "tooltip": "Return image with or without background", + }, + ), + "size": ( + IO.COMBO, + { + "options": ["auto", "1024x1024", "1024x1536", "1536x1024"], + "default": "auto", + "tooltip": "Image size", + }, + ), + "n": ( + IO.INT, + { + "default": 1, + "min": 1, + "max": 8, + "step": 1, + "display": "number", + "tooltip": "How many images to generate", + }, + ), + "image": ( + IO.IMAGE, + { + "default": None, + "tooltip": "Optional reference image for image editing.", + }, + ), + "mask": ( + IO.MASK, + { + "default": None, + "tooltip": "Optional mask for inpainting (white areas will be replaced)", + }, + ), }, - "hidden": { - "auth_token": "AUTH_TOKEN_COMFY_ORG" - } + "hidden": {"auth_token": "AUTH_TOKEN_COMFY_ORG"}, } RETURN_TYPES = (IO.IMAGE,) @@ -585,7 +658,18 @@ class OpenAIGPTImage1(ComfyNodeABC): DESCRIPTION = cleandoc(__doc__ or "") API_NODE = True - def api_call(self, prompt, seed=0, quality="low", background="opaque", image=None, mask=None, n=1, size="1024x1024", auth_token=None): + def api_call( + self, + prompt, + seed=0, + quality="low", + background="opaque", + image=None, + mask=None, + n=1, + size="1024x1024", + auth_token=None, + ): model = "gpt-image-1" path = "/proxy/openai/images/generations" request_class = OpenAIImageGenerationRequest @@ -599,15 +683,14 @@ class OpenAIGPTImage1(ComfyNodeABC): batch_size = image.shape[0] - for i in range(batch_size): - single_image = image[i:i+1] + single_image = image[i : i + 1] scaled_image = downscale_input(single_image).squeeze() image_np = (scaled_image.numpy() * 255).astype(np.uint8) img = Image.fromarray(image_np) img_byte_arr = io.BytesIO() - img.save(img_byte_arr, format='PNG') + img.save(img_byte_arr, format="PNG") img_byte_arr.seek(0) img_binary = img_byte_arr img_binary.name = f"image_{i}.png" @@ -627,27 +710,26 @@ class OpenAIGPTImage1(ComfyNodeABC): raise Exception("Mask and Image must be the same size") batch, height, width = mask.shape rgba_mask = torch.zeros(height, width, 4, device="cpu") - rgba_mask[:,:,3] = (1-mask.squeeze().cpu()) + rgba_mask[:, :, 3] = 1 - mask.squeeze().cpu() scaled_mask = downscale_input(rgba_mask.unsqueeze(0)).squeeze() mask_np = (scaled_mask.numpy() * 255).astype(np.uint8) mask_img = Image.fromarray(mask_np) mask_img_byte_arr = io.BytesIO() - mask_img.save(mask_img_byte_arr, format='PNG') + mask_img.save(mask_img_byte_arr, format="PNG") mask_img_byte_arr.seek(0) mask_binary = mask_img_byte_arr mask_binary.name = "mask.png" files.append(("mask", mask_binary)) - # Build the operation operation = SynchronousOperation( endpoint=ApiEndpoint( path=path, method=HttpMethod.POST, request_model=request_class, - response_model=OpenAIImageGenerationResponse + response_model=OpenAIImageGenerationResponse, ), request=request_class( model=model, @@ -659,7 +741,7 @@ class OpenAIGPTImage1(ComfyNodeABC): size=size, ), files=files if files else None, - auth_token=auth_token + auth_token=auth_token, ) response = operation.execute() @@ -674,77 +756,129 @@ class IdeogramTextToImage(ComfyNodeABC): Images links are available for a limited period of time; if you would like to keep the image, you must download it. """ + def __init__(self): pass @classmethod def INPUT_TYPES(cls) -> InputTypeDict: """ - Return a dictionary which contains config for all input fields. - Some types (string): "MODEL", "VAE", "CLIP", "CONDITIONING", "LATENT", "IMAGE", "INT", "STRING", "FLOAT". - Input types "INT", "STRING" or "FLOAT" are special values for fields on the node. - The type can be a list for selection. + Return a dictionary which contains config for all input fields. + Some types (string): "MODEL", "VAE", "CLIP", "CONDITIONING", "LATENT", "IMAGE", "INT", "STRING", "FLOAT". + Input types "INT", "STRING" or "FLOAT" are special values for fields on the node. + The type can be a list for selection. - Returns: `dict`: - - Key input_fields_group (`string`): Can be either required, hidden or optional. A node class must have property `required` - - Value input_fields (`dict`): Contains input fields config: - * Key field_name (`string`): Name of a entry-point method's argument - * Value field_config (`tuple`): - + First value is a string indicate the type of field or a list for selection. - + Secound value is a config for type "INT", "STRING" or "FLOAT". + Returns: `dict`: + - Key input_fields_group (`string`): Can be either required, hidden or optional. A node class must have property `required` + - Value input_fields (`dict`): Contains input fields config: + * Key field_name (`string`): Name of a entry-point method's argument + * Value field_config (`tuple`): + + First value is a string indicate the type of field or a list for selection. + + Secound value is a config for type "INT", "STRING" or "FLOAT". """ return { "required": { - "prompt": (IO.STRING, { - "multiline": True, - "default": "", - "tooltip": "Prompt for the image generation", - }), - "model": (IO.COMBO, { "options": ["V_2", "V_2_TURBO", "V_1", "V_1_TURBO"], "default": "V_2", "tooltip": "Model to use for image generation"}), + "prompt": ( + IO.STRING, + { + "multiline": True, + "default": "", + "tooltip": "Prompt for the image generation", + }, + ), + "model": ( + IO.COMBO, + { + "options": ["V_2", "V_2_TURBO", "V_1", "V_1_TURBO"], + "default": "V_2", + "tooltip": "Model to use for image generation", + }, + ), }, "optional": { - "aspect_ratio": (IO.COMBO, { "options": ["ASPECT_1_1", "ASPECT_4_3", "ASPECT_3_4", "ASPECT_16_9", "ASPECT_9_16", "ASPECT_2_1", "ASPECT_1_2", "ASPECT_3_2", "ASPECT_2_3", "ASPECT_4_5", "ASPECT_5_4"], "default": "ASPECT_1_1", "tooltip": "The aspect ratio for image generation. Cannot be used with resolution" - }), - "resolution": (IO.COMBO, { "options": ["1024x1024", "1024x1792", "1792x1024"], - "default": "1024x1024", - "tooltip": "The resolution for image generation (V2 only). Cannot be used with aspect_ratio" - }), - "magic_prompt_option": (IO.COMBO, { "options": ["AUTO", "ON", "OFF"], - "default": "AUTO", - "tooltip": "Determine if MagicPrompt should be used in generation" - }), - "seed": (IO.INT, { - "default": 0, - "min": 0, - "max": 2147483647, - "step": 1, - "display": "number" - }), - "style_type": (IO.COMBO, { "options": ["NONE", "ANIME", "CINEMATIC", "CREATIVE", "DIGITAL_ART", "PHOTOGRAPHIC"], - "default": "NONE", - "tooltip": "Style type for generation (V2+ only)" - }), - "negative_prompt": (IO.STRING, { - "multiline": True, - "default": "", - "tooltip": "Description of what to exclude from the image (V1/V2 only)" - }), - "num_images": (IO.INT, { - "default": 1, - "min": 1, - "max": 8, - "step": 1, - "display": "number" - }), - "color_palette": (IO.STRING, { - "multiline": False, - "default": "", - "tooltip": "Color palette preset name or hex colors with weights (V2/V2_TURBO only)" - }), + "aspect_ratio": ( + IO.COMBO, + { + "options": [ + "ASPECT_1_1", + "ASPECT_4_3", + "ASPECT_3_4", + "ASPECT_16_9", + "ASPECT_9_16", + "ASPECT_2_1", + "ASPECT_1_2", + "ASPECT_3_2", + "ASPECT_2_3", + "ASPECT_4_5", + "ASPECT_5_4", + ], + "default": "ASPECT_1_1", + "tooltip": "The aspect ratio for image generation. Cannot be used with resolution", + }, + ), + "resolution": ( + IO.COMBO, + { + "options": ["1024x1024", "1024x1792", "1792x1024"], + "default": "1024x1024", + "tooltip": "The resolution for image generation (V2 only). Cannot be used with aspect_ratio", + }, + ), + "magic_prompt_option": ( + IO.COMBO, + { + "options": ["AUTO", "ON", "OFF"], + "default": "AUTO", + "tooltip": "Determine if MagicPrompt should be used in generation", + }, + ), + "seed": ( + IO.INT, + { + "default": 0, + "min": 0, + "max": 2147483647, + "step": 1, + "display": "number", + }, + ), + "style_type": ( + IO.COMBO, + { + "options": [ + "NONE", + "ANIME", + "CINEMATIC", + "CREATIVE", + "DIGITAL_ART", + "PHOTOGRAPHIC", + ], + "default": "NONE", + "tooltip": "Style type for generation (V2+ only)", + }, + ), + "negative_prompt": ( + IO.STRING, + { + "multiline": True, + "default": "", + "tooltip": "Description of what to exclude from the image (V1/V2 only)", + }, + ), + "num_images": ( + IO.INT, + {"default": 1, "min": 1, "max": 8, "step": 1, "display": "number"}, + ), + "color_palette": ( + IO.STRING, + { + "multiline": False, + "default": "", + "tooltip": "Color palette preset name or hex colors with weights (V2/V2_TURBO only)", + }, + ), }, - "hidden": { - "auth_token": "AUTH_TOKEN_COMFY_ORG" - } + "hidden": {"auth_token": "AUTH_TOKEN_COMFY_ORG"}, } RETURN_TYPES = (IO.IMAGE,) @@ -753,9 +887,20 @@ class IdeogramTextToImage(ComfyNodeABC): API_NODE = True CATEGORY = "Example" - def api_call(self, prompt, model, aspect_ratio=None, resolution=None, - magic_prompt_option="AUTO", seed=0, style_type="NONE", - negative_prompt="", num_images=1, color_palette="", auth_token=None): + def api_call( + self, + prompt, + model, + aspect_ratio=None, + resolution=None, + magic_prompt_option="AUTO", + seed=0, + style_type="NONE", + negative_prompt="", + num_images=1, + color_palette="", + auth_token=None, + ): import torch from PIL import Image import io @@ -767,23 +912,25 @@ class IdeogramTextToImage(ComfyNodeABC): path="/proxy/ideogram/generate", method=HttpMethod.POST, request_model=IdeogramGenerateRequest, - response_model=IdeogramGenerateResponse + response_model=IdeogramGenerateResponse, ), request=IdeogramGenerateRequest( image_request=ImageRequest( - prompt=prompt, - model=model, - num_images=num_images, - seed=seed, - aspect_ratio=aspect_ratio if aspect_ratio != "ASPECT_1_1" else None, - resolution=resolution if resolution != "1024x1024" else None, - magic_prompt_option=magic_prompt_option if magic_prompt_option != "AUTO" else None, - style_type=style_type if style_type != "NONE" else None, - negative_prompt=negative_prompt if negative_prompt else None, - color_palette=None + prompt=prompt, + model=model, + num_images=num_images, + seed=seed, + aspect_ratio=aspect_ratio if aspect_ratio != "ASPECT_1_1" else None, + resolution=resolution if resolution != "1024x1024" else None, + magic_prompt_option=( + magic_prompt_option if magic_prompt_option != "AUTO" else None + ), + style_type=style_type if style_type != "NONE" else None, + negative_prompt=negative_prompt if negative_prompt else None, + color_palette=None, ) ), - auth_token=auth_token + auth_token=auth_token, ) response = operation.execute() @@ -817,16 +964,18 @@ class IdeogramTextToImage(ComfyNodeABC): This method is used in the core repo for the LoadImage node where they return the image hash as a string, if the image hash changes between executions the LoadImage node is executed again. """ - #@classmethod - #def IS_CHANGED(s, image, string_field, int_field, float_field, print_to_screen): + # @classmethod + # def IS_CHANGED(s, image, string_field, int_field, float_field, print_to_screen): # return "" + class FluxProUltraImageNode(ComfyNodeABC): """ Generates images synchronously based on prompt and resolution. """ - MINIMUM_RATIO = 1/4 - MAXIMUM_RATIO = 4/1 + + MINIMUM_RATIO = 1 / 4 + MAXIMUM_RATIO = 4 / 1 MINIMUM_RATIO_STR = "1:4" MAXIMUM_RATIO_STR = "4:1" @@ -834,51 +983,74 @@ class FluxProUltraImageNode(ComfyNodeABC): def INPUT_TYPES(s): return { "required": { - "prompt": (IO.STRING, { - "multiline": True, - "default": "", - "tooltip": "Prompt for the image generation", - }), - "prompt_upsampling": (IO.BOOLEAN, { - "default": False, - "tooltip": "Whether to perform upsampling on the prompt. If active, automatically modifies the prompt for more creative generation, but results are nondeterministic (same seed will not produce exactly the same result)." - }), - "seed": (IO.INT, { - "default": 0, - "min": 0, - "max": 0xFFFFFFFFFFFFFFFF, - "control_after_generate": True, - "tooltip": "The random seed used for creating the noise.", - }), - "aspect_ratio": (IO.STRING, { - "default": "16:9", - "tooltip": "Aspect ratio of image; must be between 1:4 and 4:1.", - }), - "raw": (IO.BOOLEAN, { - "default": False, - "tooltip": "When True, generate less processed, more natural-looking images." - }), + "prompt": ( + IO.STRING, + { + "multiline": True, + "default": "", + "tooltip": "Prompt for the image generation", + }, + ), + "prompt_upsampling": ( + IO.BOOLEAN, + { + "default": False, + "tooltip": "Whether to perform upsampling on the prompt. If active, automatically modifies the prompt for more creative generation, but results are nondeterministic (same seed will not produce exactly the same result).", + }, + ), + "seed": ( + IO.INT, + { + "default": 0, + "min": 0, + "max": 0xFFFFFFFFFFFFFFFF, + "control_after_generate": True, + "tooltip": "The random seed used for creating the noise.", + }, + ), + "aspect_ratio": ( + IO.STRING, + { + "default": "16:9", + "tooltip": "Aspect ratio of image; must be between 1:4 and 4:1.", + }, + ), + "raw": ( + IO.BOOLEAN, + { + "default": False, + "tooltip": "When True, generate less processed, more natural-looking images.", + }, + ), }, "optional": { - "image_prompt": (IO.IMAGE, ), - "image_prompt_strength": (IO.FLOAT, { - "default": 0.1, - "min": 0.0, - "max": 1.0, - "step": 0.01, - "tooltip": "Blend between the prompt and the image prompt.", - }), + "image_prompt": (IO.IMAGE,), + "image_prompt_strength": ( + IO.FLOAT, + { + "default": 0.1, + "min": 0.0, + "max": 1.0, + "step": 0.01, + "tooltip": "Blend between the prompt and the image prompt.", + }, + ), }, "hidden": { "auth_token": "AUTH_TOKEN_COMFY_ORG", - } + }, } @classmethod def VALIDATE_INPUTS(cls, aspect_ratio: str): try: - validate_aspect_ratio(aspect_ratio, minimum_ratio=cls.MINIMUM_RATIO, maximum_ratio=cls.MAXIMUM_RATIO, - minimum_ratio_str=cls.MINIMUM_RATIO_STR, maximum_ratio_str=cls.MAXIMUM_RATIO_STR) + validate_aspect_ratio( + aspect_ratio, + minimum_ratio=cls.MINIMUM_RATIO, + maximum_ratio=cls.MAXIMUM_RATIO, + minimum_ratio_str=cls.MINIMUM_RATIO_STR, + maximum_ratio_str=cls.MAXIMUM_RATIO_STR, + ) except Exception as e: return str(e) return True @@ -889,32 +1061,58 @@ class FluxProUltraImageNode(ComfyNodeABC): API_NODE = True CATEGORY = "api node" - def api_call(self, prompt: str, aspect_ratio: str, prompt_upsampling=False, raw=False, seed=0, image_prompt=None, image_prompt_strength=0.1, auth_token=None, **kwargs): + def api_call( + self, + prompt: str, + aspect_ratio: str, + prompt_upsampling=False, + raw=False, + seed=0, + image_prompt=None, + image_prompt_strength=0.1, + auth_token=None, + **kwargs, + ): operation = SynchronousOperation( endpoint=ApiEndpoint( path="/proxy/bfl/flux-pro-1.1-ultra/generate", method=HttpMethod.POST, request_model=BFLFluxProGenerateRequest, - response_model=BFLFluxProGenerateResponse + response_model=BFLFluxProGenerateResponse, ), request=BFLFluxProGenerateRequest( prompt=prompt, prompt_upsampling=prompt_upsampling, seed=seed, - aspect_ratio=validate_aspect_ratio(aspect_ratio, minimum_ratio=self.MINIMUM_RATIO, maximum_ratio=self.MAXIMUM_RATIO, - minimum_ratio_str=self.MINIMUM_RATIO_STR, maximum_ratio_str=self.MAXIMUM_RATIO_STR), + aspect_ratio=validate_aspect_ratio( + aspect_ratio, + minimum_ratio=self.MINIMUM_RATIO, + maximum_ratio=self.MAXIMUM_RATIO, + minimum_ratio_str=self.MINIMUM_RATIO_STR, + maximum_ratio_str=self.MAXIMUM_RATIO_STR, + ), raw=raw, - image_prompt=image_prompt if image_prompt is None else self._convert_image_to_base64(image_prompt), - image_prompt_strength=None if image_prompt is None else round(image_prompt_strength, 2), + image_prompt=( + image_prompt + if image_prompt is None + else self._convert_image_to_base64(image_prompt) + ), + image_prompt_strength=( + None if image_prompt is None else round(image_prompt_strength, 2) + ), ), - auth_token=auth_token + auth_token=auth_token, ) output_image = self._handle_bfl_synchronous_operation(operation) return (output_image,) - def _handle_bfl_synchronous_operation(self, operation: SynchronousOperation, timeout_bfl_calls=360): + def _handle_bfl_synchronous_operation( + self, operation: SynchronousOperation, timeout_bfl_calls=360 + ): response_api: BFLFluxProGenerateResponse = operation.execute() - return self._poll_until_generated(response_api.polling_url, timeout=timeout_bfl_calls) + return self._poll_until_generated( + response_api.polling_url, timeout=timeout_bfl_calls + ) def _poll_until_generated(self, polling_url: str, timeout=360): # used bfl-comfy-nodes to verify code implementation: @@ -935,9 +1133,14 @@ class FluxProUltraImageNode(ComfyNodeABC): img_url = result["result"]["sample"] img_response = requests.get(img_url) return process_image_response(img_response) - elif result["status"] in [BFLStatus.request_moderated, BFLStatus.content_moderated]: + elif result["status"] in [ + BFLStatus.request_moderated, + BFLStatus.content_moderated, + ]: status = result["status"] - raise Exception(f"BFL API did not return an image due to: {status}.") + raise Exception( + f"BFL API did not return an image due to: {status}." + ) elif result["status"] == BFLStatus.error: raise Exception(f"BFL API encountered an error: {result}.") elif result["status"] == BFLStatus.pending: @@ -948,29 +1151,34 @@ class FluxProUltraImageNode(ComfyNodeABC): retries_404 += 1 time.sleep(retry_404_seconds) continue - raise Exception(f"BFL API could not find task after {max_retries_404} tries.") + raise Exception( + f"BFL API could not find task after {max_retries_404} tries." + ) elif response.status_code == 202: time.sleep(retry_202_seconds) elif time.time() - start_time > timeout: - raise Exception(f"BFL API experienced a timeout; could not return request under {timeout} seconds.") + raise Exception( + f"BFL API experienced a timeout; could not return request under {timeout} seconds." + ) else: raise Exception(f"BFL API encountered an error: {response.json()}") def _convert_image_to_base64(self, image: torch.Tensor): - scaled_image = downscale_input(image, total_pixels=2048*2048) + scaled_image = downscale_input(image, total_pixels=2048 * 2048) # remove batch dimension if present if len(scaled_image.shape) > 3: scaled_image = scaled_image[0] image_np = (scaled_image.numpy() * 255).astype(np.uint8) img = Image.fromarray(image_np) img_byte_arr = io.BytesIO() - img.save(img_byte_arr, format='PNG') + img.save(img_byte_arr, format="PNG") return base64.b64encode(img_byte_arr.getvalue()).decode() class LumaReferenceNode(ComfyNodeABC): """ Holds an image and weight for use with Luma Generate Image node. """ + RETURN_TYPES = (LumaIO.LUMA_REF,) RETURN_NAMES = ("luma_ref",) DESCRIPTION = cleandoc(__doc__ or "") # Handle potential None value @@ -981,29 +1189,36 @@ class LumaReferenceNode(ComfyNodeABC): def INPUT_TYPES(s): return { "required": { - "image": (IO.IMAGE, { - "tooltip": "Image to use as reference.", - }), - "weight": (IO.FLOAT, { - "default": 1.0, - "min": 0.0, - "max": 1.0, - "step": 0.01, - "tooltip": "Weight of image reference.", - }), + "image": ( + IO.IMAGE, + { + "tooltip": "Image to use as reference.", + }, + ), + "weight": ( + IO.FLOAT, + { + "default": 1.0, + "min": 0.0, + "max": 1.0, + "step": 0.01, + "tooltip": "Weight of image reference.", + }, + ), }, - "optional": { - "luma_ref": (LumaIO.LUMA_REF,) - } + "optional": {"luma_ref": (LumaIO.LUMA_REF,)}, } - def create_luma_reference(self, image: torch.Tensor, weight: float, luma_ref: LumaReferenceChain=None): + def create_luma_reference( + self, image: torch.Tensor, weight: float, luma_ref: LumaReferenceChain = None + ): if luma_ref is not None: luma_ref = luma_ref.clone() else: luma_ref = LumaReferenceChain() luma_ref.add(LumaReference(image=image, weight=round(weight, 2))) - return (luma_ref, ) + return (luma_ref,) + class LumaConceptsNode(ComfyNodeABC): """ @@ -1041,6 +1256,7 @@ class LumaImageGenerationNode(ComfyNodeABC): """ Generates images synchronously based on prompt and aspect ratio. """ + RETURN_TYPES = (IO.IMAGE,) DESCRIPTION = cleandoc(__doc__ or "") # Handle potential None value FUNCTION = "api_call" @@ -1051,69 +1267,106 @@ class LumaImageGenerationNode(ComfyNodeABC): def INPUT_TYPES(s): return { "required": { - "prompt": (IO.STRING, { - "multiline": True, - "default": "", - "tooltip": "Prompt for the image generation", - }), + "prompt": ( + IO.STRING, + { + "multiline": True, + "default": "", + "tooltip": "Prompt for the image generation", + }, + ), "model": ([model.value for model in LumaImageModel],), - "aspect_ratio": ([ratio.value for ratio in LumaAspectRatio], { - "default": LumaAspectRatio.ratio_16_9, - }), - "seed": (IO.INT, { - "default": 0, - "min": 0, - "max": 0xFFFFFFFFFFFFFFFF, - "control_after_generate": True, - "tooltip": "Seed to determine if node should re-run; actual results are nondeterministic regardless of seed.", - }), - "style_image_weight": (IO.FLOAT, { - "default": 1.0, - "min": 0.0, - "max": 1.0, - "step": 0.01, - "tooltip": "Weight of style image. Ignored if no style_image provided.", - }), + "aspect_ratio": ( + [ratio.value for ratio in LumaAspectRatio], + { + "default": LumaAspectRatio.ratio_16_9, + }, + ), + "seed": ( + IO.INT, + { + "default": 0, + "min": 0, + "max": 0xFFFFFFFFFFFFFFFF, + "control_after_generate": True, + "tooltip": "Seed to determine if node should re-run; actual results are nondeterministic regardless of seed.", + }, + ), + "style_image_weight": ( + IO.FLOAT, + { + "default": 1.0, + "min": 0.0, + "max": 1.0, + "step": 0.01, + "tooltip": "Weight of style image. Ignored if no style_image provided.", + }, + ), }, "optional": { - "image_luma_ref": (LumaIO.LUMA_REF, { - "tooltip": "Luma Reference node connection to influence generation with input images; up to 4 images can be considered." - }), - "style_image": (IO.IMAGE, { - "tooltip": "Style reference image; only 1 image will be used." - }), - "character_image": (IO.IMAGE, { - "tooltip": "Character reference images; can be a batch of multiple, up to 4 images can be considered." - }) + "image_luma_ref": ( + LumaIO.LUMA_REF, + { + "tooltip": "Luma Reference node connection to influence generation with input images; up to 4 images can be considered." + }, + ), + "style_image": ( + IO.IMAGE, + {"tooltip": "Style reference image; only 1 image will be used."}, + ), + "character_image": ( + IO.IMAGE, + { + "tooltip": "Character reference images; can be a batch of multiple, up to 4 images can be considered." + }, + ), }, "hidden": { "auth_token": "AUTH_TOKEN_COMFY_ORG", - } + }, } - def api_call(self, prompt: str, model: str, aspect_ratio: str, seed, style_image_weight: float, - image_luma_ref: LumaReferenceChain=None, style_image: torch.Tensor=None, character_image: torch.Tensor=None, - auth_token=None, **kwargs): + def api_call( + self, + prompt: str, + model: str, + aspect_ratio: str, + seed, + style_image_weight: float, + image_luma_ref: LumaReferenceChain = None, + style_image: torch.Tensor = None, + character_image: torch.Tensor = None, + auth_token=None, + **kwargs, + ): # handle image_luma_ref api_image_ref = None if image_luma_ref is not None: - api_image_ref = self._convert_luma_refs(image_luma_ref, max_refs=4, auth_token=auth_token) + api_image_ref = self._convert_luma_refs( + image_luma_ref, max_refs=4, auth_token=auth_token + ) # handle style_luma_ref api_style_ref = None if style_image is not None: - api_style_ref = self._convert_style_image(style_image, weight=style_image_weight, auth_token=auth_token) + api_style_ref = self._convert_style_image( + style_image, weight=style_image_weight, auth_token=auth_token + ) # handle character_ref images character_ref = None if character_image is not None: - download_urls = upload_images_to_comfyapi(character_image, max_images=4, auth_token=auth_token) - character_ref = LumaCharacterRef(identity0=LumaImageIdentity(images=download_urls)) + download_urls = upload_images_to_comfyapi( + character_image, max_images=4, auth_token=auth_token + ) + character_ref = LumaCharacterRef( + identity0=LumaImageIdentity(images=download_urls) + ) operation = SynchronousOperation( endpoint=ApiEndpoint( path="/proxy/luma/generations/image", method=HttpMethod.POST, request_model=LumaImageGenerationRequest, - response_model=LumaGeneration + response_model=LumaGeneration, ), request=LumaImageGenerationRequest( prompt=prompt, @@ -1123,7 +1376,7 @@ class LumaImageGenerationNode(ComfyNodeABC): style_ref=api_style_ref, character_ref=character_ref, ), - auth_token=auth_token + auth_token=auth_token, ) response_api: LumaGeneration = operation.execute() @@ -1145,25 +1398,34 @@ class LumaImageGenerationNode(ComfyNodeABC): img = process_image_response(img_response) return (img,) - def _convert_luma_refs(self, luma_ref: LumaReferenceChain, max_refs: int, auth_token=None): + def _convert_luma_refs( + self, luma_ref: LumaReferenceChain, max_refs: int, auth_token=None + ): luma_urls = [] ref_count = 0 for ref in luma_ref.refs: - download_urls = upload_images_to_comfyapi(ref.image, max_images=1, auth_token=auth_token) + download_urls = upload_images_to_comfyapi( + ref.image, max_images=1, auth_token=auth_token + ) luma_urls.append(download_urls[0]) ref_count += 1 if ref_count >= max_refs: break return luma_ref.create_api_model(download_urls=luma_urls, max_refs=max_refs) - def _convert_style_image(self, style_image: torch.Tensor, weight: float, auth_token=None): - chain = LumaReferenceChain(first_ref=LumaReference(image=style_image, weight=weight)) + def _convert_style_image( + self, style_image: torch.Tensor, weight: float, auth_token=None + ): + chain = LumaReferenceChain( + first_ref=LumaReference(image=style_image, weight=weight) + ) return self._convert_luma_refs(chain, max_refs=1, auth_token=auth_token) class LumaImageModifyNode(ComfyNodeABC): """ Modifies images synchronously based on prompt and aspect ratio. """ + RETURN_TYPES = (IO.IMAGE,) DESCRIPTION = cleandoc(__doc__ or "") # Handle potential None value FUNCTION = "api_call" @@ -1175,37 +1437,56 @@ class LumaImageModifyNode(ComfyNodeABC): return { "required": { "image": (IO.IMAGE,), - "prompt": (IO.STRING, { - "multiline": True, - "default": "", - "tooltip": "Prompt for the image generation", - }), - "image_weight": (IO.FLOAT, { - "default": 1.0, - "min": 0.0, - "max": 1.0, - "step": 0.01, - "tooltip": "Weight of the image; the closer to 0.0, the less the image will be modified." - }), + "prompt": ( + IO.STRING, + { + "multiline": True, + "default": "", + "tooltip": "Prompt for the image generation", + }, + ), + "image_weight": ( + IO.FLOAT, + { + "default": 1.0, + "min": 0.0, + "max": 1.0, + "step": 0.01, + "tooltip": "Weight of the image; the closer to 0.0, the less the image will be modified.", + }, + ), "model": ([model.value for model in LumaImageModel],), - "seed": (IO.INT, { - "default": 0, - "min": 0, - "max": 0xFFFFFFFFFFFFFFFF, - "control_after_generate": True, - "tooltip": "Seed to determine if node should re-run; actual results are nondeterministic regardless of seed.", - }), - }, - "optional": { + "seed": ( + IO.INT, + { + "default": 0, + "min": 0, + "max": 0xFFFFFFFFFFFFFFFF, + "control_after_generate": True, + "tooltip": "Seed to determine if node should re-run; actual results are nondeterministic regardless of seed.", + }, + ), }, + "optional": {}, "hidden": { "auth_token": "AUTH_TOKEN_COMFY_ORG", - } + }, } - def api_call(self, prompt: str, model: str, image: torch.Tensor, image_weight: float, seed, auth_token=None, **kwargs): + def api_call( + self, + prompt: str, + model: str, + image: torch.Tensor, + image_weight: float, + seed, + auth_token=None, + **kwargs, + ): # first, upload image - download_urls = upload_images_to_comfyapi(image, max_images=1, auth_token=auth_token) + download_urls = upload_images_to_comfyapi( + image, max_images=1, auth_token=auth_token + ) image_url = download_urls[0] # next, make Luma call with download url provided operation = SynchronousOperation( @@ -1213,17 +1494,16 @@ class LumaImageModifyNode(ComfyNodeABC): path="/proxy/luma/generations/image", method=HttpMethod.POST, request_model=LumaImageGenerationRequest, - response_model=LumaGeneration + response_model=LumaGeneration, ), request=LumaImageGenerationRequest( prompt=prompt, model=model, modify_image_ref=LumaModifyImageRef( - url=image_url, - weight=round(image_weight, 2) + url=image_url, weight=round(image_weight, 2) ), ), - auth_token=auth_token + auth_token=auth_token, ) response_api: LumaGeneration = operation.execute() @@ -1249,6 +1529,7 @@ class LumaTextToVideoGenerationNode(ComfyNodeABC): """ Generates videos synchronously based on prompt and output_size. """ + def __init__(self): self.output_dir = folder_paths.get_output_directory() self.type: Literal["output"] = "output" @@ -1263,18 +1544,27 @@ class LumaTextToVideoGenerationNode(ComfyNodeABC): def INPUT_TYPES(s): return { "required": { - "prompt": (IO.STRING, { - "multiline": True, - "default": "", - "tooltip": "Prompt for the video generation", - }), + "prompt": ( + IO.STRING, + { + "multiline": True, + "default": "", + "tooltip": "Prompt for the video generation", + }, + ), "model": ([model.value for model in LumaVideoModel],), - "aspect_ratio": ([ratio.value for ratio in LumaAspectRatio], { - "default": LumaAspectRatio.ratio_16_9, - }), - "resolution": ([resolution.value for resolution in LumaVideoOutputResolution], { - "default": LumaVideoOutputResolution.res_540p, - }), + "aspect_ratio": ( + [ratio.value for ratio in LumaAspectRatio], + { + "default": LumaAspectRatio.ratio_16_9, + }, + ), + "resolution": ( + [resolution.value for resolution in LumaVideoOutputResolution], + { + "default": LumaVideoOutputResolution.res_540p, + }, + ), "duration": ([dur.value for dur in LumaVideoModelOutputDuration],), "loop": (IO.BOOLEAN, { "default": False, @@ -1294,7 +1584,7 @@ class LumaTextToVideoGenerationNode(ComfyNodeABC): }, "hidden": { "auth_token": "AUTH_TOKEN_COMFY_ORG", - } + }, } def api_call(self, prompt: str, model: str, aspect_ratio: str, resolution: str, duration: str, loop: bool, seed, @@ -1304,7 +1594,7 @@ class LumaTextToVideoGenerationNode(ComfyNodeABC): path="/proxy/luma/generations", method=HttpMethod.POST, request_model=LumaGenerationRequest, - response_model=LumaGeneration + response_model=LumaGeneration, ), request=LumaGenerationRequest( prompt=prompt, @@ -1315,7 +1605,7 @@ class LumaTextToVideoGenerationNode(ComfyNodeABC): loop=loop, concepts=luma_concepts.create_api_model() if luma_concepts else None ), - auth_token=auth_token + auth_token=auth_token, ) response_api: LumaGeneration = operation.execute() @@ -1334,12 +1624,14 @@ class LumaTextToVideoGenerationNode(ComfyNodeABC): response_poll = operation.execute() vid_response = requests.get(response_poll.assets.video) - return (VideoFromFile(BytesIO(vid_response.content)), ) + return (VideoFromFile(BytesIO(vid_response.content)),) + class LumaImageToVideoGenerationNode(ComfyNodeABC): """ Generates videos synchronously based on prompt, input images, and output_size. """ + def __init__(self): self.output_dir = folder_paths.get_output_directory() self.type: Literal["output"] = "output" @@ -1354,29 +1646,41 @@ class LumaImageToVideoGenerationNode(ComfyNodeABC): def INPUT_TYPES(s): return { "required": { - "prompt": (IO.STRING, { - "multiline": True, - "default": "", - "tooltip": "Prompt for the video generation", - }), + "prompt": ( + IO.STRING, + { + "multiline": True, + "default": "", + "tooltip": "Prompt for the video generation", + }, + ), "model": ([model.value for model in LumaVideoModel],), # "aspect_ratio": ([ratio.value for ratio in LumaAspectRatio], { # "default": LumaAspectRatio.ratio_16_9, # }), - "resolution": ([resolution.value for resolution in LumaVideoOutputResolution], { - "default": LumaVideoOutputResolution.res_540p, - }), + "resolution": ( + [resolution.value for resolution in LumaVideoOutputResolution], + { + "default": LumaVideoOutputResolution.res_540p, + }, + ), "duration": ([dur.value for dur in LumaVideoModelOutputDuration],), - "loop": (IO.BOOLEAN, { - "default": False, - }), - "seed": (IO.INT, { - "default": 0, - "min": 0, - "max": 0xFFFFFFFFFFFFFFFF, - "control_after_generate": True, - "tooltip": "Seed to determine if node should re-run; actual results are nondeterministic regardless of seed.", - }), + "loop": ( + IO.BOOLEAN, + { + "default": False, + }, + ), + "seed": ( + IO.INT, + { + "default": 0, + "min": 0, + "max": 0xFFFFFFFFFFFFFFFF, + "control_after_generate": True, + "tooltip": "Seed to determine if node should re-run; actual results are nondeterministic regardless of seed.", + }, + ), }, "optional": { "first_image": (IO.IMAGE, { @@ -1391,14 +1695,16 @@ class LumaImageToVideoGenerationNode(ComfyNodeABC): }, "hidden": { "auth_token": "AUTH_TOKEN_COMFY_ORG", - } + }, } def api_call(self, prompt: str, model: str, resolution: str, duration: str, loop: bool, seed, first_image: torch.Tensor=None, last_image: torch.Tensor=None, luma_concepts: LumaConceptChain=None, auth_token=None, **kwargs): if first_image is None and last_image is None: - raise Exception("At least one of first_image and last_image requires an input.") + raise Exception( + "At least one of first_image and last_image requires an input." + ) keyframes = self._convert_to_keyframes(first_image, last_image, auth_token) operation = SynchronousOperation( @@ -1406,7 +1712,7 @@ class LumaImageToVideoGenerationNode(ComfyNodeABC): path="/proxy/luma/generations", method=HttpMethod.POST, request_model=LumaGenerationRequest, - response_model=LumaGeneration + response_model=LumaGeneration, ), request=LumaGenerationRequest( prompt=prompt, @@ -1418,7 +1724,7 @@ class LumaImageToVideoGenerationNode(ComfyNodeABC): keyframes=keyframes, concepts=luma_concepts.create_api_model() if luma_concepts else None ), - auth_token=auth_token + auth_token=auth_token, ) response_api: LumaGeneration = operation.execute() @@ -1437,19 +1743,28 @@ class LumaImageToVideoGenerationNode(ComfyNodeABC): response_poll = operation.execute() vid_response = requests.get(response_poll.assets.video) - return (VideoFromFile(BytesIO(vid_response.content)), ) + return (VideoFromFile(BytesIO(vid_response.content)),) - def _convert_to_keyframes(self, first_image: torch.Tensor=None, last_image: torch.Tensor=None, auth_token=None): + def _convert_to_keyframes( + self, + first_image: torch.Tensor = None, + last_image: torch.Tensor = None, + auth_token=None, + ): if first_image is None and last_image is None: return None frame0 = None frame1 = None if first_image is not None: - download_urls = upload_images_to_comfyapi(first_image, max_images=1, auth_token=auth_token) - frame0 = LumaImageReference(type='image', url=download_urls[0]) + download_urls = upload_images_to_comfyapi( + first_image, max_images=1, auth_token=auth_token + ) + frame0 = LumaImageReference(type="image", url=download_urls[0]) if last_image is not None: - download_urls = upload_images_to_comfyapi(last_image, max_images=1, auth_token=auth_token) - frame1 = LumaImageReference(type='image', url=download_urls[0]) + download_urls = upload_images_to_comfyapi( + last_image, max_images=1, auth_token=auth_token + ) + frame1 = LumaImageReference(type="image", url=download_urls[0]) return LumaKeyframes(frame0=frame0, frame1=frame1) @@ -1478,24 +1793,31 @@ class RecraftStyleV3RealisticImageNode: substyle = None return (RecraftStyle(self.RECRAFT_STYLE, substyle),) + class RecraftStyleV3DigitalIllustrationNode(RecraftStyleV3RealisticImageNode): """ Select digital_illustration style and optional substyle. """ + RECRAFT_STYLE = RecraftStyleV3.digital_illustration + class RecraftStyleV3VectorIllustrationNode(RecraftStyleV3RealisticImageNode): """ Select vector_illustration style and optional substyle. """ + RECRAFT_STYLE = RecraftStyleV3.vector_illustration + class RecraftStyleV3LogoRasterNode(RecraftStyleV3RealisticImageNode): """ Select vector_illustration style and optional substyle. """ + RECRAFT_STYLE = RecraftStyleV3.logo_raster + class RecraftTextToImageNode: """ Generates images synchronously based on prompt and resolution. @@ -1511,43 +1833,68 @@ class RecraftTextToImageNode: def INPUT_TYPES(s): return { "required": { - "prompt": (IO.STRING, { - "multiline": True, - "default": "", - "tooltip": "Prompt for the image generation.", - }), - "size": ([res.value for res in RecraftImageSize], { - "default": RecraftImageSize.res_1024x1024, - "tooltip": "The size of the generated image." - }), - "n": (IO.INT, { - "default": 1, - "min": 1, - "max": 6, - "tooltip": "The number of images to generate." - }), - "seed": (IO.INT, { - "default": 0, - "min": 0, - "max": 0xFFFFFFFFFFFFFFFF, - "control_after_generate": True, - "tooltip": "Seed to determine if node should re-run; actual results are nondeterministic regardless of seed.", - }), + "prompt": ( + IO.STRING, + { + "multiline": True, + "default": "", + "tooltip": "Prompt for the image generation.", + }, + ), + "size": ( + [res.value for res in RecraftImageSize], + { + "default": RecraftImageSize.res_1024x1024, + "tooltip": "The size of the generated image.", + }, + ), + "n": ( + IO.INT, + { + "default": 1, + "min": 1, + "max": 6, + "tooltip": "The number of images to generate.", + }, + ), + "seed": ( + IO.INT, + { + "default": 0, + "min": 0, + "max": 0xFFFFFFFFFFFFFFFF, + "control_after_generate": True, + "tooltip": "Seed to determine if node should re-run; actual results are nondeterministic regardless of seed.", + }, + ), }, "optional": { "recraft_style": (RecraftIO.STYLEV3,), - "negative_prompt": (IO.STRING, { - "default": "", - "forceInput": True, - "tooltip": "An optional text description of undesired elements on an image." - }), + "negative_prompt": ( + IO.STRING, + { + "default": "", + "forceInput": True, + "tooltip": "An optional text description of undesired elements on an image.", + }, + ), }, "hidden": { "auth_token": "AUTH_TOKEN_COMFY_ORG", - } + }, } - def api_call(self, prompt: str, size: str, n: int, seed, recraft_style: RecraftStyle=None, negative_prompt: str=None, auth_token=None, **kwargs): + def api_call( + self, + prompt: str, + size: str, + n: int, + seed, + recraft_style: RecraftStyle = None, + negative_prompt: str = None, + auth_token=None, + **kwargs, + ): default_style = RecraftStyle(RecraftStyleV3.digital_illustration) if recraft_style is None: recraft_style = default_style @@ -1560,7 +1907,7 @@ class RecraftTextToImageNode: path="/proxy/recraft/image_generation", method=HttpMethod.POST, request_model=RecraftImageGenerationRequest, - response_model=RecraftImageGenerationResponse + response_model=RecraftImageGenerationResponse, ), request=RecraftImageGenerationRequest( prompt=prompt, @@ -1569,20 +1916,23 @@ class RecraftTextToImageNode: size=size, n=n, style=recraft_style.style, - substyle=recraft_style.substyle + substyle=recraft_style.substyle, ), - auth_token=auth_token + auth_token=auth_token, ) response: RecraftImageGenerationResponse = operation.execute() images = [] for data in response.data: - image = bytesio_to_image_tensor(download_url_to_bytesio(data.url, timeout=1024)) + image = bytesio_to_image_tensor( + download_url_to_bytesio(data.url, timeout=1024) + ) if len(image.shape) < 4: image = image.unsqueeze(0) images.append(image) output_image = torch.cat(images, dim=0) - return (output_image, ) + return (output_image,) + class MinimaxTextToVideoNode: """ @@ -1710,47 +2060,18 @@ class MinimaxTextToVideoNode: file_url = file_result.file.download_url if file_url is None: - raise Exception(f"No video was found in the response. Full response: {file_result.model_dump()}") + raise Exception( + f"No video was found in the response. Full response: {file_result.model_dump()}" + ) logging.info(f"Generated video URL: {file_url}") - # Construct the save path - full_output_folder, filename, counter, subfolder, filename_prefix = ( - folder_paths.get_save_image_path(filename_prefix, self.output_dir) - ) - file_basename = f"{filename}_{counter:05}_.mp4" - save_path = os.path.join(full_output_folder, file_basename) + video_io = download_url_to_bytesio(file_url) + if video_io is None: + error_msg = f"Failed to download video from {file_url}" + logging.error(error_msg) + raise Exception(error_msg) + return (VideoFromFile(video_io),) - # Download the video data - video_response = requests.get(file_url) - video_data = video_response.content - - # Save the video data to a file - with open(save_path, "wb") as video_file: - video_file.write(video_data) - - # Add workflow metadata to the video container - if prompt is not None or extra_pnginfo is not None: - try: - container = av.open(save_path, mode="r+") - if prompt is not None: - container.metadata["prompt"] = json.dumps(prompt) - if extra_pnginfo is not None: - for x in extra_pnginfo: - container.metadata[x] = json.dumps(extra_pnginfo[x]) - container.close() - except Exception as e: - logging.warning(f"Failed to add metadata to video: {e}") - - # Create a FileLocator for the frontend to use for the preview - results: list[FileLocator] = [ - { - "filename": file_basename, - "subfolder": subfolder, - "type": self.type, - } - ] - - return {"ui": {"images": results, "animated": (True,)}} # A dictionary that contains all nodes you want to export with their names # NOTE: names should be globally unique @@ -1767,10 +2088,10 @@ NODE_CLASS_MAPPINGS = { "LumaReferenceNode": LumaReferenceNode, "LumaConceptsNode": LumaConceptsNode, "RecraftTextToImageNode": RecraftTextToImageNode, - #"RecraftStyleV3RealisticImage": RecraftStyleV3RealisticImageNode, + # "RecraftStyleV3RealisticImage": RecraftStyleV3RealisticImageNode, "RecraftStyleV3DigitalIllustration": RecraftStyleV3DigitalIllustrationNode, - #"RecraftStyleV3VectorIllustration": RecraftStyleV3VectorIllustrationNode, - #"RecraftStyleV3LogoRaster": RecraftStyleV3LogoRasterNode, + # "RecraftStyleV3VectorIllustration": RecraftStyleV3VectorIllustrationNode, + # "RecraftStyleV3LogoRaster": RecraftStyleV3LogoRasterNode, "MinimaxTextToVideoNode": MinimaxTextToVideoNode, } From fb78ec14777264707af2d5408446a26094151a68 Mon Sep 17 00:00:00 2001 From: Christian Byrne Date: Tue, 29 Apr 2025 15:32:44 -0700 Subject: [PATCH 030/121] Standard `CATEGORY` system for api nodes (#35) --- comfy_api_nodes/nodes_api.py | 28 ++++++++++++++-------------- comfy_api_nodes/nodes_kling.py | 2 +- comfy_api_nodes/nodes_runway.py | 2 +- comfy_api_nodes/nodes_veo2.py | 2 +- 4 files changed, 17 insertions(+), 17 deletions(-) diff --git a/comfy_api_nodes/nodes_api.py b/comfy_api_nodes/nodes_api.py index 725995b3e..e423e80ca 100644 --- a/comfy_api_nodes/nodes_api.py +++ b/comfy_api_nodes/nodes_api.py @@ -381,7 +381,7 @@ class OpenAIDalle2(ComfyNodeABC): RETURN_TYPES = (IO.IMAGE,) FUNCTION = "api_call" - CATEGORY = "api node" + CATEGORY = "api node/image/openai" DESCRIPTION = cleandoc(__doc__ or "") API_NODE = True @@ -522,7 +522,7 @@ class OpenAIDalle3(ComfyNodeABC): RETURN_TYPES = (IO.IMAGE,) FUNCTION = "api_call" - CATEGORY = "api node" + CATEGORY = "api node/image/openai" DESCRIPTION = cleandoc(__doc__ or "") API_NODE = True @@ -654,7 +654,7 @@ class OpenAIGPTImage1(ComfyNodeABC): RETURN_TYPES = (IO.IMAGE,) FUNCTION = "api_call" - CATEGORY = "api node" + CATEGORY = "api node/image/openai" DESCRIPTION = cleandoc(__doc__ or "") API_NODE = True @@ -885,7 +885,7 @@ class IdeogramTextToImage(ComfyNodeABC): DESCRIPTION = cleandoc(__doc__ or "") # Handle potential None value FUNCTION = "api_call" API_NODE = True - CATEGORY = "Example" + CATEGORY = "api node/image/ideogram" def api_call( self, @@ -1059,7 +1059,7 @@ class FluxProUltraImageNode(ComfyNodeABC): DESCRIPTION = cleandoc(__doc__ or "") # Handle potential None value FUNCTION = "api_call" API_NODE = True - CATEGORY = "api node" + CATEGORY = "api node/image/bfl" def api_call( self, @@ -1183,7 +1183,7 @@ class LumaReferenceNode(ComfyNodeABC): RETURN_NAMES = ("luma_ref",) DESCRIPTION = cleandoc(__doc__ or "") # Handle potential None value FUNCTION = "create_luma_reference" - CATEGORY = "api node/Luma" + CATEGORY = "api node/image/Luma" @classmethod def INPUT_TYPES(s): @@ -1228,7 +1228,7 @@ class LumaConceptsNode(ComfyNodeABC): RETURN_NAMES = ("luma_concepts",) DESCRIPTION = cleandoc(__doc__ or "") # Handle potential None value FUNCTION = "create_concepts" - CATEGORY = "api node/Luma" + CATEGORY = "api node/image/Luma" @classmethod def INPUT_TYPES(s): @@ -1261,7 +1261,7 @@ class LumaImageGenerationNode(ComfyNodeABC): DESCRIPTION = cleandoc(__doc__ or "") # Handle potential None value FUNCTION = "api_call" API_NODE = True - CATEGORY = "api node" + CATEGORY = "api node/image/Luma" @classmethod def INPUT_TYPES(s): @@ -1430,7 +1430,7 @@ class LumaImageModifyNode(ComfyNodeABC): DESCRIPTION = cleandoc(__doc__ or "") # Handle potential None value FUNCTION = "api_call" API_NODE = True - CATEGORY = "api node" + CATEGORY = "api node/image/Luma" @classmethod def INPUT_TYPES(s): @@ -1538,7 +1538,7 @@ class LumaTextToVideoGenerationNode(ComfyNodeABC): DESCRIPTION = cleandoc(__doc__ or "") # Handle potential None value FUNCTION = "api_call" API_NODE = True - CATEGORY = "api node" + CATEGORY = "api node/image/Luma" @classmethod def INPUT_TYPES(s): @@ -1640,7 +1640,7 @@ class LumaImageToVideoGenerationNode(ComfyNodeABC): DESCRIPTION = cleandoc(__doc__ or "") # Handle potential None value FUNCTION = "api_call" API_NODE = True - CATEGORY = "api node" + CATEGORY = "api node/image/Luma" @classmethod def INPUT_TYPES(s): @@ -1776,7 +1776,7 @@ class RecraftStyleV3RealisticImageNode: RETURN_TYPES = (RecraftIO.STYLEV3,) RETURN_NAMES = ("recraft_style",) FUNCTION = "create_style" - CATEGORY = "api node/Recraft" + CATEGORY = "api node/image/Recraft" RECRAFT_STYLE = RecraftStyleV3.realistic_image @@ -1827,7 +1827,7 @@ class RecraftTextToImageNode: DESCRIPTION = cleandoc(__doc__ or "") # Handle potential None value FUNCTION = "api_call" API_NODE = True - CATEGORY = "api node" + CATEGORY = "api node/image/Recraft" @classmethod def INPUT_TYPES(s): @@ -1992,7 +1992,7 @@ class MinimaxTextToVideoNode: RETURN_TYPES = ("VIDEO",) DESCRIPTION = "Generates videos from prompts using Minimax's API" FUNCTION = "generate_video" - CATEGORY = "video" + CATEGORY = "api node/video/Minimax" API_NODE = True OUTPUT_NODE = True diff --git a/comfy_api_nodes/nodes_kling.py b/comfy_api_nodes/nodes_kling.py index 258ae23b0..0b8c6c5b4 100644 --- a/comfy_api_nodes/nodes_kling.py +++ b/comfy_api_nodes/nodes_kling.py @@ -174,7 +174,7 @@ class KlingNodeABC(ComfyNodeABC): DESCRIPTION = cleandoc(__doc__ or "") FUNCTION = "api_call" - CATEGORY = "api node/video/kling" + CATEGORY = "api node/video/Kling" API_NODE = True diff --git a/comfy_api_nodes/nodes_runway.py b/comfy_api_nodes/nodes_runway.py index faa364314..25abdc150 100644 --- a/comfy_api_nodes/nodes_runway.py +++ b/comfy_api_nodes/nodes_runway.py @@ -141,7 +141,7 @@ class RunwayImageToVideoNode(ComfyNodeABC): RETURN_TYPES = ("VIDEO",) FUNCTION = "api_call" - CATEGORY = "api node/video/runway" + CATEGORY = "api node/video/Runway" DESCRIPTION = cleandoc(__doc__ or "") API_NODE = True diff --git a/comfy_api_nodes/nodes_veo2.py b/comfy_api_nodes/nodes_veo2.py index c6defe00a..5a2ea7eee 100644 --- a/comfy_api_nodes/nodes_veo2.py +++ b/comfy_api_nodes/nodes_veo2.py @@ -123,7 +123,7 @@ class VeoVideoGenerationNode(ComfyNodeABC): RETURN_TYPES = (IO.VIDEO,) FUNCTION = "generate_video" - CATEGORY = "api node/video" + CATEGORY = "api node/video/Veo" DESCRIPTION = "Generates videos from text prompts using Google's Veo API" API_NODE = True From cc9ce9ad9b93a043e46107313f0ebac7d6d784ca Mon Sep 17 00:00:00 2001 From: Christian Byrne Date: Tue, 29 Apr 2025 16:15:50 -0700 Subject: [PATCH 031/121] Set `Content-Type` header when uploading files (#36) --- comfy_api_nodes/apis/__init__.py | 6 +- comfy_api_nodes/apis/client.py | 14 +- comfy_api_nodes/nodes_api.py | 233 ++++++++++++++++++++++--------- comfy_api_nodes/nodes_runway.py | 2 +- 4 files changed, 178 insertions(+), 77 deletions(-) diff --git a/comfy_api_nodes/apis/__init__.py b/comfy_api_nodes/apis/__init__.py index 7c41e936d..2b95fe23a 100644 --- a/comfy_api_nodes/apis/__init__.py +++ b/comfy_api_nodes/apis/__init__.py @@ -622,7 +622,7 @@ class Position(str, Enum): class RunwayPromptImageDetailedObject(BaseModel): - uri: AnyUrl = Field( + uri: str = Field( ..., description='A HTTPS URL or data URI containing an encoded image.' ) position: Position = Field( @@ -648,9 +648,9 @@ class RunwayAspectRatioEnum(str, Enum): class RunwayPromptImageObject( - RootModel[Union[AnyUrl, List[RunwayPromptImageDetailedObject]]] + RootModel[Union[str, List[RunwayPromptImageDetailedObject]]] ): - root: Union[AnyUrl, List[RunwayPromptImageDetailedObject]] = Field( + root: Union[str, List[RunwayPromptImageDetailedObject]] = Field( ..., description='Image(s) to use for the video generation. Can be a single URI or an array of image objects with positions.', ) diff --git a/comfy_api_nodes/apis/client.py b/comfy_api_nodes/apis/client.py index 412fe0e44..b9575b83a 100644 --- a/comfy_api_nodes/apis/client.py +++ b/comfy_api_nodes/apis/client.py @@ -127,7 +127,7 @@ class EmptyRequest(BaseModel): class UploadRequest(BaseModel): filename: str = Field(..., description="Filename to upload") - + mime_type: str = Field(..., description="Mime type of the file. For example: image/png, image/jpeg, video/mp4, etc.") class UploadResponse(BaseModel): download_url: str = Field(..., description='URL to GET uploaded file') @@ -297,22 +297,28 @@ class ApiClient: def upload_file( upload_url: str, file: io.BytesIO | str, + mime_type: str | None = None, ): """Upload a file to the API. Make sure the file has a filename equal to what the url expects. Args: upload_url: The URL to upload to file: Either a file path string, BytesIO object, or tuple of (file_path, filename) - mime_type: The mime type of the file + mime_type: Optional mime type to set for the upload """ + headers = {} + if mime_type: + headers["Content-Type"] = mime_type + if isinstance(file, io.BytesIO): file.seek(0) # Ensure we're at the start of the file data = file.read() - return requests.put(upload_url, data=data) + return requests.put(upload_url, data=data, headers=headers) elif isinstance(file, str): with open(file, "rb") as f: data = f.read() - return requests.put(upload_url, data=data) + return requests.put(upload_url, data=data, headers=headers) + class ApiEndpoint(Generic[T, R]): """Defines an API endpoint with its request and response types""" diff --git a/comfy_api_nodes/nodes_api.py b/comfy_api_nodes/nodes_api.py index e423e80ca..ad5ebd330 100644 --- a/comfy_api_nodes/nodes_api.py +++ b/comfy_api_nodes/nodes_api.py @@ -51,7 +51,16 @@ from comfy_api_nodes.apis.recraft_api import ( RecraftIO, get_v3_substyles, ) -from comfy_api_nodes.apis.client import ApiClient, ApiEndpoint, HttpMethod, SynchronousOperation, PollingOperation, EmptyRequest, UploadRequest, UploadResponse +from comfy_api_nodes.apis.client import ( + ApiClient, + ApiEndpoint, + HttpMethod, + SynchronousOperation, + PollingOperation, + EmptyRequest, + UploadRequest, + UploadResponse, +) import numpy as np from PIL import Image @@ -65,9 +74,10 @@ import uuid import folder_paths from io import BytesIO -def downscale_input(image, total_pixels=1536*1024): - samples = image.movedim(-1,1) - #downscaling input images to roughly the same size as the outputs + +def downscale_input(image, total_pixels=1536 * 1024): + samples = image.movedim(-1, 1) + # downscaling input images to roughly the same size as the outputs total = int(total_pixels) scale_by = math.sqrt(total / (samples.shape[3] * samples.shape[2])) if scale_by >= 1: @@ -76,9 +86,10 @@ def downscale_input(image, total_pixels=1536*1024): height = round(samples.shape[2] * scale_by) s = common_upscale(samples, width, height, "lanczos", "disabled") - s = s.movedim(1,-1) + s = s.movedim(1, -1) return s + def validate_and_cast_response(response): # validate raw JSON response data = response.data @@ -117,29 +128,46 @@ def validate_and_cast_response(response): return torch.stack(image_tensors, dim=0) -def validate_aspect_ratio(aspect_ratio: str, minimum_ratio: float, maximum_ratio: float, minimum_ratio_str: str, maximum_ratio_str: str): + +def validate_aspect_ratio( + aspect_ratio: str, + minimum_ratio: float, + maximum_ratio: float, + minimum_ratio_str: str, + maximum_ratio_str: str, +): # get ratio values - numbers = aspect_ratio.split(':') + numbers = aspect_ratio.split(":") if len(numbers) != 2: - raise Exception(f"Aspect ratio must be in the format X:Y, such as 16:9, but was {aspect_ratio}.") + raise Exception( + f"Aspect ratio must be in the format X:Y, such as 16:9, but was {aspect_ratio}." + ) try: numerator = int(numbers[0]) denominator = int(numbers[1]) except ValueError: - raise Exception(f"Aspect ratio must contain numbers separated by ':', such as 16:9, but was {aspect_ratio}.") - calculated_ratio = numerator/denominator + raise Exception( + f"Aspect ratio must contain numbers separated by ':', such as 16:9, but was {aspect_ratio}." + ) + calculated_ratio = numerator / denominator # if not close to minimum and maximum, check bounds - if not math.isclose(calculated_ratio, minimum_ratio) or not math.isclose(calculated_ratio, maximum_ratio): + if not math.isclose(calculated_ratio, minimum_ratio) or not math.isclose( + calculated_ratio, maximum_ratio + ): if calculated_ratio < minimum_ratio: - raise Exception(f"Aspect ratio cannot reduce to any less than {minimum_ratio_str} ({minimum_ratio}), but was {aspect_ratio} ({calculated_ratio}).") + raise Exception( + f"Aspect ratio cannot reduce to any less than {minimum_ratio_str} ({minimum_ratio}), but was {aspect_ratio} ({calculated_ratio})." + ) elif calculated_ratio > maximum_ratio: - raise Exception(f"Aspect ratio cannot reduce to any greater than {maximum_ratio_str} ({maximum_ratio}), but was {aspect_ratio} ({calculated_ratio}).") + raise Exception( + f"Aspect ratio cannot reduce to any greater than {maximum_ratio_str} ({maximum_ratio}), but was {aspect_ratio} ({calculated_ratio})." + ) return aspect_ratio def mimetype_to_extension(mime_type: str) -> str: """Converts a MIME type to a file extension.""" - return mime_type.split('/')[-1].lower() + return mime_type.split("/")[-1].lower() def download_url_to_bytesio(url: str, timeout: int = None) -> BytesIO: @@ -153,7 +181,7 @@ def download_url_to_bytesio(url: str, timeout: int = None) -> BytesIO: BytesIO object containing the downloaded content. """ response = requests.get(url, stream=True, timeout=timeout) - response.raise_for_status() # Raises HTTPError for bad responses (4XX or 5XX) + response.raise_for_status() # Raises HTTPError for bad responses (4XX or 5XX) return BytesIO(response.content) @@ -178,35 +206,42 @@ def bytesio_to_image_tensor(image_bytesio: BytesIO, mode: str = "RGBA") -> torch def process_image_response(response: requests.Response): - '''Uses content from a Response object and converts it to a torch.Tensor''' + """Uses content from a Response object and converts it to a torch.Tensor""" return bytesio_to_image_tensor(BytesIO(response.content)) -def _tensor_to_pil(image: torch.Tensor, total_pixels: int = 2048*2048) -> Image.Image: +def _tensor_to_pil(image: torch.Tensor, total_pixels: int = 2048 * 2048) -> Image.Image: """Converts a single torch.Tensor image [H, W, C] to a PIL Image, optionally downscaling.""" if len(image.shape) > 3: image = image[0] # TODO: remove alpha if not allowed and present input_tensor = image.cpu() - input_tensor = downscale_input(input_tensor.unsqueeze(0), total_pixels=total_pixels).squeeze() + input_tensor = downscale_input( + input_tensor.unsqueeze(0), total_pixels=total_pixels + ).squeeze() image_np = (input_tensor.numpy() * 255).astype(np.uint8) img = Image.fromarray(image_np) return img -def _pil_to_bytesio(img: Image.Image, mime_type: str = 'image/png') -> BytesIO: +def _pil_to_bytesio(img: Image.Image, mime_type: str = "image/png") -> BytesIO: """Converts a PIL Image to a BytesIO object.""" img_byte_arr = io.BytesIO() # Derive PIL format from MIME type (e.g., 'image/png' -> 'PNG') - pil_format = mime_type.split('/')[-1].upper() - if pil_format == 'JPG': - pil_format = 'JPEG' + pil_format = mime_type.split("/")[-1].upper() + if pil_format == "JPG": + pil_format = "JPEG" img.save(img_byte_arr, format=pil_format) img_byte_arr.seek(0) return img_byte_arr -def tensor_to_bytesio(image: torch.Tensor, name: Optional[str] = None, total_pixels: int = 2048*2048, mime_type: str = 'image/png') -> BytesIO: +def tensor_to_bytesio( + image: torch.Tensor, + name: Optional[str] = None, + total_pixels: int = 2048 * 2048, + mime_type: str = "image/png", +) -> BytesIO: """Converts a torch.Tensor image to a named BytesIO object. Args: @@ -220,11 +255,17 @@ def tensor_to_bytesio(image: torch.Tensor, name: Optional[str] = None, total_pix """ pil_image = _tensor_to_pil(image, total_pixels=total_pixels) img_binary = _pil_to_bytesio(pil_image, mime_type=mime_type) - img_binary.name = f"{name if name else uuid.uuid4()}.{mimetype_to_extension(mime_type)}" + img_binary.name = ( + f"{name if name else uuid.uuid4()}.{mimetype_to_extension(mime_type)}" + ) return img_binary -def tensor_to_base64_string(image_tensor: torch.Tensor, total_pixels: int = 2048*2048, mime_type: str = 'image/png') -> str: +def tensor_to_base64_string( + image_tensor: torch.Tensor, + total_pixels: int = 2048 * 2048, + mime_type: str = "image/png", +) -> str: """Convert [B, H, W, C] or [H, W, C] tensor to a base64 string. Args: @@ -243,7 +284,11 @@ def tensor_to_base64_string(image_tensor: torch.Tensor, total_pixels: int = 2048 return base64_encoded_string -def tensor_to_data_uri(image_tensor: torch.Tensor, total_pixels: int = 2048 * 2048, mime_type: str = 'image/png') -> str: +def tensor_to_data_uri( + image_tensor: torch.Tensor, + total_pixels: int = 2048 * 2048, + mime_type: str = "image/png", +) -> str: """Converts a tensor image to a Data URI string. Args: @@ -258,7 +303,9 @@ def tensor_to_data_uri(image_tensor: torch.Tensor, total_pixels: int = 2048 * 20 return f"data:{mime_type};base64,{base64_string}" -def upload_images_to_comfyapi(image: torch.Tensor, max_images=8, auth_token=None) -> list[str]: +def upload_images_to_comfyapi( + image: torch.Tensor, max_images=8, auth_token=None, mime_type: str = "image/png" +) -> list[str]: # if batch, try to upload each file if max_images is greater than 0 idx_image = 0 download_urls: list[str] = [] @@ -271,7 +318,7 @@ def upload_images_to_comfyapi(image: torch.Tensor, max_images=8, auth_token=None if len(image.shape) > 3: curr_image = image[idx_image] # get BytesIO version of image - img_binary = tensor_to_bytesio(curr_image) + img_binary = tensor_to_bytesio(curr_image, mime_type=mime_type) # first, request upload/download urls from comfy API operation = SynchronousOperation( endpoint=ApiEndpoint( @@ -280,13 +327,13 @@ def upload_images_to_comfyapi(image: torch.Tensor, max_images=8, auth_token=None request_model=UploadRequest, response_model=UploadResponse, ), - request=UploadRequest(filename=img_binary.name), + request=UploadRequest(filename=img_binary.name, mime_type=mime_type), auth_token=auth_token, ) response = operation.execute() upload_response = ApiClient.upload_file( - response.upload_url, img_binary + response.upload_url, img_binary, mime_type=mime_type ) # verify success try: @@ -1174,6 +1221,7 @@ class FluxProUltraImageNode(ComfyNodeABC): img.save(img_byte_arr, format="PNG") return base64.b64encode(img_byte_arr.getvalue()).decode() + class LumaReferenceNode(ComfyNodeABC): """ Holds an image and weight for use with Luma Generate Image node. @@ -1224,6 +1272,7 @@ class LumaConceptsNode(ComfyNodeABC): """ Holds one or more Camera Concepts for use with Luma Text to Video and Luma Image to Video nodes. """ + RETURN_TYPES = (LumaIO.LUMA_CONCEPTS,) RETURN_NAMES = ("luma_concepts",) DESCRIPTION = cleandoc(__doc__ or "") # Handle potential None value @@ -1234,24 +1283,35 @@ class LumaConceptsNode(ComfyNodeABC): def INPUT_TYPES(s): return { "required": { - "concept1": (get_luma_concepts(include_none=True), ), - "concept2": (get_luma_concepts(include_none=True), ), - "concept3": (get_luma_concepts(include_none=True), ), - "concept4": (get_luma_concepts(include_none=True), ), + "concept1": (get_luma_concepts(include_none=True),), + "concept2": (get_luma_concepts(include_none=True),), + "concept3": (get_luma_concepts(include_none=True),), + "concept4": (get_luma_concepts(include_none=True),), }, "optional": { - "luma_concepts": (LumaIO.LUMA_CONCEPTS, { - "tooltip": "Optional Camera Concepts to add to the ones chosen here." - }), - } + "luma_concepts": ( + LumaIO.LUMA_CONCEPTS, + { + "tooltip": "Optional Camera Concepts to add to the ones chosen here." + }, + ), + }, } - def create_concepts(self, concept1: str, concept2: str, concept3: str, concept4: str, luma_concepts: LumaConceptChain=None): + def create_concepts( + self, + concept1: str, + concept2: str, + concept3: str, + concept4: str, + luma_concepts: LumaConceptChain = None, + ): chain = LumaConceptChain(str_list=[concept1, concept2, concept3, concept4]) if luma_concepts is not None: chain = luma_concepts.clone_and_merge(chain) return (chain,) + class LumaImageGenerationNode(ComfyNodeABC): """ Generates images synchronously based on prompt and aspect ratio. @@ -1421,6 +1481,7 @@ class LumaImageGenerationNode(ComfyNodeABC): ) return self._convert_luma_refs(chain, max_refs=1, auth_token=auth_token) + class LumaImageModifyNode(ComfyNodeABC): """ Modifies images synchronously based on prompt and aspect ratio. @@ -1525,6 +1586,7 @@ class LumaImageModifyNode(ComfyNodeABC): img = process_image_response(img_response) return (img,) + class LumaTextToVideoGenerationNode(ComfyNodeABC): """ Generates videos synchronously based on prompt and output_size. @@ -1566,29 +1628,49 @@ class LumaTextToVideoGenerationNode(ComfyNodeABC): }, ), "duration": ([dur.value for dur in LumaVideoModelOutputDuration],), - "loop": (IO.BOOLEAN, { - "default": False, - }), - "seed": (IO.INT, { - "default": 0, - "min": 0, - "max": 0xFFFFFFFFFFFFFFFF, - "control_after_generate": True, - "tooltip": "Seed to determine if node should re-run; actual results are nondeterministic regardless of seed.", - }), + "loop": ( + IO.BOOLEAN, + { + "default": False, + }, + ), + "seed": ( + IO.INT, + { + "default": 0, + "min": 0, + "max": 0xFFFFFFFFFFFFFFFF, + "control_after_generate": True, + "tooltip": "Seed to determine if node should re-run; actual results are nondeterministic regardless of seed.", + }, + ), }, "optional": { - "luma_concepts": (LumaIO.LUMA_CONCEPTS, { - "tooltip": "Optional Camera Concepts to dictate camera motion via the Luma Concepts node." - }), + "luma_concepts": ( + LumaIO.LUMA_CONCEPTS, + { + "tooltip": "Optional Camera Concepts to dictate camera motion via the Luma Concepts node." + }, + ), }, "hidden": { "auth_token": "AUTH_TOKEN_COMFY_ORG", }, } - def api_call(self, prompt: str, model: str, aspect_ratio: str, resolution: str, duration: str, loop: bool, seed, - luma_concepts: LumaConceptChain=None, auth_token=None, **kwargs): + def api_call( + self, + prompt: str, + model: str, + aspect_ratio: str, + resolution: str, + duration: str, + loop: bool, + seed, + luma_concepts: LumaConceptChain = None, + auth_token=None, + **kwargs, + ): operation = SynchronousOperation( endpoint=ApiEndpoint( path="/proxy/luma/generations", @@ -1603,7 +1685,7 @@ class LumaTextToVideoGenerationNode(ComfyNodeABC): aspect_ratio=aspect_ratio, duration=duration, loop=loop, - concepts=luma_concepts.create_api_model() if luma_concepts else None + concepts=luma_concepts.create_api_model() if luma_concepts else None, ), auth_token=auth_token, ) @@ -1683,24 +1765,37 @@ class LumaImageToVideoGenerationNode(ComfyNodeABC): ), }, "optional": { - "first_image": (IO.IMAGE, { - "tooltip": "First frame of generated video." - }), - "last_image": (IO.IMAGE, { - "tooltip": "Last frame of generated video." - }), - "luma_concepts": (LumaIO.LUMA_CONCEPTS, { - "tooltip": "Optional Camera Concepts to dictate camera motion via the Luma Concepts node." - }), + "first_image": ( + IO.IMAGE, + {"tooltip": "First frame of generated video."}, + ), + "last_image": (IO.IMAGE, {"tooltip": "Last frame of generated video."}), + "luma_concepts": ( + LumaIO.LUMA_CONCEPTS, + { + "tooltip": "Optional Camera Concepts to dictate camera motion via the Luma Concepts node." + }, + ), }, "hidden": { "auth_token": "AUTH_TOKEN_COMFY_ORG", }, } - def api_call(self, prompt: str, model: str, resolution: str, duration: str, loop: bool, seed, - first_image: torch.Tensor=None, last_image: torch.Tensor=None, luma_concepts: LumaConceptChain=None, - auth_token=None, **kwargs): + def api_call( + self, + prompt: str, + model: str, + resolution: str, + duration: str, + loop: bool, + seed, + first_image: torch.Tensor = None, + last_image: torch.Tensor = None, + luma_concepts: LumaConceptChain = None, + auth_token=None, + **kwargs, + ): if first_image is None and last_image is None: raise Exception( "At least one of first_image and last_image requires an input." @@ -1717,12 +1812,12 @@ class LumaImageToVideoGenerationNode(ComfyNodeABC): request=LumaGenerationRequest( prompt=prompt, model=model, - aspect_ratio=LumaAspectRatio.ratio_16_9, # ignored, but still needed by the API for some reason + aspect_ratio=LumaAspectRatio.ratio_16_9, # ignored, but still needed by the API for some reason resolution=resolution, duration=duration, loop=loop, keyframes=keyframes, - concepts=luma_concepts.create_api_model() if luma_concepts else None + concepts=luma_concepts.create_api_model() if luma_concepts else None, ), auth_token=auth_token, ) diff --git a/comfy_api_nodes/nodes_runway.py b/comfy_api_nodes/nodes_runway.py index 25abdc150..dd4db4a5f 100644 --- a/comfy_api_nodes/nodes_runway.py +++ b/comfy_api_nodes/nodes_runway.py @@ -193,7 +193,7 @@ class RunwayImageToVideoNode(ComfyNodeABC): prompt_images_tensor = torch.cat(prompt_images_tensors, dim=0) download_urls = upload_images_to_comfyapi( - prompt_images_tensor, max_images=2, auth_token=auth_token + prompt_images_tensor, max_images=2, auth_token=auth_token, mime_type="image/png" ) # Create a list of detailed image objects From 122a4b42886205b19c00cce8b60d18943e65021f Mon Sep 17 00:00:00 2001 From: thot experiment <94414189+thot-experiment@users.noreply.github.com> Date: Tue, 29 Apr 2025 17:28:18 -0700 Subject: [PATCH 032/121] add better error propagation to veo2 (#37) --- comfy_api_nodes/apis/PixverseController.py | 4 +- comfy_api_nodes/apis/PixverseDto.py | 12 +- comfy_api_nodes/apis/__init__.py | 1900 +++++++++++++------- comfy_api_nodes/nodes_veo2.py | 22 +- 4 files changed, 1311 insertions(+), 627 deletions(-) diff --git a/comfy_api_nodes/apis/PixverseController.py b/comfy_api_nodes/apis/PixverseController.py index 6f87c3d2c..310c0f546 100644 --- a/comfy_api_nodes/apis/PixverseController.py +++ b/comfy_api_nodes/apis/PixverseController.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: -# filename: https://stagingapi.comfy.org/openapi -# timestamp: 2025-04-29T03:13:19+00:00 +# filename: filtered-openapi.yaml +# timestamp: 2025-04-29T23:44:54+00:00 from __future__ import annotations diff --git a/comfy_api_nodes/apis/PixverseDto.py b/comfy_api_nodes/apis/PixverseDto.py index 0b08fa2aa..323c38e96 100644 --- a/comfy_api_nodes/apis/PixverseDto.py +++ b/comfy_api_nodes/apis/PixverseDto.py @@ -1,12 +1,12 @@ # generated by datamodel-codegen: -# filename: https://stagingapi.comfy.org/openapi -# timestamp: 2025-04-29T03:13:19+00:00 +# filename: filtered-openapi.yaml +# timestamp: 2025-04-29T23:44:54+00:00 from __future__ import annotations from typing import Optional -from pydantic import BaseModel, Field, constr +from pydantic import BaseModel, Field class V2OpenAPII2VResp(BaseModel): @@ -30,10 +30,10 @@ class V2OpenAPIT2VReq(BaseModel): description='Motion mode (normal, fast, --fast only available when duration=5; --quality=1080p does not support fast)', examples=['normal'], ) - negative_prompt: Optional[constr(max_length=2048)] = Field( - None, description='Negative prompt\n' + negative_prompt: Optional[str] = Field( + None, description='Negative prompt\n', max_length=2048 ) - prompt: constr(max_length=2048) = Field(..., description='Prompt') + prompt: str = Field(..., description='Prompt', max_length=2048) quality: str = Field( ..., description='Video quality ("360p"(Turbo model), "540p", "720p", "1080p")', diff --git a/comfy_api_nodes/apis/__init__.py b/comfy_api_nodes/apis/__init__.py index 2b95fe23a..9a4852f43 100644 --- a/comfy_api_nodes/apis/__init__.py +++ b/comfy_api_nodes/apis/__init__.py @@ -1,33 +1,108 @@ # generated by datamodel-codegen: # filename: filtered-openapi.yaml -# timestamp: 2025-04-29T19:38:18+00:00 +# timestamp: 2025-04-29T23:44:54+00:00 from __future__ import annotations from datetime import datetime from enum import Enum -from typing import Any, Dict, List, Optional, Union +from typing import Any, Dict, List, Literal, Optional, Union +from uuid import UUID from pydantic import AnyUrl, BaseModel, Field, RootModel +class BFLFluxProGenerateRequest(BaseModel): + guidance_scale: Optional[float] = Field( + None, description='The guidance scale for generation.', ge=1.0, le=20.0 + ) + height: int = Field( + ..., description='The height of the image to generate.', ge=64, le=2048 + ) + negative_prompt: Optional[str] = Field( + None, description='The negative prompt for image generation.' + ) + num_images: Optional[int] = Field( + None, description='The number of images to generate.', ge=1, le=4 + ) + num_inference_steps: Optional[int] = Field( + None, description='The number of inference steps.', ge=1, le=100 + ) + prompt: str = Field(..., description='The text prompt for image generation.') + seed: Optional[int] = Field(None, description='The seed value for reproducibility.') + width: int = Field( + ..., description='The width of the image to generate.', ge=64, le=2048 + ) + + +class BFLFluxProGenerateResponse(BaseModel): + id: str = Field(..., description='The unique identifier for the generation task.') + polling_url: str = Field(..., description='URL to poll for the generation result.') + + +class Customer(BaseModel): + createdAt: Optional[datetime] = Field( + None, description='The date and time the user was created' + ) + email: Optional[str] = Field(None, description='The email address for this user') + id: str = Field(..., description='The firebase UID of the user') + name: Optional[str] = Field(None, description='The name for this user') + updatedAt: Optional[datetime] = Field( + None, description='The date and time the user was last updated' + ) + + +class CustomerStorageResourceResponse(BaseModel): + download_url: Optional[str] = Field( + None, + description='The signed URL to use for downloading the file from the specified path', + ) + existing_file: Optional[bool] = Field( + None, description='Whether an existing file with the same hash was found' + ) + expires_at: Optional[datetime] = Field( + None, description='When the signed URL will expire' + ) + upload_url: Optional[str] = Field( + None, + description='The signed URL to use for uploading the file to the specified path', + ) + + class ErrorResponse(BaseModel): error: str message: str class ImageRequest(BaseModel): - prompt: str = Field( - ..., description='Required. The prompt to use to generate the image.' - ) aspect_ratio: Optional[str] = Field( None, description="Optional. The aspect ratio (e.g., 'ASPECT_16_9', 'ASPECT_1_1'). Cannot be used with resolution. Defaults to 'ASPECT_1_1' if unspecified.", ) - model: str = Field(..., description="The model used (e.g., 'V_2', 'V_2A_TURBO')") + color_palette: Optional[Dict[str, Any]] = Field( + None, description='Optional. Color palette object. Only for V_2, V_2_TURBO.' + ) magic_prompt_option: Optional[str] = Field( None, description="Optional. MagicPrompt usage ('AUTO', 'ON', 'OFF')." ) + model: str = Field(..., description="The model used (e.g., 'V_2', 'V_2A_TURBO')") + negative_prompt: Optional[str] = Field( + None, + description='Optional. Description of what to exclude. Only for V_1, V_1_TURBO, V_2, V_2_TURBO.', + ) + num_images: Optional[int] = Field( + 1, + description='Optional. Number of images to generate (1-8). Defaults to 1.', + ge=1, + le=8, + ) + prompt: str = Field( + ..., description='Required. The prompt to use to generate the image.' + ) + resolution: Optional[str] = Field( + None, + description="Optional. Resolution (e.g., 'RESOLUTION_1024_1024'). Only for model V_2. Cannot be used with aspect_ratio.", + ) seed: Optional[int] = Field( None, description='Optional. A number between 0 and 2147483647.', @@ -38,23 +113,6 @@ class ImageRequest(BaseModel): None, description="Optional. Style type ('AUTO', 'GENERAL', 'REALISTIC', 'DESIGN', 'RENDER_3D', 'ANIME'). Only for models V_2 and above.", ) - negative_prompt: Optional[str] = Field( - None, - description='Optional. Description of what to exclude. Only for V_1, V_1_TURBO, V_2, V_2_TURBO.', - ) - num_images: Optional[int] = Field( - 1, - description='Optional. Number of images to generate (1-8). Defaults to 1.', - ge=1, - le=8, - ) - resolution: Optional[str] = Field( - None, - description="Optional. Resolution (e.g., 'RESOLUTION_1024_1024'). Only for model V_2. Cannot be used with aspect_ratio.", - ) - color_palette: Optional[Dict[str, Any]] = Field( - None, description='Optional. Color palette object. Only for V_2, V_2_TURBO.' - ) class IdeogramGenerateRequest(BaseModel): @@ -64,23 +122,23 @@ class IdeogramGenerateRequest(BaseModel): class Datum(BaseModel): + is_image_safe: Optional[bool] = Field( + None, description='Indicates whether the image is considered safe.' + ) prompt: Optional[str] = Field( None, description='The prompt used to generate this image.' ) resolution: Optional[str] = Field( None, description="The resolution of the generated image (e.g., '1024x1024')." ) - is_image_safe: Optional[bool] = Field( - None, description='Indicates whether the image is considered safe.' - ) seed: Optional[int] = Field( None, description='The seed value used for this generation.' ) - url: Optional[str] = Field(None, description='URL to the generated image.') style_type: Optional[str] = Field( None, description="The style type used for generation (e.g., 'REALISTIC', 'ANIME').", ) + url: Optional[str] = Field(None, description='URL to the generated image.') class IdeogramGenerateResponse(BaseModel): @@ -92,9 +150,9 @@ class IdeogramGenerateResponse(BaseModel): ) -class ModelName(str, Enum): - kling_v1 = 'kling-v1' - kling_v1_6 = 'kling-v1-6' +class Duration(str, Enum): + field_5 = '5' + field_10 = '10' class Mode(str, Enum): @@ -102,6 +160,89 @@ class Mode(str, Enum): pro = 'pro' +class ModelName(str, Enum): + kling_v1 = 'kling-v1' + kling_v1_5 = 'kling-v1-5' + kling_v1_6 = 'kling-v1-6' + + +class KlingDualCharacterEffectInput(BaseModel): + duration: Duration = Field( + ..., + description='Video Length in seconds. Both 5 and 10-second videos are supported.', + ) + images: List[str] = Field( + ..., + description='Reference Image Group. Must contain exactly 2 images. First image will be positioned on left side, second on right side of the composite. Each image follows the same requirements as single image effects.', + max_length=2, + min_length=2, + ) + mode: Optional[Mode] = Field( + 'std', + description='Video generation mode. std (Standard Mode) is cost-effective, pro (Professional Mode) generates videos with longer duration and higher quality.', + ) + model_name: Optional[ModelName] = Field( + 'kling-v1', + description='Model Name. Can be kling-v1, kling-v1-5, or kling-v1-6.', + ) + + +class KlingErrorResponse(BaseModel): + code: int = Field( + ..., + description='- 1000: Authentication failed\n- 1001: Authorization is empty\n- 1002: Authorization is invalid\n- 1003: Authorization is not yet valid\n- 1004: Authorization has expired\n- 1100: Account exception\n- 1101: Account in arrears (postpaid scenario)\n- 1102: Resource pack depleted or expired (prepaid scenario)\n- 1103: Unauthorized access to requested resource\n- 1200: Invalid request parameters\n- 1201: Invalid parameters\n- 1202: Invalid request method\n- 1203: Requested resource does not exist\n- 1300: Trigger platform strategy\n- 1301: Trigger content security policy\n- 1302: API request too frequent\n- 1303: Concurrency/QPS exceeds limit\n- 1304: Trigger IP whitelist policy\n- 5000: Internal server error\n- 5001: Service temporarily unavailable\n- 5002: Server internal timeout\n', + ) + message: str = Field(..., description='Human-readable error message') + request_id: str = Field( + ..., description='Request ID for tracking and troubleshooting' + ) + + +class AspectRatio(str, Enum): + field_16_9 = '16:9' + field_9_16 = '9:16' + field_1_1 = '1:1' + + +class Config(BaseModel): + horizontal: Optional[float] = Field( + None, + description="Controls camera's movement along horizontal axis (x-axis). Negative indicates left, positive indicates right.", + ge=-10.0, + le=10.0, + ) + pan: Optional[float] = Field( + None, + description="Controls camera's rotation in vertical plane (x-axis). Negative indicates downward rotation, positive indicates upward rotation.", + ge=-10.0, + le=10.0, + ) + roll: Optional[float] = Field( + None, + description="Controls camera's rolling amount (z-axis). Negative indicates counterclockwise, positive indicates clockwise.", + ge=-10.0, + le=10.0, + ) + tilt: Optional[float] = Field( + None, + description="Controls camera's rotation in horizontal plane (y-axis). Negative indicates left rotation, positive indicates right rotation.", + ge=-10.0, + le=10.0, + ) + vertical: Optional[float] = Field( + None, + description="Controls camera's movement along vertical axis (y-axis). Negative indicates downward, positive indicates upward.", + ge=-10.0, + le=10.0, + ) + zoom: Optional[float] = Field( + None, + description="Controls change in camera's focal length. Negative indicates narrower field of view, positive indicates wider field of view.", + ge=-10.0, + le=10.0, + ) + + class Type(str, Enum): simple = 'simple' down_back = 'down_back' @@ -110,93 +251,12 @@ class Type(str, Enum): left_turn_forward = 'left_turn_forward' -class Config(BaseModel): - horizontal: Optional[float] = Field(None, ge=-10.0, le=10.0) - vertical: Optional[float] = Field(None, ge=-10.0, le=10.0) - pan: Optional[float] = Field(None, ge=-10.0, le=10.0) - tilt: Optional[float] = Field(None, ge=-10.0, le=10.0) - roll: Optional[float] = Field(None, ge=-10.0, le=10.0) - zoom: Optional[float] = Field(None, ge=-10.0, le=10.0) - - class CameraControl(BaseModel): - type: Optional[Type] = Field(None, description='Predefined camera movements type') config: Optional[Config] = None - - -class AspectRatio(str, Enum): - field_16_9 = '16:9' - field_9_16 = '9:16' - field_1_1 = '1:1' - - -class Duration(str, Enum): - field_5 = '5' - field_10 = '10' - - -class KlingText2VideoRequest(BaseModel): - model_name: Optional[ModelName] = Field('kling-v1', description='Model Name') - prompt: Optional[str] = Field( - None, description='Positive text prompt', max_length=2500 + type: Optional[Type] = Field( + None, + description='Predefined camera movements type. simple: Customizable camera movement. down_back: Camera descends and moves backward. forward_up: Camera moves forward and tilts up. right_turn_forward: Rotate right and move forward. left_turn_forward: Rotate left and move forward.', ) - negative_prompt: Optional[str] = Field( - None, description='Negative text prompt', max_length=2500 - ) - cfg_scale: Optional[float] = Field( - 0.5, description='Flexibility in video generation', ge=0.0, le=1.0 - ) - mode: Optional[Mode] = Field('std', description='Video generation mode') - camera_control: Optional[CameraControl] = None - aspect_ratio: Optional[AspectRatio] = '16:9' - duration: Optional[Duration] = '5' - callback_url: Optional[AnyUrl] = Field( - None, description='The callback notification address' - ) - external_task_id: Optional[str] = Field(None, description='Customized Task ID') - - -class TaskStatus(str, Enum): - submitted = 'submitted' - processing = 'processing' - succeed = 'succeed' - failed = 'failed' - - -class TaskInfo(BaseModel): - external_task_id: Optional[str] = None - - -class Video(BaseModel): - id: Optional[str] = Field(None, description='Generated video ID') - url: Optional[AnyUrl] = Field(None, description='URL for generated video') - duration: Optional[str] = Field(None, description='Total video duration') - - -class TaskResult(BaseModel): - videos: Optional[List[Video]] = None - - -class Data(BaseModel): - task_id: Optional[str] = Field(None, description='Task ID') - task_status: Optional[TaskStatus] = None - task_info: Optional[TaskInfo] = None - created_at: Optional[int] = Field(None, description='Task creation time') - updated_at: Optional[int] = Field(None, description='Task update time') - task_result: Optional[TaskResult] = None - - -class KlingText2VideoResponse(BaseModel): - code: Optional[int] = Field(None, description='Error code') - message: Optional[str] = Field(None, description='Error message') - request_id: Optional[str] = Field(None, description='Request ID') - data: Optional[Data] = None - - -class ModelName1(str, Enum): - kling_v1 = 'kling-v1' - kling_v1_5 = 'kling-v1-5' - kling_v1_6 = 'kling-v1-6' class Trajectory(BaseModel): @@ -218,55 +278,28 @@ class DynamicMask(BaseModel): trajectories: Optional[List[Trajectory]] = None -class Config1(BaseModel): - horizontal: Optional[float] = Field( - None, - description="Controls camera's movement along horizontal axis (x-axis). Negative indicates left, positive indicates right.", - ge=-10.0, - le=10.0, - ) - vertical: Optional[float] = Field( - None, - description="Controls camera's movement along vertical axis (y-axis). Negative indicates downward, positive indicates upward.", - ge=-10.0, - le=10.0, - ) - pan: Optional[float] = Field( - None, - description="Controls camera's rotation in vertical plane (x-axis). Negative indicates downward rotation, positive indicates upward rotation.", - ge=-10.0, - le=10.0, - ) - tilt: Optional[float] = Field( - None, - description="Controls camera's rotation in horizontal plane (y-axis). Negative indicates left rotation, positive indicates right rotation.", - ge=-10.0, - le=10.0, - ) - roll: Optional[float] = Field( - None, - description="Controls camera's rolling amount (z-axis). Negative indicates counterclockwise, positive indicates clockwise.", - ge=-10.0, - le=10.0, - ) - zoom: Optional[float] = Field( - None, - description="Controls change in camera's focal length. Negative indicates narrower field of view, positive indicates wider field of view.", - ge=-10.0, - le=10.0, - ) - - -class CameraControl1(BaseModel): - type: Optional[Type] = Field( - None, - description='Predefined camera movements type. simple: Customizable camera movement. down_back: Camera descends and moves backward. forward_up: Camera moves forward and tilts up. right_turn_forward: Rotate right and move forward. left_turn_forward: Rotate left and move forward.', - ) - config: Optional[Config1] = None - - class KlingImage2VideoRequest(BaseModel): - model_name: Optional[ModelName1] = Field('kling-v1', description='Model Name') + aspect_ratio: Optional[AspectRatio] = '16:9' + callback_url: Optional[AnyUrl] = Field( + None, + description='The callback notification address. Server will notify when the task status changes.', + ) + camera_control: Optional[CameraControl] = None + cfg_scale: Optional[float] = Field( + 0.5, + description="Flexibility in video generation. The higher the value, the lower the model's degree of flexibility, and the stronger the relevance to the user's prompt.", + ge=0.0, + le=1.0, + ) + duration: Optional[Duration] = Field('5', description='Video length in seconds') + dynamic_masks: Optional[List[DynamicMask]] = Field( + None, + description='Dynamic Brush Configuration List (up to 6 groups). For 5-second videos, trajectory length must not exceed 77 coordinates.', + ) + external_task_id: Optional[str] = Field( + None, + description='Customized Task ID. Must be unique within a single user account.', + ) image: Optional[str] = Field( None, description='Reference Image - URL or Base64 encoded string, cannot exceed 10MB, resolution not less than 300*300px, aspect ratio between 1:2.5 ~ 2.5:1. Base64 should not include data:image prefix.', @@ -275,61 +308,710 @@ class KlingImage2VideoRequest(BaseModel): None, description='Reference Image - End frame control. URL or Base64 encoded string, cannot exceed 10MB, resolution not less than 300*300px. Base64 should not include data:image prefix.', ) - prompt: Optional[str] = Field( - None, description='Positive text prompt', max_length=2500 - ) - negative_prompt: Optional[str] = Field( - None, description='Negative text prompt', max_length=2500 - ) - cfg_scale: Optional[float] = Field( - 0.5, - description="Flexibility in video generation. The higher the value, the lower the model's degree of flexibility, and the stronger the relevance to the user's prompt.", - ge=0.0, - le=1.0, - ) mode: Optional[Mode] = Field( 'std', description='Video generation mode. std: Standard Mode, which is cost-effective. pro: Professional Mode, generates videos with longer duration but higher quality output.', ) + model_name: Optional[ModelName] = Field('kling-v1', description='Model Name') + negative_prompt: Optional[str] = Field( + None, description='Negative text prompt', max_length=2500 + ) + prompt: Optional[str] = Field( + None, description='Positive text prompt', max_length=2500 + ) static_mask: Optional[AnyUrl] = Field( None, description='Static Brush Application Area (Mask image created by users using the motion brush). The aspect ratio must match the input image.', ) - dynamic_masks: Optional[List[DynamicMask]] = Field( - None, - description='Dynamic Brush Configuration List (up to 6 groups). For 5-second videos, trajectory length must not exceed 77 coordinates.', + + +class TaskInfo(BaseModel): + external_task_id: Optional[str] = None + + +class Video(BaseModel): + duration: Optional[str] = Field(None, description='Total video duration') + id: Optional[str] = Field(None, description='Generated video ID') + url: Optional[AnyUrl] = Field(None, description='URL for generated video') + + +class TaskResult(BaseModel): + videos: Optional[List[Video]] = None + + +class TaskStatus(str, Enum): + submitted = 'submitted' + processing = 'processing' + succeed = 'succeed' + failed = 'failed' + + +class Data(BaseModel): + created_at: Optional[int] = Field(None, description='Task creation time') + task_id: Optional[str] = Field(None, description='Task ID') + task_info: Optional[TaskInfo] = None + task_result: Optional[TaskResult] = None + task_status: Optional[TaskStatus] = None + updated_at: Optional[int] = Field(None, description='Task update time') + + +class KlingImage2VideoResponse(BaseModel): + code: Optional[int] = Field(None, description='Error code') + data: Optional[Data] = None + message: Optional[str] = Field(None, description='Error message') + request_id: Optional[str] = Field(None, description='Request ID') + + +class AspectRatio1(str, Enum): + field_16_9 = '16:9' + field_9_16 = '9:16' + field_1_1 = '1:1' + field_4_3 = '4:3' + field_3_4 = '3:4' + field_3_2 = '3:2' + field_2_3 = '2:3' + field_21_9 = '21:9' + + +class ImageReference(str, Enum): + subject = 'subject' + face = 'face' + + +class ModelName2(str, Enum): + kling_v1 = 'kling-v1' + kling_v1_5 = 'kling-v1-5' + + +class KlingImageGenerationsRequest(BaseModel): + aspect_ratio: Optional[AspectRatio1] = Field( + '16:9', description='Aspect ratio of the generated images' ) - camera_control: Optional[CameraControl1] = None - aspect_ratio: Optional[AspectRatio] = '16:9' - duration: Optional[Duration] = Field('5', description='Video length in seconds') + callback_url: Optional[AnyUrl] = Field( + None, description='The callback notification address' + ) + human_fidelity: Optional[float] = Field( + 0.45, description='Subject reference similarity', ge=0.0, le=1.0 + ) + image: Optional[str] = Field( + None, description='Reference Image - Base64 encoded string or image URL' + ) + image_fidelity: Optional[float] = Field( + 0.5, description='Reference intensity for user-uploaded images', ge=0.0, le=1.0 + ) + image_reference: Optional[ImageReference] = Field( + None, description='Image reference type' + ) + model_name: Optional[ModelName2] = Field('kling-v1', description='Model Name') + n: Optional[int] = Field(1, description='Number of generated images', ge=1, le=9) + negative_prompt: Optional[str] = Field( + None, description='Negative text prompt', max_length=200 + ) + prompt: str = Field(..., description='Positive text prompt', max_length=500) + + +class Image(BaseModel): + index: Optional[int] = Field(None, description='Image Number (0-9)') + url: Optional[AnyUrl] = Field(None, description='URL for generated image') + + +class TaskResult1(BaseModel): + images: Optional[List[Image]] = None + + +class Data1(BaseModel): + created_at: Optional[int] = Field(None, description='Task creation time') + task_id: Optional[str] = Field(None, description='Task ID') + task_result: Optional[TaskResult1] = None + task_status: Optional[TaskStatus] = None + task_status_msg: Optional[str] = Field(None, description='Task status information') + updated_at: Optional[int] = Field(None, description='Task update time') + + +class KlingImageGenerationsResponse(BaseModel): + code: Optional[int] = Field(None, description='Error code') + data: Optional[Data1] = None + message: Optional[str] = Field(None, description='Error message') + request_id: Optional[str] = Field(None, description='Request ID') + + +class AudioType(str, Enum): + file = 'file' + url = 'url' + + +class Mode2(str, Enum): + text2video = 'text2video' + audio2video = 'audio2video' + + +class VoiceLanguage(str, Enum): + zh = 'zh' + en = 'en' + + +class Input(BaseModel): + audio_file: Optional[str] = Field( + None, + description='Local Path of Audio File. Supported formats: .mp3/.wav/.m4a/.aac, maximum file size of 5MB. Base64 code.', + ) + audio_type: Optional[AudioType] = Field( + None, + description='Method of Transmitting Audio Files for Lip-Sync. Required when mode is audio2video.', + ) + audio_url: Optional[AnyUrl] = Field( + None, + description='Audio File Download URL. Supported formats: .mp3/.wav/.m4a/.aac, maximum file size of 5MB.', + ) + mode: Mode2 = Field( + ..., + description='Video Generation Mode. text2video: Text-to-video generation mode; audio2video: Audio-to-video generation mode', + ) + text: Optional[str] = Field( + None, + description='Text Content for Lip-Sync Video Generation. Required when mode is text2video. Maximum length is 120 characters.', + ) + video_id: Optional[str] = Field( + None, + description='The ID of the video generated by Kling AI. Only supports 5-second and 10-second videos generated within the last 30 days.', + ) + video_url: Optional[AnyUrl] = Field( + None, + description='Get link for uploaded video. Video files support .mp4/.mov, file size does not exceed 100MB, video length between 2-10s.', + ) + voice_id: Optional[str] = Field( + None, + description='Voice ID. Required when mode is text2video. The system offers a variety of voice options to choose from.', + ) + voice_language: Optional[VoiceLanguage] = Field( + 'zh', description='The voice language corresponds to the Voice ID.' + ) + voice_speed: Optional[float] = Field( + 1, + description='Speech Rate. Valid range: 0.8~2.0, accurate to one decimal place.', + ge=0.8, + le=2.0, + ) + + +class KlingLipSyncRequest(BaseModel): callback_url: Optional[AnyUrl] = Field( None, description='The callback notification address. Server will notify when the task status changes.', ) + input: Input + + +class TaskResult2(BaseModel): + videos: Optional[List[Video]] = None + + +class Data2(BaseModel): + created_at: Optional[int] = Field(None, description='Task creation time') + task_id: Optional[str] = Field(None, description='Task ID') + task_info: Optional[TaskInfo] = None + task_result: Optional[TaskResult2] = None + task_status: Optional[TaskStatus] = None + updated_at: Optional[int] = Field(None, description='Task update time') + + +class KlingLipSyncResponse(BaseModel): + code: Optional[int] = Field(None, description='Error code') + data: Optional[Data2] = None + message: Optional[str] = Field(None, description='Error message') + request_id: Optional[str] = Field(None, description='Request ID') + + +class ResourcePackType(str, Enum): + decreasing_total = 'decreasing_total' + constant_period = 'constant_period' + + +class Status(str, Enum): + toBeOnline = 'toBeOnline' + online = 'online' + expired = 'expired' + runOut = 'runOut' + + +class ResourcePackSubscribeInfo(BaseModel): + effective_time: Optional[int] = Field( + None, description='Effective time, Unix timestamp in ms' + ) + invalid_time: Optional[int] = Field( + None, description='Expiration time, Unix timestamp in ms' + ) + purchase_time: Optional[int] = Field( + None, description='Purchase time, Unix timestamp in ms' + ) + remaining_quantity: Optional[float] = Field( + None, description='Remaining quantity (updated with a 12-hour delay)' + ) + resource_pack_id: Optional[str] = Field(None, description='Resource package ID') + resource_pack_name: Optional[str] = Field(None, description='Resource package name') + resource_pack_type: Optional[ResourcePackType] = Field( + None, + description='Resource package type (decreasing_total=decreasing total, constant_period=constant periodicity)', + ) + status: Optional[Status] = Field(None, description='Resource Package Status') + total_quantity: Optional[float] = Field(None, description='Total quantity') + + +class Data3(BaseModel): + code: Optional[int] = Field(None, description='Error code; 0 indicates success') + msg: Optional[str] = Field(None, description='Error information') + resource_pack_subscribe_infos: Optional[List[ResourcePackSubscribeInfo]] = Field( + None, description='Resource package list' + ) + + +class KlingResourcePackageResponse(BaseModel): + code: Optional[int] = Field(None, description='Error code; 0 indicates success') + data: Optional[Data3] = None + message: Optional[str] = Field(None, description='Error information') + request_id: Optional[str] = Field( + None, + description='Request ID, generated by the system, used to track requests and troubleshoot problems', + ) + + +class Duration2(str, Enum): + field_5 = '5' + + +class ModelName3(str, Enum): + kling_v1_6 = 'kling-v1-6' + + +class KlingSingleImageEffectInput(BaseModel): + duration: Duration2 = Field( + ..., description='Video Length in seconds. Only 5-second videos are supported.' + ) + image: str = Field( + ..., + description='Reference Image. URL or Base64 encoded string (without data:image prefix). File size cannot exceed 10MB, resolution not less than 300*300px, aspect ratio between 1:2.5 ~ 2.5:1.', + ) + model_name: ModelName3 = Field( + ..., + description='Model Name. Only kling-v1-6 is supported for single image effects.', + ) + + +class AspectRatio2(str, Enum): + field_16_9 = '16:9' + field_9_16 = '9:16' + field_1_1 = '1:1' + + +class Config1(BaseModel): + horizontal: Optional[float] = Field(None, ge=-10.0, le=10.0) + pan: Optional[float] = Field(None, ge=-10.0, le=10.0) + roll: Optional[float] = Field(None, ge=-10.0, le=10.0) + tilt: Optional[float] = Field(None, ge=-10.0, le=10.0) + vertical: Optional[float] = Field(None, ge=-10.0, le=10.0) + zoom: Optional[float] = Field(None, ge=-10.0, le=10.0) + + +class CameraControl1(BaseModel): + config: Optional[Config1] = None + type: Optional[Type] = Field(None, description='Predefined camera movements type') + + +class Duration3(str, Enum): + field_5 = '5' + field_10 = '10' + + +class Mode3(str, Enum): + std = 'std' + pro = 'pro' + + +class ModelName4(str, Enum): + kling_v1 = 'kling-v1' + kling_v1_6 = 'kling-v1-6' + + +class KlingText2VideoRequest(BaseModel): + aspect_ratio: Optional[AspectRatio2] = '16:9' + callback_url: Optional[AnyUrl] = Field( + None, description='The callback notification address' + ) + camera_control: Optional[CameraControl1] = None + cfg_scale: Optional[float] = Field( + 0.5, description='Flexibility in video generation', ge=0.0, le=1.0 + ) + duration: Optional[Duration3] = '5' + external_task_id: Optional[str] = Field(None, description='Customized Task ID') + mode: Optional[Mode3] = Field('std', description='Video generation mode') + model_name: Optional[ModelName4] = Field('kling-v1', description='Model Name') + negative_prompt: Optional[str] = Field( + None, description='Negative text prompt', max_length=2500 + ) + prompt: Optional[str] = Field( + None, description='Positive text prompt', max_length=2500 + ) + + +class TaskResult3(BaseModel): + videos: Optional[List[Video]] = None + + +class Data4(BaseModel): + created_at: Optional[int] = Field(None, description='Task creation time') + task_id: Optional[str] = Field(None, description='Task ID') + task_info: Optional[TaskInfo] = None + task_result: Optional[TaskResult3] = None + task_status: Optional[TaskStatus] = None + updated_at: Optional[int] = Field(None, description='Task update time') + + +class KlingText2VideoResponse(BaseModel): + code: Optional[int] = Field(None, description='Error code') + data: Optional[Data4] = None + message: Optional[str] = Field(None, description='Error message') + request_id: Optional[str] = Field(None, description='Request ID') + + +class KlingVideoEffectsInput( + RootModel[Union[KlingSingleImageEffectInput, KlingDualCharacterEffectInput]] +): + root: Union[KlingSingleImageEffectInput, KlingDualCharacterEffectInput] + + +class EffectScene(str, Enum): + bloombloom = 'bloombloom' + dizzydizzy = 'dizzydizzy' + fuzzyfuzzy = 'fuzzyfuzzy' + squish = 'squish' + expansion = 'expansion' + hug = 'hug' + kiss = 'kiss' + heart_gesture = 'heart_gesture' + + +class KlingVideoEffectsRequest(BaseModel): + callback_url: Optional[AnyUrl] = Field( + None, + description='The callback notification address for the result of this task.', + ) + effect_scene: EffectScene = Field( + ..., + description='Scene Name. Single Image Effects (bloombloom, dizzydizzy, fuzzyfuzzy, squish, expansion) or Dual-character Effects (hug, kiss, heart_gesture).', + ) external_task_id: Optional[str] = Field( None, description='Customized Task ID. Must be unique within a single user account.', ) + input: Optional[KlingVideoEffectsInput] = None -class TaskResult1(BaseModel): +class TaskResult4(BaseModel): videos: Optional[List[Video]] = None -class Data1(BaseModel): - task_id: Optional[str] = Field(None, description='Task ID') - task_status: Optional[TaskStatus] = None - task_info: Optional[TaskInfo] = None +class Data5(BaseModel): created_at: Optional[int] = Field(None, description='Task creation time') + task_id: Optional[str] = Field(None, description='Task ID') + task_info: Optional[TaskInfo] = None + task_result: Optional[TaskResult4] = None + task_status: Optional[TaskStatus] = None updated_at: Optional[int] = Field(None, description='Task update time') - task_result: Optional[TaskResult1] = None -class KlingImage2VideoResponse(BaseModel): +class KlingVideoEffectsResponse(BaseModel): code: Optional[int] = Field(None, description='Error code') + data: Optional[Data5] = None message: Optional[str] = Field(None, description='Error message') request_id: Optional[str] = Field(None, description='Request ID') - data: Optional[Data1] = None + + +class KlingVideoExtendRequest(BaseModel): + callback_url: Optional[AnyUrl] = Field( + None, + description='The callback notification address. Server will notify when the task status changes.', + ) + cfg_scale: Optional[float] = Field( + 0.5, + description="Flexibility in video generation. The higher the value, the lower the model's flexibility and the stronger the relevance to the user's prompt.", + ge=0.0, + le=1.0, + ) + negative_prompt: Optional[str] = Field( + None, + description='Negative text prompt for elements to avoid in the extended video', + max_length=2500, + ) + prompt: Optional[str] = Field( + None, + description='Positive text prompt for guiding the video extension', + max_length=2500, + ) + video_id: Optional[str] = Field( + None, + description='The ID of the video to be extended. Supports videos generated by text-to-video, image-to-video, and previous video extension operations. Cannot exceed 3 minutes total duration after extension.', + ) + + +class TaskResult5(BaseModel): + videos: Optional[List[Video]] = None + + +class Data6(BaseModel): + created_at: Optional[int] = Field(None, description='Task creation time') + task_id: Optional[str] = Field(None, description='Task ID') + task_info: Optional[TaskInfo] = None + task_result: Optional[TaskResult5] = None + task_status: Optional[TaskStatus] = None + updated_at: Optional[int] = Field(None, description='Task update time') + + +class KlingVideoExtendResponse(BaseModel): + code: Optional[int] = Field(None, description='Error code') + data: Optional[Data6] = None + message: Optional[str] = Field(None, description='Error message') + request_id: Optional[str] = Field(None, description='Request ID') + + +class ModelName5(str, Enum): + kolors_virtual_try_on_v1 = 'kolors-virtual-try-on-v1' + kolors_virtual_try_on_v1_5 = 'kolors-virtual-try-on-v1-5' + + +class KlingVirtualTryOnRequest(BaseModel): + callback_url: Optional[AnyUrl] = Field( + None, description='The callback notification address' + ) + cloth_image: Optional[str] = Field( + None, + description='Reference clothing image - Base64 encoded string or image URL', + ) + human_image: str = Field( + ..., description='Reference human image - Base64 encoded string or image URL' + ) + model_name: Optional[ModelName5] = Field( + 'kolors-virtual-try-on-v1', description='Model Name' + ) + + +class Image1(BaseModel): + index: Optional[int] = Field(None, description='Image Number') + url: Optional[AnyUrl] = Field(None, description='URL for generated image') + + +class TaskResult6(BaseModel): + images: Optional[List[Image1]] = None + + +class Data7(BaseModel): + created_at: Optional[int] = Field(None, description='Task creation time') + task_id: Optional[str] = Field(None, description='Task ID') + task_result: Optional[TaskResult6] = None + task_status: Optional[TaskStatus] = None + task_status_msg: Optional[str] = Field(None, description='Task status information') + updated_at: Optional[int] = Field(None, description='Task update time') + + +class KlingVirtualTryOnResponse(BaseModel): + code: Optional[int] = Field(None, description='Error code') + data: Optional[Data7] = None + message: Optional[str] = Field(None, description='Error message') + request_id: Optional[str] = Field(None, description='Request ID') + + +class LumaAspectRatio(str, Enum): + field_1_1 = '1:1' + field_16_9 = '16:9' + field_9_16 = '9:16' + field_4_3 = '4:3' + field_3_4 = '3:4' + field_21_9 = '21:9' + field_9_21 = '9:21' + + +class LumaAssets(BaseModel): + image: Optional[AnyUrl] = Field(None, description='The URL of the image') + progress_video: Optional[AnyUrl] = Field( + None, description='The URL of the progress video' + ) + video: Optional[AnyUrl] = Field(None, description='The URL of the video') + + +class GenerationType(str, Enum): + add_audio = 'add_audio' + + +class LumaAudioGenerationRequest(BaseModel): + callback_url: Optional[AnyUrl] = Field( + None, description='The callback URL for the audio' + ) + generation_type: Optional[GenerationType] = 'add_audio' + negative_prompt: Optional[str] = Field( + None, description='The negative prompt of the audio' + ) + prompt: Optional[str] = Field(None, description='The prompt of the audio') + + +class LumaError(BaseModel): + detail: Optional[str] = Field(None, description='The error message') + + +class Type2(str, Enum): + generation = 'generation' + + +class LumaGenerationReference(BaseModel): + id: UUID = Field(..., description='The ID of the generation') + type: Literal['generation'] + + +class GenerationType1(str, Enum): + video = 'video' + + +class LumaGenerationType(str, Enum): + video = 'video' + image = 'image' + + +class GenerationType2(str, Enum): + image = 'image' + + +class LumaImageIdentity(BaseModel): + images: Optional[List[AnyUrl]] = Field( + None, description='The URLs of the image identity' + ) + + +class LumaImageModel(str, Enum): + photon_1 = 'photon-1' + photon_flash_1 = 'photon-flash-1' + + +class LumaImageRef(BaseModel): + url: Optional[AnyUrl] = Field(None, description='The URL of the image reference') + weight: Optional[float] = Field( + None, description='The weight of the image reference' + ) + + +class Type3(str, Enum): + image = 'image' + + +class LumaImageReference(BaseModel): + type: Literal['image'] + url: AnyUrl = Field(..., description='The URL of the image') + + +class LumaKeyframe(RootModel[Union[LumaGenerationReference, LumaImageReference]]): + root: Union[LumaGenerationReference, LumaImageReference] = Field( + ..., + description='A keyframe can be either a Generation reference, an Image, or a Video', + discriminator='type', + ) + + +class LumaKeyframes(BaseModel): + frame0: Optional[LumaKeyframe] = None + frame1: Optional[LumaKeyframe] = None + + +class LumaModifyImageRef(BaseModel): + url: Optional[AnyUrl] = Field(None, description='The URL of the image reference') + weight: Optional[float] = Field( + None, description='The weight of the modify image reference' + ) + + +class LumaState(str, Enum): + queued = 'queued' + dreaming = 'dreaming' + completed = 'completed' + failed = 'failed' + + +class GenerationType3(str, Enum): + upscale_video = 'upscale_video' + + +class LumaVideoModel(str, Enum): + ray_2 = 'ray-2' + ray_2_flash = 'ray-2-flash' + + +class LumaVideoModelOutputDuration1(str, Enum): + field_5s = '5s' + field_9s = '9s' + + +class LumaVideoModelOutputDuration( + RootModel[Union[LumaVideoModelOutputDuration1, str]] +): + root: Union[LumaVideoModelOutputDuration1, str] + + +class LumaVideoModelOutputResolution1(str, Enum): + field_540p = '540p' + field_720p = '720p' + field_1080p = '1080p' + field_4k = '4k' + + +class LumaVideoModelOutputResolution( + RootModel[Union[LumaVideoModelOutputResolution1, str]] +): + root: Union[LumaVideoModelOutputResolution1, str] + + +class MinimaxBaseResponse(BaseModel): + status_code: int = Field( + ..., + description='Status code. 0 indicates success, other values indicate errors.', + ) + status_msg: str = Field( + ..., description='Specific error details or success message.' + ) + + +class File(BaseModel): + bytes: Optional[int] = Field(None, description='File size in bytes') + created_at: Optional[int] = Field( + None, description='Unix timestamp when the file was created, in seconds' + ) + download_url: Optional[str] = Field( + None, description='The URL to download the video' + ) + file_id: Optional[int] = Field(None, description='Unique identifier for the file') + filename: Optional[str] = Field(None, description='The name of the file') + purpose: Optional[str] = Field(None, description='The purpose of using the file') + + +class MinimaxFileRetrieveResponse(BaseModel): + base_resp: MinimaxBaseResponse + file: File + + +class Status1(str, Enum): + Queueing = 'Queueing' + Preparing = 'Preparing' + Processing = 'Processing' + Success = 'Success' + Fail = 'Fail' + + +class MinimaxTaskResultResponse(BaseModel): + base_resp: MinimaxBaseResponse + file_id: Optional[str] = Field( + None, + description='After the task status changes to Success, this field returns the file ID corresponding to the generated video.', + ) + status: Status1 = Field( + ..., + description="Task status: 'Queueing' (in queue), 'Preparing' (task is preparing), 'Processing' (generating), 'Success' (task completed successfully), or 'Fail' (task failed).", + ) + task_id: str = Field(..., description='The task ID being queried.') class Model(str, Enum): @@ -352,6 +1034,14 @@ class SubjectReferenceItem(BaseModel): class MinimaxVideoGenerationRequest(BaseModel): + callback_url: Optional[str] = Field( + None, + description='Optional. URL to receive real-time status updates about the video generation task.', + ) + first_frame_image: Optional[str] = Field( + None, + description='URL or base64 encoding of the first frame image. Required when model is I2V-01, I2V-01-Director, or I2V-01-live.', + ) model: Model = Field( ..., description='Required. ID of model. Options: T2V-01-Director, I2V-01-Director, S2V-01, I2V-01, I2V-01-live, T2V-01', @@ -365,122 +1055,257 @@ class MinimaxVideoGenerationRequest(BaseModel): True, description='If true (default), the model will automatically optimize the prompt. Set to false for more precise control.', ) - first_frame_image: Optional[str] = Field( - None, - description='URL or base64 encoding of the first frame image. Required when model is I2V-01, I2V-01-Director, or I2V-01-live.', - ) subject_reference: Optional[List[SubjectReferenceItem]] = Field( None, description='Only available when model is S2V-01. The model will generate a video based on the subject uploaded through this parameter.', ) - callback_url: Optional[str] = Field( - None, - description='Optional. URL to receive real-time status updates about the video generation task.', - ) - - -class MinimaxBaseResponse(BaseModel): - status_code: int = Field( - ..., - description='Status code. 0 indicates success, other values indicate errors.', - ) - status_msg: str = Field( - ..., description='Specific error details or success message.' - ) class MinimaxVideoGenerationResponse(BaseModel): + base_resp: MinimaxBaseResponse task_id: str = Field( ..., description='The task ID for the asynchronous video generation task.' ) - base_resp: MinimaxBaseResponse -class File(BaseModel): - file_id: Optional[int] = Field(None, description='Unique identifier for the file') - bytes: Optional[int] = Field(None, description='File size in bytes') - created_at: Optional[int] = Field( - None, description='Unix timestamp when the file was created, in seconds' +class Moderation(str, Enum): + low = 'low' + auto = 'auto' + + +class OutputFormat(str, Enum): + png = 'png' + webp = 'webp' + jpeg = 'jpeg' + + +class OpenAIImageEditRequest(BaseModel): + background: Optional[str] = Field( + None, description='Background transparency', examples=['opaque'] ) - filename: Optional[str] = Field(None, description='The name of the file') - purpose: Optional[str] = Field(None, description='The purpose of using the file') - download_url: Optional[str] = Field( - None, description='The URL to download the video' + model: str = Field( + ..., description='The model to use for image editing', examples=['gpt-image-1'] ) - - -class MinimaxFileRetrieveResponse(BaseModel): - file: File - base_resp: MinimaxBaseResponse - - -class Status(str, Enum): - Queueing = 'Queueing' - Preparing = 'Preparing' - Processing = 'Processing' - Success = 'Success' - Fail = 'Fail' - - -class MinimaxTaskResultResponse(BaseModel): - task_id: str = Field(..., description='The task ID being queried.') - status: Status = Field( + moderation: Optional[Moderation] = Field( + None, description='Content moderation setting', examples=['auto'] + ) + n: Optional[int] = Field( + None, description='The number of images to generate', examples=[1] + ) + output_compression: Optional[int] = Field( + None, description='Compression level for JPEG or WebP (0-100)', examples=[100] + ) + output_format: Optional[OutputFormat] = Field( + None, description='Format of the output image', examples=['png'] + ) + prompt: str = Field( ..., - description="Task status: 'Queueing' (in queue), 'Preparing' (task is preparing), 'Processing' (generating), 'Success' (task completed successfully), or 'Fail' (task failed).", + description='A text description of the desired edit', + examples=['Give the rocketship rainbow coloring'], ) - file_id: Optional[str] = Field( + quality: Optional[str] = Field( + None, description='The quality of the edited image', examples=['low'] + ) + size: Optional[str] = Field( + None, description='Size of the output image', examples=['1024x1024'] + ) + user: Optional[str] = Field( None, - description='After the task status changes to Success, this field returns the file ID corresponding to the generated video.', - ) - base_resp: MinimaxBaseResponse - - -class BFLFluxProGenerateRequest(BaseModel): - prompt: str = Field(..., description='The text prompt for image generation.') - negative_prompt: Optional[str] = Field( - None, description='The negative prompt for image generation.' - ) - width: int = Field( - ..., description='The width of the image to generate.', ge=64, le=2048 - ) - height: int = Field( - ..., description='The height of the image to generate.', ge=64, le=2048 - ) - num_inference_steps: Optional[int] = Field( - None, description='The number of inference steps.', ge=1, le=100 - ) - guidance_scale: Optional[float] = Field( - None, description='The guidance scale for generation.', ge=1.0, le=20.0 - ) - seed: Optional[int] = Field(None, description='The seed value for reproducibility.') - num_images: Optional[int] = Field( - None, description='The number of images to generate.', ge=1, le=4 + description='A unique identifier for end-user monitoring', + examples=['user-1234'], ) -class BFLFluxProGenerateResponse(BaseModel): - id: str = Field(..., description='The unique identifier for the generation task.') - polling_url: str = Field(..., description='URL to poll for the generation result.') +class Background(str, Enum): + transparent = 'transparent' + opaque = 'opaque' + + +class Quality(str, Enum): + low = 'low' + medium = 'medium' + high = 'high' + standard = 'standard' + hd = 'hd' + + +class ResponseFormat(str, Enum): + url = 'url' + b64_json = 'b64_json' + + +class Style(str, Enum): + vivid = 'vivid' + natural = 'natural' + + +class OpenAIImageGenerationRequest(BaseModel): + background: Optional[Background] = Field( + None, description='Background transparency', examples=['opaque'] + ) + model: Optional[str] = Field( + None, description='The model to use for image generation', examples=['dall-e-3'] + ) + moderation: Optional[Moderation] = Field( + None, description='Content moderation setting', examples=['auto'] + ) + n: Optional[int] = Field( + None, + description='The number of images to generate (1-10). Only 1 supported for dall-e-3.', + examples=[1], + ) + output_compression: Optional[int] = Field( + None, description='Compression level for JPEG or WebP (0-100)', examples=[100] + ) + output_format: Optional[OutputFormat] = Field( + None, description='Format of the output image', examples=['png'] + ) + prompt: str = Field( + ..., + description='A text description of the desired image', + examples=['Draw a rocket in front of a blackhole in deep space'], + ) + quality: Optional[Quality] = Field( + None, description='The quality of the generated image', examples=['high'] + ) + response_format: Optional[ResponseFormat] = Field( + None, description='Response format of image data', examples=['b64_json'] + ) + size: Optional[str] = Field( + None, + description='Size of the image (e.g., 1024x1024, 1536x1024, auto)', + examples=['1024x1536'], + ) + style: Optional[Style] = Field( + None, description='Style of the image (only for dall-e-3)', examples=['vivid'] + ) + user: Optional[str] = Field( + None, + description='A unique identifier for end-user monitoring', + examples=['user-1234'], + ) + + +class Datum1(BaseModel): + b64_json: Optional[str] = Field(None, description='Base64 encoded image data') + revised_prompt: Optional[str] = Field(None, description='Revised prompt') + url: Optional[str] = Field(None, description='URL of the image') + + +class InputTokensDetails(BaseModel): + image_tokens: Optional[int] = None + text_tokens: Optional[int] = None + + +class Usage(BaseModel): + input_tokens: Optional[int] = None + input_tokens_details: Optional[InputTokensDetails] = None + output_tokens: Optional[int] = None + total_tokens: Optional[int] = None + + +class OpenAIImageGenerationResponse(BaseModel): + data: Optional[List[Datum1]] = None + usage: Optional[Usage] = None + + +class AspectRatio3(RootModel[float]): + root: float = Field( + ..., + description='Aspect ratio (width / height)', + ge=0.4, + le=2.5, + title='Aspectratio', + ) + + +class IngredientsMode(str, Enum): + creative = 'creative' + precise = 'precise' + +bytes_aliased = bytes + +class PikaBodyGenerate22C2vGenerate22PikascenesPost(BaseModel): + aspectRatio: Optional[AspectRatio3] = Field( + None, description='Aspect ratio (width / height)', title='Aspectratio' + ) + duration: Optional[int] = Field(5, title='Duration') + images: List[bytes_aliased] = Field( + ..., description='Array of images to process', title='Images' + ) + ingredientsMode: IngredientsMode = Field(..., title='Ingredientsmode') + negativePrompt: Optional[str] = Field(None, title='Negativeprompt') + promptText: Optional[str] = Field(None, title='Prompttext') + resolution: Optional[str] = Field('1080p', title='Resolution') + seed: Optional[int] = Field(None, title='Seed') + + +class PikaBodyGenerate22I2vGenerate22I2vPost(BaseModel): + duration: Optional[int] = Field(5, title='Duration') + image: bytes_aliased = Field(..., title='Image') + negativePrompt: Optional[str] = Field(None, title='Negativeprompt') + promptText: Optional[str] = Field(None, title='Prompttext') + resolution: Optional[str] = Field('1080p', title='Resolution') + seed: Optional[int] = Field(None, title='Seed') + + +class PikaBodyGenerate22KeyframeGenerate22PikaframesPost(BaseModel): + duration: Optional[int] = Field(5, title='Duration') + keyFrames: List[bytes_aliased] = Field( + ..., description='Array of keyframe images', title='Keyframes' + ) + negativePrompt: Optional[str] = Field(None, title='Negativeprompt') + promptText: str = Field(..., title='Prompttext') + resolution: Optional[str] = Field('1080p', title='Resolution') + seed: Optional[int] = Field(None, title='Seed') + + +class PikaBodyGenerate22T2vGenerate22T2vPost(BaseModel): + aspectRatio: Optional[AspectRatio3] = Field( + None, description='Aspect ratio (width / height)', title='Aspectratio' + ) + duration: Optional[int] = Field(5, title='Duration') + negativePrompt: Optional[str] = Field(None, title='Negativeprompt') + promptText: str = Field(..., title='Prompttext') + resolution: Optional[str] = Field('1080p', title='Resolution') + seed: Optional[int] = Field(None, title='Seed') + + +class PikaGenerateResponse(BaseModel): + video_id: str = Field(..., title='Video Id') + + +class PikaValidationError(BaseModel): + loc: List[Union[str, int]] = Field(..., title='Location') + msg: str = Field(..., title='Message') + type: str = Field(..., title='Error Type') + + +class PikaVideoResponse(BaseModel): + id: str = Field(..., title='Id') + progress: int = Field(..., title='Progress') + status: str = Field(..., title='Status') + url: str = Field(..., title='Url') class RecraftImageGenerationRequest(BaseModel): + model: str = Field( + ..., description='The model to use for generation (e.g., "recraftv3")' + ) + n: int = Field(..., description='The number of images to generate', ge=1, le=4) prompt: str = Field( ..., description='The text prompt describing the image to generate' ) - model: str = Field( - ..., description='The model to use for generation (e.g., "recraftv3")' + size: str = Field( + ..., description='The size of the generated image (e.g., "1024x1024")' ) style: Optional[str] = Field( None, description='The style to apply to the generated image (e.g., "digital_illustration")', ) - size: str = Field( - ..., description='The size of the generated image (e.g., "1024x1024")' - ) - n: int = Field(..., description='The number of images to generate', ge=1, le=4) -class Datum1(BaseModel): +class Datum2(BaseModel): image_id: Optional[str] = Field( None, description='Unique identifier for the generated image' ) @@ -492,37 +1317,142 @@ class RecraftImageGenerationResponse(BaseModel): ..., description='Unix timestamp when the generation was created' ) credits: int = Field(..., description='Number of credits used for the generation') - data: List[Datum1] = Field(..., description='Array of generated image information') + data: List[Datum2] = Field(..., description='Array of generated image information') -class KlingErrorResponse(BaseModel): - code: int = Field( +class RunwayAspectRatioEnum(str, Enum): + field_1280_720 = '1280:720' + field_720_1280 = '720:1280' + field_1104_832 = '1104:832' + field_832_1104 = '832:1104' + field_960_960 = '960:960' + field_1584_672 = '1584:672' + field_1280_768 = '1280:768' + field_768_1280 = '768:1280' + + +class RunwayDurationEnum(int, Enum): + integer_5 = 5 + integer_10 = 10 + + +class RunwayImageToVideoResponse(BaseModel): + id: Optional[str] = Field(None, description='Task ID') + + +class RunwayModelEnum(str, Enum): + gen4_turbo = 'gen4_turbo' + gen3a_turbo = 'gen3a_turbo' + + +class Position(str, Enum): + first = 'first' + last = 'last' + + +class RunwayPromptImageDetailedObject(BaseModel): + position: Position = Field( ..., - description='- 1000: Authentication failed\n- 1001: Authorization is empty\n- 1002: Authorization is invalid\n- 1003: Authorization is not yet valid\n- 1004: Authorization has expired\n- 1100: Account exception\n- 1101: Account in arrears (postpaid scenario)\n- 1102: Resource pack depleted or expired (prepaid scenario)\n- 1103: Unauthorized access to requested resource\n- 1200: Invalid request parameters\n- 1201: Invalid parameters\n- 1202: Invalid request method\n- 1203: Requested resource does not exist\n- 1300: Trigger platform strategy\n- 1301: Trigger content security policy\n- 1302: API request too frequent\n- 1303: Concurrency/QPS exceeds limit\n- 1304: Trigger IP whitelist policy\n- 5000: Internal server error\n- 5001: Service temporarily unavailable\n- 5002: Server internal timeout\n', + description="The position of the image in the output video. 'last' is currently supported for gen3a_turbo only.", ) - message: str = Field(..., description='Human-readable error message') - request_id: str = Field( - ..., description='Request ID for tracking and troubleshooting' + uri: AnyUrl = Field( + ..., description='A HTTPS URL or data URI containing an encoded image.' ) -class Image(BaseModel): +class RunwayPromptImageObject( + RootModel[Union[AnyUrl, List[RunwayPromptImageDetailedObject]]] +): + root: Union[AnyUrl, List[RunwayPromptImageDetailedObject]] = Field( + ..., + description='Image(s) to use for the video generation. Can be a single URI or an array of image objects with positions.', + ) + + +class RunwayTaskStatusEnum(str, Enum): + SUCCEEDED = 'SUCCEEDED' + RUNNING = 'RUNNING' + FAILED = 'FAILED' + PENDING = 'PENDING' + CANCELLED = 'CANCELLED' + THROTTLED = 'THROTTLED' + + +class RunwayTaskStatusResponse(BaseModel): + createdAt: Optional[datetime] = Field(None, description='Task creation timestamp') + id: Optional[str] = Field(None, description='Task ID') + output: Optional[List[str]] = Field(None, description='Array of output video URLs') + status: Optional[RunwayTaskStatusEnum] = None + + +class Veo2GenVidPollRequest(BaseModel): + operationName: str = Field( + ..., + description='Full operation name (from predict response)', + examples=[ + 'projects/PROJECT_ID/locations/us-central1/publishers/google/models/MODEL_ID/operations/OPERATION_ID' + ], + ) + + +class Error(BaseModel): + code: Optional[int] = Field(None, description='Error code') + message: Optional[str] = Field(None, description='Error message') + + +class Video5(BaseModel): + bytesBase64Encoded: Optional[str] = Field( + None, description='Base64-encoded video content' + ) + gcsUri: Optional[str] = Field(None, description='Cloud Storage URI of the video') + mimeType: Optional[str] = Field(None, description='Video MIME type') + + +class Response(BaseModel): + field_type: Optional[str] = Field( + None, + alias='@type', + examples=[ + 'type.googleapis.com/cloud.ai.large_models.vision.GenerateVideoResponse' + ], + ) + raiMediaFilteredCount: Optional[int] = Field( + None, description='Count of media filtered by responsible AI policies' + ) + raiMediaFilteredReasons: Optional[List[str]] = Field( + None, description='Reasons why media was filtered by responsible AI policies' + ) + videos: Optional[List[Video5]] = None + + +class Veo2GenVidPollResponse(BaseModel): + done: Optional[bool] = None + error: Optional[Error] = Field( + None, description='Error details if operation failed' + ) + name: Optional[str] = None + response: Optional[Response] = Field( + None, description='The actual prediction response if done is true' + ) + + +class Image2(BaseModel): bytesBase64Encoded: str gcsUri: Optional[str] = None mimeType: Optional[str] = None -class Image1(BaseModel): +class Image3(BaseModel): bytesBase64Encoded: Optional[str] = None gcsUri: str mimeType: Optional[str] = None class Instance(BaseModel): - prompt: str = Field(..., description='Text description of the video') - image: Optional[Union[Image, Image1]] = Field( + image: Optional[Union[Image2, Image3]] = Field( None, description='Optional image to guide video generation' ) + prompt: str = Field(..., description='Text description of the video') class PersonGeneration(str, Enum): @@ -532,6 +1462,8 @@ class PersonGeneration(str, Enum): class Parameters(BaseModel): aspectRatio: Optional[str] = Field(None, examples=['16:9']) + durationSeconds: Optional[int] = None + enhancePrompt: Optional[bool] = None negativePrompt: Optional[str] = None personGeneration: Optional[PersonGeneration] = None sampleCount: Optional[int] = None @@ -539,8 +1471,6 @@ class Parameters(BaseModel): storageUri: Optional[str] = Field( None, description='Optional Cloud Storage URI to upload the video' ) - durationSeconds: Optional[int] = None - enhancePrompt: Optional[bool] = None class Veo2GenVidRequest(BaseModel): @@ -558,347 +1488,81 @@ class Veo2GenVidResponse(BaseModel): ) -class Veo2GenVidPollRequest(BaseModel): - operationName: str = Field( - ..., - description='Full operation name (from predict response)', - examples=[ - 'projects/PROJECT_ID/locations/us-central1/publishers/google/models/MODEL_ID/operations/OPERATION_ID' - ], - ) - - -class Video2(BaseModel): - gcsUri: Optional[str] = Field(None, description='Cloud Storage URI of the video') - bytesBase64Encoded: Optional[str] = Field( - None, description='Base64-encoded video content' - ) - mimeType: Optional[str] = Field(None, description='Video MIME type') - - -class Response(BaseModel): - field_type: Optional[str] = Field( +class LumaGenerationRequest(BaseModel): + aspect_ratio: LumaAspectRatio + callback_url: Optional[AnyUrl] = Field( None, - alias='@type', - examples=[ - 'type.googleapis.com/cloud.ai.large_models.vision.GenerateVideoResponse' - ], + description='The callback URL of the generation, a POST request with Generation object will be sent to the callback URL when the generation is dreaming, completed, or failed', ) - raiMediaFilteredCount: Optional[int] = Field( - None, description='Count of media filtered by responsible AI policies' + duration: LumaVideoModelOutputDuration + generation_type: Optional[GenerationType1] = 'video' + keyframes: Optional[LumaKeyframes] = None + loop: Optional[bool] = Field(None, description='Whether to loop the video') + model: LumaVideoModel + prompt: str = Field(..., description='The prompt of the generation') + resolution: LumaVideoModelOutputResolution + + +class CharacterRef(BaseModel): + identity0: Optional[LumaImageIdentity] = None + + +class LumaImageGenerationRequest(BaseModel): + aspect_ratio: Optional[LumaAspectRatio] = '16:9' + callback_url: Optional[AnyUrl] = Field( + None, description='The callback URL for the generation' ) - videos: Optional[List[Video2]] = None + character_ref: Optional[CharacterRef] = None + generation_type: Optional[GenerationType2] = 'image' + image_ref: Optional[List[LumaImageRef]] = None + model: Optional[LumaImageModel] = 'photon-1' + modify_image_ref: Optional[LumaModifyImageRef] = None + prompt: Optional[str] = Field(None, description='The prompt of the generation') + style_ref: Optional[List[LumaImageRef]] = None -class Veo2GenVidPollResponse(BaseModel): - name: Optional[str] = None - done: Optional[bool] = None - response: Optional[Response] = Field( - None, description='The actual prediction response if done is true' +class LumaUpscaleVideoGenerationRequest(BaseModel): + callback_url: Optional[AnyUrl] = Field( + None, description='The callback URL for the upscale' ) - - -class RunwayImageToVideoResponse(BaseModel): - id: Optional[str] = Field(None, description='Task ID') - - -class RunwayTaskStatusEnum(str, Enum): - SUCCEEDED = 'SUCCEEDED' - RUNNING = 'RUNNING' - FAILED = 'FAILED' - PENDING = 'PENDING' - CANCELLED = 'CANCELLED' - THROTTLED = 'THROTTLED' - - -class RunwayModelEnum(str, Enum): - gen4_turbo = 'gen4_turbo' - gen3a_turbo = 'gen3a_turbo' - - -class Position(str, Enum): - first = 'first' - last = 'last' - - -class RunwayPromptImageDetailedObject(BaseModel): - uri: str = Field( - ..., description='A HTTPS URL or data URI containing an encoded image.' - ) - position: Position = Field( - ..., - description="The position of the image in the output video. 'last' is currently supported for gen3a_turbo only.", - ) - - -class RunwayDurationEnum(int, Enum): - integer_5 = 5 - integer_10 = 10 - - -class RunwayAspectRatioEnum(str, Enum): - field_1280_720 = '1280:720' - field_720_1280 = '720:1280' - field_1104_832 = '1104:832' - field_832_1104 = '832:1104' - field_960_960 = '960:960' - field_1584_672 = '1584:672' - field_1280_768 = '1280:768' - field_768_1280 = '768:1280' - - -class RunwayPromptImageObject( - RootModel[Union[str, List[RunwayPromptImageDetailedObject]]] -): - root: Union[str, List[RunwayPromptImageDetailedObject]] = Field( - ..., - description='Image(s) to use for the video generation. Can be a single URI or an array of image objects with positions.', - ) - - -class Datum2(BaseModel): - b64_json: Optional[str] = Field(None, description='Base64 encoded image data') - url: Optional[str] = Field(None, description='URL of the image') - revised_prompt: Optional[str] = Field(None, description='Revised prompt') - - -class InputTokensDetails(BaseModel): - text_tokens: Optional[int] = None - image_tokens: Optional[int] = None - - -class Usage(BaseModel): - input_tokens: Optional[int] = None - input_tokens_details: Optional[InputTokensDetails] = None - output_tokens: Optional[int] = None - total_tokens: Optional[int] = None - - -class OpenAIImageGenerationResponse(BaseModel): - data: Optional[List[Datum2]] = None - usage: Optional[Usage] = None - - -class Quality(str, Enum): - low = 'low' - medium = 'medium' - high = 'high' - standard = 'standard' - hd = 'hd' - - -class OutputFormat(str, Enum): - png = 'png' - webp = 'webp' - jpeg = 'jpeg' - - -class Moderation(str, Enum): - low = 'low' - auto = 'auto' - - -class Background(str, Enum): - transparent = 'transparent' - opaque = 'opaque' - - -class ResponseFormat(str, Enum): - url = 'url' - b64_json = 'b64_json' - - -class Style(str, Enum): - vivid = 'vivid' - natural = 'natural' - - -class OpenAIImageGenerationRequest(BaseModel): - model: Optional[str] = Field( - None, description='The model to use for image generation', examples=['dall-e-3'] - ) - prompt: str = Field( - ..., - description='A text description of the desired image', - examples=['Draw a rocket in front of a blackhole in deep space'], - ) - n: Optional[int] = Field( - None, - description='The number of images to generate (1-10). Only 1 supported for dall-e-3.', - examples=[1], - ) - quality: Optional[Quality] = Field( - None, description='The quality of the generated image', examples=['high'] - ) - size: Optional[str] = Field( - None, - description='Size of the image (e.g., 1024x1024, 1536x1024, auto)', - examples=['1024x1536'], - ) - output_format: Optional[OutputFormat] = Field( - None, description='Format of the output image', examples=['png'] - ) - output_compression: Optional[int] = Field( - None, description='Compression level for JPEG or WebP (0-100)', examples=[100] - ) - moderation: Optional[Moderation] = Field( - None, description='Content moderation setting', examples=['auto'] - ) - background: Optional[Background] = Field( - None, description='Background transparency', examples=['opaque'] - ) - response_format: Optional[ResponseFormat] = Field( - None, description='Response format of image data', examples=['b64_json'] - ) - style: Optional[Style] = Field( - None, description='Style of the image (only for dall-e-3)', examples=['vivid'] - ) - user: Optional[str] = Field( - None, - description='A unique identifier for end-user monitoring', - examples=['user-1234'], - ) - - -class OpenAIImageEditRequest(BaseModel): - model: str = Field( - ..., description='The model to use for image editing', examples=['gpt-image-1'] - ) - prompt: str = Field( - ..., - description='A text description of the desired edit', - examples=['Give the rocketship rainbow coloring'], - ) - n: Optional[int] = Field( - None, description='The number of images to generate', examples=[1] - ) - quality: Optional[str] = Field( - None, description='The quality of the edited image', examples=['low'] - ) - size: Optional[str] = Field( - None, description='Size of the output image', examples=['1024x1024'] - ) - output_format: Optional[OutputFormat] = Field( - None, description='Format of the output image', examples=['png'] - ) - output_compression: Optional[int] = Field( - None, description='Compression level for JPEG or WebP (0-100)', examples=[100] - ) - moderation: Optional[Moderation] = Field( - None, description='Content moderation setting', examples=['auto'] - ) - background: Optional[str] = Field( - None, description='Background transparency', examples=['opaque'] - ) - user: Optional[str] = Field( - None, - description='A unique identifier for end-user monitoring', - examples=['user-1234'], - ) - - -class AspectRatio2(RootModel[float]): - root: float = Field( - ..., - description='Aspect ratio (width / height)', - ge=0.4, - le=2.5, - title='Aspectratio', - ) - - -class PikaBodyGenerate22T2vGenerate22T2vPost(BaseModel): - promptText: str = Field(..., title='Prompttext') - negativePrompt: Optional[str] = Field(None, title='Negativeprompt') - seed: Optional[int] = Field(None, title='Seed') - resolution: Optional[str] = Field('1080p', title='Resolution') - duration: Optional[int] = Field(5, title='Duration') - aspectRatio: Optional[AspectRatio2] = Field( - None, description='Aspect ratio (width / height)', title='Aspectratio' - ) - - -class PikaGenerateResponse(BaseModel): - video_id: str = Field(..., title='Video Id') - - -class PikaBodyGenerate22I2vGenerate22I2vPost(BaseModel): - image: bytes = Field(..., title='Image') - promptText: Optional[str] = Field(None, title='Prompttext') - negativePrompt: Optional[str] = Field(None, title='Negativeprompt') - seed: Optional[int] = Field(None, title='Seed') - resolution: Optional[str] = Field('1080p', title='Resolution') - duration: Optional[int] = Field(5, title='Duration') - - -class IngredientsMode(str, Enum): - creative = 'creative' - precise = 'precise' - - -class PikaBodyGenerate22C2vGenerate22PikascenesPost(BaseModel): - images: List[bytes] = Field( - ..., description='Array of images to process', title='Images' - ) - ingredientsMode: IngredientsMode = Field(..., title='Ingredientsmode') - promptText: Optional[str] = Field(None, title='Prompttext') - negativePrompt: Optional[str] = Field(None, title='Negativeprompt') - seed: Optional[int] = Field(None, title='Seed') - resolution: Optional[str] = Field('1080p', title='Resolution') - duration: Optional[int] = Field(5, title='Duration') - aspectRatio: Optional[AspectRatio2] = Field( - None, description='Aspect ratio (width / height)', title='Aspectratio' - ) - - -class PikaBodyGenerate22KeyframeGenerate22PikaframesPost(BaseModel): - keyFrames: List[bytes] = Field( - ..., description='Array of keyframe images', title='Keyframes' - ) - promptText: str = Field(..., title='Prompttext') - negativePrompt: Optional[str] = Field(None, title='Negativeprompt') - seed: Optional[int] = Field(None, title='Seed') - resolution: Optional[str] = Field('1080p', title='Resolution') - duration: Optional[int] = Field(5, title='Duration') - - -class PikaVideoResponse(BaseModel): - id: str = Field(..., title='Id') - status: str = Field(..., title='Status') - url: str = Field(..., title='Url') - progress: int = Field(..., title='Progress') - - -class PikaValidationError(BaseModel): - loc: List[Union[str, int]] = Field(..., title='Location') - msg: str = Field(..., title='Message') - type: str = Field(..., title='Error Type') - - -class RunwayImageToVideoRequest(BaseModel): - promptImage: RunwayPromptImageObject - seed: int = Field( - ..., description='Random seed for generation', ge=0, le=4294967295 - ) - model: RunwayModelEnum = Field(..., description='Model to use for generation') - promptText: Optional[str] = Field( - None, description='Text prompt for the generation', max_length=1000 - ) - duration: RunwayDurationEnum = Field( - ..., description='The number of seconds of duration for the output video.' - ) - ratio: RunwayAspectRatioEnum = Field( - ..., - description='The resolution (aspect ratio) of the output video. Allowable values depend on the selected model. 1280:768 and 768:1280 are only supported for gen3a_turbo.', - ) - - -class RunwayTaskStatusResponse(BaseModel): - id: Optional[str] = Field(None, description='Task ID') - status: Optional[RunwayTaskStatusEnum] = Field(None, description='Task status') - createdAt: Optional[datetime] = Field(None, description='Task creation timestamp') - output: Optional[List[str]] = Field(None, description='Array of output video URLs') + generation_type: Optional[GenerationType3] = 'upscale_video' + resolution: Optional[LumaVideoModelOutputResolution] = None class PikaHTTPValidationError(BaseModel): detail: Optional[List[PikaValidationError]] = Field(None, title='Detail') + + +class RunwayImageToVideoRequest(BaseModel): + duration: RunwayDurationEnum + model: RunwayModelEnum + promptImage: RunwayPromptImageObject + promptText: Optional[str] = Field( + None, description='Text prompt for the generation', max_length=1000 + ) + ratio: RunwayAspectRatioEnum + seed: int = Field( + ..., description='Random seed for generation', ge=0, le=4294967295 + ) + + +class LumaGeneration(BaseModel): + assets: Optional[LumaAssets] = None + created_at: Optional[datetime] = Field( + None, description='The date and time when the generation was created' + ) + failure_reason: Optional[str] = Field( + None, description='The reason for the state of the generation' + ) + generation_type: Optional[LumaGenerationType] = None + id: Optional[UUID] = Field(None, description='The ID of the generation') + model: Optional[str] = Field(None, description='The model used for the generation') + request: Optional[ + Union[ + LumaGenerationRequest, + LumaImageGenerationRequest, + LumaUpscaleVideoGenerationRequest, + LumaAudioGenerationRequest, + ] + ] = Field(None, description='The request of the generation') + state: Optional[LumaState] = None diff --git a/comfy_api_nodes/nodes_veo2.py b/comfy_api_nodes/nodes_veo2.py index 5a2ea7eee..206480c89 100644 --- a/comfy_api_nodes/nodes_veo2.py +++ b/comfy_api_nodes/nodes_veo2.py @@ -228,8 +228,28 @@ class VeoVideoGenerationNode(ComfyNodeABC): poll_response = poll_operation.execute() + # Check for error in poll response + if hasattr(poll_response, 'error') and poll_response.error: + error_message = f"Veo API error: {poll_response.error.message} (code: {poll_response.error.code})" + logging.error(error_message) + raise Exception(error_message) + if poll_response.done: - if poll_response.response and poll_response.response.videos and len(poll_response.response.videos) > 0: + # Check for RAI filtered content + if (hasattr(poll_response.response, 'raiMediaFilteredCount') and + poll_response.response.raiMediaFilteredCount > 0): + + # Extract reason message if available + if (hasattr(poll_response.response, 'raiMediaFilteredReasons') and + poll_response.response.raiMediaFilteredReasons): + reason = poll_response.response.raiMediaFilteredReasons[0] + error_message = f"Content filtered by Google's Responsible AI practices: {reason} ({poll_response.response.raiMediaFilteredCount} videos filtered.)" + + logging.error(error_message) + raise Exception(error_message) + + # Process successful response + if poll_response.response and hasattr(poll_response.response, 'videos') and poll_response.response.videos and len(poll_response.response.videos) > 0: video = poll_response.response.videos[0] # Check if video is provided as base64 or URL From d10a1259fa70a081019ca68f9a39b00e55f3cf15 Mon Sep 17 00:00:00 2001 From: Jedrzej Kosinski Date: Tue, 29 Apr 2025 19:30:45 -0500 Subject: [PATCH 033/121] Add Realistic Image and Logo Raster styles for Recraft v3 (#38) --- comfy_api_nodes/nodes_api.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/comfy_api_nodes/nodes_api.py b/comfy_api_nodes/nodes_api.py index ad5ebd330..3789ee236 100644 --- a/comfy_api_nodes/nodes_api.py +++ b/comfy_api_nodes/nodes_api.py @@ -1910,6 +1910,14 @@ class RecraftStyleV3LogoRasterNode(RecraftStyleV3RealisticImageNode): Select vector_illustration style and optional substyle. """ + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "substyle": (get_v3_substyles(s.RECRAFT_STYLE, include_none=False),), + } + } + RECRAFT_STYLE = RecraftStyleV3.logo_raster @@ -2183,10 +2191,10 @@ NODE_CLASS_MAPPINGS = { "LumaReferenceNode": LumaReferenceNode, "LumaConceptsNode": LumaConceptsNode, "RecraftTextToImageNode": RecraftTextToImageNode, - # "RecraftStyleV3RealisticImage": RecraftStyleV3RealisticImageNode, + "RecraftStyleV3RealisticImage": RecraftStyleV3RealisticImageNode, "RecraftStyleV3DigitalIllustration": RecraftStyleV3DigitalIllustrationNode, + "RecraftStyleV3LogoRaster": RecraftStyleV3LogoRasterNode, # "RecraftStyleV3VectorIllustration": RecraftStyleV3VectorIllustrationNode, - # "RecraftStyleV3LogoRaster": RecraftStyleV3LogoRasterNode, "MinimaxTextToVideoNode": MinimaxTextToVideoNode, } From 2d4d2f0dfeb0f240595f6c71963dde26b116af9d Mon Sep 17 00:00:00 2001 From: Christian Byrne Date: Tue, 29 Apr 2025 18:05:00 -0700 Subject: [PATCH 034/121] Fix runway image upload and progress polling (#39) --- comfy_api_nodes/apis/__init__.py | 2 +- comfy_api_nodes/apis/client.py | 21 ++++++++++++--------- comfy_api_nodes/nodes_api.py | 4 ++-- comfy_api_nodes/nodes_runway.py | 24 +++++++++++++++++------- 4 files changed, 32 insertions(+), 19 deletions(-) diff --git a/comfy_api_nodes/apis/__init__.py b/comfy_api_nodes/apis/__init__.py index 9a4852f43..f6da69112 100644 --- a/comfy_api_nodes/apis/__init__.py +++ b/comfy_api_nodes/apis/__init__.py @@ -1355,7 +1355,7 @@ class RunwayPromptImageDetailedObject(BaseModel): ..., description="The position of the image in the output video. 'last' is currently supported for gen3a_turbo only.", ) - uri: AnyUrl = Field( + uri: str = Field( ..., description='A HTTPS URL or data URI containing an encoded image.' ) diff --git a/comfy_api_nodes/apis/client.py b/comfy_api_nodes/apis/client.py index b9575b83a..7d19b4cf7 100644 --- a/comfy_api_nodes/apis/client.py +++ b/comfy_api_nodes/apis/client.py @@ -105,7 +105,7 @@ from typing import ( TypeVar, Generic, ) -from pydantic import BaseModel, Field +from pydantic import BaseModel, Field, HttpUrl from enum import Enum import json import requests @@ -127,11 +127,11 @@ class EmptyRequest(BaseModel): class UploadRequest(BaseModel): filename: str = Field(..., description="Filename to upload") - mime_type: str = Field(..., description="Mime type of the file. For example: image/png, image/jpeg, video/mp4, etc.") + content_type: str = Field(..., description="Mime type of the file. For example: image/png, image/jpeg, video/mp4, etc.") class UploadResponse(BaseModel): - download_url: str = Field(..., description='URL to GET uploaded file') - upload_url: str = Field(..., description='URL to PUT file to upload') + download_url: HttpUrl = Field(..., description='URL to GET uploaded file') + upload_url: HttpUrl = Field(..., description='URL to PUT file to upload') class HttpMethod(str, Enum): @@ -297,7 +297,7 @@ class ApiClient: def upload_file( upload_url: str, file: io.BytesIO | str, - mime_type: str | None = None, + content_type: str | None = None, ): """Upload a file to the API. Make sure the file has a filename equal to what the url expects. @@ -307,8 +307,8 @@ class ApiClient: mime_type: Optional mime type to set for the upload """ headers = {} - if mime_type: - headers["Content-Type"] = mime_type + if content_type: + headers["Content-Type"] = content_type if isinstance(file, io.BytesIO): file.seek(0) # Ensure we're at the start of the file @@ -503,7 +503,6 @@ class PollingOperation(Generic[T, R]): def _poll_until_complete(self, client: ApiClient) -> R: """Poll until the task is complete""" poll_count = 0 - progress = 0 if self.progress_extractor: progress = utils.ProgressBar(100) @@ -542,11 +541,15 @@ class PollingOperation(Generic[T, R]): # If progress extractor is provided, extract progress if self.progress_extractor: - progress.update(self.progress_extractor(response_obj)) + new_progress = self.progress_extractor(response_obj) + if new_progress is not None: + progress.update(new_progress) if status == TaskStatus.COMPLETED: logging.debug("[DEBUG] Task completed successfully") self.final_response = response_obj + if self.progress_extractor: + progress.update(100) return self.final_response elif status == TaskStatus.FAILED: logging.debug(f"[DEBUG] Task failed: {json.dumps(resp)}") diff --git a/comfy_api_nodes/nodes_api.py b/comfy_api_nodes/nodes_api.py index 3789ee236..4c8ab1f59 100644 --- a/comfy_api_nodes/nodes_api.py +++ b/comfy_api_nodes/nodes_api.py @@ -327,13 +327,13 @@ def upload_images_to_comfyapi( request_model=UploadRequest, response_model=UploadResponse, ), - request=UploadRequest(filename=img_binary.name, mime_type=mime_type), + request=UploadRequest(filename=img_binary.name, content_type=mime_type), auth_token=auth_token, ) response = operation.execute() upload_response = ApiClient.upload_file( - response.upload_url, img_binary, mime_type=mime_type + response.upload_url, img_binary, content_type=mime_type ) # verify success try: diff --git a/comfy_api_nodes/nodes_runway.py b/comfy_api_nodes/nodes_runway.py index dd4db4a5f..dc6f544cb 100644 --- a/comfy_api_nodes/nodes_runway.py +++ b/comfy_api_nodes/nodes_runway.py @@ -30,6 +30,7 @@ from comfy_api.input_impl import VideoFromFile from comfy_api_nodes.mapper_utils import model_field_to_node_input PATH_IMAGE_TO_VIDEO = "/proxy/runway/image-to-video" +PATH_GET_TASK_STATUS = "/proxy/runway/tasks" class RunwayApiError(Exception): @@ -38,6 +39,12 @@ class RunwayApiError(Exception): pass +def extract_progress_from_task_status(response: TaskStatusResponse) -> float: + if hasattr(response, "progress") and response.progress is not None: + return response.progress * 100 + return None + + class RunwayImageToVideoNode(ComfyNodeABC): """ Runway Image to Video Node. @@ -86,7 +93,7 @@ class RunwayImageToVideoNode(ComfyNodeABC): """ polling_operation = PollingOperation( poll_endpoint=ApiEndpoint( - path=f"{PATH_IMAGE_TO_VIDEO}/{task_id}", + path=f"{PATH_GET_TASK_STATUS}/{task_id}", method=HttpMethod.GET, request_model=EmptyRequest, response_model=TaskStatusResponse, @@ -98,7 +105,7 @@ class RunwayImageToVideoNode(ComfyNodeABC): TaskStatus.FAILED.value, TaskStatus.CANCELLED.value, ], - progress_extractor=lambda response: (response.progress * 100), + progress_extractor=extract_progress_from_task_status, status_extractor=lambda response: (response.status.value), auth_token=auth_token, ) @@ -193,7 +200,10 @@ class RunwayImageToVideoNode(ComfyNodeABC): prompt_images_tensor = torch.cat(prompt_images_tensors, dim=0) download_urls = upload_images_to_comfyapi( - prompt_images_tensor, max_images=2, auth_token=auth_token, mime_type="image/png" + prompt_images_tensor, + max_images=2, + auth_token=auth_token, + mime_type="image/png", ) # Create a list of detailed image objects @@ -202,15 +212,15 @@ class RunwayImageToVideoNode(ComfyNodeABC): ] if len(download_urls) > 1: prompt_image_details.append( - RunwayPromptImageDetailedObject(uri=str(download_urls[1]), position="last") + RunwayPromptImageDetailedObject( + uri=str(download_urls[1]), position="last" + ) ) # Wrap the list in the main object if details exist prompt_image_object: Optional[RunwayPromptImageObject] = None if prompt_image_details: - prompt_image_object = RunwayPromptImageObject( - root=prompt_image_details - ) + prompt_image_object = RunwayPromptImageObject(root=prompt_image_details) initial_operation = SynchronousOperation( endpoint=ApiEndpoint( From c42295c5792a374d1ae071a9824060812246ec50 Mon Sep 17 00:00:00 2001 From: Christian Byrne Date: Tue, 29 Apr 2025 18:31:09 -0700 Subject: [PATCH 035/121] Fix image upload for Luma: only include `Content-Type` header field if it's set explicitly (#40) --- comfy_api_nodes/apis/client.py | 12 ++++++++---- comfy_api_nodes/nodes_api.py | 16 ++++++++++++++-- 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/comfy_api_nodes/apis/client.py b/comfy_api_nodes/apis/client.py index 7d19b4cf7..c9d7ef8f8 100644 --- a/comfy_api_nodes/apis/client.py +++ b/comfy_api_nodes/apis/client.py @@ -105,7 +105,7 @@ from typing import ( TypeVar, Generic, ) -from pydantic import BaseModel, Field, HttpUrl +from pydantic import BaseModel, Field from enum import Enum import json import requests @@ -127,11 +127,15 @@ class EmptyRequest(BaseModel): class UploadRequest(BaseModel): filename: str = Field(..., description="Filename to upload") - content_type: str = Field(..., description="Mime type of the file. For example: image/png, image/jpeg, video/mp4, etc.") + content_type: str | None = Field( + None, + description="Mime type of the file. For example: image/png, image/jpeg, video/mp4, etc.", + ) + class UploadResponse(BaseModel): - download_url: HttpUrl = Field(..., description='URL to GET uploaded file') - upload_url: HttpUrl = Field(..., description='URL to PUT file to upload') + download_url: str = Field(..., description="URL to GET uploaded file") + upload_url: str = Field(..., description="URL to PUT file to upload") class HttpMethod(str, Enum): diff --git a/comfy_api_nodes/nodes_api.py b/comfy_api_nodes/nodes_api.py index 4c8ab1f59..978b48ff8 100644 --- a/comfy_api_nodes/nodes_api.py +++ b/comfy_api_nodes/nodes_api.py @@ -226,6 +226,9 @@ def _tensor_to_pil(image: torch.Tensor, total_pixels: int = 2048 * 2048) -> Imag def _pil_to_bytesio(img: Image.Image, mime_type: str = "image/png") -> BytesIO: """Converts a PIL Image to a BytesIO object.""" + if not mime_type: + mime_type = "image/png" + img_byte_arr = io.BytesIO() # Derive PIL format from MIME type (e.g., 'image/png' -> 'PNG') pil_format = mime_type.split("/")[-1].upper() @@ -253,6 +256,9 @@ def tensor_to_bytesio( Returns: Named BytesIO object containing the image data. """ + if not mime_type: + mime_type = "image/png" + pil_image = _tensor_to_pil(image, total_pixels=total_pixels) img_binary = _pil_to_bytesio(pil_image, mime_type=mime_type) img_binary.name = ( @@ -304,7 +310,7 @@ def tensor_to_data_uri( def upload_images_to_comfyapi( - image: torch.Tensor, max_images=8, auth_token=None, mime_type: str = "image/png" + image: torch.Tensor, max_images=8, auth_token=None, mime_type: Optional[str] = None ) -> list[str]: # if batch, try to upload each file if max_images is greater than 0 idx_image = 0 @@ -320,6 +326,12 @@ def upload_images_to_comfyapi( # get BytesIO version of image img_binary = tensor_to_bytesio(curr_image, mime_type=mime_type) # first, request upload/download urls from comfy API + if not mime_type: + request_object = UploadRequest(filename=img_binary.name) + else: + request_object = UploadRequest( + filename=img_binary.name, content_type=mime_type + ) operation = SynchronousOperation( endpoint=ApiEndpoint( path="/customers/storage", @@ -327,7 +339,7 @@ def upload_images_to_comfyapi( request_model=UploadRequest, response_model=UploadResponse, ), - request=UploadRequest(filename=img_binary.name, content_type=mime_type), + request=request_object, auth_token=auth_token, ) response = operation.execute() From 48a07964e3f2168b7d4e514f29cb74975a0856a7 Mon Sep 17 00:00:00 2001 From: Jedrzej Kosinski Date: Wed, 30 Apr 2025 00:20:31 -0500 Subject: [PATCH 036/121] Moved Luma nodes to nodes_luma.py (#47) --- comfy_api_nodes/nodes_api.py | 675 --------------------------------- comfy_api_nodes/nodes_luma.py | 694 ++++++++++++++++++++++++++++++++++ nodes.py | 1 + 3 files changed, 695 insertions(+), 675 deletions(-) create mode 100644 comfy_api_nodes/nodes_luma.py diff --git a/comfy_api_nodes/nodes_api.py b/comfy_api_nodes/nodes_api.py index 978b48ff8..6018a6f6b 100644 --- a/comfy_api_nodes/nodes_api.py +++ b/comfy_api_nodes/nodes_api.py @@ -20,27 +20,6 @@ from comfy_api_nodes.apis import ( Model ) from comfy_api_nodes.apis.BFLPolling import BFLStatus -from comfy_api_nodes.apis.luma_api import ( - LumaImageModel, - LumaVideoModel, - LumaVideoOutputResolution, - LumaVideoModelOutputDuration, - LumaAspectRatio, - LumaState, - LumaImageGenerationRequest, - LumaGenerationRequest, - LumaGeneration, - LumaCharacterRef, - LumaModifyImageRef, - LumaImageIdentity, - LumaReference, - LumaReferenceChain, - LumaImageReference, - LumaKeyframes, - LumaConceptChain, - LumaIO, - get_luma_concepts, -) from comfy_api_nodes.apis.recraft_api import ( RecraftImageGenerationRequest, RecraftImageGenerationResponse, @@ -1233,648 +1212,6 @@ class FluxProUltraImageNode(ComfyNodeABC): img.save(img_byte_arr, format="PNG") return base64.b64encode(img_byte_arr.getvalue()).decode() - -class LumaReferenceNode(ComfyNodeABC): - """ - Holds an image and weight for use with Luma Generate Image node. - """ - - RETURN_TYPES = (LumaIO.LUMA_REF,) - RETURN_NAMES = ("luma_ref",) - DESCRIPTION = cleandoc(__doc__ or "") # Handle potential None value - FUNCTION = "create_luma_reference" - CATEGORY = "api node/image/Luma" - - @classmethod - def INPUT_TYPES(s): - return { - "required": { - "image": ( - IO.IMAGE, - { - "tooltip": "Image to use as reference.", - }, - ), - "weight": ( - IO.FLOAT, - { - "default": 1.0, - "min": 0.0, - "max": 1.0, - "step": 0.01, - "tooltip": "Weight of image reference.", - }, - ), - }, - "optional": {"luma_ref": (LumaIO.LUMA_REF,)}, - } - - def create_luma_reference( - self, image: torch.Tensor, weight: float, luma_ref: LumaReferenceChain = None - ): - if luma_ref is not None: - luma_ref = luma_ref.clone() - else: - luma_ref = LumaReferenceChain() - luma_ref.add(LumaReference(image=image, weight=round(weight, 2))) - return (luma_ref,) - - -class LumaConceptsNode(ComfyNodeABC): - """ - Holds one or more Camera Concepts for use with Luma Text to Video and Luma Image to Video nodes. - """ - - RETURN_TYPES = (LumaIO.LUMA_CONCEPTS,) - RETURN_NAMES = ("luma_concepts",) - DESCRIPTION = cleandoc(__doc__ or "") # Handle potential None value - FUNCTION = "create_concepts" - CATEGORY = "api node/image/Luma" - - @classmethod - def INPUT_TYPES(s): - return { - "required": { - "concept1": (get_luma_concepts(include_none=True),), - "concept2": (get_luma_concepts(include_none=True),), - "concept3": (get_luma_concepts(include_none=True),), - "concept4": (get_luma_concepts(include_none=True),), - }, - "optional": { - "luma_concepts": ( - LumaIO.LUMA_CONCEPTS, - { - "tooltip": "Optional Camera Concepts to add to the ones chosen here." - }, - ), - }, - } - - def create_concepts( - self, - concept1: str, - concept2: str, - concept3: str, - concept4: str, - luma_concepts: LumaConceptChain = None, - ): - chain = LumaConceptChain(str_list=[concept1, concept2, concept3, concept4]) - if luma_concepts is not None: - chain = luma_concepts.clone_and_merge(chain) - return (chain,) - - -class LumaImageGenerationNode(ComfyNodeABC): - """ - Generates images synchronously based on prompt and aspect ratio. - """ - - RETURN_TYPES = (IO.IMAGE,) - DESCRIPTION = cleandoc(__doc__ or "") # Handle potential None value - FUNCTION = "api_call" - API_NODE = True - CATEGORY = "api node/image/Luma" - - @classmethod - def INPUT_TYPES(s): - return { - "required": { - "prompt": ( - IO.STRING, - { - "multiline": True, - "default": "", - "tooltip": "Prompt for the image generation", - }, - ), - "model": ([model.value for model in LumaImageModel],), - "aspect_ratio": ( - [ratio.value for ratio in LumaAspectRatio], - { - "default": LumaAspectRatio.ratio_16_9, - }, - ), - "seed": ( - IO.INT, - { - "default": 0, - "min": 0, - "max": 0xFFFFFFFFFFFFFFFF, - "control_after_generate": True, - "tooltip": "Seed to determine if node should re-run; actual results are nondeterministic regardless of seed.", - }, - ), - "style_image_weight": ( - IO.FLOAT, - { - "default": 1.0, - "min": 0.0, - "max": 1.0, - "step": 0.01, - "tooltip": "Weight of style image. Ignored if no style_image provided.", - }, - ), - }, - "optional": { - "image_luma_ref": ( - LumaIO.LUMA_REF, - { - "tooltip": "Luma Reference node connection to influence generation with input images; up to 4 images can be considered." - }, - ), - "style_image": ( - IO.IMAGE, - {"tooltip": "Style reference image; only 1 image will be used."}, - ), - "character_image": ( - IO.IMAGE, - { - "tooltip": "Character reference images; can be a batch of multiple, up to 4 images can be considered." - }, - ), - }, - "hidden": { - "auth_token": "AUTH_TOKEN_COMFY_ORG", - }, - } - - def api_call( - self, - prompt: str, - model: str, - aspect_ratio: str, - seed, - style_image_weight: float, - image_luma_ref: LumaReferenceChain = None, - style_image: torch.Tensor = None, - character_image: torch.Tensor = None, - auth_token=None, - **kwargs, - ): - # handle image_luma_ref - api_image_ref = None - if image_luma_ref is not None: - api_image_ref = self._convert_luma_refs( - image_luma_ref, max_refs=4, auth_token=auth_token - ) - # handle style_luma_ref - api_style_ref = None - if style_image is not None: - api_style_ref = self._convert_style_image( - style_image, weight=style_image_weight, auth_token=auth_token - ) - # handle character_ref images - character_ref = None - if character_image is not None: - download_urls = upload_images_to_comfyapi( - character_image, max_images=4, auth_token=auth_token - ) - character_ref = LumaCharacterRef( - identity0=LumaImageIdentity(images=download_urls) - ) - - operation = SynchronousOperation( - endpoint=ApiEndpoint( - path="/proxy/luma/generations/image", - method=HttpMethod.POST, - request_model=LumaImageGenerationRequest, - response_model=LumaGeneration, - ), - request=LumaImageGenerationRequest( - prompt=prompt, - model=model, - aspect_ratio=aspect_ratio, - image_ref=api_image_ref, - style_ref=api_style_ref, - character_ref=character_ref, - ), - auth_token=auth_token, - ) - response_api: LumaGeneration = operation.execute() - - operation = PollingOperation( - poll_endpoint=ApiEndpoint( - path=f"/proxy/luma/generations/{response_api.id}", - method=HttpMethod.GET, - request_model=EmptyRequest, - response_model=LumaGeneration, - ), - completed_statuses=[LumaState.completed], - failed_statuses=[LumaState.failed], - status_extractor=lambda x: x.state, - auth_token=auth_token, - ) - response_poll = operation.execute() - - img_response = requests.get(response_poll.assets.image) - img = process_image_response(img_response) - return (img,) - - def _convert_luma_refs( - self, luma_ref: LumaReferenceChain, max_refs: int, auth_token=None - ): - luma_urls = [] - ref_count = 0 - for ref in luma_ref.refs: - download_urls = upload_images_to_comfyapi( - ref.image, max_images=1, auth_token=auth_token - ) - luma_urls.append(download_urls[0]) - ref_count += 1 - if ref_count >= max_refs: - break - return luma_ref.create_api_model(download_urls=luma_urls, max_refs=max_refs) - - def _convert_style_image( - self, style_image: torch.Tensor, weight: float, auth_token=None - ): - chain = LumaReferenceChain( - first_ref=LumaReference(image=style_image, weight=weight) - ) - return self._convert_luma_refs(chain, max_refs=1, auth_token=auth_token) - - -class LumaImageModifyNode(ComfyNodeABC): - """ - Modifies images synchronously based on prompt and aspect ratio. - """ - - RETURN_TYPES = (IO.IMAGE,) - DESCRIPTION = cleandoc(__doc__ or "") # Handle potential None value - FUNCTION = "api_call" - API_NODE = True - CATEGORY = "api node/image/Luma" - - @classmethod - def INPUT_TYPES(s): - return { - "required": { - "image": (IO.IMAGE,), - "prompt": ( - IO.STRING, - { - "multiline": True, - "default": "", - "tooltip": "Prompt for the image generation", - }, - ), - "image_weight": ( - IO.FLOAT, - { - "default": 1.0, - "min": 0.0, - "max": 1.0, - "step": 0.01, - "tooltip": "Weight of the image; the closer to 0.0, the less the image will be modified.", - }, - ), - "model": ([model.value for model in LumaImageModel],), - "seed": ( - IO.INT, - { - "default": 0, - "min": 0, - "max": 0xFFFFFFFFFFFFFFFF, - "control_after_generate": True, - "tooltip": "Seed to determine if node should re-run; actual results are nondeterministic regardless of seed.", - }, - ), - }, - "optional": {}, - "hidden": { - "auth_token": "AUTH_TOKEN_COMFY_ORG", - }, - } - - def api_call( - self, - prompt: str, - model: str, - image: torch.Tensor, - image_weight: float, - seed, - auth_token=None, - **kwargs, - ): - # first, upload image - download_urls = upload_images_to_comfyapi( - image, max_images=1, auth_token=auth_token - ) - image_url = download_urls[0] - # next, make Luma call with download url provided - operation = SynchronousOperation( - endpoint=ApiEndpoint( - path="/proxy/luma/generations/image", - method=HttpMethod.POST, - request_model=LumaImageGenerationRequest, - response_model=LumaGeneration, - ), - request=LumaImageGenerationRequest( - prompt=prompt, - model=model, - modify_image_ref=LumaModifyImageRef( - url=image_url, weight=round(image_weight, 2) - ), - ), - auth_token=auth_token, - ) - response_api: LumaGeneration = operation.execute() - - operation = PollingOperation( - poll_endpoint=ApiEndpoint( - path=f"/proxy/luma/generations/{response_api.id}", - method=HttpMethod.GET, - request_model=EmptyRequest, - response_model=LumaGeneration, - ), - completed_statuses=[LumaState.completed], - failed_statuses=[LumaState.failed], - status_extractor=lambda x: x.state, - auth_token=auth_token, - ) - response_poll = operation.execute() - - img_response = requests.get(response_poll.assets.image) - img = process_image_response(img_response) - return (img,) - - -class LumaTextToVideoGenerationNode(ComfyNodeABC): - """ - Generates videos synchronously based on prompt and output_size. - """ - - def __init__(self): - self.output_dir = folder_paths.get_output_directory() - self.type: Literal["output"] = "output" - - RETURN_TYPES = (IO.VIDEO,) - DESCRIPTION = cleandoc(__doc__ or "") # Handle potential None value - FUNCTION = "api_call" - API_NODE = True - CATEGORY = "api node/image/Luma" - - @classmethod - def INPUT_TYPES(s): - return { - "required": { - "prompt": ( - IO.STRING, - { - "multiline": True, - "default": "", - "tooltip": "Prompt for the video generation", - }, - ), - "model": ([model.value for model in LumaVideoModel],), - "aspect_ratio": ( - [ratio.value for ratio in LumaAspectRatio], - { - "default": LumaAspectRatio.ratio_16_9, - }, - ), - "resolution": ( - [resolution.value for resolution in LumaVideoOutputResolution], - { - "default": LumaVideoOutputResolution.res_540p, - }, - ), - "duration": ([dur.value for dur in LumaVideoModelOutputDuration],), - "loop": ( - IO.BOOLEAN, - { - "default": False, - }, - ), - "seed": ( - IO.INT, - { - "default": 0, - "min": 0, - "max": 0xFFFFFFFFFFFFFFFF, - "control_after_generate": True, - "tooltip": "Seed to determine if node should re-run; actual results are nondeterministic regardless of seed.", - }, - ), - }, - "optional": { - "luma_concepts": ( - LumaIO.LUMA_CONCEPTS, - { - "tooltip": "Optional Camera Concepts to dictate camera motion via the Luma Concepts node." - }, - ), - }, - "hidden": { - "auth_token": "AUTH_TOKEN_COMFY_ORG", - }, - } - - def api_call( - self, - prompt: str, - model: str, - aspect_ratio: str, - resolution: str, - duration: str, - loop: bool, - seed, - luma_concepts: LumaConceptChain = None, - auth_token=None, - **kwargs, - ): - operation = SynchronousOperation( - endpoint=ApiEndpoint( - path="/proxy/luma/generations", - method=HttpMethod.POST, - request_model=LumaGenerationRequest, - response_model=LumaGeneration, - ), - request=LumaGenerationRequest( - prompt=prompt, - model=model, - resolution=resolution, - aspect_ratio=aspect_ratio, - duration=duration, - loop=loop, - concepts=luma_concepts.create_api_model() if luma_concepts else None, - ), - auth_token=auth_token, - ) - response_api: LumaGeneration = operation.execute() - - operation = PollingOperation( - poll_endpoint=ApiEndpoint( - path=f"/proxy/luma/generations/{response_api.id}", - method=HttpMethod.GET, - request_model=EmptyRequest, - response_model=LumaGeneration, - ), - completed_statuses=[LumaState.completed], - failed_statuses=[LumaState.failed], - status_extractor=lambda x: x.state, - auth_token=auth_token, - ) - response_poll = operation.execute() - - vid_response = requests.get(response_poll.assets.video) - return (VideoFromFile(BytesIO(vid_response.content)),) - - -class LumaImageToVideoGenerationNode(ComfyNodeABC): - """ - Generates videos synchronously based on prompt, input images, and output_size. - """ - - def __init__(self): - self.output_dir = folder_paths.get_output_directory() - self.type: Literal["output"] = "output" - - RETURN_TYPES = (IO.VIDEO,) - DESCRIPTION = cleandoc(__doc__ or "") # Handle potential None value - FUNCTION = "api_call" - API_NODE = True - CATEGORY = "api node/image/Luma" - - @classmethod - def INPUT_TYPES(s): - return { - "required": { - "prompt": ( - IO.STRING, - { - "multiline": True, - "default": "", - "tooltip": "Prompt for the video generation", - }, - ), - "model": ([model.value for model in LumaVideoModel],), - # "aspect_ratio": ([ratio.value for ratio in LumaAspectRatio], { - # "default": LumaAspectRatio.ratio_16_9, - # }), - "resolution": ( - [resolution.value for resolution in LumaVideoOutputResolution], - { - "default": LumaVideoOutputResolution.res_540p, - }, - ), - "duration": ([dur.value for dur in LumaVideoModelOutputDuration],), - "loop": ( - IO.BOOLEAN, - { - "default": False, - }, - ), - "seed": ( - IO.INT, - { - "default": 0, - "min": 0, - "max": 0xFFFFFFFFFFFFFFFF, - "control_after_generate": True, - "tooltip": "Seed to determine if node should re-run; actual results are nondeterministic regardless of seed.", - }, - ), - }, - "optional": { - "first_image": ( - IO.IMAGE, - {"tooltip": "First frame of generated video."}, - ), - "last_image": (IO.IMAGE, {"tooltip": "Last frame of generated video."}), - "luma_concepts": ( - LumaIO.LUMA_CONCEPTS, - { - "tooltip": "Optional Camera Concepts to dictate camera motion via the Luma Concepts node." - }, - ), - }, - "hidden": { - "auth_token": "AUTH_TOKEN_COMFY_ORG", - }, - } - - def api_call( - self, - prompt: str, - model: str, - resolution: str, - duration: str, - loop: bool, - seed, - first_image: torch.Tensor = None, - last_image: torch.Tensor = None, - luma_concepts: LumaConceptChain = None, - auth_token=None, - **kwargs, - ): - if first_image is None and last_image is None: - raise Exception( - "At least one of first_image and last_image requires an input." - ) - keyframes = self._convert_to_keyframes(first_image, last_image, auth_token) - - operation = SynchronousOperation( - endpoint=ApiEndpoint( - path="/proxy/luma/generations", - method=HttpMethod.POST, - request_model=LumaGenerationRequest, - response_model=LumaGeneration, - ), - request=LumaGenerationRequest( - prompt=prompt, - model=model, - aspect_ratio=LumaAspectRatio.ratio_16_9, # ignored, but still needed by the API for some reason - resolution=resolution, - duration=duration, - loop=loop, - keyframes=keyframes, - concepts=luma_concepts.create_api_model() if luma_concepts else None, - ), - auth_token=auth_token, - ) - response_api: LumaGeneration = operation.execute() - - operation = PollingOperation( - poll_endpoint=ApiEndpoint( - path=f"/proxy/luma/generations/{response_api.id}", - method=HttpMethod.GET, - request_model=EmptyRequest, - response_model=LumaGeneration, - ), - completed_statuses=[LumaState.completed], - failed_statuses=[LumaState.failed], - status_extractor=lambda x: x.state, - auth_token=auth_token, - ) - response_poll = operation.execute() - - vid_response = requests.get(response_poll.assets.video) - return (VideoFromFile(BytesIO(vid_response.content)),) - - def _convert_to_keyframes( - self, - first_image: torch.Tensor = None, - last_image: torch.Tensor = None, - auth_token=None, - ): - if first_image is None and last_image is None: - return None - frame0 = None - frame1 = None - if first_image is not None: - download_urls = upload_images_to_comfyapi( - first_image, max_images=1, auth_token=auth_token - ) - frame0 = LumaImageReference(type="image", url=download_urls[0]) - if last_image is not None: - download_urls = upload_images_to_comfyapi( - last_image, max_images=1, auth_token=auth_token - ) - frame1 = LumaImageReference(type="image", url=download_urls[0]) - return LumaKeyframes(frame0=frame0, frame1=frame1) - - class RecraftStyleV3RealisticImageNode: """ Select realistic_image style and optional substyle. @@ -2196,12 +1533,6 @@ NODE_CLASS_MAPPINGS = { "OpenAIGPTImage1": OpenAIGPTImage1, "IdeogramTextToImage": IdeogramTextToImage, "FluxProUltraImageNode": FluxProUltraImageNode, - "LumaImageNode": LumaImageGenerationNode, - "LumaImageModifyNode": LumaImageModifyNode, - "LumaVideoNode": LumaTextToVideoGenerationNode, - "LumaImageToVideoNode": LumaImageToVideoGenerationNode, - "LumaReferenceNode": LumaReferenceNode, - "LumaConceptsNode": LumaConceptsNode, "RecraftTextToImageNode": RecraftTextToImageNode, "RecraftStyleV3RealisticImage": RecraftStyleV3RealisticImageNode, "RecraftStyleV3DigitalIllustration": RecraftStyleV3DigitalIllustrationNode, @@ -2217,12 +1548,6 @@ NODE_DISPLAY_NAME_MAPPINGS = { "OpenAIGPTImage1": "OpenAI GPT Image 1", "IdeogramTextToImage": "Ideogram Text to Image", "FluxProUltraImageNode": "Flux 1.1 [pro] Ultra Image", - "LumaImageNode": "Luma Text to Image", - "LumaImageModifyNode": "Luma Image to Image", - "LumaVideoNode": "Luma Text to Video", - "LumaImageToVideoNode": "Luma Image to Video", - "LumaReferenceNode": "Luma Reference", - "LumaConceptsNode": "Luma Concepts", "RecraftTextToImageNode": "Recraft Text to Image", "RecraftStyleV3RealisticImage": "Recraft Style - Realistic Image", "RecraftStyleV3DigitalIllustration": "Recraft Style - Digital Illustration", diff --git a/comfy_api_nodes/nodes_luma.py b/comfy_api_nodes/nodes_luma.py new file mode 100644 index 000000000..a846c38b6 --- /dev/null +++ b/comfy_api_nodes/nodes_luma.py @@ -0,0 +1,694 @@ +from inspect import cleandoc +from comfy.comfy_types.node_typing import IO, ComfyNodeABC +from comfy_api.input_impl.video_types import VideoFromFile +from comfy_api_nodes.apis.luma_api import ( + LumaImageModel, + LumaVideoModel, + LumaVideoOutputResolution, + LumaVideoModelOutputDuration, + LumaAspectRatio, + LumaState, + LumaImageGenerationRequest, + LumaGenerationRequest, + LumaGeneration, + LumaCharacterRef, + LumaModifyImageRef, + LumaImageIdentity, + LumaReference, + LumaReferenceChain, + LumaImageReference, + LumaKeyframes, + LumaConceptChain, + LumaIO, + get_luma_concepts, +) +from comfy_api_nodes.apis.client import ( + ApiEndpoint, + HttpMethod, + SynchronousOperation, + PollingOperation, + EmptyRequest, +) +from comfy_api_nodes.nodes_api import ( + upload_images_to_comfyapi, + process_image_response, +) + +import requests +import torch +from io import BytesIO + + +class LumaReferenceNode(ComfyNodeABC): + """ + Holds an image and weight for use with Luma Generate Image node. + """ + + RETURN_TYPES = (LumaIO.LUMA_REF,) + RETURN_NAMES = ("luma_ref",) + DESCRIPTION = cleandoc(__doc__ or "") # Handle potential None value + FUNCTION = "create_luma_reference" + CATEGORY = "api node/image/Luma" + + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "image": ( + IO.IMAGE, + { + "tooltip": "Image to use as reference.", + }, + ), + "weight": ( + IO.FLOAT, + { + "default": 1.0, + "min": 0.0, + "max": 1.0, + "step": 0.01, + "tooltip": "Weight of image reference.", + }, + ), + }, + "optional": {"luma_ref": (LumaIO.LUMA_REF,)}, + } + + def create_luma_reference( + self, image: torch.Tensor, weight: float, luma_ref: LumaReferenceChain = None + ): + if luma_ref is not None: + luma_ref = luma_ref.clone() + else: + luma_ref = LumaReferenceChain() + luma_ref.add(LumaReference(image=image, weight=round(weight, 2))) + return (luma_ref,) + + +class LumaConceptsNode(ComfyNodeABC): + """ + Holds one or more Camera Concepts for use with Luma Text to Video and Luma Image to Video nodes. + """ + + RETURN_TYPES = (LumaIO.LUMA_CONCEPTS,) + RETURN_NAMES = ("luma_concepts",) + DESCRIPTION = cleandoc(__doc__ or "") # Handle potential None value + FUNCTION = "create_concepts" + CATEGORY = "api node/image/Luma" + + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "concept1": (get_luma_concepts(include_none=True),), + "concept2": (get_luma_concepts(include_none=True),), + "concept3": (get_luma_concepts(include_none=True),), + "concept4": (get_luma_concepts(include_none=True),), + }, + "optional": { + "luma_concepts": ( + LumaIO.LUMA_CONCEPTS, + { + "tooltip": "Optional Camera Concepts to add to the ones chosen here." + }, + ), + }, + } + + def create_concepts( + self, + concept1: str, + concept2: str, + concept3: str, + concept4: str, + luma_concepts: LumaConceptChain = None, + ): + chain = LumaConceptChain(str_list=[concept1, concept2, concept3, concept4]) + if luma_concepts is not None: + chain = luma_concepts.clone_and_merge(chain) + return (chain,) + + +class LumaImageGenerationNode(ComfyNodeABC): + """ + Generates images synchronously based on prompt and aspect ratio. + """ + + RETURN_TYPES = (IO.IMAGE,) + DESCRIPTION = cleandoc(__doc__ or "") # Handle potential None value + FUNCTION = "api_call" + API_NODE = True + CATEGORY = "api node/image/Luma" + + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "prompt": ( + IO.STRING, + { + "multiline": True, + "default": "", + "tooltip": "Prompt for the image generation", + }, + ), + "model": ([model.value for model in LumaImageModel],), + "aspect_ratio": ( + [ratio.value for ratio in LumaAspectRatio], + { + "default": LumaAspectRatio.ratio_16_9, + }, + ), + "seed": ( + IO.INT, + { + "default": 0, + "min": 0, + "max": 0xFFFFFFFFFFFFFFFF, + "control_after_generate": True, + "tooltip": "Seed to determine if node should re-run; actual results are nondeterministic regardless of seed.", + }, + ), + "style_image_weight": ( + IO.FLOAT, + { + "default": 1.0, + "min": 0.0, + "max": 1.0, + "step": 0.01, + "tooltip": "Weight of style image. Ignored if no style_image provided.", + }, + ), + }, + "optional": { + "image_luma_ref": ( + LumaIO.LUMA_REF, + { + "tooltip": "Luma Reference node connection to influence generation with input images; up to 4 images can be considered." + }, + ), + "style_image": ( + IO.IMAGE, + {"tooltip": "Style reference image; only 1 image will be used."}, + ), + "character_image": ( + IO.IMAGE, + { + "tooltip": "Character reference images; can be a batch of multiple, up to 4 images can be considered." + }, + ), + }, + "hidden": { + "auth_token": "AUTH_TOKEN_COMFY_ORG", + }, + } + + def api_call( + self, + prompt: str, + model: str, + aspect_ratio: str, + seed, + style_image_weight: float, + image_luma_ref: LumaReferenceChain = None, + style_image: torch.Tensor = None, + character_image: torch.Tensor = None, + auth_token=None, + **kwargs, + ): + # handle image_luma_ref + api_image_ref = None + if image_luma_ref is not None: + api_image_ref = self._convert_luma_refs( + image_luma_ref, max_refs=4, auth_token=auth_token + ) + # handle style_luma_ref + api_style_ref = None + if style_image is not None: + api_style_ref = self._convert_style_image( + style_image, weight=style_image_weight, auth_token=auth_token + ) + # handle character_ref images + character_ref = None + if character_image is not None: + download_urls = upload_images_to_comfyapi( + character_image, max_images=4, auth_token=auth_token + ) + character_ref = LumaCharacterRef( + identity0=LumaImageIdentity(images=download_urls) + ) + + operation = SynchronousOperation( + endpoint=ApiEndpoint( + path="/proxy/luma/generations/image", + method=HttpMethod.POST, + request_model=LumaImageGenerationRequest, + response_model=LumaGeneration, + ), + request=LumaImageGenerationRequest( + prompt=prompt, + model=model, + aspect_ratio=aspect_ratio, + image_ref=api_image_ref, + style_ref=api_style_ref, + character_ref=character_ref, + ), + auth_token=auth_token, + ) + response_api: LumaGeneration = operation.execute() + + operation = PollingOperation( + poll_endpoint=ApiEndpoint( + path=f"/proxy/luma/generations/{response_api.id}", + method=HttpMethod.GET, + request_model=EmptyRequest, + response_model=LumaGeneration, + ), + completed_statuses=[LumaState.completed], + failed_statuses=[LumaState.failed], + status_extractor=lambda x: x.state, + auth_token=auth_token, + ) + response_poll = operation.execute() + + img_response = requests.get(response_poll.assets.image) + img = process_image_response(img_response) + return (img,) + + def _convert_luma_refs( + self, luma_ref: LumaReferenceChain, max_refs: int, auth_token=None + ): + luma_urls = [] + ref_count = 0 + for ref in luma_ref.refs: + download_urls = upload_images_to_comfyapi( + ref.image, max_images=1, auth_token=auth_token + ) + luma_urls.append(download_urls[0]) + ref_count += 1 + if ref_count >= max_refs: + break + return luma_ref.create_api_model(download_urls=luma_urls, max_refs=max_refs) + + def _convert_style_image( + self, style_image: torch.Tensor, weight: float, auth_token=None + ): + chain = LumaReferenceChain( + first_ref=LumaReference(image=style_image, weight=weight) + ) + return self._convert_luma_refs(chain, max_refs=1, auth_token=auth_token) + + +class LumaImageModifyNode(ComfyNodeABC): + """ + Modifies images synchronously based on prompt and aspect ratio. + """ + + RETURN_TYPES = (IO.IMAGE,) + DESCRIPTION = cleandoc(__doc__ or "") # Handle potential None value + FUNCTION = "api_call" + API_NODE = True + CATEGORY = "api node/image/Luma" + + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "image": (IO.IMAGE,), + "prompt": ( + IO.STRING, + { + "multiline": True, + "default": "", + "tooltip": "Prompt for the image generation", + }, + ), + "image_weight": ( + IO.FLOAT, + { + "default": 1.0, + "min": 0.0, + "max": 1.0, + "step": 0.01, + "tooltip": "Weight of the image; the closer to 0.0, the less the image will be modified.", + }, + ), + "model": ([model.value for model in LumaImageModel],), + "seed": ( + IO.INT, + { + "default": 0, + "min": 0, + "max": 0xFFFFFFFFFFFFFFFF, + "control_after_generate": True, + "tooltip": "Seed to determine if node should re-run; actual results are nondeterministic regardless of seed.", + }, + ), + }, + "optional": {}, + "hidden": { + "auth_token": "AUTH_TOKEN_COMFY_ORG", + }, + } + + def api_call( + self, + prompt: str, + model: str, + image: torch.Tensor, + image_weight: float, + seed, + auth_token=None, + **kwargs, + ): + # first, upload image + download_urls = upload_images_to_comfyapi( + image, max_images=1, auth_token=auth_token + ) + image_url = download_urls[0] + # next, make Luma call with download url provided + operation = SynchronousOperation( + endpoint=ApiEndpoint( + path="/proxy/luma/generations/image", + method=HttpMethod.POST, + request_model=LumaImageGenerationRequest, + response_model=LumaGeneration, + ), + request=LumaImageGenerationRequest( + prompt=prompt, + model=model, + modify_image_ref=LumaModifyImageRef( + url=image_url, weight=round(image_weight, 2) + ), + ), + auth_token=auth_token, + ) + response_api: LumaGeneration = operation.execute() + + operation = PollingOperation( + poll_endpoint=ApiEndpoint( + path=f"/proxy/luma/generations/{response_api.id}", + method=HttpMethod.GET, + request_model=EmptyRequest, + response_model=LumaGeneration, + ), + completed_statuses=[LumaState.completed], + failed_statuses=[LumaState.failed], + status_extractor=lambda x: x.state, + auth_token=auth_token, + ) + response_poll = operation.execute() + + img_response = requests.get(response_poll.assets.image) + img = process_image_response(img_response) + return (img,) + + +class LumaTextToVideoGenerationNode(ComfyNodeABC): + """ + Generates videos synchronously based on prompt and output_size. + """ + + RETURN_TYPES = (IO.VIDEO,) + DESCRIPTION = cleandoc(__doc__ or "") # Handle potential None value + FUNCTION = "api_call" + API_NODE = True + CATEGORY = "api node/image/Luma" + + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "prompt": ( + IO.STRING, + { + "multiline": True, + "default": "", + "tooltip": "Prompt for the video generation", + }, + ), + "model": ([model.value for model in LumaVideoModel],), + "aspect_ratio": ( + [ratio.value for ratio in LumaAspectRatio], + { + "default": LumaAspectRatio.ratio_16_9, + }, + ), + "resolution": ( + [resolution.value for resolution in LumaVideoOutputResolution], + { + "default": LumaVideoOutputResolution.res_540p, + }, + ), + "duration": ([dur.value for dur in LumaVideoModelOutputDuration],), + "loop": ( + IO.BOOLEAN, + { + "default": False, + }, + ), + "seed": ( + IO.INT, + { + "default": 0, + "min": 0, + "max": 0xFFFFFFFFFFFFFFFF, + "control_after_generate": True, + "tooltip": "Seed to determine if node should re-run; actual results are nondeterministic regardless of seed.", + }, + ), + }, + "optional": { + "luma_concepts": ( + LumaIO.LUMA_CONCEPTS, + { + "tooltip": "Optional Camera Concepts to dictate camera motion via the Luma Concepts node." + }, + ), + }, + "hidden": { + "auth_token": "AUTH_TOKEN_COMFY_ORG", + }, + } + + def api_call( + self, + prompt: str, + model: str, + aspect_ratio: str, + resolution: str, + duration: str, + loop: bool, + seed, + luma_concepts: LumaConceptChain = None, + auth_token=None, + **kwargs, + ): + operation = SynchronousOperation( + endpoint=ApiEndpoint( + path="/proxy/luma/generations", + method=HttpMethod.POST, + request_model=LumaGenerationRequest, + response_model=LumaGeneration, + ), + request=LumaGenerationRequest( + prompt=prompt, + model=model, + resolution=resolution, + aspect_ratio=aspect_ratio, + duration=duration, + loop=loop, + concepts=luma_concepts.create_api_model() if luma_concepts else None, + ), + auth_token=auth_token, + ) + response_api: LumaGeneration = operation.execute() + + operation = PollingOperation( + poll_endpoint=ApiEndpoint( + path=f"/proxy/luma/generations/{response_api.id}", + method=HttpMethod.GET, + request_model=EmptyRequest, + response_model=LumaGeneration, + ), + completed_statuses=[LumaState.completed], + failed_statuses=[LumaState.failed], + status_extractor=lambda x: x.state, + auth_token=auth_token, + ) + response_poll = operation.execute() + + vid_response = requests.get(response_poll.assets.video) + return (VideoFromFile(BytesIO(vid_response.content)),) + + +class LumaImageToVideoGenerationNode(ComfyNodeABC): + """ + Generates videos synchronously based on prompt, input images, and output_size. + """ + + RETURN_TYPES = (IO.VIDEO,) + DESCRIPTION = cleandoc(__doc__ or "") # Handle potential None value + FUNCTION = "api_call" + API_NODE = True + CATEGORY = "api node/image/Luma" + + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "prompt": ( + IO.STRING, + { + "multiline": True, + "default": "", + "tooltip": "Prompt for the video generation", + }, + ), + "model": ([model.value for model in LumaVideoModel],), + # "aspect_ratio": ([ratio.value for ratio in LumaAspectRatio], { + # "default": LumaAspectRatio.ratio_16_9, + # }), + "resolution": ( + [resolution.value for resolution in LumaVideoOutputResolution], + { + "default": LumaVideoOutputResolution.res_540p, + }, + ), + "duration": ([dur.value for dur in LumaVideoModelOutputDuration],), + "loop": ( + IO.BOOLEAN, + { + "default": False, + }, + ), + "seed": ( + IO.INT, + { + "default": 0, + "min": 0, + "max": 0xFFFFFFFFFFFFFFFF, + "control_after_generate": True, + "tooltip": "Seed to determine if node should re-run; actual results are nondeterministic regardless of seed.", + }, + ), + }, + "optional": { + "first_image": ( + IO.IMAGE, + {"tooltip": "First frame of generated video."}, + ), + "last_image": (IO.IMAGE, {"tooltip": "Last frame of generated video."}), + "luma_concepts": ( + LumaIO.LUMA_CONCEPTS, + { + "tooltip": "Optional Camera Concepts to dictate camera motion via the Luma Concepts node." + }, + ), + }, + "hidden": { + "auth_token": "AUTH_TOKEN_COMFY_ORG", + }, + } + + def api_call( + self, + prompt: str, + model: str, + resolution: str, + duration: str, + loop: bool, + seed, + first_image: torch.Tensor = None, + last_image: torch.Tensor = None, + luma_concepts: LumaConceptChain = None, + auth_token=None, + **kwargs, + ): + if first_image is None and last_image is None: + raise Exception( + "At least one of first_image and last_image requires an input." + ) + keyframes = self._convert_to_keyframes(first_image, last_image, auth_token) + + operation = SynchronousOperation( + endpoint=ApiEndpoint( + path="/proxy/luma/generations", + method=HttpMethod.POST, + request_model=LumaGenerationRequest, + response_model=LumaGeneration, + ), + request=LumaGenerationRequest( + prompt=prompt, + model=model, + aspect_ratio=LumaAspectRatio.ratio_16_9, # ignored, but still needed by the API for some reason + resolution=resolution, + duration=duration, + loop=loop, + keyframes=keyframes, + concepts=luma_concepts.create_api_model() if luma_concepts else None, + ), + auth_token=auth_token, + ) + response_api: LumaGeneration = operation.execute() + + operation = PollingOperation( + poll_endpoint=ApiEndpoint( + path=f"/proxy/luma/generations/{response_api.id}", + method=HttpMethod.GET, + request_model=EmptyRequest, + response_model=LumaGeneration, + ), + completed_statuses=[LumaState.completed], + failed_statuses=[LumaState.failed], + status_extractor=lambda x: x.state, + auth_token=auth_token, + ) + response_poll = operation.execute() + + vid_response = requests.get(response_poll.assets.video) + return (VideoFromFile(BytesIO(vid_response.content)),) + + def _convert_to_keyframes( + self, + first_image: torch.Tensor = None, + last_image: torch.Tensor = None, + auth_token=None, + ): + if first_image is None and last_image is None: + return None + frame0 = None + frame1 = None + if first_image is not None: + download_urls = upload_images_to_comfyapi( + first_image, max_images=1, auth_token=auth_token + ) + frame0 = LumaImageReference(type="image", url=download_urls[0]) + if last_image is not None: + download_urls = upload_images_to_comfyapi( + last_image, max_images=1, auth_token=auth_token + ) + frame1 = LumaImageReference(type="image", url=download_urls[0]) + return LumaKeyframes(frame0=frame0, frame1=frame1) + + +# A dictionary that contains all nodes you want to export with their names +# NOTE: names should be globally unique +NODE_CLASS_MAPPINGS = { + "LumaImageNode": LumaImageGenerationNode, + "LumaImageModifyNode": LumaImageModifyNode, + "LumaVideoNode": LumaTextToVideoGenerationNode, + "LumaImageToVideoNode": LumaImageToVideoGenerationNode, + "LumaReferenceNode": LumaReferenceNode, + "LumaConceptsNode": LumaConceptsNode, +} + +# A dictionary that contains the friendly/humanly readable titles for the nodes +NODE_DISPLAY_NAME_MAPPINGS = { + "LumaImageNode": "Luma Text to Image", + "LumaImageModifyNode": "Luma Image to Image", + "LumaVideoNode": "Luma Text to Video", + "LumaImageToVideoNode": "Luma Image to Video", + "LumaReferenceNode": "Luma Reference", + "LumaConceptsNode": "Luma Concepts", +} diff --git a/nodes.py b/nodes.py index 240f4c248..d5d514469 100644 --- a/nodes.py +++ b/nodes.py @@ -2266,6 +2266,7 @@ def init_builtin_extra_nodes(): "nodes_veo2.py", "nodes_kling.py", "nodes_runway.py", + "nodes_luma.py", ] import_failed = [] From 75611aa449d7fbab906f8443e816dd4a0cd6511d Mon Sep 17 00:00:00 2001 From: Jedrzej Kosinski Date: Wed, 30 Apr 2025 00:37:54 -0500 Subject: [PATCH 037/121] Moved Recraft nodes to nodes_recraft.py (#48) --- comfy_api_nodes/nodes_api.py | 194 --------------------------- comfy_api_nodes/nodes_recraft.py | 217 +++++++++++++++++++++++++++++++ nodes.py | 1 + 3 files changed, 218 insertions(+), 194 deletions(-) create mode 100644 comfy_api_nodes/nodes_recraft.py diff --git a/comfy_api_nodes/nodes_api.py b/comfy_api_nodes/nodes_api.py index 6018a6f6b..f47de0bfb 100644 --- a/comfy_api_nodes/nodes_api.py +++ b/comfy_api_nodes/nodes_api.py @@ -20,16 +20,6 @@ from comfy_api_nodes.apis import ( Model ) from comfy_api_nodes.apis.BFLPolling import BFLStatus -from comfy_api_nodes.apis.recraft_api import ( - RecraftImageGenerationRequest, - RecraftImageGenerationResponse, - RecraftImageSize, - RecraftModel, - RecraftStyle, - RecraftStyleV3, - RecraftIO, - get_v3_substyles, -) from comfy_api_nodes.apis.client import ( ApiClient, ApiEndpoint, @@ -1212,180 +1202,6 @@ class FluxProUltraImageNode(ComfyNodeABC): img.save(img_byte_arr, format="PNG") return base64.b64encode(img_byte_arr.getvalue()).decode() -class RecraftStyleV3RealisticImageNode: - """ - Select realistic_image style and optional substyle. - """ - - RETURN_TYPES = (RecraftIO.STYLEV3,) - RETURN_NAMES = ("recraft_style",) - FUNCTION = "create_style" - CATEGORY = "api node/image/Recraft" - - RECRAFT_STYLE = RecraftStyleV3.realistic_image - - @classmethod - def INPUT_TYPES(s): - return { - "required": { - "substyle": (get_v3_substyles(s.RECRAFT_STYLE),), - } - } - - def create_style(self, substyle: str): - if substyle == "None": - substyle = None - return (RecraftStyle(self.RECRAFT_STYLE, substyle),) - - -class RecraftStyleV3DigitalIllustrationNode(RecraftStyleV3RealisticImageNode): - """ - Select digital_illustration style and optional substyle. - """ - - RECRAFT_STYLE = RecraftStyleV3.digital_illustration - - -class RecraftStyleV3VectorIllustrationNode(RecraftStyleV3RealisticImageNode): - """ - Select vector_illustration style and optional substyle. - """ - - RECRAFT_STYLE = RecraftStyleV3.vector_illustration - - -class RecraftStyleV3LogoRasterNode(RecraftStyleV3RealisticImageNode): - """ - Select vector_illustration style and optional substyle. - """ - - @classmethod - def INPUT_TYPES(s): - return { - "required": { - "substyle": (get_v3_substyles(s.RECRAFT_STYLE, include_none=False),), - } - } - - RECRAFT_STYLE = RecraftStyleV3.logo_raster - - -class RecraftTextToImageNode: - """ - Generates images synchronously based on prompt and resolution. - """ - - RETURN_TYPES = (IO.IMAGE,) - DESCRIPTION = cleandoc(__doc__ or "") # Handle potential None value - FUNCTION = "api_call" - API_NODE = True - CATEGORY = "api node/image/Recraft" - - @classmethod - def INPUT_TYPES(s): - return { - "required": { - "prompt": ( - IO.STRING, - { - "multiline": True, - "default": "", - "tooltip": "Prompt for the image generation.", - }, - ), - "size": ( - [res.value for res in RecraftImageSize], - { - "default": RecraftImageSize.res_1024x1024, - "tooltip": "The size of the generated image.", - }, - ), - "n": ( - IO.INT, - { - "default": 1, - "min": 1, - "max": 6, - "tooltip": "The number of images to generate.", - }, - ), - "seed": ( - IO.INT, - { - "default": 0, - "min": 0, - "max": 0xFFFFFFFFFFFFFFFF, - "control_after_generate": True, - "tooltip": "Seed to determine if node should re-run; actual results are nondeterministic regardless of seed.", - }, - ), - }, - "optional": { - "recraft_style": (RecraftIO.STYLEV3,), - "negative_prompt": ( - IO.STRING, - { - "default": "", - "forceInput": True, - "tooltip": "An optional text description of undesired elements on an image.", - }, - ), - }, - "hidden": { - "auth_token": "AUTH_TOKEN_COMFY_ORG", - }, - } - - def api_call( - self, - prompt: str, - size: str, - n: int, - seed, - recraft_style: RecraftStyle = None, - negative_prompt: str = None, - auth_token=None, - **kwargs, - ): - default_style = RecraftStyle(RecraftStyleV3.digital_illustration) - if recraft_style is None: - recraft_style = default_style - - if not negative_prompt: - negative_prompt = None - - operation = SynchronousOperation( - endpoint=ApiEndpoint( - path="/proxy/recraft/image_generation", - method=HttpMethod.POST, - request_model=RecraftImageGenerationRequest, - response_model=RecraftImageGenerationResponse, - ), - request=RecraftImageGenerationRequest( - prompt=prompt, - negative_prompts=negative_prompt, - model=RecraftModel.recraftv3, - size=size, - n=n, - style=recraft_style.style, - substyle=recraft_style.substyle, - ), - auth_token=auth_token, - ) - response: RecraftImageGenerationResponse = operation.execute() - images = [] - for data in response.data: - image = bytesio_to_image_tensor( - download_url_to_bytesio(data.url, timeout=1024) - ) - if len(image.shape) < 4: - image = image.unsqueeze(0) - images.append(image) - output_image = torch.cat(images, dim=0) - - return (output_image,) - - class MinimaxTextToVideoNode: """ Generates videos synchronously based on a prompt, and optional parameters using Minimax's API. @@ -1533,11 +1349,6 @@ NODE_CLASS_MAPPINGS = { "OpenAIGPTImage1": OpenAIGPTImage1, "IdeogramTextToImage": IdeogramTextToImage, "FluxProUltraImageNode": FluxProUltraImageNode, - "RecraftTextToImageNode": RecraftTextToImageNode, - "RecraftStyleV3RealisticImage": RecraftStyleV3RealisticImageNode, - "RecraftStyleV3DigitalIllustration": RecraftStyleV3DigitalIllustrationNode, - "RecraftStyleV3LogoRaster": RecraftStyleV3LogoRasterNode, - # "RecraftStyleV3VectorIllustration": RecraftStyleV3VectorIllustrationNode, "MinimaxTextToVideoNode": MinimaxTextToVideoNode, } @@ -1548,10 +1359,5 @@ NODE_DISPLAY_NAME_MAPPINGS = { "OpenAIGPTImage1": "OpenAI GPT Image 1", "IdeogramTextToImage": "Ideogram Text to Image", "FluxProUltraImageNode": "Flux 1.1 [pro] Ultra Image", - "RecraftTextToImageNode": "Recraft Text to Image", - "RecraftStyleV3RealisticImage": "Recraft Style - Realistic Image", - "RecraftStyleV3DigitalIllustration": "Recraft Style - Digital Illustration", - "RecraftStyleV3VectorIllustration": "Recraft Style - Vector Illustration", - "RecraftStyleV3LogoRaster": "Recraft Style - Logo Raster", "MinimaxTextToVideoNode": "Minimax Text to Video", } diff --git a/comfy_api_nodes/nodes_recraft.py b/comfy_api_nodes/nodes_recraft.py new file mode 100644 index 000000000..85ebe3eb1 --- /dev/null +++ b/comfy_api_nodes/nodes_recraft.py @@ -0,0 +1,217 @@ +from inspect import cleandoc +from comfy.comfy_types.node_typing import IO +from comfy_api_nodes.apis.recraft_api import ( + RecraftImageGenerationRequest, + RecraftImageGenerationResponse, + RecraftImageSize, + RecraftModel, + RecraftStyle, + RecraftStyleV3, + RecraftIO, + get_v3_substyles, +) +from comfy_api_nodes.apis.client import ( + ApiEndpoint, + HttpMethod, + SynchronousOperation, +) +from comfy_api_nodes.nodes_api import ( + bytesio_to_image_tensor, + download_url_to_bytesio, +) + +import torch + + +class RecraftStyleV3RealisticImageNode: + """ + Select realistic_image style and optional substyle. + """ + + RETURN_TYPES = (RecraftIO.STYLEV3,) + RETURN_NAMES = ("recraft_style",) + FUNCTION = "create_style" + CATEGORY = "api node/image/Recraft" + + RECRAFT_STYLE = RecraftStyleV3.realistic_image + + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "substyle": (get_v3_substyles(s.RECRAFT_STYLE),), + } + } + + def create_style(self, substyle: str): + if substyle == "None": + substyle = None + return (RecraftStyle(self.RECRAFT_STYLE, substyle),) + + +class RecraftStyleV3DigitalIllustrationNode(RecraftStyleV3RealisticImageNode): + """ + Select digital_illustration style and optional substyle. + """ + + RECRAFT_STYLE = RecraftStyleV3.digital_illustration + + +class RecraftStyleV3VectorIllustrationNode(RecraftStyleV3RealisticImageNode): + """ + Select vector_illustration style and optional substyle. + """ + + RECRAFT_STYLE = RecraftStyleV3.vector_illustration + + +class RecraftStyleV3LogoRasterNode(RecraftStyleV3RealisticImageNode): + """ + Select vector_illustration style and optional substyle. + """ + + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "substyle": (get_v3_substyles(s.RECRAFT_STYLE, include_none=False),), + } + } + + RECRAFT_STYLE = RecraftStyleV3.logo_raster + + +class RecraftTextToImageNode: + """ + Generates images synchronously based on prompt and resolution. + """ + + RETURN_TYPES = (IO.IMAGE,) + DESCRIPTION = cleandoc(__doc__ or "") # Handle potential None value + FUNCTION = "api_call" + API_NODE = True + CATEGORY = "api node/image/Recraft" + + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "prompt": ( + IO.STRING, + { + "multiline": True, + "default": "", + "tooltip": "Prompt for the image generation.", + }, + ), + "size": ( + [res.value for res in RecraftImageSize], + { + "default": RecraftImageSize.res_1024x1024, + "tooltip": "The size of the generated image.", + }, + ), + "n": ( + IO.INT, + { + "default": 1, + "min": 1, + "max": 6, + "tooltip": "The number of images to generate.", + }, + ), + "seed": ( + IO.INT, + { + "default": 0, + "min": 0, + "max": 0xFFFFFFFFFFFFFFFF, + "control_after_generate": True, + "tooltip": "Seed to determine if node should re-run; actual results are nondeterministic regardless of seed.", + }, + ), + }, + "optional": { + "recraft_style": (RecraftIO.STYLEV3,), + "negative_prompt": ( + IO.STRING, + { + "default": "", + "forceInput": True, + "tooltip": "An optional text description of undesired elements on an image.", + }, + ), + }, + "hidden": { + "auth_token": "AUTH_TOKEN_COMFY_ORG", + }, + } + + def api_call( + self, + prompt: str, + size: str, + n: int, + seed, + recraft_style: RecraftStyle = None, + negative_prompt: str = None, + auth_token=None, + **kwargs, + ): + default_style = RecraftStyle(RecraftStyleV3.digital_illustration) + if recraft_style is None: + recraft_style = default_style + + if not negative_prompt: + negative_prompt = None + + operation = SynchronousOperation( + endpoint=ApiEndpoint( + path="/proxy/recraft/image_generation", + method=HttpMethod.POST, + request_model=RecraftImageGenerationRequest, + response_model=RecraftImageGenerationResponse, + ), + request=RecraftImageGenerationRequest( + prompt=prompt, + negative_prompts=negative_prompt, + model=RecraftModel.recraftv3, + size=size, + n=n, + style=recraft_style.style, + substyle=recraft_style.substyle, + ), + auth_token=auth_token, + ) + response: RecraftImageGenerationResponse = operation.execute() + images = [] + for data in response.data: + image = bytesio_to_image_tensor( + download_url_to_bytesio(data.url, timeout=1024) + ) + if len(image.shape) < 4: + image = image.unsqueeze(0) + images.append(image) + output_image = torch.cat(images, dim=0) + + return (output_image,) + + +# A dictionary that contains all nodes you want to export with their names +# NOTE: names should be globally unique +NODE_CLASS_MAPPINGS = { + "RecraftTextToImageNode": RecraftTextToImageNode, + "RecraftStyleV3RealisticImage": RecraftStyleV3RealisticImageNode, + "RecraftStyleV3DigitalIllustration": RecraftStyleV3DigitalIllustrationNode, + "RecraftStyleV3LogoRaster": RecraftStyleV3LogoRasterNode, + # "RecraftStyleV3VectorIllustration": RecraftStyleV3VectorIllustrationNode, +} + +# A dictionary that contains the friendly/humanly readable titles for the nodes +NODE_DISPLAY_NAME_MAPPINGS = { + "RecraftTextToImageNode": "Recraft Text to Image", + "RecraftStyleV3RealisticImage": "Recraft Style - Realistic Image", + "RecraftStyleV3DigitalIllustration": "Recraft Style - Digital Illustration", + "RecraftStyleV3VectorIllustration": "Recraft Style - Vector Illustration", + "RecraftStyleV3LogoRaster": "Recraft Style - Logo Raster", +} diff --git a/nodes.py b/nodes.py index d5d514469..ddce24af9 100644 --- a/nodes.py +++ b/nodes.py @@ -2267,6 +2267,7 @@ def init_builtin_extra_nodes(): "nodes_kling.py", "nodes_runway.py", "nodes_luma.py", + "nodes_recraft.py", ] import_failed = [] From f73c7028292178e645b6b30558c9bec4022ac84b Mon Sep 17 00:00:00 2001 From: Jedrzej Kosinski Date: Wed, 30 Apr 2025 01:03:04 -0500 Subject: [PATCH 038/121] Add Pixverse nodes (#46) --- comfy_api_nodes/apis/pixverse_api.py | 93 ++++++++++++++++ comfy_api_nodes/nodes_pixverse.py | 154 +++++++++++++++++++++++++++ nodes.py | 1 + 3 files changed, 248 insertions(+) create mode 100644 comfy_api_nodes/apis/pixverse_api.py create mode 100644 comfy_api_nodes/nodes_pixverse.py diff --git a/comfy_api_nodes/apis/pixverse_api.py b/comfy_api_nodes/apis/pixverse_api.py new file mode 100644 index 000000000..174d50965 --- /dev/null +++ b/comfy_api_nodes/apis/pixverse_api.py @@ -0,0 +1,93 @@ +from __future__ import annotations + +from enum import Enum +from typing import Optional + +from pydantic import BaseModel, Field + + +class PixverseStatus(int, Enum): + successful = 1 + generating = 5 + contents_moderation = 7 + failed = 8 + + +class PixverseAspectRatio(str, Enum): + ratio_16_9 = "16:9" + ratio_4_3 = "4:3" + ratio_1_1 = "1:1" + ratio_3_4 = "3:4" + ratio_9_16 = "9:16" + + +class PixverseQuality(str, Enum): + res_360p = "360p" + res_540p = "540p" + res_720p = "720p" + res_1080p = "1080p" + + +class PixverseDuration(int, Enum): + dur_5 = 5 + dur_8 = 8 + + +class PixverseMotionMode(str, Enum): + normal = "normal" + fast = "fast" + + +class PixverseStyle(str, Enum): + anime = "anime" + animation_3d = "3d_animation" + clay = "clay" + comic = "comic" + cyberpunk = "cyberpunk" + + +# NOTE: forgoing descriptions for now in return for dev speed +class PixverseDto_V2OpenAPIT2VReq(BaseModel): + aspect_ratio: PixverseAspectRatio = Field(...) + quality: PixverseQuality = Field(...) + duration: PixverseDuration = Field(...) + model: Optional[str] = Field("v3.5") + motion_mode: Optional[PixverseMotionMode] = Field(PixverseMotionMode.normal) + prompt: str = Field(...) + negative_prompt: Optional[str] = Field(None) + seed: Optional[int] = Field(None) + style: Optional[str] = Field(None) + template_id: Optional[str] = Field(None) + water_mark: Optional[bool] = Field(None) + + +class PixverseController_ResponseData(BaseModel): + ErrCode: Optional[int] = Field(None) + ErrMsg: Optional[str] = Field(None) + Resp: Optional[PixverseDto_V2OpenAPII2VResp] = Field(None) + + +class PixverseDto_V2OpenAPII2VResp(BaseModel): + video_id: int = Field(..., description='Video_id') + + +class PixverseGenerationStatusResponse(BaseModel): + ErrCode: Optional[int] = Field(None) + ErrMsg: Optional[str] = Field(None) + Resp: Optional[PixverseDto_GetOpenapiMediaDetailResp] = Field(None) + + +class PixverseDto_GetOpenapiMediaDetailResp(BaseModel): + create_time: Optional[str] = Field(None) + id: Optional[int] = Field(None) + modify_time: Optional[str] = Field(None) + negative_prompt: Optional[str] = Field(None) + outputHeight: Optional[int] = Field(None) + outputWidth: Optional[int] = Field(None) + prompt: Optional[str] = Field(None) + resolution_ratio: Optional[int] = Field(None) + seed: Optional[int] = Field(None) + size: Optional[int] = Field(None) + status: Optional[int] = Field(None) + style: Optional[str] = Field(None) + url: Optional[str] = Field(None) diff --git a/comfy_api_nodes/nodes_pixverse.py b/comfy_api_nodes/nodes_pixverse.py new file mode 100644 index 000000000..b90601deb --- /dev/null +++ b/comfy_api_nodes/nodes_pixverse.py @@ -0,0 +1,154 @@ +from inspect import cleandoc + +from comfy_api_nodes.apis.pixverse_api import ( + PixverseDto_V2OpenAPIT2VReq, + PixverseController_ResponseData, + PixverseGenerationStatusResponse, + PixverseAspectRatio, + PixverseQuality, + PixverseDuration, + PixverseMotionMode, + PixverseStatus, +) +from comfy_api_nodes.apis.client import ( + ApiEndpoint, + HttpMethod, + SynchronousOperation, + PollingOperation, + EmptyRequest, +) +from comfy.comfy_types.node_typing import IO, ComfyNodeABC +from comfy_api.input_impl import VideoFromFile + +import requests +from io import BytesIO + + +class PixverseTextToVideoNode(ComfyNodeABC): + """ + Generates videos synchronously based on prompt and output_size. + """ + + RETURN_TYPES = (IO.VIDEO,) + DESCRIPTION = cleandoc(__doc__ or "") # Handle potential None value + FUNCTION = "api_call" + API_NODE = True + CATEGORY = "api node/video/Pixverse" + + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "prompt": ( + IO.STRING, + { + "multiline": True, + "default": "", + "tooltip": "Prompt for the video generation", + }, + ), + "aspect_ratio": ( + [ratio.value for ratio in PixverseAspectRatio], + ), + "quality": ( + [resolution.value for resolution in PixverseQuality], + { + "default": PixverseQuality.res_540p, + }, + ), + "duration_seconds": ([dur.value for dur in PixverseDuration],), + "motion_mode": ([mode.value for mode in PixverseMotionMode],), + "seed": ( + IO.INT, + { + "default": 0, + "min": 0, + "max": 2147483647, + "control_after_generate": True, + "tooltip": "Seed for video generation.", + }, + ), + }, + "optional": { + "negative_prompt": ( + IO.STRING, + { + "default": "", + "forceInput": True, + "tooltip": "An optional text description of undesired elements on an image.", + }, + ), + }, + "hidden": { + "auth_token": "AUTH_TOKEN_COMFY_ORG", + }, + } + + def api_call( + self, + prompt: str, + aspect_ratio: str, + quality: str, + duration_seconds: int, + motion_mode: str, + seed, + negative_prompt: str=None, + auth_token=None, + **kwargs, + ): + # 1080p is limited to 5 seconds duration + # only normal motion_mode supported for 1080p or for non-5 second duration + if quality == PixverseQuality.res_1080p: + motion_mode = PixverseMotionMode.normal + duration_seconds = PixverseDuration.dur_5 + elif duration_seconds != PixverseDuration.dur_5: + motion_mode = PixverseMotionMode.normal + + operation = SynchronousOperation( + endpoint=ApiEndpoint( + path="/proxy/pixverse/video/text/generate", + method=HttpMethod.POST, + request_model=PixverseDto_V2OpenAPIT2VReq, + response_model=PixverseController_ResponseData, + ), + request=PixverseDto_V2OpenAPIT2VReq( + prompt=prompt, + aspect_ratio=aspect_ratio, + quality=quality, + duration=duration_seconds, + motion_mode=motion_mode, + negative_prompt=negative_prompt if negative_prompt else None, + seed=seed, + ), + auth_token=auth_token, + ) + response_api = operation.execute() + + if response_api.Resp is None: + raise Exception(f"Pixverse request failed: '{response_api.ErrMsg}'") + + operation = PollingOperation( + poll_endpoint=ApiEndpoint( + path=f"/proxy/pixverse/video/result/{response_api.Resp.video_id}", + method=HttpMethod.GET, + request_model=EmptyRequest, + response_model=PixverseGenerationStatusResponse, + ), + completed_statuses=[PixverseStatus.successful], + failed_statuses=[PixverseStatus.contents_moderation, PixverseStatus.failed], + status_extractor=lambda x: x.Resp.status, + auth_token=auth_token, + ) + response_poll = operation.execute() + + vid_response = requests.get(response_poll.Resp.url) + return (VideoFromFile(BytesIO(vid_response.content)),) + + +NODE_CLASS_MAPPINGS = { + "PixverseTextToVideoNode": PixverseTextToVideoNode +} + +NODE_DISPLAY_NAME_MAPPINGS = { + "PixverseTextToVideoNode": "Pixverse Text to Video" +} diff --git a/nodes.py b/nodes.py index ddce24af9..46991590e 100644 --- a/nodes.py +++ b/nodes.py @@ -2268,6 +2268,7 @@ def init_builtin_extra_nodes(): "nodes_runway.py", "nodes_luma.py", "nodes_recraft.py", + "nodes_pixverse.py", ] import_failed = [] From b121fac8a4c6837ad6b55bf96fe1360b29e4314b Mon Sep 17 00:00:00 2001 From: Jedrzej Kosinski Date: Wed, 30 Apr 2025 01:22:32 -0500 Subject: [PATCH 039/121] Move and fix BFL nodes to node_bfl.py (#49) --- comfy_api_nodes/apis/BFLPolling.py | 29 ---- comfy_api_nodes/apis/bfl_api.py | 59 +++++++ comfy_api_nodes/nodes_api.py | 212 ------------------------- comfy_api_nodes/nodes_bfl.py | 243 +++++++++++++++++++++++++++++ nodes.py | 1 + 5 files changed, 303 insertions(+), 241 deletions(-) delete mode 100644 comfy_api_nodes/apis/BFLPolling.py create mode 100644 comfy_api_nodes/apis/bfl_api.py create mode 100644 comfy_api_nodes/nodes_bfl.py diff --git a/comfy_api_nodes/apis/BFLPolling.py b/comfy_api_nodes/apis/BFLPolling.py deleted file mode 100644 index e40bf3344..000000000 --- a/comfy_api_nodes/apis/BFLPolling.py +++ /dev/null @@ -1,29 +0,0 @@ -from __future__ import annotations - -from enum import Enum -from typing import Any, Dict, Optional - -from pydantic import BaseModel, Field, confloat - - -class BFLStatus(str, Enum): - task_not_found = "Task not found" - pending = "Pending" - request_moderated = "Request Moderated" - content_moderated = "Content Moderated" - ready = "Ready" - error = "Error" - - -class BFLFluxProStatusResponse(BaseModel): - id: str = Field(..., description="The unique identifier for the generation task.") - status: BFLStatus = Field(..., description="The status of the task.") - result: Optional[Dict[str, Any]] = Field( - None, description="The result of the task (null if not completed)." - ) - progress: confloat(ge=0.0, le=1.0) = Field( - ..., description="The progress of the task (0.0 to 1.0)." - ) - details: Optional[Dict[str, Any]] = Field( - None, description="Additional details about the task (null if not available)." - ) diff --git a/comfy_api_nodes/apis/bfl_api.py b/comfy_api_nodes/apis/bfl_api.py new file mode 100644 index 000000000..722f75eb3 --- /dev/null +++ b/comfy_api_nodes/apis/bfl_api.py @@ -0,0 +1,59 @@ +from __future__ import annotations + +from enum import Enum +from typing import Any, Dict, Optional + +from pydantic import BaseModel, Field, confloat, conint + + +class BFLOutputFormat(str, Enum): + png = 'png' + jpeg = 'jpeg' + + +class BFLFluxProGenerateRequest(BaseModel): + prompt: str = Field(..., description='The text prompt for image generation.') + prompt_upsampling: Optional[bool] = Field( + None, description='Whether to perform upsampling on the prompt. If active, automatically modifies the prompt for more creative generation.' + ) + seed: Optional[int] = Field(None, description='The seed value for reproducibility.') + aspect_ratio: Optional[str] = Field(None, description='Aspect ratio of the image between 21:9 and 9:21.') + safety_tolerance: Optional[conint(ge=0, le=6)] = Field( + 6, description='Tolerance level for input and output moderation. Between 0 and 6, 0 being most strict, 6 being least strict. Defaults to 2.' + ) + output_format: Optional[BFLOutputFormat] = Field( + BFLOutputFormat.png, description="Output format for the generated image. Can be 'jpeg' or 'png'.", examples=['png'] + ) + raw: Optional[bool] = Field(None, description='Generate less processed, more natural-looking images.') + image_prompt: Optional[str] = Field(None, description='Optional image to remix in base64 format') + image_prompt_strength: Optional[confloat(ge=0.0, le=1.0)] = Field( + None, description='Blend between the prompt and the image prompt.' + ) + + +class BFLFluxProGenerateResponse(BaseModel): + id: str = Field(..., description='The unique identifier for the generation task.') + polling_url: str = Field(..., description='URL to poll for the generation result.') + + +class BFLStatus(str, Enum): + task_not_found = "Task not found" + pending = "Pending" + request_moderated = "Request Moderated" + content_moderated = "Content Moderated" + ready = "Ready" + error = "Error" + + +class BFLFluxProStatusResponse(BaseModel): + id: str = Field(..., description="The unique identifier for the generation task.") + status: BFLStatus = Field(..., description="The status of the task.") + result: Optional[Dict[str, Any]] = Field( + None, description="The result of the task (null if not completed)." + ) + progress: confloat(ge=0.0, le=1.0) = Field( + ..., description="The progress of the task (0.0 to 1.0)." + ) + details: Optional[Dict[str, Any]] = Field( + None, description="Additional details about the task (null if not available)." + ) diff --git a/comfy_api_nodes/nodes_api.py b/comfy_api_nodes/nodes_api.py index f47de0bfb..d1055059e 100644 --- a/comfy_api_nodes/nodes_api.py +++ b/comfy_api_nodes/nodes_api.py @@ -14,12 +14,9 @@ from comfy_api_nodes.apis import ( MinimaxTaskResultResponse, IdeogramGenerateRequest, IdeogramGenerateResponse, - BFLFluxProGenerateRequest, - BFLFluxProGenerateResponse, ImageRequest, Model ) -from comfy_api_nodes.apis.BFLPolling import BFLStatus from comfy_api_nodes.apis.client import ( ApiClient, ApiEndpoint, @@ -38,7 +35,6 @@ import torch import math import base64 import logging -import time import uuid import folder_paths from io import BytesIO @@ -996,212 +992,6 @@ class IdeogramTextToImage(ComfyNodeABC): # def IS_CHANGED(s, image, string_field, int_field, float_field, print_to_screen): # return "" - -class FluxProUltraImageNode(ComfyNodeABC): - """ - Generates images synchronously based on prompt and resolution. - """ - - MINIMUM_RATIO = 1 / 4 - MAXIMUM_RATIO = 4 / 1 - MINIMUM_RATIO_STR = "1:4" - MAXIMUM_RATIO_STR = "4:1" - - @classmethod - def INPUT_TYPES(s): - return { - "required": { - "prompt": ( - IO.STRING, - { - "multiline": True, - "default": "", - "tooltip": "Prompt for the image generation", - }, - ), - "prompt_upsampling": ( - IO.BOOLEAN, - { - "default": False, - "tooltip": "Whether to perform upsampling on the prompt. If active, automatically modifies the prompt for more creative generation, but results are nondeterministic (same seed will not produce exactly the same result).", - }, - ), - "seed": ( - IO.INT, - { - "default": 0, - "min": 0, - "max": 0xFFFFFFFFFFFFFFFF, - "control_after_generate": True, - "tooltip": "The random seed used for creating the noise.", - }, - ), - "aspect_ratio": ( - IO.STRING, - { - "default": "16:9", - "tooltip": "Aspect ratio of image; must be between 1:4 and 4:1.", - }, - ), - "raw": ( - IO.BOOLEAN, - { - "default": False, - "tooltip": "When True, generate less processed, more natural-looking images.", - }, - ), - }, - "optional": { - "image_prompt": (IO.IMAGE,), - "image_prompt_strength": ( - IO.FLOAT, - { - "default": 0.1, - "min": 0.0, - "max": 1.0, - "step": 0.01, - "tooltip": "Blend between the prompt and the image prompt.", - }, - ), - }, - "hidden": { - "auth_token": "AUTH_TOKEN_COMFY_ORG", - }, - } - - @classmethod - def VALIDATE_INPUTS(cls, aspect_ratio: str): - try: - validate_aspect_ratio( - aspect_ratio, - minimum_ratio=cls.MINIMUM_RATIO, - maximum_ratio=cls.MAXIMUM_RATIO, - minimum_ratio_str=cls.MINIMUM_RATIO_STR, - maximum_ratio_str=cls.MAXIMUM_RATIO_STR, - ) - except Exception as e: - return str(e) - return True - - RETURN_TYPES = (IO.IMAGE,) - DESCRIPTION = cleandoc(__doc__ or "") # Handle potential None value - FUNCTION = "api_call" - API_NODE = True - CATEGORY = "api node/image/bfl" - - def api_call( - self, - prompt: str, - aspect_ratio: str, - prompt_upsampling=False, - raw=False, - seed=0, - image_prompt=None, - image_prompt_strength=0.1, - auth_token=None, - **kwargs, - ): - operation = SynchronousOperation( - endpoint=ApiEndpoint( - path="/proxy/bfl/flux-pro-1.1-ultra/generate", - method=HttpMethod.POST, - request_model=BFLFluxProGenerateRequest, - response_model=BFLFluxProGenerateResponse, - ), - request=BFLFluxProGenerateRequest( - prompt=prompt, - prompt_upsampling=prompt_upsampling, - seed=seed, - aspect_ratio=validate_aspect_ratio( - aspect_ratio, - minimum_ratio=self.MINIMUM_RATIO, - maximum_ratio=self.MAXIMUM_RATIO, - minimum_ratio_str=self.MINIMUM_RATIO_STR, - maximum_ratio_str=self.MAXIMUM_RATIO_STR, - ), - raw=raw, - image_prompt=( - image_prompt - if image_prompt is None - else self._convert_image_to_base64(image_prompt) - ), - image_prompt_strength=( - None if image_prompt is None else round(image_prompt_strength, 2) - ), - ), - auth_token=auth_token, - ) - output_image = self._handle_bfl_synchronous_operation(operation) - return (output_image,) - - def _handle_bfl_synchronous_operation( - self, operation: SynchronousOperation, timeout_bfl_calls=360 - ): - response_api: BFLFluxProGenerateResponse = operation.execute() - return self._poll_until_generated( - response_api.polling_url, timeout=timeout_bfl_calls - ) - - def _poll_until_generated(self, polling_url: str, timeout=360): - # used bfl-comfy-nodes to verify code implementation: - # https://github.com/black-forest-labs/bfl-comfy-nodes/tree/main - start_time = time.time() - retries_404 = 0 - max_retries_404 = 5 - retry_404_seconds = 2 - retry_202_seconds = 2 - retry_pending_seconds = 1 - request = requests.Request(method=HttpMethod.GET, url=polling_url) - # NOTE: should True loop be replaced with checking if workflow has been interrupted? - while True: - response = requests.Session().send(request.prepare()) - if response.status_code == 200: - result = response.json() - if result["status"] == BFLStatus.ready: - img_url = result["result"]["sample"] - img_response = requests.get(img_url) - return process_image_response(img_response) - elif result["status"] in [ - BFLStatus.request_moderated, - BFLStatus.content_moderated, - ]: - status = result["status"] - raise Exception( - f"BFL API did not return an image due to: {status}." - ) - elif result["status"] == BFLStatus.error: - raise Exception(f"BFL API encountered an error: {result}.") - elif result["status"] == BFLStatus.pending: - time.sleep(retry_pending_seconds) - continue - elif response.status_code == 404: - if retries_404 < max_retries_404: - retries_404 += 1 - time.sleep(retry_404_seconds) - continue - raise Exception( - f"BFL API could not find task after {max_retries_404} tries." - ) - elif response.status_code == 202: - time.sleep(retry_202_seconds) - elif time.time() - start_time > timeout: - raise Exception( - f"BFL API experienced a timeout; could not return request under {timeout} seconds." - ) - else: - raise Exception(f"BFL API encountered an error: {response.json()}") - - def _convert_image_to_base64(self, image: torch.Tensor): - scaled_image = downscale_input(image, total_pixels=2048 * 2048) - # remove batch dimension if present - if len(scaled_image.shape) > 3: - scaled_image = scaled_image[0] - image_np = (scaled_image.numpy() * 255).astype(np.uint8) - img = Image.fromarray(image_np) - img_byte_arr = io.BytesIO() - img.save(img_byte_arr, format="PNG") - return base64.b64encode(img_byte_arr.getvalue()).decode() - class MinimaxTextToVideoNode: """ Generates videos synchronously based on a prompt, and optional parameters using Minimax's API. @@ -1348,7 +1138,6 @@ NODE_CLASS_MAPPINGS = { "OpenAIDalle3": OpenAIDalle3, "OpenAIGPTImage1": OpenAIGPTImage1, "IdeogramTextToImage": IdeogramTextToImage, - "FluxProUltraImageNode": FluxProUltraImageNode, "MinimaxTextToVideoNode": MinimaxTextToVideoNode, } @@ -1358,6 +1147,5 @@ NODE_DISPLAY_NAME_MAPPINGS = { "OpenAIDalle3": "OpenAI DALL·E 3", "OpenAIGPTImage1": "OpenAI GPT Image 1", "IdeogramTextToImage": "Ideogram Text to Image", - "FluxProUltraImageNode": "Flux 1.1 [pro] Ultra Image", "MinimaxTextToVideoNode": "Minimax Text to Video", } diff --git a/comfy_api_nodes/nodes_bfl.py b/comfy_api_nodes/nodes_bfl.py new file mode 100644 index 000000000..66b5dea79 --- /dev/null +++ b/comfy_api_nodes/nodes_bfl.py @@ -0,0 +1,243 @@ +import io +from inspect import cleandoc +from comfy.comfy_types.node_typing import IO, ComfyNodeABC +from comfy_api_nodes.apis.bfl_api import ( + BFLStatus, + BFLFluxProGenerateRequest, + BFLFluxProGenerateResponse, +) +from comfy_api_nodes.apis.client import ( + ApiEndpoint, + HttpMethod, + SynchronousOperation, +) +from comfy_api_nodes.nodes_api import ( + downscale_input, + validate_aspect_ratio, + process_image_response, +) + +import numpy as np +from PIL import Image +import requests +import torch +import base64 +import time + + +class FluxProUltraImageNode(ComfyNodeABC): + """ + Generates images synchronously based on prompt and resolution. + """ + + MINIMUM_RATIO = 1 / 4 + MAXIMUM_RATIO = 4 / 1 + MINIMUM_RATIO_STR = "1:4" + MAXIMUM_RATIO_STR = "4:1" + + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "prompt": ( + IO.STRING, + { + "multiline": True, + "default": "", + "tooltip": "Prompt for the image generation", + }, + ), + "prompt_upsampling": ( + IO.BOOLEAN, + { + "default": False, + "tooltip": "Whether to perform upsampling on the prompt. If active, automatically modifies the prompt for more creative generation, but results are nondeterministic (same seed will not produce exactly the same result).", + }, + ), + "seed": ( + IO.INT, + { + "default": 0, + "min": 0, + "max": 0xFFFFFFFFFFFFFFFF, + "control_after_generate": True, + "tooltip": "The random seed used for creating the noise.", + }, + ), + "aspect_ratio": ( + IO.STRING, + { + "default": "16:9", + "tooltip": "Aspect ratio of image; must be between 1:4 and 4:1.", + }, + ), + "raw": ( + IO.BOOLEAN, + { + "default": False, + "tooltip": "When True, generate less processed, more natural-looking images.", + }, + ), + }, + "optional": { + "image_prompt": (IO.IMAGE,), + "image_prompt_strength": ( + IO.FLOAT, + { + "default": 0.1, + "min": 0.0, + "max": 1.0, + "step": 0.01, + "tooltip": "Blend between the prompt and the image prompt.", + }, + ), + }, + "hidden": { + "auth_token": "AUTH_TOKEN_COMFY_ORG", + }, + } + + @classmethod + def VALIDATE_INPUTS(cls, aspect_ratio: str): + try: + validate_aspect_ratio( + aspect_ratio, + minimum_ratio=cls.MINIMUM_RATIO, + maximum_ratio=cls.MAXIMUM_RATIO, + minimum_ratio_str=cls.MINIMUM_RATIO_STR, + maximum_ratio_str=cls.MAXIMUM_RATIO_STR, + ) + except Exception as e: + return str(e) + return True + + RETURN_TYPES = (IO.IMAGE,) + DESCRIPTION = cleandoc(__doc__ or "") # Handle potential None value + FUNCTION = "api_call" + API_NODE = True + CATEGORY = "api node/image/bfl" + + def api_call( + self, + prompt: str, + aspect_ratio: str, + prompt_upsampling=False, + raw=False, + seed=0, + image_prompt=None, + image_prompt_strength=0.1, + auth_token=None, + **kwargs, + ): + operation = SynchronousOperation( + endpoint=ApiEndpoint( + path="/proxy/bfl/flux-pro-1.1-ultra/generate", + method=HttpMethod.POST, + request_model=BFLFluxProGenerateRequest, + response_model=BFLFluxProGenerateResponse, + ), + request=BFLFluxProGenerateRequest( + prompt=prompt, + prompt_upsampling=prompt_upsampling, + seed=seed, + aspect_ratio=validate_aspect_ratio( + aspect_ratio, + minimum_ratio=self.MINIMUM_RATIO, + maximum_ratio=self.MAXIMUM_RATIO, + minimum_ratio_str=self.MINIMUM_RATIO_STR, + maximum_ratio_str=self.MAXIMUM_RATIO_STR, + ), + raw=raw, + image_prompt=( + image_prompt + if image_prompt is None + else self._convert_image_to_base64(image_prompt) + ), + image_prompt_strength=( + None if image_prompt is None else round(image_prompt_strength, 2) + ), + ), + auth_token=auth_token, + ) + output_image = self._handle_bfl_synchronous_operation(operation) + return (output_image,) + + def _handle_bfl_synchronous_operation( + self, operation: SynchronousOperation, timeout_bfl_calls=360 + ): + response_api: BFLFluxProGenerateResponse = operation.execute() + return self._poll_until_generated( + response_api.polling_url, timeout=timeout_bfl_calls + ) + + def _poll_until_generated(self, polling_url: str, timeout=360): + # used bfl-comfy-nodes to verify code implementation: + # https://github.com/black-forest-labs/bfl-comfy-nodes/tree/main + start_time = time.time() + retries_404 = 0 + max_retries_404 = 5 + retry_404_seconds = 2 + retry_202_seconds = 2 + retry_pending_seconds = 1 + request = requests.Request(method=HttpMethod.GET, url=polling_url) + # NOTE: should True loop be replaced with checking if workflow has been interrupted? + while True: + response = requests.Session().send(request.prepare()) + if response.status_code == 200: + result = response.json() + if result["status"] == BFLStatus.ready: + img_url = result["result"]["sample"] + img_response = requests.get(img_url) + return process_image_response(img_response) + elif result["status"] in [ + BFLStatus.request_moderated, + BFLStatus.content_moderated, + ]: + status = result["status"] + raise Exception( + f"BFL API did not return an image due to: {status}." + ) + elif result["status"] == BFLStatus.error: + raise Exception(f"BFL API encountered an error: {result}.") + elif result["status"] == BFLStatus.pending: + time.sleep(retry_pending_seconds) + continue + elif response.status_code == 404: + if retries_404 < max_retries_404: + retries_404 += 1 + time.sleep(retry_404_seconds) + continue + raise Exception( + f"BFL API could not find task after {max_retries_404} tries." + ) + elif response.status_code == 202: + time.sleep(retry_202_seconds) + elif time.time() - start_time > timeout: + raise Exception( + f"BFL API experienced a timeout; could not return request under {timeout} seconds." + ) + else: + raise Exception(f"BFL API encountered an error: {response.json()}") + + def _convert_image_to_base64(self, image: torch.Tensor): + scaled_image = downscale_input(image, total_pixels=2048 * 2048) + # remove batch dimension if present + if len(scaled_image.shape) > 3: + scaled_image = scaled_image[0] + image_np = (scaled_image.numpy() * 255).astype(np.uint8) + img = Image.fromarray(image_np) + img_byte_arr = io.BytesIO() + img.save(img_byte_arr, format="PNG") + return base64.b64encode(img_byte_arr.getvalue()).decode() + + +# A dictionary that contains all nodes you want to export with their names +# NOTE: names should be globally unique +NODE_CLASS_MAPPINGS = { + "FluxProUltraImageNode": FluxProUltraImageNode, +} + +# A dictionary that contains the friendly/humanly readable titles for the nodes +NODE_DISPLAY_NAME_MAPPINGS = { + "FluxProUltraImageNode": "Flux 1.1 [pro] Ultra Image", +} diff --git a/nodes.py b/nodes.py index 46991590e..c93b459f2 100644 --- a/nodes.py +++ b/nodes.py @@ -2266,6 +2266,7 @@ def init_builtin_extra_nodes(): "nodes_veo2.py", "nodes_kling.py", "nodes_runway.py", + "nodes_bfl.py", "nodes_luma.py", "nodes_recraft.py", "nodes_pixverse.py", From 31a33a53871689278fcf60c4de965961dc67120f Mon Sep 17 00:00:00 2001 From: Jedrzej Kosinski Date: Wed, 30 Apr 2025 01:51:57 -0500 Subject: [PATCH 040/121] Move and edit Minimax node to nodes_minimax.py (#50) --- comfy_api_nodes/nodes_api.py | 151 +-------------------------- comfy_api_nodes/nodes_minimax.py | 170 +++++++++++++++++++++++++++++++ nodes.py | 1 + 3 files changed, 172 insertions(+), 150 deletions(-) create mode 100644 comfy_api_nodes/nodes_minimax.py diff --git a/comfy_api_nodes/nodes_api.py b/comfy_api_nodes/nodes_api.py index d1055059e..645c19016 100644 --- a/comfy_api_nodes/nodes_api.py +++ b/comfy_api_nodes/nodes_api.py @@ -1,29 +1,21 @@ import io from inspect import cleandoc -from typing import Literal, Optional +from typing import Optional from comfy.utils import common_upscale from comfy.comfy_types.node_typing import IO, ComfyNodeABC, InputTypeDict -from comfy_api.input_impl.video_types import VideoFromFile from comfy_api_nodes.apis import ( OpenAIImageGenerationRequest, OpenAIImageEditRequest, OpenAIImageGenerationResponse, - MinimaxVideoGenerationRequest, - MinimaxVideoGenerationResponse, - MinimaxFileRetrieveResponse, - MinimaxTaskResultResponse, IdeogramGenerateRequest, IdeogramGenerateResponse, ImageRequest, - Model ) from comfy_api_nodes.apis.client import ( ApiClient, ApiEndpoint, HttpMethod, SynchronousOperation, - PollingOperation, - EmptyRequest, UploadRequest, UploadResponse, ) @@ -34,7 +26,6 @@ import requests import torch import math import base64 -import logging import uuid import folder_paths from io import BytesIO @@ -992,144 +983,6 @@ class IdeogramTextToImage(ComfyNodeABC): # def IS_CHANGED(s, image, string_field, int_field, float_field, print_to_screen): # return "" -class MinimaxTextToVideoNode: - """ - Generates videos synchronously based on a prompt, and optional parameters using Minimax's API. - """ - - def __init__(self): - self.output_dir = folder_paths.get_output_directory() - self.type: Literal["output"] = "output" - - @classmethod - def INPUT_TYPES(s): - return { - "required": { - "prompt_text": ( - "STRING", - { - "multiline": True, - "default": "", - "tooltip": "Text prompt to guide the video generation", - }, - ), - "filename_prefix": ("STRING", {"default": "ComfyUI"}), - "model": ( - [ - "T2V-01", - "I2V-01-Director", - "S2V-01", - "I2V-01", - "I2V-01-live", - ], - { - "default": "T2V-01", - "tooltip": "Model to use for video generation", - }, - ), - }, - "optional": { - "seed": ( - IO.INT, - { - "default": 0, - "min": 0, - "max": 0xFFFFFFFFFFFFFFFF, - "control_after_generate": True, - "tooltip": "The random seed used for creating the noise.", - }, - ), - }, - "hidden": { - "prompt": "PROMPT", - "extra_pnginfo": "EXTRA_PNGINFO", - "auth_token": "AUTH_TOKEN_COMFY_ORG", - }, - } - - RETURN_TYPES = ("VIDEO",) - DESCRIPTION = "Generates videos from prompts using Minimax's API" - FUNCTION = "generate_video" - CATEGORY = "api node/video/Minimax" - API_NODE = True - OUTPUT_NODE = True - - def generate_video( - self, - prompt_text, - filename_prefix, - seed=0, - model="T2V-01", - prompt=None, - extra_pnginfo=None, - auth_token=None, - ): - video_generate_operation = SynchronousOperation( - endpoint=ApiEndpoint( - path="/proxy/minimax/video_generation", - method=HttpMethod.POST, - request_model=MinimaxVideoGenerationRequest, - response_model=MinimaxVideoGenerationResponse, - ), - request=MinimaxVideoGenerationRequest( - model=Model(model), - prompt=prompt_text, - callback_url=None, - first_frame_image=None, - subject_reference=None, - prompt_optimizer=None, - ), - auth_token=auth_token, - ) - response = video_generate_operation.execute() - - task_id = response.task_id - - video_generate_operation = PollingOperation( - poll_endpoint=ApiEndpoint( - path="/proxy/minimax/query/video_generation", - method=HttpMethod.GET, - request_model=EmptyRequest, - response_model=MinimaxTaskResultResponse, - query_params={"task_id": task_id}, - ), - completed_statuses=["Success"], - failed_statuses=["Fail"], - status_extractor=lambda x: x.status.value, - auth_token=auth_token, - ) - task_result = video_generate_operation.execute() - - file_id = task_result.file_id - if file_id is None: - raise Exception("Request was not successful. Missing file ID.") - file_retrieve_operation = SynchronousOperation( - endpoint=ApiEndpoint( - path="/proxy/minimax/files/retrieve", - method=HttpMethod.GET, - request_model=EmptyRequest, - response_model=MinimaxFileRetrieveResponse, - query_params={"file_id": int(file_id)}, - ), - request=EmptyRequest(), - auth_token=auth_token, - ) - file_result = file_retrieve_operation.execute() - - file_url = file_result.file.download_url - if file_url is None: - raise Exception( - f"No video was found in the response. Full response: {file_result.model_dump()}" - ) - logging.info(f"Generated video URL: {file_url}") - - video_io = download_url_to_bytesio(file_url) - if video_io is None: - error_msg = f"Failed to download video from {file_url}" - logging.error(error_msg) - raise Exception(error_msg) - return (VideoFromFile(video_io),) - # A dictionary that contains all nodes you want to export with their names # NOTE: names should be globally unique @@ -1138,7 +991,6 @@ NODE_CLASS_MAPPINGS = { "OpenAIDalle3": OpenAIDalle3, "OpenAIGPTImage1": OpenAIGPTImage1, "IdeogramTextToImage": IdeogramTextToImage, - "MinimaxTextToVideoNode": MinimaxTextToVideoNode, } # A dictionary that contains the friendly/humanly readable titles for the nodes @@ -1147,5 +999,4 @@ NODE_DISPLAY_NAME_MAPPINGS = { "OpenAIDalle3": "OpenAI DALL·E 3", "OpenAIGPTImage1": "OpenAI GPT Image 1", "IdeogramTextToImage": "Ideogram Text to Image", - "MinimaxTextToVideoNode": "Minimax Text to Video", } diff --git a/comfy_api_nodes/nodes_minimax.py b/comfy_api_nodes/nodes_minimax.py new file mode 100644 index 000000000..4c259f54f --- /dev/null +++ b/comfy_api_nodes/nodes_minimax.py @@ -0,0 +1,170 @@ +from typing import Literal +from comfy.comfy_types.node_typing import IO +from comfy_api.input_impl.video_types import VideoFromFile +from comfy_api_nodes.apis import ( + MinimaxVideoGenerationRequest, + MinimaxVideoGenerationResponse, + MinimaxFileRetrieveResponse, + MinimaxTaskResultResponse, + Model +) +from comfy_api_nodes.apis.client import ( + ApiEndpoint, + HttpMethod, + SynchronousOperation, + PollingOperation, + EmptyRequest, +) +from comfy_api_nodes.nodes_api import ( + download_url_to_bytesio, +) + +import logging +import folder_paths + + +class MinimaxTextToVideoNode: + """ + Generates videos synchronously based on a prompt, and optional parameters using Minimax's API. + """ + + def __init__(self): + self.output_dir = folder_paths.get_output_directory() + self.type: Literal["output"] = "output" + + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "prompt_text": ( + "STRING", + { + "multiline": True, + "default": "", + "tooltip": "Text prompt to guide the video generation", + }, + ), + "model": ( + [ + "T2V-01", + "I2V-01-Director", + "S2V-01", + "I2V-01", + "I2V-01-live", + ], + { + "default": "T2V-01", + "tooltip": "Model to use for video generation", + }, + ), + }, + "optional": { + "seed": ( + IO.INT, + { + "default": 0, + "min": 0, + "max": 0xFFFFFFFFFFFFFFFF, + "control_after_generate": True, + "tooltip": "The random seed used for creating the noise.", + }, + ), + }, + "hidden": { + "auth_token": "AUTH_TOKEN_COMFY_ORG", + }, + } + + RETURN_TYPES = ("VIDEO",) + DESCRIPTION = "Generates videos from prompts using Minimax's API" + FUNCTION = "generate_video" + CATEGORY = "api node/video/Minimax" + API_NODE = True + OUTPUT_NODE = True + + def generate_video( + self, + prompt_text, + seed=0, + model="T2V-01", + auth_token=None, + ): + video_generate_operation = SynchronousOperation( + endpoint=ApiEndpoint( + path="/proxy/minimax/video_generation", + method=HttpMethod.POST, + request_model=MinimaxVideoGenerationRequest, + response_model=MinimaxVideoGenerationResponse, + ), + request=MinimaxVideoGenerationRequest( + model=Model(model), + prompt=prompt_text, + callback_url=None, + first_frame_image=None, + subject_reference=None, + prompt_optimizer=None, + ), + auth_token=auth_token, + ) + response = video_generate_operation.execute() + + task_id = response.task_id + if not task_id: + raise Exception(f"Minimax generation failed: {response.base_resp}") + + video_generate_operation = PollingOperation( + poll_endpoint=ApiEndpoint( + path="/proxy/minimax/query/video_generation", + method=HttpMethod.GET, + request_model=EmptyRequest, + response_model=MinimaxTaskResultResponse, + query_params={"task_id": task_id}, + ), + completed_statuses=["Success"], + failed_statuses=["Fail"], + status_extractor=lambda x: x.status.value, + auth_token=auth_token, + ) + task_result = video_generate_operation.execute() + + file_id = task_result.file_id + if file_id is None: + raise Exception("Request was not successful. Missing file ID.") + file_retrieve_operation = SynchronousOperation( + endpoint=ApiEndpoint( + path="/proxy/minimax/files/retrieve", + method=HttpMethod.GET, + request_model=EmptyRequest, + response_model=MinimaxFileRetrieveResponse, + query_params={"file_id": int(file_id)}, + ), + request=EmptyRequest(), + auth_token=auth_token, + ) + file_result = file_retrieve_operation.execute() + + file_url = file_result.file.download_url + if file_url is None: + raise Exception( + f"No video was found in the response. Full response: {file_result.model_dump()}" + ) + logging.info(f"Generated video URL: {file_url}") + + video_io = download_url_to_bytesio(file_url) + if video_io is None: + error_msg = f"Failed to download video from {file_url}" + logging.error(error_msg) + raise Exception(error_msg) + return (VideoFromFile(video_io),) + + +# A dictionary that contains all nodes you want to export with their names +# NOTE: names should be globally unique +NODE_CLASS_MAPPINGS = { + "MinimaxTextToVideoNode": MinimaxTextToVideoNode, +} + +# A dictionary that contains the friendly/humanly readable titles for the nodes +NODE_DISPLAY_NAME_MAPPINGS = { + "MinimaxTextToVideoNode": "Minimax Text to Video", +} diff --git a/nodes.py b/nodes.py index c93b459f2..61c872c5f 100644 --- a/nodes.py +++ b/nodes.py @@ -2263,6 +2263,7 @@ def init_builtin_extra_nodes(): api_nodes_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), "comfy_api_nodes") api_nodes_files = [ "nodes_api.py", + "nodes_minimax.py", "nodes_veo2.py", "nodes_kling.py", "nodes_runway.py", From a90dd38634d72f7061c2850b1d42383a5b6e0bc6 Mon Sep 17 00:00:00 2001 From: Jedrzej Kosinski Date: Wed, 30 Apr 2025 02:59:27 -0500 Subject: [PATCH 041/121] Add Minimax Image to Video node + Cleanup (#51) --- comfy_api_nodes/nodes_minimax.py | 157 ++++++++++++++++++++++++++++--- 1 file changed, 145 insertions(+), 12 deletions(-) diff --git a/comfy_api_nodes/nodes_minimax.py b/comfy_api_nodes/nodes_minimax.py index 4c259f54f..33930e88b 100644 --- a/comfy_api_nodes/nodes_minimax.py +++ b/comfy_api_nodes/nodes_minimax.py @@ -1,4 +1,3 @@ -from typing import Literal from comfy.comfy_types.node_typing import IO from comfy_api.input_impl.video_types import VideoFromFile from comfy_api_nodes.apis import ( @@ -6,6 +5,7 @@ from comfy_api_nodes.apis import ( MinimaxVideoGenerationResponse, MinimaxFileRetrieveResponse, MinimaxTaskResultResponse, + SubjectReferenceItem, Model ) from comfy_api_nodes.apis.client import ( @@ -17,10 +17,11 @@ from comfy_api_nodes.apis.client import ( ) from comfy_api_nodes.nodes_api import ( download_url_to_bytesio, + upload_images_to_comfyapi, ) +import torch import logging -import folder_paths class MinimaxTextToVideoNode: @@ -28,10 +29,6 @@ class MinimaxTextToVideoNode: Generates videos synchronously based on a prompt, and optional parameters using Minimax's API. """ - def __init__(self): - self.output_dir = folder_paths.get_output_directory() - self.type: Literal["output"] = "output" - @classmethod def INPUT_TYPES(s): return { @@ -47,10 +44,7 @@ class MinimaxTextToVideoNode: "model": ( [ "T2V-01", - "I2V-01-Director", - "S2V-01", - "I2V-01", - "I2V-01-live", + "T2V-01-Director", ], { "default": "T2V-01", @@ -87,8 +81,25 @@ class MinimaxTextToVideoNode: prompt_text, seed=0, model="T2V-01", + image: torch.Tensor=None, # used for ImageToVideo + subject: torch.Tensor=None, # used for SubjectToVideo auth_token=None, ): + ''' + Function used between Minimax nodes - supports T2V, I2V, and S2V, based on provided arguments. + ''' + # upload image, if passed in + image_url = None + if image is not None: + image_url = upload_images_to_comfyapi(image, max_images=1, auth_token=auth_token)[0] + + # TODO: figure out how to deal with subject properly, API returns invalid params when using S2V-01 model + subject_reference = None + if subject is not None: + subject_url = upload_images_to_comfyapi(subject, max_images=1, auth_token=auth_token)[0] + subject_reference = [SubjectReferenceItem(image=subject_url)] + + video_generate_operation = SynchronousOperation( endpoint=ApiEndpoint( path="/proxy/minimax/video_generation", @@ -100,8 +111,8 @@ class MinimaxTextToVideoNode: model=Model(model), prompt=prompt_text, callback_url=None, - first_frame_image=None, - subject_reference=None, + first_frame_image=image_url, + subject_reference=subject_reference, prompt_optimizer=None, ), auth_token=auth_token, @@ -158,13 +169,135 @@ class MinimaxTextToVideoNode: return (VideoFromFile(video_io),) +class MinimaxImageToVideoNode(MinimaxTextToVideoNode): + """ + Generates videos synchronously based on an image and prompt, and optional parameters using Minimax's API. + """ + + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "image": ( + IO.IMAGE, + { + "tooltip": "Image to use as first frame of video generation" + }, + ), + "prompt_text": ( + "STRING", + { + "multiline": True, + "default": "", + "tooltip": "Text prompt to guide the video generation", + }, + ), + "model": ( + [ + "I2V-01-Director", + "I2V-01", + "I2V-01-live", + ], + { + "default": "I2V-01", + "tooltip": "Model to use for video generation", + }, + ), + }, + "optional": { + "seed": ( + IO.INT, + { + "default": 0, + "min": 0, + "max": 0xFFFFFFFFFFFFFFFF, + "control_after_generate": True, + "tooltip": "The random seed used for creating the noise.", + }, + ), + }, + "hidden": { + "auth_token": "AUTH_TOKEN_COMFY_ORG", + }, + } + + RETURN_TYPES = ("VIDEO",) + DESCRIPTION = "Generates videos from an image and prompts using Minimax's API" + FUNCTION = "generate_video" + CATEGORY = "api node/video/Minimax" + API_NODE = True + OUTPUT_NODE = True + + +class MinimaxSubjectToVideoNode(MinimaxTextToVideoNode): + """ + Generates videos synchronously based on an image and prompt, and optional parameters using Minimax's API. + """ + + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "subject": ( + IO.IMAGE, + { + "tooltip": "Image of subject to reference video generation" + }, + ), + "prompt_text": ( + "STRING", + { + "multiline": True, + "default": "", + "tooltip": "Text prompt to guide the video generation", + }, + ), + "model": ( + [ + "S2V-01", + ], + { + "default": "S2V-01", + "tooltip": "Model to use for video generation", + }, + ), + }, + "optional": { + "seed": ( + IO.INT, + { + "default": 0, + "min": 0, + "max": 0xFFFFFFFFFFFFFFFF, + "control_after_generate": True, + "tooltip": "The random seed used for creating the noise.", + }, + ), + }, + "hidden": { + "auth_token": "AUTH_TOKEN_COMFY_ORG", + }, + } + + RETURN_TYPES = ("VIDEO",) + DESCRIPTION = "Generates videos from an image and prompts using Minimax's API" + FUNCTION = "generate_video" + CATEGORY = "api node/video/Minimax" + API_NODE = True + OUTPUT_NODE = True + + # A dictionary that contains all nodes you want to export with their names # NOTE: names should be globally unique NODE_CLASS_MAPPINGS = { "MinimaxTextToVideoNode": MinimaxTextToVideoNode, + "MinimaxImageToVideoNode": MinimaxImageToVideoNode, + # "MinimaxSubjectToVideoNode": MinimaxSubjectToVideoNode, } # A dictionary that contains the friendly/humanly readable titles for the nodes NODE_DISPLAY_NAME_MAPPINGS = { "MinimaxTextToVideoNode": "Minimax Text to Video", + "MinimaxImageToVideoNode": "Minimax Image to Video", + "MinimaxSubjectToVideoNode": "Minimax Subject to Video", } From 48a0ef6311e516a322d14e8773b3a8ec32be5587 Mon Sep 17 00:00:00 2001 From: Jedrzej Kosinski Date: Wed, 30 Apr 2025 03:55:49 -0500 Subject: [PATCH 042/121] Add Recraft Text to Vector node, add Save SVG node to handle its output (#53) --- comfy_api_nodes/apis/recraft_api.py | 3 + comfy_api_nodes/nodes_recraft.py | 183 +++++++++++++++++++++++++++- 2 files changed, 184 insertions(+), 2 deletions(-) diff --git a/comfy_api_nodes/apis/recraft_api.py b/comfy_api_nodes/apis/recraft_api.py index 7defea873..f9fec0f64 100644 --- a/comfy_api_nodes/apis/recraft_api.py +++ b/comfy_api_nodes/apis/recraft_api.py @@ -11,11 +11,14 @@ from pydantic import BaseModel, Field, conint class RecraftStyle: def __init__(self, style: str, substyle: str=None): self.style = style + if substyle == "None": + substyle = None self.substyle = substyle class RecraftIO: STYLEV3 = "RECRAFT_V3_STYLE" + SVG = "SVG" # TODO: if acceptable, move into ComfyUI's typing class class RecraftStyleV3(str, Enum): diff --git a/comfy_api_nodes/nodes_recraft.py b/comfy_api_nodes/nodes_recraft.py index 85ebe3eb1..f208e9f34 100644 --- a/comfy_api_nodes/nodes_recraft.py +++ b/comfy_api_nodes/nodes_recraft.py @@ -19,8 +19,76 @@ from comfy_api_nodes.nodes_api import ( bytesio_to_image_tensor, download_url_to_bytesio, ) +import folder_paths +import os import torch +from io import BytesIO + + +class SVG: + """ + Stores SVG representations via a list of BytesIO objects. + """ + def __init__(self, data: list[BytesIO]): + self.data = data + + +class SaveSVGNode: + """ + Save SVG files on disk. + """ + + def __init__(self): + self.output_dir = folder_paths.get_output_directory() + self.type = "output" + self.prefix_append = "" + + RETURN_TYPES = () + FUNCTION = "save_svg" + CATEGORY = "api node/image/Recraft" + OUTPUT_NODE = True + + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "svg": (RecraftIO.SVG,), + "filename_prefix": ("STRING", {"default": "svg/ComfyUI", "tooltip": "The prefix for the file to save. This may include formatting information such as %date:yyyy-MM-dd% or %Empty Latent Image.width% to include values from nodes."}) + }, + "hidden": { + "prompt": "PROMPT", + "extra_pnginfo": "EXTRA_PNGINFO" + } + } + + def save_svg(self, svg: SVG, filename_prefix="svg/ComfyUI", prompt=None, extra_pnginfo=None): + filename_prefix += self.prefix_append + full_output_folder, filename, counter, subfolder, filename_prefix = folder_paths.get_save_image_path(filename_prefix, self.output_dir) + results = list() + for batch_number, svg_bytes in enumerate(svg.data): + # NOTE: no way to do metadata for SVG right now, maybe figure this out later + # metadata = None + # if not args.disable_metadata: + # metadata = PngInfo() + # if prompt is not None: + # metadata.add_text("prompt", json.dumps(prompt)) + # if extra_pnginfo is not None: + # for x in extra_pnginfo: + # metadata.add_text(x, json.dumps(extra_pnginfo[x])) + + filename_with_batch_num = filename.replace("%batch_num%", str(batch_number)) + file = f"{filename_with_batch_num}_{counter:05}_.svg" + with open(os.path.join(full_output_folder, file), 'wb') as svg_file: + svg_bytes.seek(0) + svg_file.write(svg_bytes.read()) + results.append({ + "filename": file, + "subfolder": subfolder, + "type": self.type + }) + counter += 1 + return (None,) class RecraftStyleV3RealisticImageNode: @@ -197,21 +265,132 @@ class RecraftTextToImageNode: return (output_image,) +class RecraftTextToVectorNode: + """ + Generates SVG synchronously based on prompt and resolution. + """ + + RETURN_TYPES = (RecraftIO.SVG,) + DESCRIPTION = cleandoc(__doc__ or "") # Handle potential None value + FUNCTION = "api_call" + API_NODE = True + CATEGORY = "api node/image/Recraft" + + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "prompt": ( + IO.STRING, + { + "multiline": True, + "default": "", + "tooltip": "Prompt for the image generation.", + }, + ), + "substyle": (get_v3_substyles(RecraftStyleV3.vector_illustration),), + "size": ( + [res.value for res in RecraftImageSize], + { + "default": RecraftImageSize.res_1024x1024, + "tooltip": "The size of the generated image.", + }, + ), + "n": ( + IO.INT, + { + "default": 1, + "min": 1, + "max": 6, + "tooltip": "The number of images to generate.", + }, + ), + "seed": ( + IO.INT, + { + "default": 0, + "min": 0, + "max": 0xFFFFFFFFFFFFFFFF, + "control_after_generate": True, + "tooltip": "Seed to determine if node should re-run; actual results are nondeterministic regardless of seed.", + }, + ), + }, + "optional": { + "negative_prompt": ( + IO.STRING, + { + "default": "", + "forceInput": True, + "tooltip": "An optional text description of undesired elements on an image.", + }, + ), + }, + "hidden": { + "auth_token": "AUTH_TOKEN_COMFY_ORG", + }, + } + + def api_call( + self, + prompt: str, + substyle: str, + size: str, + n: int, + seed, + negative_prompt: str = None, + auth_token=None, + **kwargs, + ): + # create RecraftStyle so strings will be formatted properly (i.e. "None" will become None) + recraft_style = RecraftStyle(RecraftStyleV3.vector_illustration, substyle=substyle) + + if not negative_prompt: + negative_prompt = None + + operation = SynchronousOperation( + endpoint=ApiEndpoint( + path="/proxy/recraft/image_generation", + method=HttpMethod.POST, + request_model=RecraftImageGenerationRequest, + response_model=RecraftImageGenerationResponse, + ), + request=RecraftImageGenerationRequest( + prompt=prompt, + negative_prompts=negative_prompt, + model=RecraftModel.recraftv3, + size=size, + n=n, + style=recraft_style.style, + substyle=recraft_style.substyle, + ), + auth_token=auth_token, + ) + response: RecraftImageGenerationResponse = operation.execute() + svg_data = [] + for data in response.data: + svg_data.append(download_url_to_bytesio(data.url, timeout=1024)) + + return (SVG(svg_data),) + + # A dictionary that contains all nodes you want to export with their names # NOTE: names should be globally unique NODE_CLASS_MAPPINGS = { "RecraftTextToImageNode": RecraftTextToImageNode, + "RecraftTextToVectorNode": RecraftTextToVectorNode, "RecraftStyleV3RealisticImage": RecraftStyleV3RealisticImageNode, "RecraftStyleV3DigitalIllustration": RecraftStyleV3DigitalIllustrationNode, "RecraftStyleV3LogoRaster": RecraftStyleV3LogoRasterNode, - # "RecraftStyleV3VectorIllustration": RecraftStyleV3VectorIllustrationNode, + "SaveSVG": SaveSVGNode, } # A dictionary that contains the friendly/humanly readable titles for the nodes NODE_DISPLAY_NAME_MAPPINGS = { "RecraftTextToImageNode": "Recraft Text to Image", + "RecraftTextToVectorNode": "Recraft Text to Vector", "RecraftStyleV3RealisticImage": "Recraft Style - Realistic Image", "RecraftStyleV3DigitalIllustration": "Recraft Style - Digital Illustration", - "RecraftStyleV3VectorIllustration": "Recraft Style - Vector Illustration", "RecraftStyleV3LogoRaster": "Recraft Style - Logo Raster", + "SaveSVG": "Save SVG", } From 814c86d8d4fe64faa20d78b9b2acfd26e4550692 Mon Sep 17 00:00:00 2001 From: Jedrzej Kosinski Date: Wed, 30 Apr 2025 04:31:37 -0500 Subject: [PATCH 043/121] Added pixverse_template support to Pixverse Text to Video node (#54) --- comfy_api_nodes/apis/pixverse_api.py | 15 +++++++++- comfy_api_nodes/nodes_pixverse.py | 42 ++++++++++++++++++++++++++-- 2 files changed, 54 insertions(+), 3 deletions(-) diff --git a/comfy_api_nodes/apis/pixverse_api.py b/comfy_api_nodes/apis/pixverse_api.py index 174d50965..85eae2738 100644 --- a/comfy_api_nodes/apis/pixverse_api.py +++ b/comfy_api_nodes/apis/pixverse_api.py @@ -6,6 +6,19 @@ from typing import Optional from pydantic import BaseModel, Field +pixverse_templates = { + "Microwave": 324641385496960, + "Suit Swagger": 328545151283968, + "Anything, Robot": 313358700761536, + "Subject 3 Fever": 327828816843648, + "kiss kiss": 315446315336768, +} + + +class PixverseIO: + TEMPLATE = "PIXVERSE_TEMPLATE" + + class PixverseStatus(int, Enum): successful = 1 generating = 5 @@ -57,7 +70,7 @@ class PixverseDto_V2OpenAPIT2VReq(BaseModel): negative_prompt: Optional[str] = Field(None) seed: Optional[int] = Field(None) style: Optional[str] = Field(None) - template_id: Optional[str] = Field(None) + template_id: Optional[int] = Field(None) water_mark: Optional[bool] = Field(None) diff --git a/comfy_api_nodes/nodes_pixverse.py b/comfy_api_nodes/nodes_pixverse.py index b90601deb..70083c2e7 100644 --- a/comfy_api_nodes/nodes_pixverse.py +++ b/comfy_api_nodes/nodes_pixverse.py @@ -9,6 +9,8 @@ from comfy_api_nodes.apis.pixverse_api import ( PixverseDuration, PixverseMotionMode, PixverseStatus, + PixverseIO, + pixverse_templates, ) from comfy_api_nodes.apis.client import ( ApiEndpoint, @@ -24,6 +26,32 @@ import requests from io import BytesIO +class PixverseTemplateNode: + """ + Select template for Pixverse Video generation. + """ + + RETURN_TYPES = (PixverseIO.TEMPLATE,) + RETURN_NAMES = ("pixverse_template",) + FUNCTION = "create_template" + CATEGORY = "api node/video/Pixverse" + + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "template": (list(pixverse_templates.keys()), ), + } + } + + def create_template(self, template: str): + template_id = pixverse_templates.get(template, None) + if template_id is None: + raise Exception(f"Template '{template}' is not recognized.") + # just return the integer + return (template_id,) + + class PixverseTextToVideoNode(ComfyNodeABC): """ Generates videos synchronously based on prompt and output_size. @@ -78,6 +106,12 @@ class PixverseTextToVideoNode(ComfyNodeABC): "tooltip": "An optional text description of undesired elements on an image.", }, ), + "pixverse_template": ( + PixverseIO.TEMPLATE, + { + "tooltip": "An optional template to influence style of generation, created by the Pixverse Template node." + } + ) }, "hidden": { "auth_token": "AUTH_TOKEN_COMFY_ORG", @@ -93,6 +127,7 @@ class PixverseTextToVideoNode(ComfyNodeABC): motion_mode: str, seed, negative_prompt: str=None, + pixverse_template: int=None, auth_token=None, **kwargs, ): @@ -118,6 +153,7 @@ class PixverseTextToVideoNode(ComfyNodeABC): duration=duration_seconds, motion_mode=motion_mode, negative_prompt=negative_prompt if negative_prompt else None, + template_id=pixverse_template, seed=seed, ), auth_token=auth_token, @@ -146,9 +182,11 @@ class PixverseTextToVideoNode(ComfyNodeABC): NODE_CLASS_MAPPINGS = { - "PixverseTextToVideoNode": PixverseTextToVideoNode + "PixverseTextToVideoNode": PixverseTextToVideoNode, + "PixverseTemplateNode": PixverseTemplateNode, } NODE_DISPLAY_NAME_MAPPINGS = { - "PixverseTextToVideoNode": "Pixverse Text to Video" + "PixverseTextToVideoNode": "Pixverse Text to Video", + "PixverseTemplateNode": "Pixverse Template", } From 9db268849d34212b022f658eda9f85c584d865a4 Mon Sep 17 00:00:00 2001 From: Jedrzej Kosinski Date: Wed, 30 Apr 2025 05:51:14 -0500 Subject: [PATCH 044/121] Added Recraft Controls + Recraft Color RGB nodes (#57) --- comfy_api_nodes/apis/recraft_api.py | 77 ++++++++++++++++++++- comfy_api_nodes/nodes_recraft.py | 100 ++++++++++++++++++++++++++++ 2 files changed, 176 insertions(+), 1 deletion(-) diff --git a/comfy_api_nodes/apis/recraft_api.py b/comfy_api_nodes/apis/recraft_api.py index f9fec0f64..aee3d39b6 100644 --- a/comfy_api_nodes/apis/recraft_api.py +++ b/comfy_api_nodes/apis/recraft_api.py @@ -8,6 +8,68 @@ from typing import Optional from pydantic import BaseModel, Field, conint +class RecraftColor: + def __init__(self, r: int, g: int, b: int): + self.color = [r, g, b] + + def create_api_model(self): + return RecraftColorObject(rgb=self.color) + + +class RecraftColorChain: + def __init__(self): + self.colors: list[RecraftColor] = [] + + def get_first(self): + if len(self.colors) > 0: + return self.colors[0] + return None + + def add(self, color: RecraftColor): + self.colors.append(color) + + def create_api_model(self): + if not self.colors: + return None + colors_api = [x.create_api_model() for x in self.colors] + return colors_api + + def clone(self): + c = RecraftColorChain() + for color in self.colors: + c.add(color) + return c + + def clone_and_merge(self, other: RecraftColorChain): + c = self.clone() + for color in other.colors: + c.add(color) + return c + + +class RecraftControls: + def __init__(self, colors: RecraftColorChain=None, background_color: RecraftColorChain=None, + artistic_level: int=None, no_text: bool=None): + self.colors = colors + self.background_color = background_color + self.artistic_level = artistic_level + self.no_text = no_text + + def create_api_model(self): + if self.colors is None and self.background_color is None and self.artistic_level is None and self.no_text is None: + return None + colors_api = None + background_color_api = None + if self.colors: + colors_api = self.colors.create_api_model() + if self.background_color: + first_background = self.background_color.get_first() + background_color_api = first_background.create_api_model() if first_background else None + + return RecraftControlsObject(colors=colors_api, background_color=background_color_api, + artistic_level=self.artistic_level, no_text=self.no_text) + + class RecraftStyle: def __init__(self, style: str, substyle: str=None): self.style = style @@ -19,6 +81,8 @@ class RecraftStyle: class RecraftIO: STYLEV3 = "RECRAFT_V3_STYLE" SVG = "SVG" # TODO: if acceptable, move into ComfyUI's typing class + COLOR = "RECRAFT_COLOR" + CONTROLS = "RECRAFT_CONTROLS" class RecraftStyleV3(str, Enum): @@ -160,6 +224,17 @@ class RecraftImageSize(str, Enum): res_1707x1024 = '1707x1024' +class RecraftColorObject(BaseModel): + rgb: list[int] = Field(..., description='An array of 3 integer values in range of 0...255 defining RGB Color Model') + + +class RecraftControlsObject(BaseModel): + colors: Optional[list[RecraftColorObject]] = Field(None, description='An array of preferable colors') + background_color: Optional[RecraftColorObject] = Field(None, description='Use given color as a desired background color') + no_text: Optional[bool] = Field(None, description='Do not embed text layouts') + artistic_level: Optional[conint(ge=0, le=5)] = Field(None, description='Defines artistic tone of your image. At a simple level, the person looks straight at the camera in a static and clean style. Dynamic and eccentric levels introduce movement and creativity. The value should be in range [0..5].') + + class RecraftImageGenerationRequest(BaseModel): prompt: str = Field(..., description='The text prompt describing the image to generate') size: RecraftImageSize = Field(..., description='The size of the generated image (e.g., "1024x1024")') @@ -168,8 +243,8 @@ class RecraftImageGenerationRequest(BaseModel): model: Optional[RecraftModel] = Field(RecraftModel.recraftv3, description='The model to use for generation (e.g., "recraftv3")') style: Optional[str] = Field(None, description='The style to apply to the generated image (e.g., "digital_illustration")') substyle: Optional[str] = Field(None, description='The substyle to apply to the generated image, depending on the style input') + controls: Optional[RecraftControlsObject] = Field(None, description='A set of custom parameters to tweak generation process') # text_layout - # controls class RecraftReturnedObject(BaseModel): diff --git a/comfy_api_nodes/nodes_recraft.py b/comfy_api_nodes/nodes_recraft.py index f208e9f34..e2008de44 100644 --- a/comfy_api_nodes/nodes_recraft.py +++ b/comfy_api_nodes/nodes_recraft.py @@ -7,6 +7,9 @@ from comfy_api_nodes.apis.recraft_api import ( RecraftModel, RecraftStyle, RecraftStyleV3, + RecraftColor, + RecraftColorChain, + RecraftControls, RecraftIO, get_v3_substyles, ) @@ -91,6 +94,75 @@ class SaveSVGNode: return (None,) +class RecraftColorRGBNode: + """ + Create Recraft Color by choosing specific RGB values. + """ + + RETURN_TYPES = (RecraftIO.COLOR,) + RETURN_NAMES = ("recraft_color",) + FUNCTION = "create_color" + CATEGORY = "api node/image/Recraft" + + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "r": (IO.INT, { + "default": 0, + "min": 0, + "max": 255, + "tooltip": "Red value of color." + }), + "g": (IO.INT, { + "default": 0, + "min": 0, + "max": 255, + "tooltip": "Green value of color." + }), + "b": (IO.INT, { + "default": 0, + "min": 0, + "max": 255, + "tooltip": "Blue value of color." + }), + }, + "optional": { + "recraft_color": (RecraftIO.COLOR,), + } + } + + def create_color(self, r: int, g: int, b: int, recraft_color: RecraftColorChain=None): + recraft_color = recraft_color.clone() if recraft_color else RecraftColorChain() + recraft_color.add(RecraftColor(r, g, b)) + return (recraft_color, ) + + +class RecraftControlsNode: + """ + Create Recraft Controls for customizing Recraft generation. + """ + + RETURN_TYPES = (RecraftIO.CONTROLS,) + RETURN_NAMES = ("recraft_controls",) + FUNCTION = "create_controls" + CATEGORY = "api node/image/Recraft" + + @classmethod + def INPUT_TYPES(s): + return { + "required": { + }, + "optional": { + "colors": (RecraftIO.COLOR,), + "background_color": (RecraftIO.COLOR,), + } + } + + def create_controls(self, colors: RecraftColorChain=None, background_color: RecraftColorChain=None): + return (RecraftControls(colors=colors, background_color=background_color), ) + + class RecraftStyleV3RealisticImageNode: """ Select realistic_image style and optional substyle. @@ -209,6 +281,12 @@ class RecraftTextToImageNode: "tooltip": "An optional text description of undesired elements on an image.", }, ), + "recraft_controls": ( + RecraftIO.CONTROLS, + { + "tooltip": "Optional additional controls over the generation via the Recraft Controls node." + }, + ), }, "hidden": { "auth_token": "AUTH_TOKEN_COMFY_ORG", @@ -223,6 +301,7 @@ class RecraftTextToImageNode: seed, recraft_style: RecraftStyle = None, negative_prompt: str = None, + recraft_controls: RecraftControls = None, auth_token=None, **kwargs, ): @@ -230,6 +309,10 @@ class RecraftTextToImageNode: if recraft_style is None: recraft_style = default_style + controls_api = None + if recraft_controls: + controls_api = recraft_controls.create_api_model() + if not negative_prompt: negative_prompt = None @@ -248,6 +331,7 @@ class RecraftTextToImageNode: n=n, style=recraft_style.style, substyle=recraft_style.substyle, + controls=controls_api, ), auth_token=auth_token, ) @@ -325,6 +409,12 @@ class RecraftTextToVectorNode: "tooltip": "An optional text description of undesired elements on an image.", }, ), + "recraft_controls": ( + RecraftIO.CONTROLS, + { + "tooltip": "Optional additional controls over the generation via the Recraft Controls node." + }, + ), }, "hidden": { "auth_token": "AUTH_TOKEN_COMFY_ORG", @@ -339,12 +429,17 @@ class RecraftTextToVectorNode: n: int, seed, negative_prompt: str = None, + recraft_controls: RecraftControls = None, auth_token=None, **kwargs, ): # create RecraftStyle so strings will be formatted properly (i.e. "None" will become None) recraft_style = RecraftStyle(RecraftStyleV3.vector_illustration, substyle=substyle) + controls_api = None + if recraft_controls: + controls_api = recraft_controls.create_api_model() + if not negative_prompt: negative_prompt = None @@ -363,6 +458,7 @@ class RecraftTextToVectorNode: n=n, style=recraft_style.style, substyle=recraft_style.substyle, + controls=controls_api, ), auth_token=auth_token, ) @@ -382,6 +478,8 @@ NODE_CLASS_MAPPINGS = { "RecraftStyleV3RealisticImage": RecraftStyleV3RealisticImageNode, "RecraftStyleV3DigitalIllustration": RecraftStyleV3DigitalIllustrationNode, "RecraftStyleV3LogoRaster": RecraftStyleV3LogoRasterNode, + "RecraftColorRGB": RecraftColorRGBNode, + "RecraftControls": RecraftControlsNode, "SaveSVG": SaveSVGNode, } @@ -392,5 +490,7 @@ NODE_DISPLAY_NAME_MAPPINGS = { "RecraftStyleV3RealisticImage": "Recraft Style - Realistic Image", "RecraftStyleV3DigitalIllustration": "Recraft Style - Digital Illustration", "RecraftStyleV3LogoRaster": "Recraft Style - Logo Raster", + "RecraftColorRGB": "Recraft Color RGB", + "RecraftControls": "Recraft Controls", "SaveSVG": "Save SVG", } From f7c70b382244cd895aaf8dbe60dd515fcf3f1459 Mon Sep 17 00:00:00 2001 From: thot experiment <94414189+thot-experiment@users.noreply.github.com> Date: Wed, 30 Apr 2025 08:35:01 -0700 Subject: [PATCH 045/121] split remaining nodes out of nodes_api, make utility lib, refactor ideogram (#61) --- comfy_api_nodes/apinode_utils.py | 312 +++++++++ comfy_api_nodes/nodes_api.py | 1002 ----------------------------- comfy_api_nodes/nodes_bfl.py | 6 +- comfy_api_nodes/nodes_ideogram.py | 545 ++++++++++++++++ comfy_api_nodes/nodes_kling.py | 2 +- comfy_api_nodes/nodes_luma.py | 2 +- comfy_api_nodes/nodes_minimax.py | 2 +- comfy_api_nodes/nodes_openai.py | 483 ++++++++++++++ comfy_api_nodes/nodes_recraft.py | 2 +- comfy_api_nodes/nodes_runway.py | 2 +- nodes.py | 3 +- 11 files changed, 1350 insertions(+), 1011 deletions(-) create mode 100644 comfy_api_nodes/apinode_utils.py delete mode 100644 comfy_api_nodes/nodes_api.py create mode 100644 comfy_api_nodes/nodes_ideogram.py create mode 100644 comfy_api_nodes/nodes_openai.py diff --git a/comfy_api_nodes/apinode_utils.py b/comfy_api_nodes/apinode_utils.py new file mode 100644 index 000000000..e20c68a51 --- /dev/null +++ b/comfy_api_nodes/apinode_utils.py @@ -0,0 +1,312 @@ +import io +from typing import Optional +from comfy.utils import common_upscale +from comfy_api_nodes.apis.client import ( + ApiClient, + ApiEndpoint, + HttpMethod, + SynchronousOperation, + UploadRequest, + UploadResponse, +) + + +import numpy as np +from PIL import Image +import requests +import torch +import math +import base64 +import uuid +from io import BytesIO + +def downscale_image_tensor(image, total_pixels=1536 * 1024): + """Downscale input image tensor to roughly the specified total pixels.""" + samples = image.movedim(-1, 1) + total = int(total_pixels) + scale_by = math.sqrt(total / (samples.shape[3] * samples.shape[2])) + if scale_by >= 1: + return image + width = round(samples.shape[3] * scale_by) + height = round(samples.shape[2] * scale_by) + + s = common_upscale(samples, width, height, "lanczos", "disabled") + s = s.movedim(1, -1) + return s + +def validate_and_cast_response(response): + # validate raw JSON response + data = response.data + if not data or len(data) == 0: + raise Exception("No images returned from API endpoint") + + # Initialize list to store image tensors + image_tensors = [] + + # Process each image in the data array + for image_data in data: + image_url = image_data.url + b64_data = image_data.b64_json + + if not image_url and not b64_data: + raise Exception("No image was generated in the response") + + if b64_data: + img_data = base64.b64decode(b64_data) + img = Image.open(io.BytesIO(img_data)) + + elif image_url: + img_response = requests.get(image_url) + if img_response.status_code != 200: + raise Exception("Failed to download the image") + img = Image.open(io.BytesIO(img_response.content)) + + img = img.convert("RGBA") + + # Convert to numpy array, normalize to float32 between 0 and 1 + img_array = np.array(img).astype(np.float32) / 255.0 + img_tensor = torch.from_numpy(img_array) + + # Add to list of tensors + image_tensors.append(img_tensor) + + return torch.stack(image_tensors, dim=0) + + +def validate_aspect_ratio( + aspect_ratio: str, + minimum_ratio: float, + maximum_ratio: float, + minimum_ratio_str: str, + maximum_ratio_str: str, +): + # get ratio values + numbers = aspect_ratio.split(":") + if len(numbers) != 2: + raise Exception( + f"Aspect ratio must be in the format X:Y, such as 16:9, but was {aspect_ratio}." + ) + try: + numerator = int(numbers[0]) + denominator = int(numbers[1]) + except ValueError: + raise Exception( + f"Aspect ratio must contain numbers separated by ':', such as 16:9, but was {aspect_ratio}." + ) + calculated_ratio = numerator / denominator + # if not close to minimum and maximum, check bounds + if not math.isclose(calculated_ratio, minimum_ratio) or not math.isclose( + calculated_ratio, maximum_ratio + ): + if calculated_ratio < minimum_ratio: + raise Exception( + f"Aspect ratio cannot reduce to any less than {minimum_ratio_str} ({minimum_ratio}), but was {aspect_ratio} ({calculated_ratio})." + ) + elif calculated_ratio > maximum_ratio: + raise Exception( + f"Aspect ratio cannot reduce to any greater than {maximum_ratio_str} ({maximum_ratio}), but was {aspect_ratio} ({calculated_ratio})." + ) + return aspect_ratio + + +def mimetype_to_extension(mime_type: str) -> str: + """Converts a MIME type to a file extension.""" + return mime_type.split("/")[-1].lower() + + +def download_url_to_bytesio(url: str, timeout: int = None) -> BytesIO: + """Downloads content from a URL using requests and returns it as BytesIO. + + Args: + url: The URL to download. + timeout: Request timeout in seconds. Defaults to None (no timeout). + + Returns: + BytesIO object containing the downloaded content. + """ + response = requests.get(url, stream=True, timeout=timeout) + response.raise_for_status() # Raises HTTPError for bad responses (4XX or 5XX) + return BytesIO(response.content) + + +def bytesio_to_image_tensor(image_bytesio: BytesIO, mode: str = "RGBA") -> torch.Tensor: + """Converts image data from BytesIO to a torch.Tensor. + + Args: + image_bytesio: BytesIO object containing the image data. + mode: The PIL mode to convert the image to (e.g., "RGB", "RGBA"). + + Returns: + A torch.Tensor representing the image (1, H, W, C). + + Raises: + PIL.UnidentifiedImageError: If the image data cannot be identified. + ValueError: If the specified mode is invalid. + """ + image = Image.open(image_bytesio) + image = image.convert(mode) + image_array = np.array(image).astype(np.float32) / 255.0 + return torch.from_numpy(image_array).unsqueeze(0) + + +def process_image_response(response: requests.Response): + """Uses content from a Response object and converts it to a torch.Tensor""" + return bytesio_to_image_tensor(BytesIO(response.content)) + + +def _tensor_to_pil(image: torch.Tensor, total_pixels: int = 2048 * 2048) -> Image.Image: + """Converts a single torch.Tensor image [H, W, C] to a PIL Image, optionally downscaling.""" + if len(image.shape) > 3: + image = image[0] + # TODO: remove alpha if not allowed and present + input_tensor = image.cpu() + input_tensor = downscale_image_tensor( + input_tensor.unsqueeze(0), total_pixels=total_pixels + ).squeeze() + image_np = (input_tensor.numpy() * 255).astype(np.uint8) + img = Image.fromarray(image_np) + return img + + +def _pil_to_bytesio(img: Image.Image, mime_type: str = "image/png") -> BytesIO: + """Converts a PIL Image to a BytesIO object.""" + if not mime_type: + mime_type = "image/png" + + img_byte_arr = io.BytesIO() + # Derive PIL format from MIME type (e.g., 'image/png' -> 'PNG') + pil_format = mime_type.split("/")[-1].upper() + if pil_format == "JPG": + pil_format = "JPEG" + img.save(img_byte_arr, format=pil_format) + img_byte_arr.seek(0) + return img_byte_arr + + +def tensor_to_bytesio( + image: torch.Tensor, + name: Optional[str] = None, + total_pixels: int = 2048 * 2048, + mime_type: str = "image/png", +) -> BytesIO: + """Converts a torch.Tensor image to a named BytesIO object. + + Args: + image: Input torch.Tensor image. + name: Optional filename for the BytesIO object. + total_pixels: Maximum total pixels for potential downscaling. + mime_type: Target image MIME type (e.g., 'image/png', 'image/jpeg', 'image/webp', 'video/mp4'). + + Returns: + Named BytesIO object containing the image data. + """ + if not mime_type: + mime_type = "image/png" + + pil_image = _tensor_to_pil(image, total_pixels=total_pixels) + img_binary = _pil_to_bytesio(pil_image, mime_type=mime_type) + img_binary.name = ( + f"{name if name else uuid.uuid4()}.{mimetype_to_extension(mime_type)}" + ) + return img_binary + + +def tensor_to_base64_string( + image_tensor: torch.Tensor, + total_pixels: int = 2048 * 2048, + mime_type: str = "image/png", +) -> str: + """Convert [B, H, W, C] or [H, W, C] tensor to a base64 string. + + Args: + image_tensor: Input torch.Tensor image. + total_pixels: Maximum total pixels for potential downscaling. + mime_type: Target image MIME type (e.g., 'image/png', 'image/jpeg', 'image/webp', 'video/mp4'). + + Returns: + Base64 encoded string of the image. + """ + pil_image = _tensor_to_pil(image_tensor, total_pixels=total_pixels) + img_byte_arr = _pil_to_bytesio(pil_image, mime_type=mime_type) + img_bytes = img_byte_arr.getvalue() + # Encode bytes to base64 string + base64_encoded_string = base64.b64encode(img_bytes).decode("utf-8") + return base64_encoded_string + + +def tensor_to_data_uri( + image_tensor: torch.Tensor, + total_pixels: int = 2048 * 2048, + mime_type: str = "image/png", +) -> str: + """Converts a tensor image to a Data URI string. + + Args: + image_tensor: Input torch.Tensor image. + total_pixels: Maximum total pixels for potential downscaling. + mime_type: Target image MIME type (e.g., 'image/png', 'image/jpeg', 'image/webp'). + + Returns: + Data URI string (e.g., 'data:image/png;base64,...'). + """ + base64_string = tensor_to_base64_string(image_tensor, total_pixels, mime_type) + return f"data:{mime_type};base64,{base64_string}" + + +def upload_images_to_comfyapi( + image: torch.Tensor, max_images=8, auth_token=None, mime_type: Optional[str] = None +) -> list[str]: + # if batch, try to upload each file if max_images is greater than 0 + idx_image = 0 + download_urls: list[str] = [] + is_batch = len(image.shape) > 3 + batch_length = 1 + if is_batch: + batch_length = image.shape[0] + while True: + curr_image = image + if len(image.shape) > 3: + curr_image = image[idx_image] + # get BytesIO version of image + img_binary = tensor_to_bytesio(curr_image, mime_type=mime_type) + # first, request upload/download urls from comfy API + if not mime_type: + request_object = UploadRequest(filename=img_binary.name) + else: + request_object = UploadRequest( + filename=img_binary.name, content_type=mime_type + ) + operation = SynchronousOperation( + endpoint=ApiEndpoint( + path="/customers/storage", + method=HttpMethod.POST, + request_model=UploadRequest, + response_model=UploadResponse, + ), + request=request_object, + auth_token=auth_token, + ) + response = operation.execute() + + upload_response = ApiClient.upload_file( + response.upload_url, img_binary, content_type=mime_type + ) + # verify success + try: + upload_response.raise_for_status() + except requests.exceptions.HTTPError as e: + raise Exception(f"Could not upload one or more images: {e}") + # add download_url to list + download_urls.append(response.download_url) + + idx_image += 1 + # stop uploading additional files if done + if is_batch and max_images > 0: + if idx_image >= max_images: + break + if idx_image >= batch_length: + break + return download_urls + + + diff --git a/comfy_api_nodes/nodes_api.py b/comfy_api_nodes/nodes_api.py deleted file mode 100644 index 645c19016..000000000 --- a/comfy_api_nodes/nodes_api.py +++ /dev/null @@ -1,1002 +0,0 @@ -import io -from inspect import cleandoc -from typing import Optional -from comfy.utils import common_upscale -from comfy.comfy_types.node_typing import IO, ComfyNodeABC, InputTypeDict -from comfy_api_nodes.apis import ( - OpenAIImageGenerationRequest, - OpenAIImageEditRequest, - OpenAIImageGenerationResponse, - IdeogramGenerateRequest, - IdeogramGenerateResponse, - ImageRequest, -) -from comfy_api_nodes.apis.client import ( - ApiClient, - ApiEndpoint, - HttpMethod, - SynchronousOperation, - UploadRequest, - UploadResponse, -) - -import numpy as np -from PIL import Image -import requests -import torch -import math -import base64 -import uuid -import folder_paths -from io import BytesIO - - -def downscale_input(image, total_pixels=1536 * 1024): - samples = image.movedim(-1, 1) - # downscaling input images to roughly the same size as the outputs - total = int(total_pixels) - scale_by = math.sqrt(total / (samples.shape[3] * samples.shape[2])) - if scale_by >= 1: - return image - width = round(samples.shape[3] * scale_by) - height = round(samples.shape[2] * scale_by) - - s = common_upscale(samples, width, height, "lanczos", "disabled") - s = s.movedim(1, -1) - return s - - -def validate_and_cast_response(response): - # validate raw JSON response - data = response.data - if not data or len(data) == 0: - raise Exception("No images returned from API endpoint") - - # Initialize list to store image tensors - image_tensors = [] - - # Process each image in the data array - for image_data in data: - image_url = image_data.url - b64_data = image_data.b64_json - - if not image_url and not b64_data: - raise Exception("No image was generated in the response") - - if b64_data: - img_data = base64.b64decode(b64_data) - img = Image.open(io.BytesIO(img_data)) - - elif image_url: - img_response = requests.get(image_url) - if img_response.status_code != 200: - raise Exception("Failed to download the image") - img = Image.open(io.BytesIO(img_response.content)) - - img = img.convert("RGBA") - - # Convert to numpy array, normalize to float32 between 0 and 1 - img_array = np.array(img).astype(np.float32) / 255.0 - img_tensor = torch.from_numpy(img_array) - - # Add to list of tensors - image_tensors.append(img_tensor) - - return torch.stack(image_tensors, dim=0) - - -def validate_aspect_ratio( - aspect_ratio: str, - minimum_ratio: float, - maximum_ratio: float, - minimum_ratio_str: str, - maximum_ratio_str: str, -): - # get ratio values - numbers = aspect_ratio.split(":") - if len(numbers) != 2: - raise Exception( - f"Aspect ratio must be in the format X:Y, such as 16:9, but was {aspect_ratio}." - ) - try: - numerator = int(numbers[0]) - denominator = int(numbers[1]) - except ValueError: - raise Exception( - f"Aspect ratio must contain numbers separated by ':', such as 16:9, but was {aspect_ratio}." - ) - calculated_ratio = numerator / denominator - # if not close to minimum and maximum, check bounds - if not math.isclose(calculated_ratio, minimum_ratio) or not math.isclose( - calculated_ratio, maximum_ratio - ): - if calculated_ratio < minimum_ratio: - raise Exception( - f"Aspect ratio cannot reduce to any less than {minimum_ratio_str} ({minimum_ratio}), but was {aspect_ratio} ({calculated_ratio})." - ) - elif calculated_ratio > maximum_ratio: - raise Exception( - f"Aspect ratio cannot reduce to any greater than {maximum_ratio_str} ({maximum_ratio}), but was {aspect_ratio} ({calculated_ratio})." - ) - return aspect_ratio - - -def mimetype_to_extension(mime_type: str) -> str: - """Converts a MIME type to a file extension.""" - return mime_type.split("/")[-1].lower() - - -def download_url_to_bytesio(url: str, timeout: int = None) -> BytesIO: - """Downloads content from a URL using requests and returns it as BytesIO. - - Args: - url: The URL to download. - timeout: Request timeout in seconds. Defaults to None (no timeout). - - Returns: - BytesIO object containing the downloaded content. - """ - response = requests.get(url, stream=True, timeout=timeout) - response.raise_for_status() # Raises HTTPError for bad responses (4XX or 5XX) - return BytesIO(response.content) - - -def bytesio_to_image_tensor(image_bytesio: BytesIO, mode: str = "RGBA") -> torch.Tensor: - """Converts image data from BytesIO to a torch.Tensor. - - Args: - image_bytesio: BytesIO object containing the image data. - mode: The PIL mode to convert the image to (e.g., "RGB", "RGBA"). - - Returns: - A torch.Tensor representing the image (1, H, W, C). - - Raises: - PIL.UnidentifiedImageError: If the image data cannot be identified. - ValueError: If the specified mode is invalid. - """ - image = Image.open(image_bytesio) - image = image.convert(mode) - image_array = np.array(image).astype(np.float32) / 255.0 - return torch.from_numpy(image_array).unsqueeze(0) - - -def process_image_response(response: requests.Response): - """Uses content from a Response object and converts it to a torch.Tensor""" - return bytesio_to_image_tensor(BytesIO(response.content)) - - -def _tensor_to_pil(image: torch.Tensor, total_pixels: int = 2048 * 2048) -> Image.Image: - """Converts a single torch.Tensor image [H, W, C] to a PIL Image, optionally downscaling.""" - if len(image.shape) > 3: - image = image[0] - # TODO: remove alpha if not allowed and present - input_tensor = image.cpu() - input_tensor = downscale_input( - input_tensor.unsqueeze(0), total_pixels=total_pixels - ).squeeze() - image_np = (input_tensor.numpy() * 255).astype(np.uint8) - img = Image.fromarray(image_np) - return img - - -def _pil_to_bytesio(img: Image.Image, mime_type: str = "image/png") -> BytesIO: - """Converts a PIL Image to a BytesIO object.""" - if not mime_type: - mime_type = "image/png" - - img_byte_arr = io.BytesIO() - # Derive PIL format from MIME type (e.g., 'image/png' -> 'PNG') - pil_format = mime_type.split("/")[-1].upper() - if pil_format == "JPG": - pil_format = "JPEG" - img.save(img_byte_arr, format=pil_format) - img_byte_arr.seek(0) - return img_byte_arr - - -def tensor_to_bytesio( - image: torch.Tensor, - name: Optional[str] = None, - total_pixels: int = 2048 * 2048, - mime_type: str = "image/png", -) -> BytesIO: - """Converts a torch.Tensor image to a named BytesIO object. - - Args: - image: Input torch.Tensor image. - name: Optional filename for the BytesIO object. - total_pixels: Maximum total pixels for potential downscaling. - mime_type: Target image MIME type (e.g., 'image/png', 'image/jpeg', 'image/webp', 'video/mp4'). - - Returns: - Named BytesIO object containing the image data. - """ - if not mime_type: - mime_type = "image/png" - - pil_image = _tensor_to_pil(image, total_pixels=total_pixels) - img_binary = _pil_to_bytesio(pil_image, mime_type=mime_type) - img_binary.name = ( - f"{name if name else uuid.uuid4()}.{mimetype_to_extension(mime_type)}" - ) - return img_binary - - -def tensor_to_base64_string( - image_tensor: torch.Tensor, - total_pixels: int = 2048 * 2048, - mime_type: str = "image/png", -) -> str: - """Convert [B, H, W, C] or [H, W, C] tensor to a base64 string. - - Args: - image_tensor: Input torch.Tensor image. - total_pixels: Maximum total pixels for potential downscaling. - mime_type: Target image MIME type (e.g., 'image/png', 'image/jpeg', 'image/webp', 'video/mp4'). - - Returns: - Base64 encoded string of the image. - """ - pil_image = _tensor_to_pil(image_tensor, total_pixels=total_pixels) - img_byte_arr = _pil_to_bytesio(pil_image, mime_type=mime_type) - img_bytes = img_byte_arr.getvalue() - # Encode bytes to base64 string - base64_encoded_string = base64.b64encode(img_bytes).decode("utf-8") - return base64_encoded_string - - -def tensor_to_data_uri( - image_tensor: torch.Tensor, - total_pixels: int = 2048 * 2048, - mime_type: str = "image/png", -) -> str: - """Converts a tensor image to a Data URI string. - - Args: - image_tensor: Input torch.Tensor image. - total_pixels: Maximum total pixels for potential downscaling. - mime_type: Target image MIME type (e.g., 'image/png', 'image/jpeg', 'image/webp'). - - Returns: - Data URI string (e.g., 'data:image/png;base64,...'). - """ - base64_string = tensor_to_base64_string(image_tensor, total_pixels, mime_type) - return f"data:{mime_type};base64,{base64_string}" - - -def upload_images_to_comfyapi( - image: torch.Tensor, max_images=8, auth_token=None, mime_type: Optional[str] = None -) -> list[str]: - # if batch, try to upload each file if max_images is greater than 0 - idx_image = 0 - download_urls: list[str] = [] - is_batch = len(image.shape) > 3 - batch_length = 1 - if is_batch: - batch_length = image.shape[0] - while True: - curr_image = image - if len(image.shape) > 3: - curr_image = image[idx_image] - # get BytesIO version of image - img_binary = tensor_to_bytesio(curr_image, mime_type=mime_type) - # first, request upload/download urls from comfy API - if not mime_type: - request_object = UploadRequest(filename=img_binary.name) - else: - request_object = UploadRequest( - filename=img_binary.name, content_type=mime_type - ) - operation = SynchronousOperation( - endpoint=ApiEndpoint( - path="/customers/storage", - method=HttpMethod.POST, - request_model=UploadRequest, - response_model=UploadResponse, - ), - request=request_object, - auth_token=auth_token, - ) - response = operation.execute() - - upload_response = ApiClient.upload_file( - response.upload_url, img_binary, content_type=mime_type - ) - # verify success - try: - upload_response.raise_for_status() - except requests.exceptions.HTTPError as e: - raise Exception(f"Could not upload one or more images: {e}") - # add download_url to list - download_urls.append(response.download_url) - - idx_image += 1 - # stop uploading additional files if done - if is_batch and max_images > 0: - if idx_image >= max_images: - break - if idx_image >= batch_length: - break - return download_urls - - -class OpenAIDalle2(ComfyNodeABC): - """ - Generates images synchronously via OpenAI's DALL·E 2 endpoint. - - Uses the proxy at /proxy/openai/images/generations. Returned URLs are short‑lived, - so download or cache results if you need to keep them. - """ - - def __init__(self): - pass - - @classmethod - def INPUT_TYPES(cls) -> InputTypeDict: - return { - "required": { - "prompt": ( - IO.STRING, - { - "multiline": True, - "default": "", - "tooltip": "Text prompt for DALL·E", - }, - ), - }, - "optional": { - "seed": ( - IO.INT, - { - "default": 0, - "min": 0, - "max": 2**31 - 1, - "step": 1, - "display": "number", - "tooltip": "not implemented yet in backend", - }, - ), - "size": ( - IO.COMBO, - { - "options": ["256x256", "512x512", "1024x1024"], - "default": "1024x1024", - "tooltip": "Image size", - }, - ), - "n": ( - IO.INT, - { - "default": 1, - "min": 1, - "max": 8, - "step": 1, - "display": "number", - "tooltip": "How many images to generate", - }, - ), - "image": ( - IO.IMAGE, - { - "default": None, - "tooltip": "Optional reference image for image editing.", - }, - ), - "mask": ( - IO.MASK, - { - "default": None, - "tooltip": "Optional mask for inpainting (white areas will be replaced)", - }, - ), - }, - "hidden": {"auth_token": "AUTH_TOKEN_COMFY_ORG"}, - } - - RETURN_TYPES = (IO.IMAGE,) - FUNCTION = "api_call" - CATEGORY = "api node/image/openai" - DESCRIPTION = cleandoc(__doc__ or "") - API_NODE = True - - def api_call( - self, - prompt, - seed=0, - image=None, - mask=None, - n=1, - size="1024x1024", - auth_token=None, - ): - model = "dall-e-2" - path = "/proxy/openai/images/generations" - request_class = OpenAIImageGenerationRequest - img_binary = None - - if image is not None and mask is not None: - path = "/proxy/openai/images/edits" - request_class = OpenAIImageEditRequest - - input_tensor = image.squeeze().cpu() - height, width, channels = input_tensor.shape - rgba_tensor = torch.ones(height, width, 4, device="cpu") - rgba_tensor[:, :, :channels] = input_tensor - - if mask.shape[1:] != image.shape[1:-1]: - raise Exception("Mask and Image must be the same size") - rgba_tensor[:, :, 3] = 1 - mask.squeeze().cpu() - - rgba_tensor = downscale_input(rgba_tensor.unsqueeze(0)).squeeze() - - image_np = (rgba_tensor.numpy() * 255).astype(np.uint8) - img = Image.fromarray(image_np) - img_byte_arr = io.BytesIO() - img.save(img_byte_arr, format="PNG") - img_byte_arr.seek(0) - img_binary = img_byte_arr # .getvalue() - img_binary.name = "image.png" - elif image is not None or mask is not None: - raise Exception("Dall-E 2 image editing requires an image AND a mask") - - # Build the operation - operation = SynchronousOperation( - endpoint=ApiEndpoint( - path=path, - method=HttpMethod.POST, - request_model=request_class, - response_model=OpenAIImageGenerationResponse, - ), - request=request_class( - model=model, - prompt=prompt, - n=n, - size=size, - seed=seed, - ), - files=( - { - "image": img_binary, - } - if img_binary - else None - ), - auth_token=auth_token, - ) - - response = operation.execute() - - img_tensor = validate_and_cast_response(response) - return (img_tensor,) - - -class OpenAIDalle3(ComfyNodeABC): - """ - Generates images synchronously via OpenAI's DALL·E 3 endpoint. - - Uses the proxy at /proxy/openai/images/generations. Returned URLs are short‑lived, - so download or cache results if you need to keep them. - """ - - def __init__(self): - pass - - @classmethod - def INPUT_TYPES(cls) -> InputTypeDict: - return { - "required": { - "prompt": ( - IO.STRING, - { - "multiline": True, - "default": "", - "tooltip": "Text prompt for DALL·E", - }, - ), - }, - "optional": { - "seed": ( - IO.INT, - { - "default": 0, - "min": 0, - "max": 2**31 - 1, - "step": 1, - "display": "number", - "tooltip": "not implemented yet in backend", - }, - ), - "quality": ( - IO.COMBO, - { - "options": ["standard", "hd"], - "default": "standard", - "tooltip": "Image quality", - }, - ), - "style": ( - IO.COMBO, - { - "options": ["natural", "vivid"], - "default": "natural", - "tooltip": "Vivid causes the model to lean towards generating hyper-real and dramatic images. Natural causes the model to produce more natural, less hyper-real looking images.", - }, - ), - "size": ( - IO.COMBO, - { - "options": ["1024x1024", "1024x1792", "1792x1024"], - "default": "1024x1024", - "tooltip": "Image size", - }, - ), - }, - "hidden": {"auth_token": "AUTH_TOKEN_COMFY_ORG"}, - } - - RETURN_TYPES = (IO.IMAGE,) - FUNCTION = "api_call" - CATEGORY = "api node/image/openai" - DESCRIPTION = cleandoc(__doc__ or "") - API_NODE = True - - def api_call( - self, - prompt, - seed=0, - style="natural", - quality="standard", - size="1024x1024", - auth_token=None, - ): - model = "dall-e-3" - - # build the operation - operation = SynchronousOperation( - endpoint=ApiEndpoint( - path="/proxy/openai/images/generations", - method=HttpMethod.POST, - request_model=OpenAIImageGenerationRequest, - response_model=OpenAIImageGenerationResponse, - ), - request=OpenAIImageGenerationRequest( - model=model, - prompt=prompt, - quality=quality, - size=size, - style=style, - seed=seed, - ), - auth_token=auth_token, - ) - - response = operation.execute() - - img_tensor = validate_and_cast_response(response) - return (img_tensor,) - - -class OpenAIGPTImage1(ComfyNodeABC): - """ - Generates images synchronously via OpenAI's GPT Image 1 endpoint. - - Uses the proxy at /proxy/openai/images/generations. Returned URLs are short‑lived, - so download or cache results if you need to keep them. - """ - - def __init__(self): - self.output_dir = folder_paths.get_output_directory() - self.type = "output" - - @classmethod - def INPUT_TYPES(cls) -> InputTypeDict: - return { - "required": { - "prompt": ( - IO.STRING, - { - "multiline": True, - "default": "", - "tooltip": "Text prompt for GPT Image 1", - }, - ), - }, - "optional": { - "seed": ( - IO.INT, - { - "default": 0, - "min": 0, - "max": 2**31 - 1, - "step": 1, - "display": "number", - "tooltip": "not implemented yet in backend", - }, - ), - "quality": ( - IO.COMBO, - { - "options": ["low", "medium", "high"], - "default": "low", - "tooltip": "Image quality, affects cost and generation time.", - }, - ), - "background": ( - IO.COMBO, - { - "options": ["opaque", "transparent"], - "default": "opaque", - "tooltip": "Return image with or without background", - }, - ), - "size": ( - IO.COMBO, - { - "options": ["auto", "1024x1024", "1024x1536", "1536x1024"], - "default": "auto", - "tooltip": "Image size", - }, - ), - "n": ( - IO.INT, - { - "default": 1, - "min": 1, - "max": 8, - "step": 1, - "display": "number", - "tooltip": "How many images to generate", - }, - ), - "image": ( - IO.IMAGE, - { - "default": None, - "tooltip": "Optional reference image for image editing.", - }, - ), - "mask": ( - IO.MASK, - { - "default": None, - "tooltip": "Optional mask for inpainting (white areas will be replaced)", - }, - ), - }, - "hidden": {"auth_token": "AUTH_TOKEN_COMFY_ORG"}, - } - - RETURN_TYPES = (IO.IMAGE,) - FUNCTION = "api_call" - CATEGORY = "api node/image/openai" - DESCRIPTION = cleandoc(__doc__ or "") - API_NODE = True - - def api_call( - self, - prompt, - seed=0, - quality="low", - background="opaque", - image=None, - mask=None, - n=1, - size="1024x1024", - auth_token=None, - ): - model = "gpt-image-1" - path = "/proxy/openai/images/generations" - request_class = OpenAIImageGenerationRequest - img_binaries = [] - mask_binary = None - files = [] - - if image is not None: - path = "/proxy/openai/images/edits" - request_class = OpenAIImageEditRequest - - batch_size = image.shape[0] - - for i in range(batch_size): - single_image = image[i : i + 1] - scaled_image = downscale_input(single_image).squeeze() - - image_np = (scaled_image.numpy() * 255).astype(np.uint8) - img = Image.fromarray(image_np) - img_byte_arr = io.BytesIO() - img.save(img_byte_arr, format="PNG") - img_byte_arr.seek(0) - img_binary = img_byte_arr - img_binary.name = f"image_{i}.png" - - img_binaries.append(img_binary) - if batch_size == 1: - files.append(("image", img_binary)) - else: - files.append(("image[]", img_binary)) - - if mask is not None: - if image.shape[0] != 1: - raise Exception("Cannot use a mask with multiple image") - if image is None: - raise Exception("Cannot use a mask without an input image") - if mask.shape[1:] != image.shape[1:-1]: - raise Exception("Mask and Image must be the same size") - batch, height, width = mask.shape - rgba_mask = torch.zeros(height, width, 4, device="cpu") - rgba_mask[:, :, 3] = 1 - mask.squeeze().cpu() - - scaled_mask = downscale_input(rgba_mask.unsqueeze(0)).squeeze() - - mask_np = (scaled_mask.numpy() * 255).astype(np.uint8) - mask_img = Image.fromarray(mask_np) - mask_img_byte_arr = io.BytesIO() - mask_img.save(mask_img_byte_arr, format="PNG") - mask_img_byte_arr.seek(0) - mask_binary = mask_img_byte_arr - mask_binary.name = "mask.png" - files.append(("mask", mask_binary)) - - # Build the operation - operation = SynchronousOperation( - endpoint=ApiEndpoint( - path=path, - method=HttpMethod.POST, - request_model=request_class, - response_model=OpenAIImageGenerationResponse, - ), - request=request_class( - model=model, - prompt=prompt, - quality=quality, - background=background, - n=n, - seed=seed, - size=size, - ), - files=files if files else None, - auth_token=auth_token, - ) - - response = operation.execute() - - img_tensor = validate_and_cast_response(response) - return (img_tensor,) - - -class IdeogramTextToImage(ComfyNodeABC): - """ - Generates images synchronously based on a given prompt and optional parameters. - - Images links are available for a limited period of time; if you would like to keep the image, you must download it. - """ - - def __init__(self): - pass - - @classmethod - def INPUT_TYPES(cls) -> InputTypeDict: - """ - Return a dictionary which contains config for all input fields. - Some types (string): "MODEL", "VAE", "CLIP", "CONDITIONING", "LATENT", "IMAGE", "INT", "STRING", "FLOAT". - Input types "INT", "STRING" or "FLOAT" are special values for fields on the node. - The type can be a list for selection. - - Returns: `dict`: - - Key input_fields_group (`string`): Can be either required, hidden or optional. A node class must have property `required` - - Value input_fields (`dict`): Contains input fields config: - * Key field_name (`string`): Name of a entry-point method's argument - * Value field_config (`tuple`): - + First value is a string indicate the type of field or a list for selection. - + Secound value is a config for type "INT", "STRING" or "FLOAT". - """ - return { - "required": { - "prompt": ( - IO.STRING, - { - "multiline": True, - "default": "", - "tooltip": "Prompt for the image generation", - }, - ), - "model": ( - IO.COMBO, - { - "options": ["V_2", "V_2_TURBO", "V_1", "V_1_TURBO"], - "default": "V_2", - "tooltip": "Model to use for image generation", - }, - ), - }, - "optional": { - "aspect_ratio": ( - IO.COMBO, - { - "options": [ - "ASPECT_1_1", - "ASPECT_4_3", - "ASPECT_3_4", - "ASPECT_16_9", - "ASPECT_9_16", - "ASPECT_2_1", - "ASPECT_1_2", - "ASPECT_3_2", - "ASPECT_2_3", - "ASPECT_4_5", - "ASPECT_5_4", - ], - "default": "ASPECT_1_1", - "tooltip": "The aspect ratio for image generation. Cannot be used with resolution", - }, - ), - "resolution": ( - IO.COMBO, - { - "options": ["1024x1024", "1024x1792", "1792x1024"], - "default": "1024x1024", - "tooltip": "The resolution for image generation (V2 only). Cannot be used with aspect_ratio", - }, - ), - "magic_prompt_option": ( - IO.COMBO, - { - "options": ["AUTO", "ON", "OFF"], - "default": "AUTO", - "tooltip": "Determine if MagicPrompt should be used in generation", - }, - ), - "seed": ( - IO.INT, - { - "default": 0, - "min": 0, - "max": 2147483647, - "step": 1, - "display": "number", - }, - ), - "style_type": ( - IO.COMBO, - { - "options": [ - "NONE", - "ANIME", - "CINEMATIC", - "CREATIVE", - "DIGITAL_ART", - "PHOTOGRAPHIC", - ], - "default": "NONE", - "tooltip": "Style type for generation (V2+ only)", - }, - ), - "negative_prompt": ( - IO.STRING, - { - "multiline": True, - "default": "", - "tooltip": "Description of what to exclude from the image (V1/V2 only)", - }, - ), - "num_images": ( - IO.INT, - {"default": 1, "min": 1, "max": 8, "step": 1, "display": "number"}, - ), - "color_palette": ( - IO.STRING, - { - "multiline": False, - "default": "", - "tooltip": "Color palette preset name or hex colors with weights (V2/V2_TURBO only)", - }, - ), - }, - "hidden": {"auth_token": "AUTH_TOKEN_COMFY_ORG"}, - } - - RETURN_TYPES = (IO.IMAGE,) - DESCRIPTION = cleandoc(__doc__ or "") # Handle potential None value - FUNCTION = "api_call" - API_NODE = True - CATEGORY = "api node/image/ideogram" - - def api_call( - self, - prompt, - model, - aspect_ratio=None, - resolution=None, - magic_prompt_option="AUTO", - seed=0, - style_type="NONE", - negative_prompt="", - num_images=1, - color_palette="", - auth_token=None, - ): - import torch - from PIL import Image - import io - import numpy as np - import requests - - operation = SynchronousOperation( - endpoint=ApiEndpoint( - path="/proxy/ideogram/generate", - method=HttpMethod.POST, - request_model=IdeogramGenerateRequest, - response_model=IdeogramGenerateResponse, - ), - request=IdeogramGenerateRequest( - image_request=ImageRequest( - prompt=prompt, - model=model, - num_images=num_images, - seed=seed, - aspect_ratio=aspect_ratio if aspect_ratio != "ASPECT_1_1" else None, - resolution=resolution if resolution != "1024x1024" else None, - magic_prompt_option=( - magic_prompt_option if magic_prompt_option != "AUTO" else None - ), - style_type=style_type if style_type != "NONE" else None, - negative_prompt=negative_prompt if negative_prompt else None, - color_palette=None, - ) - ), - auth_token=auth_token, - ) - - response = operation.execute() - - if not response.data or len(response.data) == 0: - raise Exception("No images were generated in the response") - image_url = response.data[0].url - - if not image_url: - raise Exception("No image URL was generated in the response") - img_response = requests.get(image_url) - if img_response.status_code != 200: - raise Exception("Failed to download the image") - - img = Image.open(io.BytesIO(img_response.content)) - img = img.convert("RGB") # Ensure RGB format - - # Convert to numpy array, normalize to float32 between 0 and 1 - img_array = np.array(img).astype(np.float32) / 255.0 - - # Convert to torch tensor and add batch dimension - img_tensor = torch.from_numpy(img_array)[None,] - - return (img_tensor,) - - """ - The node will always be re executed if any of the inputs change but - this method can be used to force the node to execute again even when the inputs don't change. - You can make this node return a number or a string. This value will be compared to the one returned the last time the node was - executed, if it is different the node will be executed again. - This method is used in the core repo for the LoadImage node where they return the image hash as a string, if the image hash - changes between executions the LoadImage node is executed again. - """ - # @classmethod - # def IS_CHANGED(s, image, string_field, int_field, float_field, print_to_screen): - # return "" - - -# A dictionary that contains all nodes you want to export with their names -# NOTE: names should be globally unique -NODE_CLASS_MAPPINGS = { - "OpenAIDalle2": OpenAIDalle2, - "OpenAIDalle3": OpenAIDalle3, - "OpenAIGPTImage1": OpenAIGPTImage1, - "IdeogramTextToImage": IdeogramTextToImage, -} - -# A dictionary that contains the friendly/humanly readable titles for the nodes -NODE_DISPLAY_NAME_MAPPINGS = { - "OpenAIDalle2": "OpenAI DALL·E 2", - "OpenAIDalle3": "OpenAI DALL·E 3", - "OpenAIGPTImage1": "OpenAI GPT Image 1", - "IdeogramTextToImage": "Ideogram Text to Image", -} diff --git a/comfy_api_nodes/nodes_bfl.py b/comfy_api_nodes/nodes_bfl.py index 66b5dea79..84ff267f2 100644 --- a/comfy_api_nodes/nodes_bfl.py +++ b/comfy_api_nodes/nodes_bfl.py @@ -11,8 +11,8 @@ from comfy_api_nodes.apis.client import ( HttpMethod, SynchronousOperation, ) -from comfy_api_nodes.nodes_api import ( - downscale_input, +from comfy_api_nodes.apinode_utils import ( + downscale_image_tensor, validate_aspect_ratio, process_image_response, ) @@ -220,7 +220,7 @@ class FluxProUltraImageNode(ComfyNodeABC): raise Exception(f"BFL API encountered an error: {response.json()}") def _convert_image_to_base64(self, image: torch.Tensor): - scaled_image = downscale_input(image, total_pixels=2048 * 2048) + scaled_image = downscale_image_tensor(image, total_pixels=2048 * 2048) # remove batch dimension if present if len(scaled_image.shape) > 3: scaled_image = scaled_image[0] diff --git a/comfy_api_nodes/nodes_ideogram.py b/comfy_api_nodes/nodes_ideogram.py new file mode 100644 index 000000000..088039d85 --- /dev/null +++ b/comfy_api_nodes/nodes_ideogram.py @@ -0,0 +1,545 @@ +from comfy.comfy_types.node_typing import IO, ComfyNodeABC, InputTypeDict +from inspect import cleandoc +from comfy_api_nodes.apis import ( + IdeogramGenerateRequest, + IdeogramGenerateResponse, + ImageRequest, +) + +from comfy_api_nodes.apis.client import ( + ApiEndpoint, + HttpMethod, + SynchronousOperation, +) + +from comfy_api_nodes.apinode_utils import ( + download_url_to_bytesio, + bytesio_to_image_tensor, +) + +RESOLUTION_MAPPING = { + "Auto":"AUTO", + "512 x 1536":"RESOLUTION_512_1536", + "576 x 1408":"RESOLUTION_576_1408", + "576 x 1472":"RESOLUTION_576_1472", + "576 x 1536":"RESOLUTION_576_1536", + "640 x 1024":"RESOLUTION_640_1024", + "640 x 1344":"RESOLUTION_640_1344", + "640 x 1408":"RESOLUTION_640_1408", + "640 x 1472":"RESOLUTION_640_1472", + "640 x 1536":"RESOLUTION_640_1536", + "704 x 1152":"RESOLUTION_704_1152", + "704 x 1216":"RESOLUTION_704_1216", + "704 x 1280":"RESOLUTION_704_1280", + "704 x 1344":"RESOLUTION_704_1344", + "704 x 1408":"RESOLUTION_704_1408", + "704 x 1472":"RESOLUTION_704_1472", + "720 x 1280":"RESOLUTION_720_1280", + "736 x 1312":"RESOLUTION_736_1312", + "768 x 1024":"RESOLUTION_768_1024", + "768 x 1088":"RESOLUTION_768_1088", + "768 x 1152":"RESOLUTION_768_1152", + "768 x 1216":"RESOLUTION_768_1216", + "768 x 1232":"RESOLUTION_768_1232", + "768 x 1280":"RESOLUTION_768_1280", + "768 x 1344":"RESOLUTION_768_1344", + "832 x 960":"RESOLUTION_832_960", + "832 x 1024":"RESOLUTION_832_1024", + "832 x 1088":"RESOLUTION_832_1088", + "832 x 1152":"RESOLUTION_832_1152", + "832 x 1216":"RESOLUTION_832_1216", + "832 x 1248":"RESOLUTION_832_1248", + "864 x 1152":"RESOLUTION_864_1152", + "896 x 960":"RESOLUTION_896_960", + "896 x 1024":"RESOLUTION_896_1024", + "896 x 1088":"RESOLUTION_896_1088", + "896 x 1120":"RESOLUTION_896_1120", + "896 x 1152":"RESOLUTION_896_1152", + "960 x 832":"RESOLUTION_960_832", + "960 x 896":"RESOLUTION_960_896", + "960 x 1024":"RESOLUTION_960_1024", + "960 x 1088":"RESOLUTION_960_1088", + "1024 x 640":"RESOLUTION_1024_640", + "1024 x 768":"RESOLUTION_1024_768", + "1024 x 832":"RESOLUTION_1024_832", + "1024 x 896":"RESOLUTION_1024_896", + "1024 x 960":"RESOLUTION_1024_960", + "1024 x 1024":"RESOLUTION_1024_1024", + "1088 x 768":"RESOLUTION_1088_768", + "1088 x 832":"RESOLUTION_1088_832", + "1088 x 896":"RESOLUTION_1088_896", + "1088 x 960":"RESOLUTION_1088_960", + "1120 x 896":"RESOLUTION_1120_896", + "1152 x 704":"RESOLUTION_1152_704", + "1152 x 768":"RESOLUTION_1152_768", + "1152 x 832":"RESOLUTION_1152_832", + "1152 x 864":"RESOLUTION_1152_864", + "1152 x 896":"RESOLUTION_1152_896", + "1216 x 704":"RESOLUTION_1216_704", + "1216 x 768":"RESOLUTION_1216_768", + "1216 x 832":"RESOLUTION_1216_832", + "1232 x 768":"RESOLUTION_1232_768", + "1248 x 832":"RESOLUTION_1248_832", + "1280 x 704":"RESOLUTION_1280_704", + "1280 x 720":"RESOLUTION_1280_720", + "1280 x 768":"RESOLUTION_1280_768", + "1280 x 800":"RESOLUTION_1280_800", + "1312 x 736":"RESOLUTION_1312_736", + "1344 x 640":"RESOLUTION_1344_640", + "1344 x 704":"RESOLUTION_1344_704", + "1344 x 768":"RESOLUTION_1344_768", + "1408 x 576":"RESOLUTION_1408_576", + "1408 x 640":"RESOLUTION_1408_640", + "1408 x 704":"RESOLUTION_1408_704", + "1472 x 576":"RESOLUTION_1472_576", + "1472 x 640":"RESOLUTION_1472_640", + "1472 x 704":"RESOLUTION_1472_704", + "1536 x 512":"RESOLUTION_1536_512", + "1536 x 576":"RESOLUTION_1536_576", + "1536 x 640":"RESOLUTION_1536_640", +} + +ASPECT_RATIO_MAPPING = { + "1:1":"ASPECT_1_1", + "4:3":"ASPECT_4_3", + "3:4":"ASPECT_3_4", + "16:9":"ASPECT_16_9", + "9:16":"ASPECT_9_16", + "2:1":"ASPECT_2_1", + "1:2":"ASPECT_1_2", + "3:2":"ASPECT_3_2", + "2:3":"ASPECT_2_3", + "4:5":"ASPECT_4_5", + "5:4":"ASPECT_5_4", +} + +def download_and_process_image(image_url): + """Helper function to download and process image from URL""" + + # Using functions from apinode_utils.py to handle downloading and processing + image_bytesio = download_url_to_bytesio(image_url) # Download image content to BytesIO + img_tensor = bytesio_to_image_tensor(image_bytesio, mode="RGB") # Convert to torch.Tensor with RGB mode + + return img_tensor + +class IdeogramV1(ComfyNodeABC): + """ + Generates images synchronously using the Ideogram V1 model. + + Images links are available for a limited period of time; if you would like to keep the image, you must download it. + """ + + def __init__(self): + pass + + @classmethod + def INPUT_TYPES(cls) -> InputTypeDict: + return { + "required": { + "prompt": ( + IO.STRING, + { + "multiline": True, + "default": "", + "tooltip": "Prompt for the image generation", + }, + ), + "turbo": ( + IO.BOOLEAN, + { + "default": False, + "tooltip": "Whether to use turbo mode (faster generation, potentially lower quality)", + } + ), + }, + "optional": { + "aspect_ratio": ( + IO.COMBO, + { + "options": list(ASPECT_RATIO_MAPPING.keys()), + "default": "1:1", + "tooltip": "The aspect ratio for image generation.", + }, + ), + "magic_prompt_option": ( + IO.COMBO, + { + "options": ["AUTO", "ON", "OFF"], + "default": "AUTO", + "tooltip": "Determine if MagicPrompt should be used in generation", + }, + ), + "seed": ( + IO.INT, + { + "default": 0, + "min": 0, + "max": 2147483647, + "step": 1, + "display": "number", + }, + ), + "negative_prompt": ( + IO.STRING, + { + "multiline": True, + "default": "", + "tooltip": "Description of what to exclude from the image", + }, + ), + "num_images": ( + IO.INT, + {"default": 1, "min": 1, "max": 8, "step": 1, "display": "number"}, + ), + }, + "hidden": {"auth_token": "AUTH_TOKEN_COMFY_ORG"}, + } + + RETURN_TYPES = (IO.IMAGE,) + FUNCTION = "api_call" + CATEGORY = "api node/image/ideogram/v1" + DESCRIPTION = cleandoc(__doc__ or "") + API_NODE = True + + def api_call( + self, + prompt, + turbo=False, + aspect_ratio="1:1", + magic_prompt_option="AUTO", + seed=0, + negative_prompt="", + num_images=1, + auth_token=None, + ): + # Determine the model based on turbo setting + aspect_ratio = ASPECT_RATIO_MAPPING.get(aspect_ratio, None) + model = "V_1_TURBO" if turbo else "V_1" + + operation = SynchronousOperation( + endpoint=ApiEndpoint( + path="/proxy/ideogram/generate", + method=HttpMethod.POST, + request_model=IdeogramGenerateRequest, + response_model=IdeogramGenerateResponse, + ), + request=IdeogramGenerateRequest( + image_request=ImageRequest( + prompt=prompt, + model=model, + num_images=num_images, + seed=seed, + aspect_ratio=aspect_ratio if aspect_ratio != "ASPECT_1_1" else None, + magic_prompt_option=( + magic_prompt_option if magic_prompt_option != "AUTO" else None + ), + negative_prompt=negative_prompt if negative_prompt else None, + ) + ), + auth_token=auth_token, + ) + + response = operation.execute() + + if not response.data or len(response.data) == 0: + raise Exception("No images were generated in the response") + image_url = response.data[0].url + + if not image_url: + raise Exception("No image URL was generated in the response") + + return (download_and_process_image(image_url),) + + +class IdeogramV2(ComfyNodeABC): + """ + Generates images synchronously using the Ideogram V2 model. + + Images links are available for a limited period of time; if you would like to keep the image, you must download it. + """ + + def __init__(self): + pass + + @classmethod + def INPUT_TYPES(cls) -> InputTypeDict: + return { + "required": { + "prompt": ( + IO.STRING, + { + "multiline": True, + "default": "", + "tooltip": "Prompt for the image generation", + }, + ), + "turbo": ( + IO.BOOLEAN, + { + "default": False, + "tooltip": "Whether to use turbo mode (faster generation, potentially lower quality)", + } + ), + }, + "optional": { + "aspect_ratio": ( + IO.COMBO, + { + "options": list(ASPECT_RATIO_MAPPING.keys()), + "default": "1:1", + "tooltip": "The aspect ratio for image generation. Ignored if resolution is not set to AUTO.", + }, + ), + "resolution": ( + IO.COMBO, + { + "options": list(RESOLUTION_MAPPING.keys()), + "default": "Auto", + "tooltip": "The resolution for image generation. If not set to AUTO, this overrides the aspect_ratio setting.", + }, + ), + "magic_prompt_option": ( + IO.COMBO, + { + "options": ["AUTO", "ON", "OFF"], + "default": "AUTO", + "tooltip": "Determine if MagicPrompt should be used in generation", + }, + ), + "seed": ( + IO.INT, + { + "default": 0, + "min": 0, + "max": 2147483647, + "step": 1, + "display": "number", + }, + ), + "style_type": ( + IO.COMBO, + { + "options": ["NONE", "ANIME", "CINEMATIC", "CREATIVE", "DIGITAL_ART", "PHOTOGRAPHIC"], + "default": "NONE", + "tooltip": "Style type for generation (V2 only)", + }, + ), + "negative_prompt": ( + IO.STRING, + { + "multiline": True, + "default": "", + "tooltip": "Description of what to exclude from the image", + }, + ), + "num_images": ( + IO.INT, + {"default": 1, "min": 1, "max": 8, "step": 1, "display": "number"}, + ), + #"color_palette": ( + # IO.STRING, + # { + # "multiline": False, + # "default": "", + # "tooltip": "Color palette preset name or hex colors with weights", + # }, + #), + }, + "hidden": {"auth_token": "AUTH_TOKEN_COMFY_ORG"}, + } + + RETURN_TYPES = (IO.IMAGE,) + FUNCTION = "api_call" + CATEGORY = "api node/image/ideogram/v2" + DESCRIPTION = cleandoc(__doc__ or "") + API_NODE = True + + def api_call( + self, + prompt, + turbo=False, + aspect_ratio="1:1", + resolution="Auto", + magic_prompt_option="AUTO", + seed=0, + style_type="NONE", + negative_prompt="", + num_images=1, + color_palette="", + auth_token=None, + ): + aspect_ratio = ASPECT_RATIO_MAPPING.get(aspect_ratio, None) + resolution = RESOLUTION_MAPPING.get(resolution, None) + # Determine the model based on turbo setting + model = "V_2_TURBO" if turbo else "V_2" + + # Handle resolution vs aspect_ratio logic + # If resolution is not AUTO, it overrides aspect_ratio + final_resolution = None + final_aspect_ratio = None + + if resolution != "AUTO": + final_resolution = resolution + else: + final_aspect_ratio = aspect_ratio if aspect_ratio != "ASPECT_1_1" else None + + operation = SynchronousOperation( + endpoint=ApiEndpoint( + path="/proxy/ideogram/generate", + method=HttpMethod.POST, + request_model=IdeogramGenerateRequest, + response_model=IdeogramGenerateResponse, + ), + request=IdeogramGenerateRequest( + image_request=ImageRequest( + prompt=prompt, + model=model, + num_images=num_images, + seed=seed, + aspect_ratio=final_aspect_ratio, + resolution=final_resolution, + magic_prompt_option=( + magic_prompt_option if magic_prompt_option != "AUTO" else None + ), + style_type=style_type if style_type != "NONE" else None, + negative_prompt=negative_prompt if negative_prompt else None, + color_palette=color_palette if color_palette else None, + ) + ), + auth_token=auth_token, + ) + + response = operation.execute() + + if not response.data or len(response.data) == 0: + raise Exception("No images were generated in the response") + image_url = response.data[0].url + + if not image_url: + raise Exception("No image URL was generated in the response") + + return (download_and_process_image(image_url),) + + +class IdeogramV3(ComfyNodeABC): + """ + Generates images synchronously using the Ideogram V3 model. + + Images links are available for a limited period of time; if you would like to keep the image, you must download it. + """ + + def __init__(self): + pass + + @classmethod + def INPUT_TYPES(cls) -> InputTypeDict: + return { + "required": { + "prompt": ( + IO.STRING, + { + "multiline": True, + "default": "", + "tooltip": "Prompt for the image generation", + }, + ), + }, + "optional": { + "aspect_ratio": ( + IO.COMBO, + { + "options": list(ASPECT_RATIO_MAPPING.keys()), + "default": "1:1", + "tooltip": "The aspect ratio for image generation.", + }, + ), + "magic_prompt_option": ( + IO.COMBO, + { + "options": ["AUTO", "ON", "OFF"], + "default": "AUTO", + "tooltip": "Determine if MagicPrompt should be used in generation", + }, + ), + "seed": ( + IO.INT, + { + "default": 0, + "min": 0, + "max": 2147483647, + "step": 1, + "display": "number", + }, + ), + "num_images": ( + IO.INT, + {"default": 1, "min": 1, "max": 8, "step": 1, "display": "number"}, + ), + }, + "hidden": {"auth_token": "AUTH_TOKEN_COMFY_ORG"}, + } + + RETURN_TYPES = (IO.IMAGE,) + FUNCTION = "api_call" + CATEGORY = "api node/image/ideogram/v3" + DESCRIPTION = cleandoc(__doc__ or "") + API_NODE = True + + def api_call( + self, + prompt, + aspect_ratio="ASPECT_1_1", + magic_prompt_option="AUTO", + seed=0, + num_images=1, + auth_token=None, + ): + aspect_ratio = ASPECT_RATIO_MAPPING.get(aspect_ratio, None) + # V3 model - no turbo option + model = "V_3" + + operation = SynchronousOperation( + endpoint=ApiEndpoint( + path="/proxy/ideogram/generate", + method=HttpMethod.POST, + request_model=IdeogramGenerateRequest, + response_model=IdeogramGenerateResponse, + ), + request=IdeogramGenerateRequest( + image_request=ImageRequest( + prompt=prompt, + model=model, + num_images=num_images, + seed=seed, + aspect_ratio=aspect_ratio if aspect_ratio != "ASPECT_1_1" else None, + magic_prompt_option=( + magic_prompt_option if magic_prompt_option != "AUTO" else None + ), + ) + ), + auth_token=auth_token, + ) + + response = operation.execute() + + if not response.data or len(response.data) == 0: + raise Exception("No images were generated in the response") + image_url = response.data[0].url + + if not image_url: + raise Exception("No image URL was generated in the response") + + return (download_and_process_image(image_url),) + + +NODE_CLASS_MAPPINGS = { + "IdeogramV1": IdeogramV1, + "IdeogramV2": IdeogramV2, + #"IdeogramV3": IdeogramV3, +} + +NODE_DISPLAY_NAME_MAPPINGS = { + "IdeogramV1": "Ideogram V1", + "IdeogramV2": "Ideogram V2", + #"IdeogramV3": "Ideogram V3", +} diff --git a/comfy_api_nodes/nodes_kling.py b/comfy_api_nodes/nodes_kling.py index 0b8c6c5b4..7d0f2f48a 100644 --- a/comfy_api_nodes/nodes_kling.py +++ b/comfy_api_nodes/nodes_kling.py @@ -25,7 +25,7 @@ from comfy_api_nodes.apis.client import ( PollingOperation, EmptyRequest, ) -from comfy_api_nodes.nodes_api import ( +from comfy_api_nodes.apinode_utils import ( tensor_to_base64_string, download_url_to_bytesio, ) diff --git a/comfy_api_nodes/nodes_luma.py b/comfy_api_nodes/nodes_luma.py index a846c38b6..35f28939e 100644 --- a/comfy_api_nodes/nodes_luma.py +++ b/comfy_api_nodes/nodes_luma.py @@ -29,7 +29,7 @@ from comfy_api_nodes.apis.client import ( PollingOperation, EmptyRequest, ) -from comfy_api_nodes.nodes_api import ( +from comfy_api_nodes.apinode_utils import ( upload_images_to_comfyapi, process_image_response, ) diff --git a/comfy_api_nodes/nodes_minimax.py b/comfy_api_nodes/nodes_minimax.py index 33930e88b..449ae1473 100644 --- a/comfy_api_nodes/nodes_minimax.py +++ b/comfy_api_nodes/nodes_minimax.py @@ -15,7 +15,7 @@ from comfy_api_nodes.apis.client import ( PollingOperation, EmptyRequest, ) -from comfy_api_nodes.nodes_api import ( +from comfy_api_nodes.apinode_utils import ( download_url_to_bytesio, upload_images_to_comfyapi, ) diff --git a/comfy_api_nodes/nodes_openai.py b/comfy_api_nodes/nodes_openai.py new file mode 100644 index 000000000..17364bee2 --- /dev/null +++ b/comfy_api_nodes/nodes_openai.py @@ -0,0 +1,483 @@ +import io +from inspect import cleandoc +import numpy as np +import torch +from PIL import Image + +from comfy.comfy_types.node_typing import IO, ComfyNodeABC, InputTypeDict + + +from comfy_api_nodes.apis import ( + OpenAIImageGenerationRequest, + OpenAIImageEditRequest, + OpenAIImageGenerationResponse, +) + +from comfy_api_nodes.apis.client import ( + ApiEndpoint, + HttpMethod, + SynchronousOperation, +) + +from comfy_api_nodes.apinode_utils import ( + downscale_image_tensor, + validate_and_cast_response +) + +class OpenAIDalle2(ComfyNodeABC): + """ + Generates images synchronously via OpenAI's DALL·E 2 endpoint. + + Uses the proxy at /proxy/openai/images/generations. Returned URLs are short‑lived, + so download or cache results if you need to keep them. + """ + + def __init__(self): + pass + + @classmethod + def INPUT_TYPES(cls) -> InputTypeDict: + return { + "required": { + "prompt": ( + IO.STRING, + { + "multiline": True, + "default": "", + "tooltip": "Text prompt for DALL·E", + }, + ), + }, + "optional": { + "seed": ( + IO.INT, + { + "default": 0, + "min": 0, + "max": 2**31 - 1, + "step": 1, + "display": "number", + "tooltip": "not implemented yet in backend", + }, + ), + "size": ( + IO.COMBO, + { + "options": ["256x256", "512x512", "1024x1024"], + "default": "1024x1024", + "tooltip": "Image size", + }, + ), + "n": ( + IO.INT, + { + "default": 1, + "min": 1, + "max": 8, + "step": 1, + "display": "number", + "tooltip": "How many images to generate", + }, + ), + "image": ( + IO.IMAGE, + { + "default": None, + "tooltip": "Optional reference image for image editing.", + }, + ), + "mask": ( + IO.MASK, + { + "default": None, + "tooltip": "Optional mask for inpainting (white areas will be replaced)", + }, + ), + }, + "hidden": {"auth_token": "AUTH_TOKEN_COMFY_ORG"}, + } + + RETURN_TYPES = (IO.IMAGE,) + FUNCTION = "api_call" + CATEGORY = "api node/image/openai" + DESCRIPTION = cleandoc(__doc__ or "") + API_NODE = True + + def api_call( + self, + prompt, + seed=0, + image=None, + mask=None, + n=1, + size="1024x1024", + auth_token=None, + ): + model = "dall-e-2" + path = "/proxy/openai/images/generations" + request_class = OpenAIImageGenerationRequest + img_binary = None + + if image is not None and mask is not None: + path = "/proxy/openai/images/edits" + request_class = OpenAIImageEditRequest + + input_tensor = image.squeeze().cpu() + height, width, channels = input_tensor.shape + rgba_tensor = torch.ones(height, width, 4, device="cpu") + rgba_tensor[:, :, :channels] = input_tensor + + if mask.shape[1:] != image.shape[1:-1]: + raise Exception("Mask and Image must be the same size") + rgba_tensor[:, :, 3] = 1 - mask.squeeze().cpu() + + rgba_tensor = downscale_image_tensor(rgba_tensor.unsqueeze(0)).squeeze() + + image_np = (rgba_tensor.numpy() * 255).astype(np.uint8) + img = Image.fromarray(image_np) + img_byte_arr = io.BytesIO() + img.save(img_byte_arr, format="PNG") + img_byte_arr.seek(0) + img_binary = img_byte_arr # .getvalue() + img_binary.name = "image.png" + elif image is not None or mask is not None: + raise Exception("Dall-E 2 image editing requires an image AND a mask") + + # Build the operation + operation = SynchronousOperation( + endpoint=ApiEndpoint( + path=path, + method=HttpMethod.POST, + request_model=request_class, + response_model=OpenAIImageGenerationResponse, + ), + request=request_class( + model=model, + prompt=prompt, + n=n, + size=size, + seed=seed, + ), + files=( + { + "image": img_binary, + } + if img_binary + else None + ), + auth_token=auth_token, + ) + + response = operation.execute() + + img_tensor = validate_and_cast_response(response) + return (img_tensor,) + + +class OpenAIDalle3(ComfyNodeABC): + """ + Generates images synchronously via OpenAI's DALL·E 3 endpoint. + + Uses the proxy at /proxy/openai/images/generations. Returned URLs are short‑lived, + so download or cache results if you need to keep them. + """ + + def __init__(self): + pass + + @classmethod + def INPUT_TYPES(cls) -> InputTypeDict: + return { + "required": { + "prompt": ( + IO.STRING, + { + "multiline": True, + "default": "", + "tooltip": "Text prompt for DALL·E", + }, + ), + }, + "optional": { + "seed": ( + IO.INT, + { + "default": 0, + "min": 0, + "max": 2**31 - 1, + "step": 1, + "display": "number", + "tooltip": "not implemented yet in backend", + }, + ), + "quality": ( + IO.COMBO, + { + "options": ["standard", "hd"], + "default": "standard", + "tooltip": "Image quality", + }, + ), + "style": ( + IO.COMBO, + { + "options": ["natural", "vivid"], + "default": "natural", + "tooltip": "Vivid causes the model to lean towards generating hyper-real and dramatic images. Natural causes the model to produce more natural, less hyper-real looking images.", + }, + ), + "size": ( + IO.COMBO, + { + "options": ["1024x1024", "1024x1792", "1792x1024"], + "default": "1024x1024", + "tooltip": "Image size", + }, + ), + }, + "hidden": {"auth_token": "AUTH_TOKEN_COMFY_ORG"}, + } + + RETURN_TYPES = (IO.IMAGE,) + FUNCTION = "api_call" + CATEGORY = "api node/image/openai" + DESCRIPTION = cleandoc(__doc__ or "") + API_NODE = True + + def api_call( + self, + prompt, + seed=0, + style="natural", + quality="standard", + size="1024x1024", + auth_token=None, + ): + model = "dall-e-3" + + # build the operation + operation = SynchronousOperation( + endpoint=ApiEndpoint( + path="/proxy/openai/images/generations", + method=HttpMethod.POST, + request_model=OpenAIImageGenerationRequest, + response_model=OpenAIImageGenerationResponse, + ), + request=OpenAIImageGenerationRequest( + model=model, + prompt=prompt, + quality=quality, + size=size, + style=style, + seed=seed, + ), + auth_token=auth_token, + ) + + response = operation.execute() + + img_tensor = validate_and_cast_response(response) + return (img_tensor,) + + +class OpenAIGPTImage1(ComfyNodeABC): + """ + Generates images synchronously via OpenAI's GPT Image 1 endpoint. + + Uses the proxy at /proxy/openai/images/generations. Returned URLs are short‑lived, + so download or cache results if you need to keep them. + """ + + def __init__(self): + pass + + @classmethod + def INPUT_TYPES(cls) -> InputTypeDict: + return { + "required": { + "prompt": ( + IO.STRING, + { + "multiline": True, + "default": "", + "tooltip": "Text prompt for GPT Image 1", + }, + ), + }, + "optional": { + "seed": ( + IO.INT, + { + "default": 0, + "min": 0, + "max": 2**31 - 1, + "step": 1, + "display": "number", + "tooltip": "not implemented yet in backend", + }, + ), + "quality": ( + IO.COMBO, + { + "options": ["low", "medium", "high"], + "default": "low", + "tooltip": "Image quality, affects cost and generation time.", + }, + ), + "background": ( + IO.COMBO, + { + "options": ["opaque", "transparent"], + "default": "opaque", + "tooltip": "Return image with or without background", + }, + ), + "size": ( + IO.COMBO, + { + "options": ["auto", "1024x1024", "1024x1536", "1536x1024"], + "default": "auto", + "tooltip": "Image size", + }, + ), + "n": ( + IO.INT, + { + "default": 1, + "min": 1, + "max": 8, + "step": 1, + "display": "number", + "tooltip": "How many images to generate", + }, + ), + "image": ( + IO.IMAGE, + { + "default": None, + "tooltip": "Optional reference image for image editing.", + }, + ), + "mask": ( + IO.MASK, + { + "default": None, + "tooltip": "Optional mask for inpainting (white areas will be replaced)", + }, + ), + }, + "hidden": {"auth_token": "AUTH_TOKEN_COMFY_ORG"}, + } + + RETURN_TYPES = (IO.IMAGE,) + FUNCTION = "api_call" + CATEGORY = "api node/image/openai" + DESCRIPTION = cleandoc(__doc__ or "") + API_NODE = True + + def api_call( + self, + prompt, + seed=0, + quality="low", + background="opaque", + image=None, + mask=None, + n=1, + size="1024x1024", + auth_token=None, + ): + model = "gpt-image-1" + path = "/proxy/openai/images/generations" + request_class = OpenAIImageGenerationRequest + img_binaries = [] + mask_binary = None + files = [] + + if image is not None: + path = "/proxy/openai/images/edits" + request_class = OpenAIImageEditRequest + + batch_size = image.shape[0] + + for i in range(batch_size): + single_image = image[i : i + 1] + scaled_image = downscale_image_tensor(single_image).squeeze() + + image_np = (scaled_image.numpy() * 255).astype(np.uint8) + img = Image.fromarray(image_np) + img_byte_arr = io.BytesIO() + img.save(img_byte_arr, format="PNG") + img_byte_arr.seek(0) + img_binary = img_byte_arr + img_binary.name = f"image_{i}.png" + + img_binaries.append(img_binary) + if batch_size == 1: + files.append(("image", img_binary)) + else: + files.append(("image[]", img_binary)) + + if mask is not None: + if image.shape[0] != 1: + raise Exception("Cannot use a mask with multiple image") + if image is None: + raise Exception("Cannot use a mask without an input image") + if mask.shape[1:] != image.shape[1:-1]: + raise Exception("Mask and Image must be the same size") + batch, height, width = mask.shape + rgba_mask = torch.zeros(height, width, 4, device="cpu") + rgba_mask[:, :, 3] = 1 - mask.squeeze().cpu() + + scaled_mask = downscale_image_tensor(rgba_mask.unsqueeze(0)).squeeze() + + mask_np = (scaled_mask.numpy() * 255).astype(np.uint8) + mask_img = Image.fromarray(mask_np) + mask_img_byte_arr = io.BytesIO() + mask_img.save(mask_img_byte_arr, format="PNG") + mask_img_byte_arr.seek(0) + mask_binary = mask_img_byte_arr + mask_binary.name = "mask.png" + files.append(("mask", mask_binary)) + + # Build the operation + operation = SynchronousOperation( + endpoint=ApiEndpoint( + path=path, + method=HttpMethod.POST, + request_model=request_class, + response_model=OpenAIImageGenerationResponse, + ), + request=request_class( + model=model, + prompt=prompt, + quality=quality, + background=background, + n=n, + seed=seed, + size=size, + ), + files=files if files else None, + auth_token=auth_token, + ) + + response = operation.execute() + + img_tensor = validate_and_cast_response(response) + return (img_tensor,) + + +# A dictionary that contains all nodes you want to export with their names +# NOTE: names should be globally unique +NODE_CLASS_MAPPINGS = { + "OpenAIDalle2": OpenAIDalle2, + "OpenAIDalle3": OpenAIDalle3, + "OpenAIGPTImage1": OpenAIGPTImage1, +} + +# A dictionary that contains the friendly/humanly readable titles for the nodes +NODE_DISPLAY_NAME_MAPPINGS = { + "OpenAIDalle2": "OpenAI DALL·E 2", + "OpenAIDalle3": "OpenAI DALL·E 3", + "OpenAIGPTImage1": "OpenAI GPT Image 1", +} diff --git a/comfy_api_nodes/nodes_recraft.py b/comfy_api_nodes/nodes_recraft.py index e2008de44..be75d3224 100644 --- a/comfy_api_nodes/nodes_recraft.py +++ b/comfy_api_nodes/nodes_recraft.py @@ -18,7 +18,7 @@ from comfy_api_nodes.apis.client import ( HttpMethod, SynchronousOperation, ) -from comfy_api_nodes.nodes_api import ( +from comfy_api_nodes.apinode_utils import ( bytesio_to_image_tensor, download_url_to_bytesio, ) diff --git a/comfy_api_nodes/nodes_runway.py b/comfy_api_nodes/nodes_runway.py index dc6f544cb..083b0998a 100644 --- a/comfy_api_nodes/nodes_runway.py +++ b/comfy_api_nodes/nodes_runway.py @@ -21,7 +21,7 @@ from comfy_api_nodes.apis.client import ( PollingOperation, EmptyRequest, ) -from comfy_api_nodes.nodes_api import ( +from comfy_api_nodes.apinode_utils import ( download_url_to_bytesio, upload_images_to_comfyapi, ) diff --git a/nodes.py b/nodes.py index 61c872c5f..1fc782cd4 100644 --- a/nodes.py +++ b/nodes.py @@ -2262,7 +2262,8 @@ def init_builtin_extra_nodes(): api_nodes_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), "comfy_api_nodes") api_nodes_files = [ - "nodes_api.py", + "nodes_ideogram.py", + "nodes_openai.py", "nodes_minimax.py", "nodes_veo2.py", "nodes_kling.py", From 1bae2de0d8439c84387c5f28b5d52b4d8cb3ca9f Mon Sep 17 00:00:00 2001 From: Christian Byrne Date: Wed, 30 Apr 2025 08:56:32 -0700 Subject: [PATCH 046/121] Add types and doctstrings to utils file (#64) --- comfy_api_nodes/apinode_utils.py | 95 +++++++++++++++++++++++++------- 1 file changed, 76 insertions(+), 19 deletions(-) diff --git a/comfy_api_nodes/apinode_utils.py b/comfy_api_nodes/apinode_utils.py index e20c68a51..5a507a6ac 100644 --- a/comfy_api_nodes/apinode_utils.py +++ b/comfy_api_nodes/apinode_utils.py @@ -1,6 +1,8 @@ import io +import logging from typing import Optional from comfy.utils import common_upscale +from comfy_api.input_impl import VideoFromFile from comfy_api_nodes.apis.client import ( ApiClient, ApiEndpoint, @@ -20,7 +22,27 @@ import base64 import uuid from io import BytesIO -def downscale_image_tensor(image, total_pixels=1536 * 1024): + +def download_url_to_video_output( + video_url: str, timeout: int = None +) -> tuple[VideoFromFile]: + """Downloads a video from a URL and returns a `VIDEO` output. + + Args: + video_url: The URL of the video to download. + + Returns: + A Comfy node `VIDEO` output. + """ + video_io = download_url_to_bytesio(video_url, timeout) + if video_io is None: + error_msg = f"Failed to download video from {video_url}" + logging.error(error_msg) + raise ValueError(error_msg) + return (VideoFromFile(video_io),) + + +def downscale_image_tensor(image, total_pixels=1536 * 1024) -> torch.Tensor: """Downscale input image tensor to roughly the specified total pixels.""" samples = image.movedim(-1, 1) total = int(total_pixels) @@ -34,14 +56,27 @@ def downscale_image_tensor(image, total_pixels=1536 * 1024): s = s.movedim(1, -1) return s -def validate_and_cast_response(response): + +def validate_and_cast_response(response, timeout: int = None) -> torch.Tensor: + """Validates and casts a response to a torch.Tensor. + + Args: + response: The response to validate and cast. + timeout: Request timeout in seconds. Defaults to None (no timeout). + + Returns: + A torch.Tensor representing the image (1, H, W, C). + + Raises: + ValueError: If the response is not valid. + """ # validate raw JSON response data = response.data if not data or len(data) == 0: - raise Exception("No images returned from API endpoint") + raise ValueError("No images returned from API endpoint") # Initialize list to store image tensors - image_tensors = [] + image_tensors: list[torch.Tensor] = [] # Process each image in the data array for image_data in data: @@ -49,16 +84,16 @@ def validate_and_cast_response(response): b64_data = image_data.b64_json if not image_url and not b64_data: - raise Exception("No image was generated in the response") + raise ValueError("No image was generated in the response") if b64_data: img_data = base64.b64decode(b64_data) img = Image.open(io.BytesIO(img_data)) elif image_url: - img_response = requests.get(image_url) + img_response = requests.get(image_url, timeout=timeout) if img_response.status_code != 200: - raise Exception("Failed to download the image") + raise ValueError("Failed to download the image") img = Image.open(io.BytesIO(img_response.content)) img = img.convert("RGBA") @@ -79,31 +114,46 @@ def validate_aspect_ratio( maximum_ratio: float, minimum_ratio_str: str, maximum_ratio_str: str, -): +) -> float: + """Validates and casts an aspect ratio string to a float. + + Args: + aspect_ratio: The aspect ratio string to validate. + minimum_ratio: The minimum aspect ratio. + maximum_ratio: The maximum aspect ratio. + minimum_ratio_str: The minimum aspect ratio string. + maximum_ratio_str: The maximum aspect ratio string. + + Returns: + The validated and cast aspect ratio. + + Raises: + Exception: If the aspect ratio is not valid. + """ # get ratio values numbers = aspect_ratio.split(":") if len(numbers) != 2: - raise Exception( + raise TypeError( f"Aspect ratio must be in the format X:Y, such as 16:9, but was {aspect_ratio}." ) try: numerator = int(numbers[0]) denominator = int(numbers[1]) - except ValueError: - raise Exception( + except ValueError as exc: + raise TypeError( f"Aspect ratio must contain numbers separated by ':', such as 16:9, but was {aspect_ratio}." - ) + ) from exc calculated_ratio = numerator / denominator # if not close to minimum and maximum, check bounds if not math.isclose(calculated_ratio, minimum_ratio) or not math.isclose( calculated_ratio, maximum_ratio ): if calculated_ratio < minimum_ratio: - raise Exception( + raise TypeError( f"Aspect ratio cannot reduce to any less than {minimum_ratio_str} ({minimum_ratio}), but was {aspect_ratio} ({calculated_ratio})." ) elif calculated_ratio > maximum_ratio: - raise Exception( + raise TypeError( f"Aspect ratio cannot reduce to any greater than {maximum_ratio_str} ({maximum_ratio}), but was {aspect_ratio} ({calculated_ratio})." ) return aspect_ratio @@ -149,7 +199,7 @@ def bytesio_to_image_tensor(image_bytesio: BytesIO, mode: str = "RGBA") -> torch return torch.from_numpy(image_array).unsqueeze(0) -def process_image_response(response: requests.Response): +def process_image_response(response: requests.Response) -> torch.Tensor: """Uses content from a Response object and converts it to a torch.Tensor""" return bytesio_to_image_tensor(BytesIO(response.content)) @@ -256,6 +306,16 @@ def tensor_to_data_uri( def upload_images_to_comfyapi( image: torch.Tensor, max_images=8, auth_token=None, mime_type: Optional[str] = None ) -> list[str]: + """ + Uploads images to ComfyUI API and returns download URLs. + To upload multiple images, stack them in the batch dimension first. + + Args: + image: Input torch.Tensor image. + max_images: Maximum number of images to upload. + auth_token: Optional authentication token. + mime_type: Optional MIME type for the image. + """ # if batch, try to upload each file if max_images is greater than 0 idx_image = 0 download_urls: list[str] = [] @@ -295,7 +355,7 @@ def upload_images_to_comfyapi( try: upload_response.raise_for_status() except requests.exceptions.HTTPError as e: - raise Exception(f"Could not upload one or more images: {e}") + raise ValueError(f"Could not upload one or more images: {e}") from e # add download_url to list download_urls.append(response.download_url) @@ -307,6 +367,3 @@ def upload_images_to_comfyapi( if idx_image >= batch_length: break return download_urls - - - From 021e64db6c4985c58c11bec89359742e554a22c5 Mon Sep 17 00:00:00 2001 From: Christian Byrne Date: Wed, 30 Apr 2025 09:09:27 -0700 Subject: [PATCH 047/121] Fix: `PollingOperation` progress bar update progress by absolute value (#65) --- comfy_api_nodes/apis/client.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/comfy_api_nodes/apis/client.py b/comfy_api_nodes/apis/client.py index c9d7ef8f8..93c77931c 100644 --- a/comfy_api_nodes/apis/client.py +++ b/comfy_api_nodes/apis/client.py @@ -117,6 +117,8 @@ T = TypeVar("T", bound=BaseModel) R = TypeVar("R", bound=BaseModel) P = TypeVar("P", bound=BaseModel) # For poll response +PROGRESS_BAR_MAX = 100 + class EmptyRequest(BaseModel): """Base class for empty request bodies. @@ -508,7 +510,7 @@ class PollingOperation(Generic[T, R]): """Poll until the task is complete""" poll_count = 0 if self.progress_extractor: - progress = utils.ProgressBar(100) + progress = utils.ProgressBar(PROGRESS_BAR_MAX) while True: try: @@ -547,7 +549,7 @@ class PollingOperation(Generic[T, R]): if self.progress_extractor: new_progress = self.progress_extractor(response_obj) if new_progress is not None: - progress.update(new_progress) + progress.update_absolute(new_progress, total=PROGRESS_BAR_MAX) if status == TaskStatus.COMPLETED: logging.debug("[DEBUG] Task completed successfully") From d8d215dd3af99e6ff7b807ec8a14c21370af11a9 Mon Sep 17 00:00:00 2001 From: Christian Byrne Date: Wed, 30 Apr 2025 10:39:46 -0700 Subject: [PATCH 048/121] Use common download function in kling nodes module (#67) --- comfy_api_nodes/nodes_kling.py | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/comfy_api_nodes/nodes_kling.py b/comfy_api_nodes/nodes_kling.py index 7d0f2f48a..035c4f382 100644 --- a/comfy_api_nodes/nodes_kling.py +++ b/comfy_api_nodes/nodes_kling.py @@ -27,7 +27,7 @@ from comfy_api_nodes.apis.client import ( ) from comfy_api_nodes.apinode_utils import ( tensor_to_base64_string, - download_url_to_bytesio, + download_url_to_video_output, ) from comfy.comfy_types.node_typing import IO, InputTypeOptions, ComfyNodeABC from comfy_api.input_impl import VideoFromFile @@ -128,16 +128,6 @@ def _get_camera_control_inputs() -> dict[str, tuple[IO, InputTypeOptions]]: } -def download_url_to_video_output(video_url: str) -> tuple[VideoFromFile]: - """Downloads a video from a URL and returns a VIDEO output.""" - video_io = download_url_to_bytesio(video_url) - if video_io is None: - error_msg = f"Failed to download video from {video_url}" - logging.error(error_msg) - raise KlingApiError(error_msg) - return (VideoFromFile(video_io),) - - class KlingNodeABC(ComfyNodeABC): """Base class for Kling nodes.""" From f1eb74fee9300920b434ec761df75bddbfc7feeb Mon Sep 17 00:00:00 2001 From: Christian Byrne Date: Wed, 30 Apr 2025 10:39:57 -0700 Subject: [PATCH 049/121] Fix: Luma video nodes in `api nodes/image` category (#68) --- comfy_api_nodes/nodes_luma.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/comfy_api_nodes/nodes_luma.py b/comfy_api_nodes/nodes_luma.py index 35f28939e..32f8cb392 100644 --- a/comfy_api_nodes/nodes_luma.py +++ b/comfy_api_nodes/nodes_luma.py @@ -413,7 +413,7 @@ class LumaTextToVideoGenerationNode(ComfyNodeABC): DESCRIPTION = cleandoc(__doc__ or "") # Handle potential None value FUNCTION = "api_call" API_NODE = True - CATEGORY = "api node/image/Luma" + CATEGORY = "api node/video/Luma" @classmethod def INPUT_TYPES(s): @@ -531,7 +531,7 @@ class LumaImageToVideoGenerationNode(ComfyNodeABC): DESCRIPTION = cleandoc(__doc__ or "") # Handle potential None value FUNCTION = "api_call" API_NODE = True - CATEGORY = "api node/image/Luma" + CATEGORY = "api node/video/Luma" @classmethod def INPUT_TYPES(s): From 7d3f693c00efefd4dce45a298b06b1d08c81d0d2 Mon Sep 17 00:00:00 2001 From: Christian Byrne Date: Wed, 30 Apr 2025 11:19:57 -0700 Subject: [PATCH 050/121] Set request type explicitly (#66) --- comfy_api_nodes/apis/client.py | 93 +++++++++++++++++++++++---------- comfy_api_nodes/nodes_openai.py | 1 + 2 files changed, 65 insertions(+), 29 deletions(-) diff --git a/comfy_api_nodes/apis/client.py b/comfy_api_nodes/apis/client.py index 93c77931c..195144dd9 100644 --- a/comfy_api_nodes/apis/client.py +++ b/comfy_api_nodes/apis/client.py @@ -165,6 +165,45 @@ class ApiClient: self.timeout = timeout self.verify_ssl = verify_ssl + def _create_json_payload_args( + self, + data: Optional[Dict[str, Any]] = None, + headers: Optional[Dict[str, str]] = None, + ) -> Dict[str, Any]: + return { + "json": data, + "headers": headers, + } + + def _create_form_data_args( + self, + data: Dict[str, Any], + files: Dict[str, Any], + headers: Optional[Dict[str, str]] = None, + ) -> Dict[str, Any]: + if headers: + del headers["Content-Type"] + + return { + "data": data, + "files": files, + "headers": headers, + } + + def _create_urlencoded_form_data_args( + self, + data: Dict[str, Any], + headers: Optional[Dict[str, str]] = None, + ) -> Dict[str, Any]: + headers = headers or {} + headers["Content-Type"] = "application/x-www-form-urlencoded" + + return { + "data": data, + "headers": headers, + } + + def get_headers(self) -> Dict[str, str]: """Get headers for API requests, including authentication if available""" headers = {"Content-Type": "application/json", "Accept": "application/json"} @@ -179,9 +218,10 @@ class ApiClient: method: str, path: str, params: Optional[Dict[str, Any]] = None, - json: Optional[Dict[str, Any]] = None, + data: Optional[Dict[str, Any]] = None, files: Optional[Dict[str, Any]] = None, headers: Optional[Dict[str, str]] = None, + content_type: str = "application/json", ) -> Dict[str, Any]: """ Make an HTTP request to the API @@ -190,9 +230,10 @@ class ApiClient: method: HTTP method (GET, POST, etc.) path: API endpoint path (will be joined with base_url) params: Query parameters - json: JSON body data + data: body data files: Files to upload headers: Additional headers + content_type: Content type of the request. Defaults to application/json. Returns: Parsed JSON response @@ -214,34 +255,25 @@ class ApiClient: logging.debug(f"[DEBUG] Request Headers: {request_headers}") logging.debug(f"[DEBUG] Files: {files}") logging.debug(f"[DEBUG] Params: {params}") - logging.debug(f"[DEBUG] Json: {json}") + logging.debug(f"[DEBUG] Data: {data}") + + match content_type: + case "application/x-www-form-urlencoded": + payload_args = self._create_urlencoded_form_data_args(data, request_headers) + case "multipart/form-data": + payload_args = self._create_form_data_args(data, files, request_headers) + case _: + payload_args = self._create_json_payload_args(data, request_headers) try: - # If files are present, use data parameter instead of json - if files: - form_data = {} - if json: - form_data.update(json) - response = requests.request( - method=method, - url=url, - params=params, - data=form_data, # Use data instead of json - files=files, - headers=request_headers, - timeout=self.timeout, - verify=self.verify_ssl, - ) - else: - response = requests.request( - method=method, - url=url, - params=params, - json=json, - headers=request_headers, - timeout=self.timeout, - verify=self.verify_ssl, - ) + response = requests.request( + method=method, + url=url, + params=params, + timeout=self.timeout, + verify=self.verify_ssl, + **payload_args, + ) # Raise exception for error status codes response.raise_for_status() @@ -367,6 +399,7 @@ class SynchronousOperation(Generic[T, R]): auth_token: Optional[str] = None, timeout: float = 604800.0, verify_ssl: bool = True, + content_type: str = "application/json", ): self.endpoint = endpoint self.request = request @@ -377,6 +410,7 @@ class SynchronousOperation(Generic[T, R]): self.timeout = timeout self.verify_ssl = verify_ssl self.files = files + self.content_type = content_type def execute(self, client: Optional[ApiClient] = None) -> R: """Execute the API operation using the provided client or create one""" @@ -408,9 +442,10 @@ class SynchronousOperation(Generic[T, R]): resp = client.request( method=self.endpoint.method.value, path=self.endpoint.path, - json=request_dict, + data=request_dict, params=self.endpoint.query_params, files=self.files, + content_type=self.content_type, ) # Debug log for response diff --git a/comfy_api_nodes/nodes_openai.py b/comfy_api_nodes/nodes_openai.py index 17364bee2..6296f7551 100644 --- a/comfy_api_nodes/nodes_openai.py +++ b/comfy_api_nodes/nodes_openai.py @@ -458,6 +458,7 @@ class OpenAIGPTImage1(ComfyNodeABC): size=size, ), files=files if files else None, + content_type="multipart/form-data", auth_token=auth_token, ) From 3291883d864ed6637b2e03d0512cd1aea7b969ed Mon Sep 17 00:00:00 2001 From: Christian Byrne Date: Wed, 30 Apr 2025 11:20:19 -0700 Subject: [PATCH 051/121] Add `control_after_generate` to all seed inputs (#69) --- comfy_api_nodes/nodes_ideogram.py | 3 +++ comfy_api_nodes/nodes_openai.py | 3 +++ comfy_api_nodes/nodes_runway.py | 2 +- comfy_api_nodes/nodes_veo2.py | 1 + 4 files changed, 8 insertions(+), 1 deletion(-) diff --git a/comfy_api_nodes/nodes_ideogram.py b/comfy_api_nodes/nodes_ideogram.py index 088039d85..653e797a0 100644 --- a/comfy_api_nodes/nodes_ideogram.py +++ b/comfy_api_nodes/nodes_ideogram.py @@ -176,6 +176,7 @@ class IdeogramV1(ComfyNodeABC): "min": 0, "max": 2147483647, "step": 1, + "control_after_generate": True, "display": "number", }, ), @@ -313,6 +314,7 @@ class IdeogramV2(ComfyNodeABC): "min": 0, "max": 2147483647, "step": 1, + "control_after_generate": True, "display": "number", }, ), @@ -468,6 +470,7 @@ class IdeogramV3(ComfyNodeABC): "min": 0, "max": 2147483647, "step": 1, + "control_after_generate": True, "display": "number", }, ), diff --git a/comfy_api_nodes/nodes_openai.py b/comfy_api_nodes/nodes_openai.py index 6296f7551..a734e8a95 100644 --- a/comfy_api_nodes/nodes_openai.py +++ b/comfy_api_nodes/nodes_openai.py @@ -57,6 +57,7 @@ class OpenAIDalle2(ComfyNodeABC): "max": 2**31 - 1, "step": 1, "display": "number", + "control_after_generate": True, "tooltip": "not implemented yet in backend", }, ), @@ -207,6 +208,7 @@ class OpenAIDalle3(ComfyNodeABC): "max": 2**31 - 1, "step": 1, "display": "number", + "control_after_generate": True, "tooltip": "not implemented yet in backend", }, ), @@ -313,6 +315,7 @@ class OpenAIGPTImage1(ComfyNodeABC): "max": 2**31 - 1, "step": 1, "display": "number", + "control_after_generate": True, "tooltip": "not implemented yet in backend", }, ), diff --git a/comfy_api_nodes/nodes_runway.py b/comfy_api_nodes/nodes_runway.py index 083b0998a..e472debd0 100644 --- a/comfy_api_nodes/nodes_runway.py +++ b/comfy_api_nodes/nodes_runway.py @@ -128,7 +128,7 @@ class RunwayImageToVideoNode(ComfyNodeABC): IO.COMBO, RunwayImageToVideoRequest, "ratio", enum_type=AspectRatio ), "seed": model_field_to_node_input( - IO.INT, RunwayImageToVideoRequest, "seed" + IO.INT, RunwayImageToVideoRequest, "seed", control_after_generate=True ), }, "optional": { diff --git a/comfy_api_nodes/nodes_veo2.py b/comfy_api_nodes/nodes_veo2.py index 206480c89..ae3329b8d 100644 --- a/comfy_api_nodes/nodes_veo2.py +++ b/comfy_api_nodes/nodes_veo2.py @@ -108,6 +108,7 @@ class VeoVideoGenerationNode(ComfyNodeABC): "max": 0xFFFFFFFF, "step": 1, "display": "number", + "control_after_generate": True, "tooltip": "Seed for video generation (0 for random)", }, ), From 1af6fd53bb2ab8afb4fee751c338ba1c29b62eef Mon Sep 17 00:00:00 2001 From: Christian Byrne Date: Wed, 30 Apr 2025 11:52:37 -0700 Subject: [PATCH 052/121] Fix bug: deleting `Content-Type` when property does not exist (#73) --- comfy_api_nodes/apis/client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/comfy_api_nodes/apis/client.py b/comfy_api_nodes/apis/client.py index 195144dd9..bbb70519a 100644 --- a/comfy_api_nodes/apis/client.py +++ b/comfy_api_nodes/apis/client.py @@ -181,7 +181,7 @@ class ApiClient: files: Dict[str, Any], headers: Optional[Dict[str, str]] = None, ) -> Dict[str, Any]: - if headers: + if headers and "Content-Type" in headers: del headers["Content-Type"] return { @@ -571,7 +571,7 @@ class PollingOperation(Generic[T, R]): method=self.poll_endpoint.method.value, path=self.poll_endpoint.path, params=self.poll_endpoint.query_params, - json=request_dict, + data=request_dict, ) # Parse response From ae436ac3b836cd95b31c8d5206e411774bcc94f4 Mon Sep 17 00:00:00 2001 From: Jedrzej Kosinski Date: Wed, 30 Apr 2025 14:12:21 -0500 Subject: [PATCH 053/121] Add preview to Save SVG node (#74) --- comfy_api_nodes/nodes_recraft.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/comfy_api_nodes/nodes_recraft.py b/comfy_api_nodes/nodes_recraft.py index be75d3224..fd93ea7d5 100644 --- a/comfy_api_nodes/nodes_recraft.py +++ b/comfy_api_nodes/nodes_recraft.py @@ -91,7 +91,7 @@ class SaveSVGNode: "type": self.type }) counter += 1 - return (None,) + return { "ui": { "images": results } } class RecraftColorRGBNode: From 3404de63a8b0edd589e1235fc3e35bb6e950958c Mon Sep 17 00:00:00 2001 From: thot experiment <94414189+thot-experiment@users.noreply.github.com> Date: Wed, 30 Apr 2025 13:14:02 -0700 Subject: [PATCH 054/121] change default poll interval (#76), rework veo2 --- comfy_api_nodes/apis/client.py | 2 +- comfy_api_nodes/nodes_veo2.py | 178 +++++++++++++++------------------ 2 files changed, 84 insertions(+), 96 deletions(-) diff --git a/comfy_api_nodes/apis/client.py b/comfy_api_nodes/apis/client.py index bbb70519a..22e011f47 100644 --- a/comfy_api_nodes/apis/client.py +++ b/comfy_api_nodes/apis/client.py @@ -496,7 +496,7 @@ class PollingOperation(Generic[T, R]): request: Optional[T] = None, api_base: str | None = None, auth_token: Optional[str] = None, - poll_interval: float = 1.0, + poll_interval: float = 5.0, ): self.poll_endpoint = poll_endpoint self.request = request diff --git a/comfy_api_nodes/nodes_veo2.py b/comfy_api_nodes/nodes_veo2.py index ae3329b8d..9233944b5 100644 --- a/comfy_api_nodes/nodes_veo2.py +++ b/comfy_api_nodes/nodes_veo2.py @@ -2,13 +2,9 @@ import io import logging import base64 import requests -import math import torch -import numpy as np -from PIL import Image from comfy.comfy_types.node_typing import IO, ComfyNodeABC -from comfy.utils import common_upscale from comfy_api.input_impl.video_types import VideoFromFile from comfy_api_nodes.apis import ( Veo2GenVidRequest, @@ -20,21 +16,20 @@ from comfy_api_nodes.apis.client import ( ApiEndpoint, HttpMethod, SynchronousOperation, + PollingOperation, ) -def downscale_input(image, total_pixels=1536*1024): - samples = image.movedim(-1,1) - # Downscaling input images to roughly the same size as the outputs - total = int(total_pixels) - scale_by = math.sqrt(total / (samples.shape[3] * samples.shape[2])) - if scale_by >= 1: - return image - width = round(samples.shape[3] * scale_by) - height = round(samples.shape[2] * scale_by) +from comfy_api_nodes.apinode_utils import ( + downscale_image_tensor, + tensor_to_base64_string +) - s = common_upscale(samples, width, height, "lanczos", "disabled") - s = s.movedim(1,-1) - return s +def convert_image_to_base64(image: torch.Tensor): + if image is None: + return None + + scaled_image = downscale_image_tensor(image, total_pixels=2048*2048) + return tensor_to_base64_string(scaled_image) class VeoVideoGenerationNode(ComfyNodeABC): """ @@ -128,25 +123,6 @@ class VeoVideoGenerationNode(ComfyNodeABC): DESCRIPTION = "Generates videos from text prompts using Google's Veo API" API_NODE = True - def _convert_image_to_base64(self, image: torch.Tensor): - if image is None: - return None - - scaled_image = downscale_input(image, total_pixels=2048*2048) - - # Remove batch dimension if present - if len(scaled_image.shape) > 3: - scaled_image = scaled_image[0] - - # Convert to numpy array and then to PIL Image - image_np = (scaled_image.numpy() * 255).astype(np.uint8) - img = Image.fromarray(image_np) - - # Convert to base64 - buffer = io.BytesIO() - img.save(buffer, format="PNG") - return base64.b64encode(buffer.getvalue()).decode('utf-8') - def generate_video( self, prompt, @@ -168,7 +144,7 @@ class VeoVideoGenerationNode(ComfyNodeABC): # Add image if provided if image is not None: - image_base64 = self._convert_image_to_base64(image) + image_base64 = convert_image_to_base64(image) if image_base64: instance["image"] = { "bytesBase64Encoded": image_base64, @@ -211,67 +187,79 @@ class VeoVideoGenerationNode(ComfyNodeABC): logging.info(f"Veo generation started with operation name: {operation_name}") - # Poll until operation is complete + # Define status extractor function + def status_extractor(response): + # Only return "completed" if the operation is done, regardless of success or failure + # We'll check for errors after polling completes + return "completed" if response.done else "pending" + + # Define progress extractor function + def progress_extractor(response): + # Could be enhanced if the API provides progress information + return None + + # Define the polling operation + poll_operation = PollingOperation( + poll_endpoint=ApiEndpoint( + path="/proxy/veo/poll", + method=HttpMethod.POST, + request_model=Veo2GenVidPollRequest, + response_model=Veo2GenVidPollResponse + ), + completed_statuses=["completed"], + failed_statuses=[], # No failed statuses, we'll handle errors after polling + status_extractor=status_extractor, + progress_extractor=progress_extractor, + request=Veo2GenVidPollRequest( + operationName=operation_name + ), + auth_token=auth_token, + poll_interval=5.0 + ) + + # Execute the polling operation + poll_response = poll_operation.execute() + + # Now check for errors in the final response + # Check for error in poll response + if hasattr(poll_response, 'error') and poll_response.error: + error_message = f"Veo API error: {poll_response.error.message} (code: {poll_response.error.code})" + logging.error(error_message) + raise Exception(error_message) + + # Check for RAI filtered content + if (hasattr(poll_response.response, 'raiMediaFilteredCount') and + poll_response.response.raiMediaFilteredCount > 0): + + # Extract reason message if available + if (hasattr(poll_response.response, 'raiMediaFilteredReasons') and + poll_response.response.raiMediaFilteredReasons): + reason = poll_response.response.raiMediaFilteredReasons[0] + error_message = f"Content filtered by Google's Responsible AI practices: {reason} ({poll_response.response.raiMediaFilteredCount} videos filtered.)" + else: + error_message = f"Content filtered by Google's Responsible AI practices ({poll_response.response.raiMediaFilteredCount} videos filtered.)" + + logging.error(error_message) + raise Exception(error_message) + + # Extract video data video_data = None - while True: - poll_operation = SynchronousOperation( - endpoint=ApiEndpoint( - path="/proxy/veo/poll", - method=HttpMethod.POST, - request_model=Veo2GenVidPollRequest, - response_model=Veo2GenVidPollResponse - ), - request=Veo2GenVidPollRequest( - operationName=operation_name - ), - auth_token=auth_token - ) + if poll_response.response and hasattr(poll_response.response, 'videos') and poll_response.response.videos and len(poll_response.response.videos) > 0: + video = poll_response.response.videos[0] - poll_response = poll_operation.execute() - - # Check for error in poll response - if hasattr(poll_response, 'error') and poll_response.error: - error_message = f"Veo API error: {poll_response.error.message} (code: {poll_response.error.code})" - logging.error(error_message) - raise Exception(error_message) - - if poll_response.done: - # Check for RAI filtered content - if (hasattr(poll_response.response, 'raiMediaFilteredCount') and - poll_response.response.raiMediaFilteredCount > 0): - - # Extract reason message if available - if (hasattr(poll_response.response, 'raiMediaFilteredReasons') and - poll_response.response.raiMediaFilteredReasons): - reason = poll_response.response.raiMediaFilteredReasons[0] - error_message = f"Content filtered by Google's Responsible AI practices: {reason} ({poll_response.response.raiMediaFilteredCount} videos filtered.)" - - logging.error(error_message) - raise Exception(error_message) - - # Process successful response - if poll_response.response and hasattr(poll_response.response, 'videos') and poll_response.response.videos and len(poll_response.response.videos) > 0: - video = poll_response.response.videos[0] - - # Check if video is provided as base64 or URL - if hasattr(video, 'bytesBase64Encoded') and video.bytesBase64Encoded: - # Decode base64 string to bytes - video_data = base64.b64decode(video.bytesBase64Encoded) - break - elif hasattr(video, 'gcsUri') and video.gcsUri: - # Download from URL - video_url = video.gcsUri - video_response = requests.get(video_url) - video_data = video_response.content - break - else: - raise Exception("Video returned but no data or URL was provided") - else: - raise Exception("Video generation completed but no video was returned") - - # Wait before polling again - import time - time.sleep(5) + # Check if video is provided as base64 or URL + if hasattr(video, 'bytesBase64Encoded') and video.bytesBase64Encoded: + # Decode base64 string to bytes + video_data = base64.b64decode(video.bytesBase64Encoded) + elif hasattr(video, 'gcsUri') and video.gcsUri: + # Download from URL + video_url = video.gcsUri + video_response = requests.get(video_url) + video_data = video_response.content + else: + raise Exception("Video returned but no data or URL was provided") + else: + raise Exception("Video generation completed but no video was returned") if not video_data: raise Exception("No video data was returned") From 983a1653f380c3bcb88f4ad6d754bf3cb3fa832f Mon Sep 17 00:00:00 2001 From: Christian Byrne Date: Wed, 30 Apr 2025 13:42:20 -0700 Subject: [PATCH 055/121] Add Pixverse and updated Kling types (#75) --- comfy_api_nodes/apis/__init__.py | 657 ++++++++++--------------------- comfy_api_nodes/nodes_kling.py | 10 +- 2 files changed, 207 insertions(+), 460 deletions(-) diff --git a/comfy_api_nodes/apis/__init__.py b/comfy_api_nodes/apis/__init__.py index f6da69112..77e6d34e4 100644 --- a/comfy_api_nodes/apis/__init__.py +++ b/comfy_api_nodes/apis/__init__.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: filtered-openapi.yaml -# timestamp: 2025-04-29T23:44:54+00:00 +# timestamp: 2025-04-30T19:05:50+00:00 from __future__ import annotations @@ -150,43 +150,6 @@ class IdeogramGenerateResponse(BaseModel): ) -class Duration(str, Enum): - field_5 = '5' - field_10 = '10' - - -class Mode(str, Enum): - std = 'std' - pro = 'pro' - - -class ModelName(str, Enum): - kling_v1 = 'kling-v1' - kling_v1_5 = 'kling-v1-5' - kling_v1_6 = 'kling-v1-6' - - -class KlingDualCharacterEffectInput(BaseModel): - duration: Duration = Field( - ..., - description='Video Length in seconds. Both 5 and 10-second videos are supported.', - ) - images: List[str] = Field( - ..., - description='Reference Image Group. Must contain exactly 2 images. First image will be positioned on left side, second on right side of the composite. Each image follows the same requirements as single image effects.', - max_length=2, - min_length=2, - ) - mode: Optional[Mode] = Field( - 'std', - description='Video generation mode. std (Standard Mode) is cost-effective, pro (Professional Mode) generates videos with longer duration and higher quality.', - ) - model_name: Optional[ModelName] = Field( - 'kling-v1', - description='Model Name. Can be kling-v1, kling-v1-5, or kling-v1-6.', - ) - - class KlingErrorResponse(BaseModel): code: int = Field( ..., @@ -259,6 +222,11 @@ class CameraControl(BaseModel): ) +class Duration(str, Enum): + field_5 = '5' + field_10 = '10' + + class Trajectory(BaseModel): x: Optional[int] = Field( None, @@ -278,6 +246,18 @@ class DynamicMask(BaseModel): trajectories: Optional[List[Trajectory]] = None +class Mode(str, Enum): + std = 'std' + pro = 'pro' + + +class ModelName(str, Enum): + kling_v1 = 'kling-v1' + kling_v1_5 = 'kling-v1-5' + kling_v1_6 = 'kling-v1-6' + kling_v2_master = 'kling-v2-master' + + class KlingImage2VideoRequest(BaseModel): aspect_ratio: Optional[AspectRatio] = '16:9' callback_url: Optional[AnyUrl] = Field( @@ -362,247 +342,6 @@ class KlingImage2VideoResponse(BaseModel): request_id: Optional[str] = Field(None, description='Request ID') -class AspectRatio1(str, Enum): - field_16_9 = '16:9' - field_9_16 = '9:16' - field_1_1 = '1:1' - field_4_3 = '4:3' - field_3_4 = '3:4' - field_3_2 = '3:2' - field_2_3 = '2:3' - field_21_9 = '21:9' - - -class ImageReference(str, Enum): - subject = 'subject' - face = 'face' - - -class ModelName2(str, Enum): - kling_v1 = 'kling-v1' - kling_v1_5 = 'kling-v1-5' - - -class KlingImageGenerationsRequest(BaseModel): - aspect_ratio: Optional[AspectRatio1] = Field( - '16:9', description='Aspect ratio of the generated images' - ) - callback_url: Optional[AnyUrl] = Field( - None, description='The callback notification address' - ) - human_fidelity: Optional[float] = Field( - 0.45, description='Subject reference similarity', ge=0.0, le=1.0 - ) - image: Optional[str] = Field( - None, description='Reference Image - Base64 encoded string or image URL' - ) - image_fidelity: Optional[float] = Field( - 0.5, description='Reference intensity for user-uploaded images', ge=0.0, le=1.0 - ) - image_reference: Optional[ImageReference] = Field( - None, description='Image reference type' - ) - model_name: Optional[ModelName2] = Field('kling-v1', description='Model Name') - n: Optional[int] = Field(1, description='Number of generated images', ge=1, le=9) - negative_prompt: Optional[str] = Field( - None, description='Negative text prompt', max_length=200 - ) - prompt: str = Field(..., description='Positive text prompt', max_length=500) - - -class Image(BaseModel): - index: Optional[int] = Field(None, description='Image Number (0-9)') - url: Optional[AnyUrl] = Field(None, description='URL for generated image') - - -class TaskResult1(BaseModel): - images: Optional[List[Image]] = None - - -class Data1(BaseModel): - created_at: Optional[int] = Field(None, description='Task creation time') - task_id: Optional[str] = Field(None, description='Task ID') - task_result: Optional[TaskResult1] = None - task_status: Optional[TaskStatus] = None - task_status_msg: Optional[str] = Field(None, description='Task status information') - updated_at: Optional[int] = Field(None, description='Task update time') - - -class KlingImageGenerationsResponse(BaseModel): - code: Optional[int] = Field(None, description='Error code') - data: Optional[Data1] = None - message: Optional[str] = Field(None, description='Error message') - request_id: Optional[str] = Field(None, description='Request ID') - - -class AudioType(str, Enum): - file = 'file' - url = 'url' - - -class Mode2(str, Enum): - text2video = 'text2video' - audio2video = 'audio2video' - - -class VoiceLanguage(str, Enum): - zh = 'zh' - en = 'en' - - -class Input(BaseModel): - audio_file: Optional[str] = Field( - None, - description='Local Path of Audio File. Supported formats: .mp3/.wav/.m4a/.aac, maximum file size of 5MB. Base64 code.', - ) - audio_type: Optional[AudioType] = Field( - None, - description='Method of Transmitting Audio Files for Lip-Sync. Required when mode is audio2video.', - ) - audio_url: Optional[AnyUrl] = Field( - None, - description='Audio File Download URL. Supported formats: .mp3/.wav/.m4a/.aac, maximum file size of 5MB.', - ) - mode: Mode2 = Field( - ..., - description='Video Generation Mode. text2video: Text-to-video generation mode; audio2video: Audio-to-video generation mode', - ) - text: Optional[str] = Field( - None, - description='Text Content for Lip-Sync Video Generation. Required when mode is text2video. Maximum length is 120 characters.', - ) - video_id: Optional[str] = Field( - None, - description='The ID of the video generated by Kling AI. Only supports 5-second and 10-second videos generated within the last 30 days.', - ) - video_url: Optional[AnyUrl] = Field( - None, - description='Get link for uploaded video. Video files support .mp4/.mov, file size does not exceed 100MB, video length between 2-10s.', - ) - voice_id: Optional[str] = Field( - None, - description='Voice ID. Required when mode is text2video. The system offers a variety of voice options to choose from.', - ) - voice_language: Optional[VoiceLanguage] = Field( - 'zh', description='The voice language corresponds to the Voice ID.' - ) - voice_speed: Optional[float] = Field( - 1, - description='Speech Rate. Valid range: 0.8~2.0, accurate to one decimal place.', - ge=0.8, - le=2.0, - ) - - -class KlingLipSyncRequest(BaseModel): - callback_url: Optional[AnyUrl] = Field( - None, - description='The callback notification address. Server will notify when the task status changes.', - ) - input: Input - - -class TaskResult2(BaseModel): - videos: Optional[List[Video]] = None - - -class Data2(BaseModel): - created_at: Optional[int] = Field(None, description='Task creation time') - task_id: Optional[str] = Field(None, description='Task ID') - task_info: Optional[TaskInfo] = None - task_result: Optional[TaskResult2] = None - task_status: Optional[TaskStatus] = None - updated_at: Optional[int] = Field(None, description='Task update time') - - -class KlingLipSyncResponse(BaseModel): - code: Optional[int] = Field(None, description='Error code') - data: Optional[Data2] = None - message: Optional[str] = Field(None, description='Error message') - request_id: Optional[str] = Field(None, description='Request ID') - - -class ResourcePackType(str, Enum): - decreasing_total = 'decreasing_total' - constant_period = 'constant_period' - - -class Status(str, Enum): - toBeOnline = 'toBeOnline' - online = 'online' - expired = 'expired' - runOut = 'runOut' - - -class ResourcePackSubscribeInfo(BaseModel): - effective_time: Optional[int] = Field( - None, description='Effective time, Unix timestamp in ms' - ) - invalid_time: Optional[int] = Field( - None, description='Expiration time, Unix timestamp in ms' - ) - purchase_time: Optional[int] = Field( - None, description='Purchase time, Unix timestamp in ms' - ) - remaining_quantity: Optional[float] = Field( - None, description='Remaining quantity (updated with a 12-hour delay)' - ) - resource_pack_id: Optional[str] = Field(None, description='Resource package ID') - resource_pack_name: Optional[str] = Field(None, description='Resource package name') - resource_pack_type: Optional[ResourcePackType] = Field( - None, - description='Resource package type (decreasing_total=decreasing total, constant_period=constant periodicity)', - ) - status: Optional[Status] = Field(None, description='Resource Package Status') - total_quantity: Optional[float] = Field(None, description='Total quantity') - - -class Data3(BaseModel): - code: Optional[int] = Field(None, description='Error code; 0 indicates success') - msg: Optional[str] = Field(None, description='Error information') - resource_pack_subscribe_infos: Optional[List[ResourcePackSubscribeInfo]] = Field( - None, description='Resource package list' - ) - - -class KlingResourcePackageResponse(BaseModel): - code: Optional[int] = Field(None, description='Error code; 0 indicates success') - data: Optional[Data3] = None - message: Optional[str] = Field(None, description='Error information') - request_id: Optional[str] = Field( - None, - description='Request ID, generated by the system, used to track requests and troubleshoot problems', - ) - - -class Duration2(str, Enum): - field_5 = '5' - - -class ModelName3(str, Enum): - kling_v1_6 = 'kling-v1-6' - - -class KlingSingleImageEffectInput(BaseModel): - duration: Duration2 = Field( - ..., description='Video Length in seconds. Only 5-second videos are supported.' - ) - image: str = Field( - ..., - description='Reference Image. URL or Base64 encoded string (without data:image prefix). File size cannot exceed 10MB, resolution not less than 300*300px, aspect ratio between 1:2.5 ~ 2.5:1.', - ) - model_name: ModelName3 = Field( - ..., - description='Model Name. Only kling-v1-6 is supported for single image effects.', - ) - - -class AspectRatio2(str, Enum): - field_16_9 = '16:9' - field_9_16 = '9:16' - field_1_1 = '1:1' - - class Config1(BaseModel): horizontal: Optional[float] = Field(None, ge=-10.0, le=10.0) pan: Optional[float] = Field(None, ge=-10.0, le=10.0) @@ -617,23 +356,14 @@ class CameraControl1(BaseModel): type: Optional[Type] = Field(None, description='Predefined camera movements type') -class Duration3(str, Enum): - field_5 = '5' - field_10 = '10' - - -class Mode3(str, Enum): - std = 'std' - pro = 'pro' - - -class ModelName4(str, Enum): +class ModelName1(str, Enum): kling_v1 = 'kling-v1' kling_v1_6 = 'kling-v1-6' + kling_v2_master = 'kling-v2-master' class KlingText2VideoRequest(BaseModel): - aspect_ratio: Optional[AspectRatio2] = '16:9' + aspect_ratio: Optional[AspectRatio] = '16:9' callback_url: Optional[AnyUrl] = Field( None, description='The callback notification address' ) @@ -641,10 +371,10 @@ class KlingText2VideoRequest(BaseModel): cfg_scale: Optional[float] = Field( 0.5, description='Flexibility in video generation', ge=0.0, le=1.0 ) - duration: Optional[Duration3] = '5' + duration: Optional[Duration] = '5' external_task_id: Optional[str] = Field(None, description='Customized Task ID') - mode: Optional[Mode3] = Field('std', description='Video generation mode') - model_name: Optional[ModelName4] = Field('kling-v1', description='Model Name') + mode: Optional[Mode] = Field('std', description='Video generation mode') + model_name: Optional[ModelName1] = Field('kling-v1', description='Model Name') negative_prompt: Optional[str] = Field( None, description='Negative text prompt', max_length=2500 ) @@ -653,168 +383,22 @@ class KlingText2VideoRequest(BaseModel): ) -class TaskResult3(BaseModel): +class TaskResult1(BaseModel): videos: Optional[List[Video]] = None -class Data4(BaseModel): +class Data1(BaseModel): created_at: Optional[int] = Field(None, description='Task creation time') task_id: Optional[str] = Field(None, description='Task ID') task_info: Optional[TaskInfo] = None - task_result: Optional[TaskResult3] = None + task_result: Optional[TaskResult1] = None task_status: Optional[TaskStatus] = None updated_at: Optional[int] = Field(None, description='Task update time') class KlingText2VideoResponse(BaseModel): code: Optional[int] = Field(None, description='Error code') - data: Optional[Data4] = None - message: Optional[str] = Field(None, description='Error message') - request_id: Optional[str] = Field(None, description='Request ID') - - -class KlingVideoEffectsInput( - RootModel[Union[KlingSingleImageEffectInput, KlingDualCharacterEffectInput]] -): - root: Union[KlingSingleImageEffectInput, KlingDualCharacterEffectInput] - - -class EffectScene(str, Enum): - bloombloom = 'bloombloom' - dizzydizzy = 'dizzydizzy' - fuzzyfuzzy = 'fuzzyfuzzy' - squish = 'squish' - expansion = 'expansion' - hug = 'hug' - kiss = 'kiss' - heart_gesture = 'heart_gesture' - - -class KlingVideoEffectsRequest(BaseModel): - callback_url: Optional[AnyUrl] = Field( - None, - description='The callback notification address for the result of this task.', - ) - effect_scene: EffectScene = Field( - ..., - description='Scene Name. Single Image Effects (bloombloom, dizzydizzy, fuzzyfuzzy, squish, expansion) or Dual-character Effects (hug, kiss, heart_gesture).', - ) - external_task_id: Optional[str] = Field( - None, - description='Customized Task ID. Must be unique within a single user account.', - ) - input: Optional[KlingVideoEffectsInput] = None - - -class TaskResult4(BaseModel): - videos: Optional[List[Video]] = None - - -class Data5(BaseModel): - created_at: Optional[int] = Field(None, description='Task creation time') - task_id: Optional[str] = Field(None, description='Task ID') - task_info: Optional[TaskInfo] = None - task_result: Optional[TaskResult4] = None - task_status: Optional[TaskStatus] = None - updated_at: Optional[int] = Field(None, description='Task update time') - - -class KlingVideoEffectsResponse(BaseModel): - code: Optional[int] = Field(None, description='Error code') - data: Optional[Data5] = None - message: Optional[str] = Field(None, description='Error message') - request_id: Optional[str] = Field(None, description='Request ID') - - -class KlingVideoExtendRequest(BaseModel): - callback_url: Optional[AnyUrl] = Field( - None, - description='The callback notification address. Server will notify when the task status changes.', - ) - cfg_scale: Optional[float] = Field( - 0.5, - description="Flexibility in video generation. The higher the value, the lower the model's flexibility and the stronger the relevance to the user's prompt.", - ge=0.0, - le=1.0, - ) - negative_prompt: Optional[str] = Field( - None, - description='Negative text prompt for elements to avoid in the extended video', - max_length=2500, - ) - prompt: Optional[str] = Field( - None, - description='Positive text prompt for guiding the video extension', - max_length=2500, - ) - video_id: Optional[str] = Field( - None, - description='The ID of the video to be extended. Supports videos generated by text-to-video, image-to-video, and previous video extension operations. Cannot exceed 3 minutes total duration after extension.', - ) - - -class TaskResult5(BaseModel): - videos: Optional[List[Video]] = None - - -class Data6(BaseModel): - created_at: Optional[int] = Field(None, description='Task creation time') - task_id: Optional[str] = Field(None, description='Task ID') - task_info: Optional[TaskInfo] = None - task_result: Optional[TaskResult5] = None - task_status: Optional[TaskStatus] = None - updated_at: Optional[int] = Field(None, description='Task update time') - - -class KlingVideoExtendResponse(BaseModel): - code: Optional[int] = Field(None, description='Error code') - data: Optional[Data6] = None - message: Optional[str] = Field(None, description='Error message') - request_id: Optional[str] = Field(None, description='Request ID') - - -class ModelName5(str, Enum): - kolors_virtual_try_on_v1 = 'kolors-virtual-try-on-v1' - kolors_virtual_try_on_v1_5 = 'kolors-virtual-try-on-v1-5' - - -class KlingVirtualTryOnRequest(BaseModel): - callback_url: Optional[AnyUrl] = Field( - None, description='The callback notification address' - ) - cloth_image: Optional[str] = Field( - None, - description='Reference clothing image - Base64 encoded string or image URL', - ) - human_image: str = Field( - ..., description='Reference human image - Base64 encoded string or image URL' - ) - model_name: Optional[ModelName5] = Field( - 'kolors-virtual-try-on-v1', description='Model Name' - ) - - -class Image1(BaseModel): - index: Optional[int] = Field(None, description='Image Number') - url: Optional[AnyUrl] = Field(None, description='URL for generated image') - - -class TaskResult6(BaseModel): - images: Optional[List[Image1]] = None - - -class Data7(BaseModel): - created_at: Optional[int] = Field(None, description='Task creation time') - task_id: Optional[str] = Field(None, description='Task ID') - task_result: Optional[TaskResult6] = None - task_status: Optional[TaskStatus] = None - task_status_msg: Optional[str] = Field(None, description='Task status information') - updated_at: Optional[int] = Field(None, description='Task update time') - - -class KlingVirtualTryOnResponse(BaseModel): - code: Optional[int] = Field(None, description='Error code') - data: Optional[Data7] = None + data: Optional[Data1] = None message: Optional[str] = Field(None, description='Error message') request_id: Optional[str] = Field(None, description='Request ID') @@ -938,7 +522,8 @@ class GenerationType3(str, Enum): class LumaVideoModel(str, Enum): ray_2 = 'ray-2' - ray_2_flash = 'ray-2-flash' + ray_flash_2 = 'ray-flash-2' + ray_1_6 = 'ray-1-6' class LumaVideoModelOutputDuration1(str, Enum): @@ -993,7 +578,7 @@ class MinimaxFileRetrieveResponse(BaseModel): file: File -class Status1(str, Enum): +class Status(str, Enum): Queueing = 'Queueing' Preparing = 'Preparing' Processing = 'Processing' @@ -1007,7 +592,7 @@ class MinimaxTaskResultResponse(BaseModel): None, description='After the task status changes to Success, this field returns the file ID corresponding to the generated video.', ) - status: Status1 = Field( + status: Status = Field( ..., description="Task status: 'Queueing' (in queue), 'Preparing' (task is preparing), 'Processing' (generating), 'Success' (task completed successfully), or 'Fail' (task failed).", ) @@ -1209,7 +794,7 @@ class OpenAIImageGenerationResponse(BaseModel): usage: Optional[Usage] = None -class AspectRatio3(RootModel[float]): +class AspectRatio2(RootModel[float]): root: float = Field( ..., description='Aspect ratio (width / height)', @@ -1226,7 +811,7 @@ class IngredientsMode(str, Enum): bytes_aliased = bytes class PikaBodyGenerate22C2vGenerate22PikascenesPost(BaseModel): - aspectRatio: Optional[AspectRatio3] = Field( + aspectRatio: Optional[AspectRatio2] = Field( None, description='Aspect ratio (width / height)', title='Aspectratio' ) duration: Optional[int] = Field(5, title='Duration') @@ -1261,7 +846,7 @@ class PikaBodyGenerate22KeyframeGenerate22PikaframesPost(BaseModel): class PikaBodyGenerate22T2vGenerate22T2vPost(BaseModel): - aspectRatio: Optional[AspectRatio3] = Field( + aspectRatio: Optional[AspectRatio2] = Field( None, description='Aspect ratio (width / height)', title='Aspectratio' ) duration: Optional[int] = Field(5, title='Duration') @@ -1288,7 +873,169 @@ class PikaVideoResponse(BaseModel): url: str = Field(..., title='Url') +class Resp(BaseModel): + img_id: Optional[int] = None + + +class PixverseImageUploadResponse(BaseModel): + ErrCode: Optional[int] = None + ErrMsg: Optional[str] = None + Resp_1: Optional[Resp] = Field(None, alias='Resp') + + +class Duration2(int, Enum): + integer_5 = 5 + integer_8 = 8 + + +class Model1(str, Enum): + v3_5 = 'v3.5' + + +class MotionMode(str, Enum): + normal = 'normal' + fast = 'fast' + + +class Quality1(str, Enum): + field_360p = '360p' + field_540p = '540p' + field_720p = '720p' + field_1080p = '1080p' + + +class Style1(str, Enum): + anime = 'anime' + field_3d_animation = '3d_animation' + clay = 'clay' + comic = 'comic' + cyberpunk = 'cyberpunk' + + +class PixverseImageVideoRequest(BaseModel): + duration: Duration2 + img_id: int + model: Model1 + motion_mode: Optional[MotionMode] = None + prompt: str + quality: Quality1 + seed: Optional[int] = None + style: Optional[Style1] = None + template_id: Optional[int] = None + water_mark: Optional[bool] = None + + +class AspectRatio4(str, Enum): + field_16_9 = '16:9' + field_4_3 = '4:3' + field_1_1 = '1:1' + field_3_4 = '3:4' + field_9_16 = '9:16' + + +class PixverseTextVideoRequest(BaseModel): + aspect_ratio: AspectRatio4 + duration: Duration2 + model: Model1 + motion_mode: Optional[MotionMode] = None + negative_prompt: Optional[str] = None + prompt: str + quality: Quality1 + seed: Optional[int] = None + style: Optional[Style1] = None + template_id: Optional[int] = None + water_mark: Optional[bool] = None + + +class PixverseTransitionVideoRequest(BaseModel): + duration: Duration2 + first_frame_img: int + last_frame_img: int + model: Model1 + motion_mode: MotionMode + prompt: str + quality: Quality1 + seed: int + style: Optional[Style1] = None + template_id: Optional[int] = None + water_mark: Optional[bool] = None + + +class Resp1(BaseModel): + video_id: Optional[int] = None + + +class PixverseVideoResponse(BaseModel): + ErrCode: Optional[int] = None + ErrMsg: Optional[str] = None + Resp: Optional[Resp1] = None + + +class Status1(int, Enum): + integer_1 = 1 + integer_5 = 5 + integer_6 = 6 + integer_7 = 7 + integer_8 = 8 + + +class Resp2(BaseModel): + create_time: Optional[str] = None + id: Optional[int] = None + modify_time: Optional[str] = None + negative_prompt: Optional[str] = None + outputHeight: Optional[int] = None + outputWidth: Optional[int] = None + prompt: Optional[str] = None + resolution_ratio: Optional[int] = None + seed: Optional[int] = None + size: Optional[int] = None + status: Optional[Status1] = Field( + None, + description='Video generation status codes:\n* 1 - Generation successful\n* 5 - Generating\n* 6 - Deleted\n* 7 - Contents moderation failed\n* 8 - Generation failed\n', + ) + style: Optional[str] = None + url: Optional[str] = None + + +class PixverseVideoResultResponse(BaseModel): + ErrCode: Optional[int] = None + ErrMsg: Optional[str] = None + Resp: Optional[Resp2] = None + + +class RGBColorItem(RootModel[int]): + root: int = Field(..., ge=0, le=255) + + +class RGBColor(RootModel[List[RGBColorItem]]): + root: List[RGBColorItem] = Field( + ..., + description='RGB color values', + examples=[[255, 0, 0]], + max_length=3, + min_length=3, + ) + + +class Controls(BaseModel): + artistic_level: Optional[int] = Field( + None, + description='Defines artistic tone of your image. At a simple level, the person looks straight at the camera in a static and clean style. Dynamic and eccentric levels introduce movement and creativity.', + ge=0, + le=5, + ) + background_color: Optional[RGBColor] = None + colors: Optional[List[RGBColor]] = Field( + None, description='An array of preferable colors' + ) + no_text: Optional[bool] = Field(None, description='Do not embed text layouts') + + class RecraftImageGenerationRequest(BaseModel): + controls: Optional[Controls] = Field( + None, description='The controls for the generated image' + ) model: str = Field( ..., description='The model to use for generation (e.g., "recraftv3")' ) @@ -1400,7 +1147,7 @@ class Error(BaseModel): message: Optional[str] = Field(None, description='Error message') -class Video5(BaseModel): +class Video2(BaseModel): bytesBase64Encoded: Optional[str] = Field( None, description='Base64-encoded video content' ) @@ -1422,7 +1169,7 @@ class Response(BaseModel): raiMediaFilteredReasons: Optional[List[str]] = Field( None, description='Reasons why media was filtered by responsible AI policies' ) - videos: Optional[List[Video5]] = None + videos: Optional[List[Video2]] = None class Veo2GenVidPollResponse(BaseModel): @@ -1436,20 +1183,20 @@ class Veo2GenVidPollResponse(BaseModel): ) -class Image2(BaseModel): +class Image(BaseModel): bytesBase64Encoded: str gcsUri: Optional[str] = None mimeType: Optional[str] = None -class Image3(BaseModel): +class Image1(BaseModel): bytesBase64Encoded: Optional[str] = None gcsUri: str mimeType: Optional[str] = None class Instance(BaseModel): - image: Optional[Union[Image2, Image3]] = Field( + image: Optional[Union[Image, Image1]] = Field( None, description='Optional image to guide video generation' ) prompt: str = Field(..., description='Text description of the video') diff --git a/comfy_api_nodes/nodes_kling.py b/comfy_api_nodes/nodes_kling.py index 035c4f382..ac2fa6e28 100644 --- a/comfy_api_nodes/nodes_kling.py +++ b/comfy_api_nodes/nodes_kling.py @@ -1,4 +1,3 @@ -from inspect import cleandoc from typing import Union, Optional import math import logging @@ -128,7 +127,7 @@ def _get_camera_control_inputs() -> dict[str, tuple[IO, InputTypeOptions]]: } -class KlingNodeABC(ComfyNodeABC): +class KlingNodeBase(ComfyNodeABC): """Base class for Kling nodes.""" @classmethod @@ -162,13 +161,12 @@ class KlingNodeABC(ComfyNodeABC): return "Invalid camera control configs" return True - DESCRIPTION = cleandoc(__doc__ or "") FUNCTION = "api_call" CATEGORY = "api node/video/Kling" API_NODE = True -class KlingTextToVideoNode(KlingNodeABC): +class KlingTextToVideoNode(KlingNodeBase): """ Kling Text to Video Node. """ @@ -227,6 +225,7 @@ class KlingTextToVideoNode(KlingNodeABC): } RETURN_TYPES = ("VIDEO",) + DESCRIPTION = "Kling Text to Video Node" def api_call( self, @@ -301,7 +300,7 @@ class KlingTextToVideoNode(KlingNodeABC): return download_url_to_video_output(video_url) -class KlingImage2VideoNode(KlingNodeABC): +class KlingImage2VideoNode(KlingNodeBase): """ Kling Image to Video Node. """ @@ -372,6 +371,7 @@ class KlingImage2VideoNode(KlingNodeABC): } RETURN_TYPES = ("VIDEO",) + DESCRIPTION = "Kling Image to Video Node" def api_call( self, From 9cb4a890c65f0b9ec95295fd0160a1db74f61772 Mon Sep 17 00:00:00 2001 From: Jedrzej Kosinski Date: Wed, 30 Apr 2025 16:02:46 -0500 Subject: [PATCH 056/121] Added Pixverse Image to VIdeo node (#77) --- comfy_api_nodes/apis/pixverse_api.py | 66 ++++++++++- comfy_api_nodes/nodes_pixverse.py | 170 ++++++++++++++++++++++++++- 2 files changed, 224 insertions(+), 12 deletions(-) diff --git a/comfy_api_nodes/apis/pixverse_api.py b/comfy_api_nodes/apis/pixverse_api.py index 85eae2738..9ce488e07 100644 --- a/comfy_api_nodes/apis/pixverse_api.py +++ b/comfy_api_nodes/apis/pixverse_api.py @@ -22,6 +22,7 @@ class PixverseIO: class PixverseStatus(int, Enum): successful = 1 generating = 5 + deleted = 6 contents_moderation = 7 failed = 8 @@ -60,7 +61,7 @@ class PixverseStyle(str, Enum): # NOTE: forgoing descriptions for now in return for dev speed -class PixverseDto_V2OpenAPIT2VReq(BaseModel): +class PixverseTextVideoRequest(BaseModel): aspect_ratio: PixverseAspectRatio = Field(...) quality: PixverseQuality = Field(...) duration: PixverseDuration = Field(...) @@ -74,23 +75,76 @@ class PixverseDto_V2OpenAPIT2VReq(BaseModel): water_mark: Optional[bool] = Field(None) -class PixverseController_ResponseData(BaseModel): +class PixverseImageVideoRequest(BaseModel): + quality: PixverseQuality = Field(...) + duration: PixverseDuration = Field(...) + img_id: int = Field(...) + model: Optional[str] = Field("v3.5") + motion_mode: Optional[PixverseMotionMode] = Field(PixverseMotionMode.normal) + prompt: str = Field(...) + negative_prompt: Optional[str] = Field(None) + seed: Optional[int] = Field(None) + style: Optional[str] = Field(None) + template_id: Optional[int] = Field(None) + water_mark: Optional[bool] = Field(None) + + +# class PixverseImageVideoRequest(BaseModel): +# quality: Optional[PixverseQuality] = Field(None) +# duration: Optional[PixverseDuration] = Field(None) +# img_id: int = Field(...) +# model: Optional[str] = Field("v3.5") +# motion_mode: Optional[PixverseMotionMode] = Field(PixverseMotionMode.normal) +# prompt: Optional[str ]= Field(None) +# negative_prompt: Optional[str] = Field(None) +# seed: Optional[int] = Field(None) +# style: Optional[str] = Field(None) +# template_id: Optional[int] = Field(None) +# water_mark: Optional[bool] = Field(None) + + +class PixverseTransitionVideoRequest(BaseModel): + quality: PixverseQuality = Field(...) + duration: PixverseDuration = Field(...) + first_frame_img: int = Field(...) + last_frame_img: int = Field(...) + model: Optional[str] = Field("v3.5") + motion_mode: Optional[PixverseMotionMode] = Field(PixverseMotionMode.normal) + prompt: str = Field(...) + # negative_prompt: Optional[str] = Field(None) + seed: Optional[int] = Field(None) + # style: Optional[str] = Field(None) + # template_id: Optional[int] = Field(None) + # water_mark: Optional[bool] = Field(None) + + +class PixverseImageUploadResponse(BaseModel): + ErrCode: Optional[int] = None + ErrMsg: Optional[str] = None + Resp: Optional[PixverseImgIdResponseObject] = Field(None, alias='Resp') + + +class PixverseImgIdResponseObject(BaseModel): + img_id: Optional[int] = None + + +class PixverseVideoResponse(BaseModel): ErrCode: Optional[int] = Field(None) ErrMsg: Optional[str] = Field(None) - Resp: Optional[PixverseDto_V2OpenAPII2VResp] = Field(None) + Resp: Optional[PixverseVideoIdResponseObject] = Field(None) -class PixverseDto_V2OpenAPII2VResp(BaseModel): +class PixverseVideoIdResponseObject(BaseModel): video_id: int = Field(..., description='Video_id') class PixverseGenerationStatusResponse(BaseModel): ErrCode: Optional[int] = Field(None) ErrMsg: Optional[str] = Field(None) - Resp: Optional[PixverseDto_GetOpenapiMediaDetailResp] = Field(None) + Resp: Optional[PixverseGenerationStatusResponseObject] = Field(None) -class PixverseDto_GetOpenapiMediaDetailResp(BaseModel): +class PixverseGenerationStatusResponseObject(BaseModel): create_time: Optional[str] = Field(None) id: Optional[int] = Field(None) modify_time: Optional[str] = Field(None) diff --git a/comfy_api_nodes/nodes_pixverse.py b/comfy_api_nodes/nodes_pixverse.py index 70083c2e7..5808575e4 100644 --- a/comfy_api_nodes/nodes_pixverse.py +++ b/comfy_api_nodes/nodes_pixverse.py @@ -1,8 +1,10 @@ from inspect import cleandoc from comfy_api_nodes.apis.pixverse_api import ( - PixverseDto_V2OpenAPIT2VReq, - PixverseController_ResponseData, + PixverseTextVideoRequest, + PixverseImageVideoRequest, + PixverseImageUploadResponse, + PixverseVideoResponse, PixverseGenerationStatusResponse, PixverseAspectRatio, PixverseQuality, @@ -19,9 +21,13 @@ from comfy_api_nodes.apis.client import ( PollingOperation, EmptyRequest, ) +from comfy_api_nodes.apinode_utils import ( + tensor_to_bytesio, +) from comfy.comfy_types.node_typing import IO, ComfyNodeABC from comfy_api.input_impl import VideoFromFile +import torch import requests from io import BytesIO @@ -143,10 +149,10 @@ class PixverseTextToVideoNode(ComfyNodeABC): endpoint=ApiEndpoint( path="/proxy/pixverse/video/text/generate", method=HttpMethod.POST, - request_model=PixverseDto_V2OpenAPIT2VReq, - response_model=PixverseController_ResponseData, + request_model=PixverseTextVideoRequest, + response_model=PixverseVideoResponse, ), - request=PixverseDto_V2OpenAPIT2VReq( + request=PixverseTextVideoRequest( prompt=prompt, aspect_ratio=aspect_ratio, quality=quality, @@ -171,7 +177,157 @@ class PixverseTextToVideoNode(ComfyNodeABC): response_model=PixverseGenerationStatusResponse, ), completed_statuses=[PixverseStatus.successful], - failed_statuses=[PixverseStatus.contents_moderation, PixverseStatus.failed], + failed_statuses=[PixverseStatus.contents_moderation, PixverseStatus.failed, PixverseStatus.deleted], + status_extractor=lambda x: x.Resp.status, + auth_token=auth_token, + ) + response_poll = operation.execute() + + vid_response = requests.get(response_poll.Resp.url) + return (VideoFromFile(BytesIO(vid_response.content)),) + + +class PixverseImageToVideoNode(ComfyNodeABC): + """ + Generates videos synchronously based on prompt and output_size. + """ + + RETURN_TYPES = (IO.VIDEO,) + DESCRIPTION = cleandoc(__doc__ or "") # Handle potential None value + FUNCTION = "api_call" + API_NODE = True + CATEGORY = "api node/video/Pixverse" + + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "image": ( + IO.IMAGE, + ), + "prompt": ( + IO.STRING, + { + "multiline": True, + "default": "", + "tooltip": "Prompt for the video generation", + }, + ), + "quality": ( + [resolution.value for resolution in PixverseQuality], + { + "default": PixverseQuality.res_540p, + }, + ), + "duration_seconds": ([dur.value for dur in PixverseDuration],), + "motion_mode": ([mode.value for mode in PixverseMotionMode],), + "seed": ( + IO.INT, + { + "default": 0, + "min": 0, + "max": 2147483647, + "control_after_generate": True, + "tooltip": "Seed for video generation.", + }, + ), + }, + "optional": { + "negative_prompt": ( + IO.STRING, + { + "default": "", + "forceInput": True, + "tooltip": "An optional text description of undesired elements on an image.", + }, + ), + "pixverse_template": ( + PixverseIO.TEMPLATE, + { + "tooltip": "An optional template to influence style of generation, created by the Pixverse Template node." + } + ) + }, + "hidden": { + "auth_token": "AUTH_TOKEN_COMFY_ORG", + }, + } + + def api_call( + self, + image: torch.Tensor, + prompt: str, + quality: str, + duration_seconds: int, + motion_mode: str, + seed, + negative_prompt: str=None, + pixverse_template: int=None, + auth_token=None, + **kwargs, + ): + # first, upload image to Pixverse and get image id to use in actual generation call + files = { + "image": tensor_to_bytesio(image) + } + operation = SynchronousOperation( + endpoint=ApiEndpoint( + path="/proxy/pixverse/image/upload", + method=HttpMethod.POST, + request_model=EmptyRequest, + response_model=PixverseImageUploadResponse, + ), + request=EmptyRequest(), + files=files, + content_type="multipart/form-data", + auth_token=auth_token, + ) + response_upload: PixverseImageUploadResponse = operation.execute() + + if response_upload.Resp is None: + raise Exception(f"Pixverse image upload request failed: '{response_upload.ErrMsg}'") + + # 1080p is limited to 5 seconds duration + # only normal motion_mode supported for 1080p or for non-5 second duration + if quality == PixverseQuality.res_1080p: + motion_mode = PixverseMotionMode.normal + duration_seconds = PixverseDuration.dur_5 + elif duration_seconds != PixverseDuration.dur_5: + motion_mode = PixverseMotionMode.normal + + operation = SynchronousOperation( + endpoint=ApiEndpoint( + path="/proxy/pixverse/video/img/generate", + method=HttpMethod.POST, + request_model=PixverseImageVideoRequest, + response_model=PixverseVideoResponse, + ), + request=PixverseImageVideoRequest( + img_id=response_upload.Resp.img_id, + prompt=prompt, + quality=quality, + duration=duration_seconds, + motion_mode=motion_mode, + negative_prompt=negative_prompt if negative_prompt else None, + template_id=pixverse_template, + seed=seed, + ), + auth_token=auth_token, + ) + response_api = operation.execute() + + if response_api.Resp is None: + raise Exception(f"Pixverse request failed: '{response_api.ErrMsg}'") + + operation = PollingOperation( + poll_endpoint=ApiEndpoint( + path=f"/proxy/pixverse/video/result/{response_api.Resp.video_id}", + method=HttpMethod.GET, + request_model=EmptyRequest, + response_model=PixverseGenerationStatusResponse, + ), + completed_statuses=[PixverseStatus.successful], + failed_statuses=[PixverseStatus.contents_moderation, PixverseStatus.failed, PixverseStatus.deleted], status_extractor=lambda x: x.Resp.status, auth_token=auth_token, ) @@ -183,10 +339,12 @@ class PixverseTextToVideoNode(ComfyNodeABC): NODE_CLASS_MAPPINGS = { "PixverseTextToVideoNode": PixverseTextToVideoNode, + "PixverseImageToVideoNode": PixverseImageToVideoNode, "PixverseTemplateNode": PixverseTemplateNode, } NODE_DISPLAY_NAME_MAPPINGS = { "PixverseTextToVideoNode": "Pixverse Text to Video", + "PixverseImageToVideoNode": "Pixverse Image to Video", "PixverseTemplateNode": "Pixverse Template", } From 686f2271f277000ef592c3764d862e0daae5782b Mon Sep 17 00:00:00 2001 From: Jedrzej Kosinski Date: Wed, 30 Apr 2025 17:10:14 -0500 Subject: [PATCH 057/121] Add Pixverse Transition Video node (#79) --- comfy_api_nodes/apis/pixverse_api.py | 14 -- comfy_api_nodes/nodes_pixverse.py | 188 ++++++++++++++++++++++++--- 2 files changed, 167 insertions(+), 35 deletions(-) diff --git a/comfy_api_nodes/apis/pixverse_api.py b/comfy_api_nodes/apis/pixverse_api.py index 9ce488e07..9bb29c383 100644 --- a/comfy_api_nodes/apis/pixverse_api.py +++ b/comfy_api_nodes/apis/pixverse_api.py @@ -89,20 +89,6 @@ class PixverseImageVideoRequest(BaseModel): water_mark: Optional[bool] = Field(None) -# class PixverseImageVideoRequest(BaseModel): -# quality: Optional[PixverseQuality] = Field(None) -# duration: Optional[PixverseDuration] = Field(None) -# img_id: int = Field(...) -# model: Optional[str] = Field("v3.5") -# motion_mode: Optional[PixverseMotionMode] = Field(PixverseMotionMode.normal) -# prompt: Optional[str ]= Field(None) -# negative_prompt: Optional[str] = Field(None) -# seed: Optional[int] = Field(None) -# style: Optional[str] = Field(None) -# template_id: Optional[int] = Field(None) -# water_mark: Optional[bool] = Field(None) - - class PixverseTransitionVideoRequest(BaseModel): quality: PixverseQuality = Field(...) duration: PixverseDuration = Field(...) diff --git a/comfy_api_nodes/nodes_pixverse.py b/comfy_api_nodes/nodes_pixverse.py index 5808575e4..8064b8f9f 100644 --- a/comfy_api_nodes/nodes_pixverse.py +++ b/comfy_api_nodes/nodes_pixverse.py @@ -3,6 +3,7 @@ from inspect import cleandoc from comfy_api_nodes.apis.pixverse_api import ( PixverseTextVideoRequest, PixverseImageVideoRequest, + PixverseTransitionVideoRequest, PixverseImageUploadResponse, PixverseVideoResponse, PixverseGenerationStatusResponse, @@ -32,6 +33,31 @@ import requests from io import BytesIO +def upload_image_to_pixverse(image: torch.Tensor, auth_token=None): + # first, upload image to Pixverse and get image id to use in actual generation call + files = { + "image": tensor_to_bytesio(image) + } + operation = SynchronousOperation( + endpoint=ApiEndpoint( + path="/proxy/pixverse/image/upload", + method=HttpMethod.POST, + request_model=EmptyRequest, + response_model=PixverseImageUploadResponse, + ), + request=EmptyRequest(), + files=files, + content_type="multipart/form-data", + auth_token=auth_token, + ) + response_upload: PixverseImageUploadResponse = operation.execute() + + if response_upload.Resp is None: + raise Exception(f"Pixverse image upload request failed: '{response_upload.ErrMsg}'") + + return response_upload.Resp.img_id + + class PixverseTemplateNode: """ Select template for Pixverse Video generation. @@ -266,26 +292,7 @@ class PixverseImageToVideoNode(ComfyNodeABC): auth_token=None, **kwargs, ): - # first, upload image to Pixverse and get image id to use in actual generation call - files = { - "image": tensor_to_bytesio(image) - } - operation = SynchronousOperation( - endpoint=ApiEndpoint( - path="/proxy/pixverse/image/upload", - method=HttpMethod.POST, - request_model=EmptyRequest, - response_model=PixverseImageUploadResponse, - ), - request=EmptyRequest(), - files=files, - content_type="multipart/form-data", - auth_token=auth_token, - ) - response_upload: PixverseImageUploadResponse = operation.execute() - - if response_upload.Resp is None: - raise Exception(f"Pixverse image upload request failed: '{response_upload.ErrMsg}'") + img_id = upload_image_to_pixverse(image, auth_token=auth_token) # 1080p is limited to 5 seconds duration # only normal motion_mode supported for 1080p or for non-5 second duration @@ -303,7 +310,144 @@ class PixverseImageToVideoNode(ComfyNodeABC): response_model=PixverseVideoResponse, ), request=PixverseImageVideoRequest( - img_id=response_upload.Resp.img_id, + img_id=img_id, + prompt=prompt, + quality=quality, + duration=duration_seconds, + motion_mode=motion_mode, + negative_prompt=negative_prompt if negative_prompt else None, + template_id=pixverse_template, + seed=seed, + ), + auth_token=auth_token, + ) + response_api = operation.execute() + + if response_api.Resp is None: + raise Exception(f"Pixverse request failed: '{response_api.ErrMsg}'") + + operation = PollingOperation( + poll_endpoint=ApiEndpoint( + path=f"/proxy/pixverse/video/result/{response_api.Resp.video_id}", + method=HttpMethod.GET, + request_model=EmptyRequest, + response_model=PixverseGenerationStatusResponse, + ), + completed_statuses=[PixverseStatus.successful], + failed_statuses=[PixverseStatus.contents_moderation, PixverseStatus.failed, PixverseStatus.deleted], + status_extractor=lambda x: x.Resp.status, + auth_token=auth_token, + ) + response_poll = operation.execute() + + vid_response = requests.get(response_poll.Resp.url) + return (VideoFromFile(BytesIO(vid_response.content)),) + + +class PixverseTransitionVideoNode(ComfyNodeABC): + """ + Generates videos synchronously based on prompt and output_size. + """ + + RETURN_TYPES = (IO.VIDEO,) + DESCRIPTION = cleandoc(__doc__ or "") # Handle potential None value + FUNCTION = "api_call" + API_NODE = True + CATEGORY = "api node/video/Pixverse" + + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "first_frame": ( + IO.IMAGE, + ), + "last_frame": ( + IO.IMAGE, + ), + "prompt": ( + IO.STRING, + { + "multiline": True, + "default": "", + "tooltip": "Prompt for the video generation", + }, + ), + "quality": ( + [resolution.value for resolution in PixverseQuality], + { + "default": PixverseQuality.res_540p, + }, + ), + "duration_seconds": ([dur.value for dur in PixverseDuration],), + "motion_mode": ([mode.value for mode in PixverseMotionMode],), + "seed": ( + IO.INT, + { + "default": 0, + "min": 0, + "max": 2147483647, + "control_after_generate": True, + "tooltip": "Seed for video generation.", + }, + ), + }, + "optional": { + "negative_prompt": ( + IO.STRING, + { + "default": "", + "forceInput": True, + "tooltip": "An optional text description of undesired elements on an image.", + }, + ), + "pixverse_template": ( + PixverseIO.TEMPLATE, + { + "tooltip": "An optional template to influence style of generation, created by the Pixverse Template node." + } + ) + }, + "hidden": { + "auth_token": "AUTH_TOKEN_COMFY_ORG", + }, + } + + def api_call( + self, + first_frame: torch.Tensor, + last_frame: torch.Tensor, + prompt: str, + quality: str, + duration_seconds: int, + motion_mode: str, + seed, + negative_prompt: str=None, + pixverse_template: int=None, + auth_token=None, + **kwargs, + ): + first_frame_id = upload_image_to_pixverse(first_frame, auth_token=auth_token) + last_frame_id = upload_image_to_pixverse(last_frame, auth_token=auth_token) + + # 1080p is limited to 5 seconds duration + # only normal motion_mode supported for 1080p or for non-5 second duration + if quality == PixverseQuality.res_1080p: + motion_mode = PixverseMotionMode.normal + duration_seconds = PixverseDuration.dur_5 + elif duration_seconds != PixverseDuration.dur_5: + motion_mode = PixverseMotionMode.normal + + operation = SynchronousOperation( + endpoint=ApiEndpoint( + path="/proxy/pixverse/video/transition/generate", + method=HttpMethod.POST, + request_model=PixverseTransitionVideoRequest, + response_model=PixverseVideoResponse, + ), + request=PixverseTransitionVideoRequest( + first_frame_img=first_frame_id, + last_frame_img=last_frame_id, prompt=prompt, quality=quality, duration=duration_seconds, @@ -340,11 +484,13 @@ class PixverseImageToVideoNode(ComfyNodeABC): NODE_CLASS_MAPPINGS = { "PixverseTextToVideoNode": PixverseTextToVideoNode, "PixverseImageToVideoNode": PixverseImageToVideoNode, + "PixverseTransitionVideoNode": PixverseTransitionVideoNode, "PixverseTemplateNode": PixverseTemplateNode, } NODE_DISPLAY_NAME_MAPPINGS = { "PixverseTextToVideoNode": "Pixverse Text to Video", "PixverseImageToVideoNode": "Pixverse Image to Video", + "PixverseTransitionVideoNode": "Pixverse Transition Video", "PixverseTemplateNode": "Pixverse Template", } From 69c7f1bc390386fd54408e13eecabf01073d78d5 Mon Sep 17 00:00:00 2001 From: Jedrzej Kosinski Date: Wed, 30 Apr 2025 17:25:03 -0500 Subject: [PATCH 058/121] Proper ray-1-6 support as fix has been applied in backend (#80) --- comfy_api_nodes/apis/luma_api.py | 2 +- comfy_api_nodes/nodes_luma.py | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/comfy_api_nodes/apis/luma_api.py b/comfy_api_nodes/apis/luma_api.py index 037b2415c..632c4ab96 100644 --- a/comfy_api_nodes/apis/luma_api.py +++ b/comfy_api_nodes/apis/luma_api.py @@ -234,7 +234,7 @@ class LumaImageGenerationRequest(BaseModel): class LumaGenerationRequest(BaseModel): prompt: str = Field(..., description='The prompt of the generation') model: LumaVideoModel = Field(LumaVideoModel.ray_2, description='The video model used for the generation') - duration: LumaVideoModelOutputDuration = Field(LumaVideoModelOutputDuration.dur_5s, description='The duration of the generation') + duration: Optional[LumaVideoModelOutputDuration] = Field(None, description='The duration of the generation') aspect_ratio: Optional[LumaAspectRatio] = Field(None, description='The aspect ratio of the generation') resolution: Optional[LumaVideoOutputResolution] = Field(None, description='The resolution of the generation') loop: Optional[bool] = Field(None, description='Whether to loop the video') diff --git a/comfy_api_nodes/nodes_luma.py b/comfy_api_nodes/nodes_luma.py index 32f8cb392..7b95a1f41 100644 --- a/comfy_api_nodes/nodes_luma.py +++ b/comfy_api_nodes/nodes_luma.py @@ -484,6 +484,9 @@ class LumaTextToVideoGenerationNode(ComfyNodeABC): auth_token=None, **kwargs, ): + duration = duration if model != LumaVideoModel.ray_1_6 else None + resolution = resolution if model != LumaVideoModel.ray_1_6 else None + operation = SynchronousOperation( endpoint=ApiEndpoint( path="/proxy/luma/generations", @@ -610,6 +613,8 @@ class LumaImageToVideoGenerationNode(ComfyNodeABC): "At least one of first_image and last_image requires an input." ) keyframes = self._convert_to_keyframes(first_image, last_image, auth_token) + duration = duration if model != LumaVideoModel.ray_1_6 else None + resolution = resolution if model != LumaVideoModel.ray_1_6 else None operation = SynchronousOperation( endpoint=ApiEndpoint( From d3e0c485c9a7e2ea31d0f1d55ec0af0874d94a83 Mon Sep 17 00:00:00 2001 From: Jedrzej Kosinski Date: Wed, 30 Apr 2025 18:00:09 -0500 Subject: [PATCH 059/121] Added Recraft Style - Infinite Style Library node (#82) --- comfy_api_nodes/apis/recraft_api.py | 4 +++- comfy_api_nodes/nodes_recraft.py | 37 ++++++++++++++++++++++++++++- 2 files changed, 39 insertions(+), 2 deletions(-) diff --git a/comfy_api_nodes/apis/recraft_api.py b/comfy_api_nodes/apis/recraft_api.py index aee3d39b6..da6232992 100644 --- a/comfy_api_nodes/apis/recraft_api.py +++ b/comfy_api_nodes/apis/recraft_api.py @@ -71,11 +71,12 @@ class RecraftControls: class RecraftStyle: - def __init__(self, style: str, substyle: str=None): + def __init__(self, style: str=None, substyle: str=None, style_id: str=None): self.style = style if substyle == "None": substyle = None self.substyle = substyle + self.style_id = style_id class RecraftIO: @@ -244,6 +245,7 @@ class RecraftImageGenerationRequest(BaseModel): style: Optional[str] = Field(None, description='The style to apply to the generated image (e.g., "digital_illustration")') substyle: Optional[str] = Field(None, description='The substyle to apply to the generated image, depending on the style input') controls: Optional[RecraftControlsObject] = Field(None, description='A set of custom parameters to tweak generation process') + style_id: Optional[str] = Field(None, description='Use a previously uploaded style as a reference; UUID') # text_layout diff --git a/comfy_api_nodes/nodes_recraft.py b/comfy_api_nodes/nodes_recraft.py index fd93ea7d5..80cccd969 100644 --- a/comfy_api_nodes/nodes_recraft.py +++ b/comfy_api_nodes/nodes_recraft.py @@ -48,6 +48,7 @@ class SaveSVGNode: self.prefix_append = "" RETURN_TYPES = () + DESCRIPTION = cleandoc(__doc__ or "") # Handle potential None value FUNCTION = "save_svg" CATEGORY = "api node/image/Recraft" OUTPUT_NODE = True @@ -100,6 +101,7 @@ class RecraftColorRGBNode: """ RETURN_TYPES = (RecraftIO.COLOR,) + DESCRIPTION = cleandoc(__doc__ or "") # Handle potential None value RETURN_NAMES = ("recraft_color",) FUNCTION = "create_color" CATEGORY = "api node/image/Recraft" @@ -145,6 +147,7 @@ class RecraftControlsNode: RETURN_TYPES = (RecraftIO.CONTROLS,) RETURN_NAMES = ("recraft_controls",) + DESCRIPTION = cleandoc(__doc__ or "") # Handle potential None value FUNCTION = "create_controls" CATEGORY = "api node/image/Recraft" @@ -170,6 +173,7 @@ class RecraftStyleV3RealisticImageNode: RETURN_TYPES = (RecraftIO.STYLEV3,) RETURN_NAMES = ("recraft_style",) + DESCRIPTION = cleandoc(__doc__ or "") # Handle potential None value FUNCTION = "create_style" CATEGORY = "api node/image/Recraft" @@ -221,6 +225,34 @@ class RecraftStyleV3LogoRasterNode(RecraftStyleV3RealisticImageNode): RECRAFT_STYLE = RecraftStyleV3.logo_raster +class RecraftStyleInfiniteStyleLibrary: + """ + Select style based on preexisting UUID from the Infinite Style Library. + """ + + RETURN_TYPES = (RecraftIO.STYLEV3,) + RETURN_NAMES = ("recraft_style",) + DESCRIPTION = cleandoc(__doc__ or "") # Handle potential None value + FUNCTION = "create_style" + CATEGORY = "api node/image/Recraft" + + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "style_id": (IO.STRING, { + "default": "", + "tooltip": "UUID of style from Infinite Style Library.", + }) + } + } + + def create_style(self, style_id: str): + if not style_id: + raise Exception("The style_id input cannot be empty.") + return (RecraftStyle(style_id=style_id),) + + class RecraftTextToImageNode: """ Generates images synchronously based on prompt and resolution. @@ -305,7 +337,7 @@ class RecraftTextToImageNode: auth_token=None, **kwargs, ): - default_style = RecraftStyle(RecraftStyleV3.digital_illustration) + default_style = RecraftStyle(RecraftStyleV3.realistic_image) if recraft_style is None: recraft_style = default_style @@ -331,6 +363,7 @@ class RecraftTextToImageNode: n=n, style=recraft_style.style, substyle=recraft_style.substyle, + style_id=recraft_style.style_id, controls=controls_api, ), auth_token=auth_token, @@ -478,6 +511,7 @@ NODE_CLASS_MAPPINGS = { "RecraftStyleV3RealisticImage": RecraftStyleV3RealisticImageNode, "RecraftStyleV3DigitalIllustration": RecraftStyleV3DigitalIllustrationNode, "RecraftStyleV3LogoRaster": RecraftStyleV3LogoRasterNode, + "RecraftStyleV3InfiniteStyleLibrary": RecraftStyleInfiniteStyleLibrary, "RecraftColorRGB": RecraftColorRGBNode, "RecraftControls": RecraftControlsNode, "SaveSVG": SaveSVGNode, @@ -490,6 +524,7 @@ NODE_DISPLAY_NAME_MAPPINGS = { "RecraftStyleV3RealisticImage": "Recraft Style - Realistic Image", "RecraftStyleV3DigitalIllustration": "Recraft Style - Digital Illustration", "RecraftStyleV3LogoRaster": "Recraft Style - Logo Raster", + "RecraftStyleV3InfiniteStyleLibrary": "Recraft Style - Infinite Style Library", "RecraftColorRGB": "Recraft Color RGB", "RecraftControls": "Recraft Controls", "SaveSVG": "Save SVG", From 866afa53997f4e29aa5c16d6b1b999643fc9aa97 Mon Sep 17 00:00:00 2001 From: thot experiment <94414189+thot-experiment@users.noreply.github.com> Date: Wed, 30 Apr 2025 17:48:17 -0700 Subject: [PATCH 060/121] add ideogram v3 (#83) --- comfy_api_nodes/apis/__init__.py | 882 ++++++++++++++++++++++++++---- comfy_api_nodes/apis/client.py | 5 + comfy_api_nodes/nodes_ideogram.py | 286 ++++++++-- 3 files changed, 1037 insertions(+), 136 deletions(-) diff --git a/comfy_api_nodes/apis/__init__.py b/comfy_api_nodes/apis/__init__.py index 77e6d34e4..274489c23 100644 --- a/comfy_api_nodes/apis/__init__.py +++ b/comfy_api_nodes/apis/__init__.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: -# filename: filtered-openapi.yaml -# timestamp: 2025-04-30T19:05:50+00:00 +# filename: https://stagingapi.comfy.org/openapi +# timestamp: 2025-05-01T00:26:01+00:00 from __future__ import annotations @@ -9,29 +9,29 @@ from enum import Enum from typing import Any, Dict, List, Literal, Optional, Union from uuid import UUID -from pydantic import AnyUrl, BaseModel, Field, RootModel - +from pydantic import AnyUrl, BaseModel, Field, RootModel, confloat, conint, constr +bytes_aliased=bytes class BFLFluxProGenerateRequest(BaseModel): - guidance_scale: Optional[float] = Field( - None, description='The guidance scale for generation.', ge=1.0, le=20.0 + guidance_scale: Optional[confloat(ge=1.0, le=20.0)] = Field( + None, description='The guidance scale for generation.' ) - height: int = Field( - ..., description='The height of the image to generate.', ge=64, le=2048 + height: conint(ge=64, le=2048) = Field( + ..., description='The height of the image to generate.' ) negative_prompt: Optional[str] = Field( None, description='The negative prompt for image generation.' ) - num_images: Optional[int] = Field( - None, description='The number of images to generate.', ge=1, le=4 + num_images: Optional[conint(ge=1, le=4)] = Field( + None, description='The number of images to generate.' ) - num_inference_steps: Optional[int] = Field( - None, description='The number of inference steps.', ge=1, le=100 + num_inference_steps: Optional[conint(ge=1, le=100)] = Field( + None, description='The number of inference steps.' ) prompt: str = Field(..., description='The text prompt for image generation.') seed: Optional[int] = Field(None, description='The seed value for reproducibility.') - width: int = Field( - ..., description='The width of the image to generate.', ge=64, le=2048 + width: conint(ge=64, le=2048) = Field( + ..., description='The width of the image to generate.' ) @@ -40,6 +40,47 @@ class BFLFluxProGenerateResponse(BaseModel): polling_url: str = Field(..., description='URL to poll for the generation result.') +class ComfyNode(BaseModel): + category: Optional[str] = Field( + None, + description='UI category where the node is listed, used for grouping nodes.', + ) + comfy_node_name: Optional[str] = Field( + None, description='Unique identifier for the node' + ) + deprecated: Optional[bool] = Field( + None, + description='Indicates if the node is deprecated. Deprecated nodes are hidden in the UI.', + ) + description: Optional[str] = Field( + None, description="Brief description of the node's functionality or purpose." + ) + experimental: Optional[bool] = Field( + None, + description='Indicates if the node is experimental, subject to changes or removal.', + ) + function: Optional[str] = Field( + None, description='Name of the entry-point function to execute the node.' + ) + input_types: Optional[str] = Field(None, description='Defines input parameters') + output_is_list: Optional[List[bool]] = Field( + None, description='Boolean values indicating if each output is a list.' + ) + return_names: Optional[str] = Field( + None, description='Names of the outputs for clarity in workflows.' + ) + return_types: Optional[str] = Field( + None, description='Specifies the types of outputs produced by the node.' + ) + + +class ComfyNodeCloudBuildInfo(BaseModel): + build_id: Optional[str] = None + location: Optional[str] = None + project_id: Optional[str] = None + project_number: Optional[str] = None + + class Customer(BaseModel): createdAt: Optional[datetime] = Field( None, description='The date and time the user was created' @@ -69,11 +110,64 @@ class CustomerStorageResourceResponse(BaseModel): ) +class Error(BaseModel): + details: Optional[List[str]] = Field( + None, + description='Optional detailed information about the error or hints for resolving it.', + ) + message: Optional[str] = Field( + None, description='A clear and concise description of the error.' + ) + + class ErrorResponse(BaseModel): error: str message: str +class GitCommitSummary(BaseModel): + author: Optional[str] = Field(None, description='The author of the commit') + branch_name: Optional[str] = Field( + None, description='The branch where the commit was made' + ) + commit_hash: Optional[str] = Field(None, description='The hash of the commit') + commit_name: Optional[str] = Field(None, description='The name of the commit') + status_summary: Optional[Dict[str, str]] = Field( + None, description='A map of operating system to status pairs' + ) + timestamp: Optional[datetime] = Field( + None, description='The timestamp when the commit was made' + ) + + +class IdeogramColorPalette1(BaseModel): + name: str = Field(..., description='Name of the preset color palette') + + +class Member(BaseModel): + color: Optional[constr(pattern=r'^#[0-9A-Fa-f]{6}$')] = Field( + None, description='Hexadecimal color code' + ) + weight: Optional[confloat(ge=0.0, le=1.0)] = Field( + None, description='Optional weight for the color (0-1)' + ) + + +class IdeogramColorPalette2(BaseModel): + members: List[Member] = Field( + ..., description='Array of color definitions with optional weights' + ) + + +class IdeogramColorPalette( + RootModel[Union[IdeogramColorPalette1, IdeogramColorPalette2]] +): + root: Union[IdeogramColorPalette1, IdeogramColorPalette2] = Field( + ..., + description='A color palette specification that can either use a preset name or explicit color definitions with weights', + ) + + class ImageRequest(BaseModel): aspect_ratio: Optional[str] = Field( None, @@ -90,11 +184,8 @@ class ImageRequest(BaseModel): None, description='Optional. Description of what to exclude. Only for V_1, V_1_TURBO, V_2, V_2_TURBO.', ) - num_images: Optional[int] = Field( - 1, - description='Optional. Number of images to generate (1-8). Defaults to 1.', - ge=1, - le=8, + num_images: Optional[conint(ge=1, le=8)] = Field( + 1, description='Optional. Number of images to generate (1-8). Defaults to 1.' ) prompt: str = Field( ..., description='Required. The prompt to use to generate the image.' @@ -103,11 +194,8 @@ class ImageRequest(BaseModel): None, description="Optional. Resolution (e.g., 'RESOLUTION_1024_1024'). Only for model V_2. Cannot be used with aspect_ratio.", ) - seed: Optional[int] = Field( - None, - description='Optional. A number between 0 and 2147483647.', - ge=0, - le=2147483647, + seed: Optional[conint(ge=0, le=2147483647)] = Field( + None, description='Optional. A number between 0 and 2147483647.' ) style_type: Optional[str] = Field( None, @@ -150,6 +238,19 @@ class IdeogramGenerateResponse(BaseModel): ) +class ColorPalette(BaseModel): + name: str = Field(..., description='Name of the color palette', examples=['PASTEL']) + + +class MagicPrompt(str, Enum): + ON = 'ON' + OFF = 'OFF' + + +class StyleType(str, Enum): + GENERAL = 'GENERAL' + + class KlingErrorResponse(BaseModel): code: int = Field( ..., @@ -168,41 +269,29 @@ class AspectRatio(str, Enum): class Config(BaseModel): - horizontal: Optional[float] = Field( + horizontal: Optional[confloat(ge=-10.0, le=10.0)] = Field( None, description="Controls camera's movement along horizontal axis (x-axis). Negative indicates left, positive indicates right.", - ge=-10.0, - le=10.0, ) - pan: Optional[float] = Field( + pan: Optional[confloat(ge=-10.0, le=10.0)] = Field( None, description="Controls camera's rotation in vertical plane (x-axis). Negative indicates downward rotation, positive indicates upward rotation.", - ge=-10.0, - le=10.0, ) - roll: Optional[float] = Field( + roll: Optional[confloat(ge=-10.0, le=10.0)] = Field( None, description="Controls camera's rolling amount (z-axis). Negative indicates counterclockwise, positive indicates clockwise.", - ge=-10.0, - le=10.0, ) - tilt: Optional[float] = Field( + tilt: Optional[confloat(ge=-10.0, le=10.0)] = Field( None, description="Controls camera's rotation in horizontal plane (y-axis). Negative indicates left rotation, positive indicates right rotation.", - ge=-10.0, - le=10.0, ) - vertical: Optional[float] = Field( + vertical: Optional[confloat(ge=-10.0, le=10.0)] = Field( None, description="Controls camera's movement along vertical axis (y-axis). Negative indicates downward, positive indicates upward.", - ge=-10.0, - le=10.0, ) - zoom: Optional[float] = Field( + zoom: Optional[confloat(ge=-10.0, le=10.0)] = Field( None, description="Controls change in camera's focal length. Negative indicates narrower field of view, positive indicates wider field of view.", - ge=-10.0, - le=10.0, ) @@ -265,11 +354,9 @@ class KlingImage2VideoRequest(BaseModel): description='The callback notification address. Server will notify when the task status changes.', ) camera_control: Optional[CameraControl] = None - cfg_scale: Optional[float] = Field( + cfg_scale: Optional[confloat(ge=0.0, le=1.0)] = Field( 0.5, description="Flexibility in video generation. The higher the value, the lower the model's degree of flexibility, and the stronger the relevance to the user's prompt.", - ge=0.0, - le=1.0, ) duration: Optional[Duration] = Field('5', description='Video length in seconds') dynamic_masks: Optional[List[DynamicMask]] = Field( @@ -293,11 +380,11 @@ class KlingImage2VideoRequest(BaseModel): description='Video generation mode. std: Standard Mode, which is cost-effective. pro: Professional Mode, generates videos with longer duration but higher quality output.', ) model_name: Optional[ModelName] = Field('kling-v1', description='Model Name') - negative_prompt: Optional[str] = Field( - None, description='Negative text prompt', max_length=2500 + negative_prompt: Optional[constr(max_length=2500)] = Field( + None, description='Negative text prompt' ) - prompt: Optional[str] = Field( - None, description='Positive text prompt', max_length=2500 + prompt: Optional[constr(max_length=2500)] = Field( + None, description='Positive text prompt' ) static_mask: Optional[AnyUrl] = Field( None, @@ -343,12 +430,12 @@ class KlingImage2VideoResponse(BaseModel): class Config1(BaseModel): - horizontal: Optional[float] = Field(None, ge=-10.0, le=10.0) - pan: Optional[float] = Field(None, ge=-10.0, le=10.0) - roll: Optional[float] = Field(None, ge=-10.0, le=10.0) - tilt: Optional[float] = Field(None, ge=-10.0, le=10.0) - vertical: Optional[float] = Field(None, ge=-10.0, le=10.0) - zoom: Optional[float] = Field(None, ge=-10.0, le=10.0) + horizontal: Optional[confloat(ge=-10.0, le=10.0)] = None + pan: Optional[confloat(ge=-10.0, le=10.0)] = None + roll: Optional[confloat(ge=-10.0, le=10.0)] = None + tilt: Optional[confloat(ge=-10.0, le=10.0)] = None + vertical: Optional[confloat(ge=-10.0, le=10.0)] = None + zoom: Optional[confloat(ge=-10.0, le=10.0)] = None class CameraControl1(BaseModel): @@ -368,18 +455,18 @@ class KlingText2VideoRequest(BaseModel): None, description='The callback notification address' ) camera_control: Optional[CameraControl1] = None - cfg_scale: Optional[float] = Field( - 0.5, description='Flexibility in video generation', ge=0.0, le=1.0 + cfg_scale: Optional[confloat(ge=0.0, le=1.0)] = Field( + 0.5, description='Flexibility in video generation' ) duration: Optional[Duration] = '5' external_task_id: Optional[str] = Field(None, description='Customized Task ID') mode: Optional[Mode] = Field('std', description='Video generation mode') model_name: Optional[ModelName1] = Field('kling-v1', description='Model Name') - negative_prompt: Optional[str] = Field( - None, description='Negative text prompt', max_length=2500 + negative_prompt: Optional[constr(max_length=2500)] = Field( + None, description='Negative text prompt' ) - prompt: Optional[str] = Field( - None, description='Positive text prompt', max_length=2500 + prompt: Optional[constr(max_length=2500)] = Field( + None, description='Positive text prompt' ) @@ -550,6 +637,36 @@ class LumaVideoModelOutputResolution( root: Union[LumaVideoModelOutputResolution1, str] +class MachineStats(BaseModel): + cpu_capacity: Optional[str] = Field(None, description='Total CPU on the machine.') + disk_capacity: Optional[str] = Field( + None, description='Total disk capacity on the machine.' + ) + gpu_type: Optional[str] = Field( + None, description='The GPU type. eg. NVIDIA Tesla K80' + ) + initial_cpu: Optional[str] = Field( + None, description='Initial CPU available before the job starts.' + ) + initial_disk: Optional[str] = Field( + None, description='Initial disk available before the job starts.' + ) + initial_ram: Optional[str] = Field( + None, description='Initial RAM available before the job starts.' + ) + machine_name: Optional[str] = Field(None, description='Name of the machine.') + memory_capacity: Optional[str] = Field( + None, description='Total memory on the machine.' + ) + os_version: Optional[str] = Field( + None, description='The operating system version. eg. Ubuntu Linux 20.04' + ) + pip_freeze: Optional[str] = Field(None, description='The pip freeze output') + vram_time_series: Optional[Dict[str, Any]] = Field( + None, description='Time series of VRAM usage.' + ) + + class MinimaxBaseResponse(BaseModel): status_code: int = Field( ..., @@ -631,10 +748,9 @@ class MinimaxVideoGenerationRequest(BaseModel): ..., description='Required. ID of model. Options: T2V-01-Director, I2V-01-Director, S2V-01, I2V-01, I2V-01-live, T2V-01', ) - prompt: Optional[str] = Field( + prompt: Optional[constr(max_length=2000)] = Field( None, description='Description of the video. Should be less than 2000 characters. Supports camera movement instructions in [brackets].', - max_length=2000, ) prompt_optimizer: Optional[bool] = Field( True, @@ -653,6 +769,29 @@ class MinimaxVideoGenerationResponse(BaseModel): ) +class NodeStatus(str, Enum): + NodeStatusActive = 'NodeStatusActive' + NodeStatusDeleted = 'NodeStatusDeleted' + NodeStatusBanned = 'NodeStatusBanned' + + +class NodeVersionStatus(str, Enum): + NodeVersionStatusActive = 'NodeVersionStatusActive' + NodeVersionStatusDeleted = 'NodeVersionStatusDeleted' + NodeVersionStatusBanned = 'NodeVersionStatusBanned' + NodeVersionStatusPending = 'NodeVersionStatusPending' + NodeVersionStatusFlagged = 'NodeVersionStatusFlagged' + + +class NodeVersionUpdateRequest(BaseModel): + changelog: Optional[str] = Field( + None, description='The changelog describing the version changes.' + ) + deprecated: Optional[bool] = Field( + None, description='Whether the version is deprecated.' + ) + + class Moderation(str, Enum): low = 'low' auto = 'auto' @@ -794,13 +933,22 @@ class OpenAIImageGenerationResponse(BaseModel): usage: Optional[Usage] = None -class AspectRatio2(RootModel[float]): - root: float = Field( - ..., - description='Aspect ratio (width / height)', - ge=0.4, - le=2.5, - title='Aspectratio', +class PersonalAccessToken(BaseModel): + createdAt: Optional[datetime] = Field( + None, description='[Output Only]The date and time the token was created.' + ) + description: Optional[str] = Field( + None, + description="Optional. A more detailed description of the token's intended use.", + ) + id: Optional[UUID] = Field(None, description='Unique identifier for the GitCommit') + name: Optional[str] = Field( + None, + description='Required. The name of the token. Can be a simple description.', + ) + token: Optional[str] = Field( + None, + description='[Output Only]. The personal access token. Only returned during creation.', ) @@ -808,10 +956,9 @@ class IngredientsMode(str, Enum): creative = 'creative' precise = 'precise' -bytes_aliased = bytes class PikaBodyGenerate22C2vGenerate22PikascenesPost(BaseModel): - aspectRatio: Optional[AspectRatio2] = Field( + aspectRatio: Optional[confloat(ge=0.4, le=2.5)] = Field( None, description='Aspect ratio (width / height)', title='Aspectratio' ) duration: Optional[int] = Field(5, title='Duration') @@ -846,7 +993,7 @@ class PikaBodyGenerate22KeyframeGenerate22PikaframesPost(BaseModel): class PikaBodyGenerate22T2vGenerate22T2vPost(BaseModel): - aspectRatio: Optional[AspectRatio2] = Field( + aspectRatio: Optional[confloat(ge=0.4, le=2.5)] = Field( None, description='Aspect ratio (width / height)', title='Aspectratio' ) duration: Optional[int] = Field(5, title='Duration') @@ -925,7 +1072,7 @@ class PixverseImageVideoRequest(BaseModel): water_mark: Optional[bool] = None -class AspectRatio4(str, Enum): +class AspectRatio2(str, Enum): field_16_9 = '16:9' field_4_3 = '4:3' field_1_1 = '1:1' @@ -934,7 +1081,7 @@ class AspectRatio4(str, Enum): class PixverseTextVideoRequest(BaseModel): - aspect_ratio: AspectRatio4 + aspect_ratio: AspectRatio2 duration: Duration2 model: Model1 motion_mode: Optional[MotionMode] = None @@ -1004,26 +1151,29 @@ class PixverseVideoResultResponse(BaseModel): Resp: Optional[Resp2] = None -class RGBColorItem(RootModel[int]): - root: int = Field(..., ge=0, le=255) +class PublisherStatus(str, Enum): + PublisherStatusActive = 'PublisherStatusActive' + PublisherStatusBanned = 'PublisherStatusBanned' -class RGBColor(RootModel[List[RGBColorItem]]): - root: List[RGBColorItem] = Field( - ..., - description='RGB color values', - examples=[[255, 0, 0]], - max_length=3, - min_length=3, - ) +class PublisherUser(BaseModel): + email: Optional[str] = Field(None, description='The email address for this user.') + id: Optional[str] = Field(None, description='The unique id for this user.') + name: Optional[str] = Field(None, description='The name for this user.') + + +class RgbItem(RootModel[conint(ge=0, le=255)]): + root: conint(ge=0, le=255) + + +class RGBColor(BaseModel): + rgb: List[RgbItem] = Field(..., max_length=3, min_length=3) class Controls(BaseModel): - artistic_level: Optional[int] = Field( + artistic_level: Optional[conint(ge=0, le=5)] = Field( None, description='Defines artistic tone of your image. At a simple level, the person looks straight at the camera in a static and clean style. Dynamic and eccentric levels introduce movement and creativity.', - ge=0, - le=5, ) background_color: Optional[RGBColor] = None colors: Optional[List[RGBColor]] = Field( @@ -1039,7 +1189,7 @@ class RecraftImageGenerationRequest(BaseModel): model: str = Field( ..., description='The model to use for generation (e.g., "recraftv3")' ) - n: int = Field(..., description='The number of images to generate', ge=1, le=4) + n: conint(ge=1, le=4) = Field(..., description='The number of images to generate') prompt: str = Field( ..., description='The text prompt describing the image to generate' ) @@ -1050,6 +1200,10 @@ class RecraftImageGenerationRequest(BaseModel): None, description='The style to apply to the generated image (e.g., "digital_illustration")', ) + style_id: Optional[str] = Field( + None, + description='The style ID to apply to the generated image (e.g., "123e4567-e89b-12d3-a456-426614174000"). If style_id is provided, style should not be provided.', + ) class Datum2(BaseModel): @@ -1067,6 +1221,12 @@ class RecraftImageGenerationResponse(BaseModel): data: List[Datum2] = Field(..., description='Array of generated image information') +class RenderingSpeed(str, Enum): + BALANCED = 'BALANCED' + TURBO = 'TURBO' + QUALITY = 'QUALITY' + + class RunwayAspectRatioEnum(str, Enum): field_1280_720 = '1280:720' field_720_1280 = '720:1280' @@ -1102,7 +1262,7 @@ class RunwayPromptImageDetailedObject(BaseModel): ..., description="The position of the image in the output video. 'last' is currently supported for gen3a_turbo only.", ) - uri: str = Field( + uri: AnyUrl = Field( ..., description='A HTTPS URL or data URI containing an encoded image.' ) @@ -1132,6 +1292,210 @@ class RunwayTaskStatusResponse(BaseModel): status: Optional[RunwayTaskStatusEnum] = None +class Name(str, Enum): + content_moderation = 'content_moderation' + + +class StabilityContentModerationResponse(BaseModel): + errors: List[str] = Field( + ..., + description='One or more error messages indicating what went wrong.', + examples=[['some-field: is required']], + min_length=1, + ) + id: constr(min_length=1) = Field( + ..., + description='A unique identifier associated with this error. Please include this in any [support tickets](https://kb.stability.ai/knowledge-base/kb-tickets/new) you file, as it will greatly assist us in diagnosing the root cause of the problem.', + examples=['a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4'], + ) + name: Name = Field( + ..., + description='Our content moderation system has flagged some part of your request and subsequently denied it. You were not charged for this request. While this may at times be frustrating, it is necessary to maintain the integrity of our platform and ensure a safe experience for all users. If you would like to provide feedback, please use the [Support Form](https://kb.stability.ai/knowledge-base/kb-tickets/new).', + ) + + +class StabilityStabilityClientID(RootModel[constr(max_length=256)]): + root: constr(max_length=256) = Field( + ..., + description='The name of your application, used to help us communicate app-specific debugging or moderation issues to you.', + examples=['my-awesome-app'], + ) + + +class StabilityStabilityClientUserID(RootModel[constr(max_length=256)]): + root: constr(max_length=256) = Field( + ..., + description='A unique identifier for your end user. Used to help us communicate user-specific debugging or moderation issues to you. Feel free to obfuscate this value to protect user privacy.', + examples=['DiscordUser#9999'], + ) + + +class StabilityStabilityClientVersion(RootModel[constr(max_length=256)]): + root: constr(max_length=256) = Field( + ..., + description='The version of your application, used to help us communicate version-specific debugging or moderation issues to you.', + examples=['1.2.1'], + ) + + +class StorageFile(BaseModel): + file_path: Optional[str] = Field(None, description='Path to the file in storage') + id: Optional[UUID] = Field( + None, description='Unique identifier for the storage file' + ) + public_url: Optional[str] = Field(None, description='Public URL') + + +class StripeAddress(BaseModel): + city: Optional[str] = None + country: Optional[str] = None + line1: Optional[str] = None + line2: Optional[str] = None + postal_code: Optional[str] = None + state: Optional[str] = None + + +class StripeAmountDetails(BaseModel): + tip: Optional[Dict[str, Any]] = None + + +class StripeBillingDetails(BaseModel): + address: Optional[StripeAddress] = None + email: Optional[str] = None + name: Optional[str] = None + phone: Optional[str] = None + tax_id: Optional[Any] = None + + +class Checks(BaseModel): + address_line1_check: Optional[Any] = None + address_postal_code_check: Optional[Any] = None + cvc_check: Optional[str] = None + + +class ExtendedAuthorization(BaseModel): + status: Optional[str] = None + + +class IncrementalAuthorization(BaseModel): + status: Optional[str] = None + + +class Multicapture(BaseModel): + status: Optional[str] = None + + +class NetworkToken(BaseModel): + used: Optional[bool] = None + + +class Overcapture(BaseModel): + maximum_amount_capturable: Optional[int] = None + status: Optional[str] = None + + +class StripeCardDetails(BaseModel): + amount_authorized: Optional[int] = None + authorization_code: Optional[Any] = None + brand: Optional[str] = None + checks: Optional[Checks] = None + country: Optional[str] = None + exp_month: Optional[int] = None + exp_year: Optional[int] = None + extended_authorization: Optional[ExtendedAuthorization] = None + fingerprint: Optional[str] = None + funding: Optional[str] = None + incremental_authorization: Optional[IncrementalAuthorization] = None + installments: Optional[Any] = None + last4: Optional[str] = None + mandate: Optional[Any] = None + multicapture: Optional[Multicapture] = None + network: Optional[str] = None + network_token: Optional[NetworkToken] = None + network_transaction_id: Optional[str] = None + overcapture: Optional[Overcapture] = None + regulated_status: Optional[str] = None + three_d_secure: Optional[Any] = None + wallet: Optional[Any] = None + + +class Object(str, Enum): + charge = 'charge' + + +class Object1(str, Enum): + event = 'event' + + +class Type4(str, Enum): + payment_intent_succeeded = 'payment_intent.succeeded' + + +class StripeOutcome(BaseModel): + advice_code: Optional[Any] = None + network_advice_code: Optional[Any] = None + network_decline_code: Optional[Any] = None + network_status: Optional[str] = None + reason: Optional[Any] = None + risk_level: Optional[str] = None + risk_score: Optional[int] = None + seller_message: Optional[str] = None + type: Optional[str] = None + + +class Object2(str, Enum): + payment_intent = 'payment_intent' + + +class StripePaymentMethodDetails(BaseModel): + card: Optional[StripeCardDetails] = None + type: Optional[str] = None + + +class Card(BaseModel): + installments: Optional[Any] = None + mandate_options: Optional[Any] = None + network: Optional[Any] = None + request_three_d_secure: Optional[str] = None + + +class StripePaymentMethodOptions(BaseModel): + card: Optional[Card] = None + + +class StripeRefundList(BaseModel): + data: Optional[List[Dict[str, Any]]] = None + has_more: Optional[bool] = None + object: Optional[str] = None + total_count: Optional[int] = None + url: Optional[str] = None + + +class StripeRequestInfo(BaseModel): + id: Optional[str] = None + idempotency_key: Optional[str] = None + + +class StripeShipping(BaseModel): + address: Optional[StripeAddress] = None + carrier: Optional[str] = None + name: Optional[str] = None + phone: Optional[str] = None + tracking_number: Optional[str] = None + + +class User(BaseModel): + email: Optional[str] = Field(None, description='The email address for this user.') + id: Optional[str] = Field(None, description='The unique id for this user.') + isAdmin: Optional[bool] = Field( + None, description='Indicates if the user has admin privileges.' + ) + isApproved: Optional[bool] = Field( + None, description='Indicates if the user is approved.' + ) + name: Optional[str] = Field(None, description='The name for this user.') + + class Veo2GenVidPollRequest(BaseModel): operationName: str = Field( ..., @@ -1142,7 +1506,7 @@ class Veo2GenVidPollRequest(BaseModel): ) -class Error(BaseModel): +class Error1(BaseModel): code: Optional[int] = Field(None, description='Error code') message: Optional[str] = Field(None, description='Error message') @@ -1174,7 +1538,7 @@ class Response(BaseModel): class Veo2GenVidPollResponse(BaseModel): done: Optional[bool] = None - error: Optional[Error] = Field( + error: Optional[Error1] = Field( None, description='Error details if operation failed' ) name: Optional[str] = None @@ -1235,6 +1599,125 @@ class Veo2GenVidResponse(BaseModel): ) +class WorkflowRunStatus(str, Enum): + WorkflowRunStatusStarted = 'WorkflowRunStatusStarted' + WorkflowRunStatusFailed = 'WorkflowRunStatusFailed' + WorkflowRunStatusCompleted = 'WorkflowRunStatusCompleted' + + +class ActionJobResult(BaseModel): + action_job_id: Optional[str] = Field( + None, description='Identifier of the job this result belongs to' + ) + action_run_id: Optional[str] = Field( + None, description='Identifier of the run this result belongs to' + ) + author: Optional[str] = Field(None, description='The author of the commit') + avg_vram: Optional[int] = Field( + None, description='The average VRAM used by the job' + ) + branch_name: Optional[str] = Field( + None, description='Name of the relevant git branch' + ) + comfy_run_flags: Optional[str] = Field( + None, description='The comfy run flags. E.g. `--low-vram`' + ) + commit_hash: Optional[str] = Field(None, description='The hash of the commit') + commit_id: Optional[str] = Field(None, description='The ID of the commit') + commit_message: Optional[str] = Field(None, description='The message of the commit') + commit_time: Optional[int] = Field( + None, description='The Unix timestamp when the commit was made' + ) + cuda_version: Optional[str] = Field(None, description='CUDA version used') + end_time: Optional[int] = Field( + None, description='The end time of the job as a Unix timestamp.' + ) + git_repo: Optional[str] = Field(None, description='The repository name') + id: Optional[UUID] = Field(None, description='Unique identifier for the job result') + job_trigger_user: Optional[str] = Field( + None, description='The user who triggered the job.' + ) + machine_stats: Optional[MachineStats] = None + operating_system: Optional[str] = Field(None, description='Operating system used') + peak_vram: Optional[int] = Field(None, description='The peak VRAM used by the job') + pr_number: Optional[str] = Field(None, description='The pull request number') + python_version: Optional[str] = Field(None, description='PyTorch version used') + pytorch_version: Optional[str] = Field(None, description='PyTorch version used') + start_time: Optional[int] = Field( + None, description='The start time of the job as a Unix timestamp.' + ) + status: Optional[WorkflowRunStatus] = None + storage_file: Optional[StorageFile] = None + workflow_name: Optional[str] = Field(None, description='Name of the workflow') + + +class IdeogramV3EditRequest(BaseModel): + color_palette: Optional[IdeogramColorPalette] = None + image: Optional[bytes_aliased] = Field( + None, + description='The image being edited (max size 10MB); only JPEG, WebP and PNG formats are supported at this time.', + ) + magic_prompt: Optional[str] = Field( + None, + description='Determine if MagicPrompt should be used in generating the request or not.', + ) + mask: Optional[bytes_aliased] = Field( + None, + description='A black and white image of the same size as the image being edited (max size 10MB). Black regions in the mask should match up with the regions of the image that you would like to edit; only JPEG, WebP and PNG formats are supported at this time.', + ) + num_images: Optional[int] = Field( + None, description='The number of images to generate.' + ) + prompt: str = Field( + ..., description='The prompt used to describe the edited result.' + ) + rendering_speed: RenderingSpeed + seed: Optional[int] = Field( + None, description='Random seed. Set for reproducible generation.' + ) + style_codes: Optional[List[constr(pattern=r'^[0-9A-Fa-f]{8}$')]] = Field( + None, + description='A list of 8 character hexadecimal codes representing the style of the image. Cannot be used in conjunction with style_reference_images or style_type.', + ) + style_reference_images: Optional[List[bytes_aliased]] = Field( + None, + description='A set of images to use as style references (maximum total size 10MB across all style references). The images should be in JPEG, PNG or WebP format.', + ) + + +class IdeogramV3Request(BaseModel): + aspect_ratio: Optional[str] = Field( + None, description='Aspect ratio in format WxH', examples=['1x3'] + ) + color_palette: Optional[ColorPalette] = None + magic_prompt: Optional[MagicPrompt] = Field( + None, description='Whether to enable magic prompt enhancement' + ) + negative_prompt: Optional[str] = Field( + None, description='Text prompt specifying what to avoid in the generation' + ) + num_images: Optional[conint(ge=1)] = Field( + None, description='Number of images to generate' + ) + prompt: str = Field(..., description='The text prompt for image generation') + rendering_speed: RenderingSpeed + resolution: Optional[str] = Field( + None, description='Image resolution in format WxH', examples=['1280x800'] + ) + seed: Optional[int] = Field( + None, description='Seed value for reproducible generation' + ) + style_codes: Optional[List[constr(pattern=r'^[0-9A-Fa-f]{8}$')]] = Field( + None, description='Array of style codes in hexadecimal format' + ) + style_reference_images: Optional[List[str]] = Field( + None, description='Array of reference image URLs or identifiers' + ) + style_type: Optional[StyleType] = Field( + None, description='The type of style to apply' + ) + + class LumaGenerationRequest(BaseModel): aspect_ratio: LumaAspectRatio callback_url: Optional[AnyUrl] = Field( @@ -1276,23 +1759,169 @@ class LumaUpscaleVideoGenerationRequest(BaseModel): resolution: Optional[LumaVideoModelOutputResolution] = None +class NodeVersion(BaseModel): + changelog: Optional[str] = Field( + None, description='Summary of changes made in this version' + ) + comfy_node_extract_status: Optional[str] = Field( + None, description='The status of comfy node extraction process.' + ) + createdAt: Optional[datetime] = Field( + None, description='The date and time the version was created.' + ) + dependencies: Optional[List[str]] = Field( + None, description='A list of pip dependencies required by the node.' + ) + deprecated: Optional[bool] = Field( + None, description='Indicates if this version is deprecated.' + ) + downloadUrl: Optional[str] = Field( + None, description='[Output Only] URL to download this version of the node' + ) + id: Optional[str] = None + node_id: Optional[str] = Field( + None, description='The unique identifier of the node.' + ) + status: Optional[NodeVersionStatus] = None + status_reason: Optional[str] = Field( + None, description='The reason for the status change.' + ) + version: Optional[str] = Field( + None, + description='The version identifier, following semantic versioning. Must be unique for the node.', + ) + + class PikaHTTPValidationError(BaseModel): detail: Optional[List[PikaValidationError]] = Field(None, title='Detail') +class PublisherMember(BaseModel): + id: Optional[str] = Field( + None, description='The unique identifier for the publisher member.' + ) + role: Optional[str] = Field( + None, description='The role of the user in the publisher.' + ) + user: Optional[PublisherUser] = None + + class RunwayImageToVideoRequest(BaseModel): duration: RunwayDurationEnum model: RunwayModelEnum promptImage: RunwayPromptImageObject - promptText: Optional[str] = Field( - None, description='Text prompt for the generation', max_length=1000 + promptText: Optional[constr(max_length=1000)] = Field( + None, description='Text prompt for the generation' ) ratio: RunwayAspectRatioEnum - seed: int = Field( - ..., description='Random seed for generation', ge=0, le=4294967295 + seed: conint(ge=0, le=4294967295) = Field( + ..., description='Random seed for generation' ) +class StripeCharge(BaseModel): + amount: Optional[int] = None + amount_captured: Optional[int] = None + amount_refunded: Optional[int] = None + application: Optional[str] = None + application_fee: Optional[str] = None + application_fee_amount: Optional[int] = None + balance_transaction: Optional[str] = None + billing_details: Optional[StripeBillingDetails] = None + calculated_statement_descriptor: Optional[str] = None + captured: Optional[bool] = None + created: Optional[int] = None + currency: Optional[str] = None + customer: Optional[str] = None + description: Optional[str] = None + destination: Optional[Any] = None + dispute: Optional[Any] = None + disputed: Optional[bool] = None + failure_balance_transaction: Optional[Any] = None + failure_code: Optional[Any] = None + failure_message: Optional[Any] = None + fraud_details: Optional[Dict[str, Any]] = None + id: Optional[str] = None + invoice: Optional[Any] = None + livemode: Optional[bool] = None + metadata: Optional[Dict[str, Any]] = None + object: Optional[Object] = None + on_behalf_of: Optional[Any] = None + order: Optional[Any] = None + outcome: Optional[StripeOutcome] = None + paid: Optional[bool] = None + payment_intent: Optional[str] = None + payment_method: Optional[str] = None + payment_method_details: Optional[StripePaymentMethodDetails] = None + radar_options: Optional[Dict[str, Any]] = None + receipt_email: Optional[str] = None + receipt_number: Optional[str] = None + receipt_url: Optional[str] = None + refunded: Optional[bool] = None + refunds: Optional[StripeRefundList] = None + review: Optional[Any] = None + shipping: Optional[StripeShipping] = None + source: Optional[Any] = None + source_transfer: Optional[Any] = None + statement_descriptor: Optional[Any] = None + statement_descriptor_suffix: Optional[Any] = None + status: Optional[str] = None + transfer_data: Optional[Any] = None + transfer_group: Optional[Any] = None + + +class StripeChargeList(BaseModel): + data: Optional[List[StripeCharge]] = None + has_more: Optional[bool] = None + object: Optional[str] = None + total_count: Optional[int] = None + url: Optional[str] = None + + +class StripePaymentIntent(BaseModel): + amount: Optional[int] = None + amount_capturable: Optional[int] = None + amount_details: Optional[StripeAmountDetails] = None + amount_received: Optional[int] = None + application: Optional[str] = None + application_fee_amount: Optional[int] = None + automatic_payment_methods: Optional[Any] = None + canceled_at: Optional[int] = None + cancellation_reason: Optional[str] = None + capture_method: Optional[str] = None + charges: Optional[StripeChargeList] = None + client_secret: Optional[str] = None + confirmation_method: Optional[str] = None + created: Optional[int] = None + currency: Optional[str] = None + customer: Optional[str] = None + description: Optional[str] = None + id: Optional[str] = None + invoice: Optional[str] = None + last_payment_error: Optional[Any] = None + latest_charge: Optional[str] = None + livemode: Optional[bool] = None + metadata: Optional[Dict[str, Any]] = None + next_action: Optional[Any] = None + object: Optional[Object2] = None + on_behalf_of: Optional[Any] = None + payment_method: Optional[str] = None + payment_method_configuration_details: Optional[Any] = None + payment_method_options: Optional[StripePaymentMethodOptions] = None + payment_method_types: Optional[List[str]] = None + processing: Optional[Any] = None + receipt_email: Optional[str] = None + review: Optional[Any] = None + setup_future_usage: Optional[Any] = None + shipping: Optional[StripeShipping] = None + source: Optional[Any] = None + statement_descriptor: Optional[Any] = None + statement_descriptor_suffix: Optional[Any] = None + status: Optional[str] = None + transfer_data: Optional[Any] = None + transfer_group: Optional[Any] = None + + class LumaGeneration(BaseModel): assets: Optional[LumaAssets] = None created_at: Optional[datetime] = Field( @@ -1313,3 +1942,64 @@ class LumaGeneration(BaseModel): ] ] = Field(None, description='The request of the generation') state: Optional[LumaState] = None + + +class Publisher(BaseModel): + createdAt: Optional[datetime] = Field( + None, description='The date and time the publisher was created.' + ) + description: Optional[str] = None + id: Optional[str] = Field( + None, + description="The unique identifier for the publisher. It's akin to a username. Should be lowercase.", + ) + logo: Optional[str] = Field(None, description="URL to the publisher's logo.") + members: Optional[List[PublisherMember]] = Field( + None, description='A list of members in the publisher.' + ) + name: Optional[str] = None + source_code_repo: Optional[str] = None + status: Optional[PublisherStatus] = None + support: Optional[str] = None + website: Optional[str] = None + + +class Data2(BaseModel): + object: Optional[StripePaymentIntent] = None + + +class StripeEvent(BaseModel): + api_version: Optional[str] = None + created: Optional[int] = None + data: Data2 + id: str + livemode: Optional[bool] = None + object: Object1 + pending_webhooks: Optional[int] = None + request: Optional[StripeRequestInfo] = None + type: Type4 + + +class Node(BaseModel): + author: Optional[str] = None + category: Optional[str] = Field(None, description='The category of the node.') + description: Optional[str] = None + downloads: Optional[int] = Field( + None, description='The number of downloads of the node.' + ) + icon: Optional[str] = Field(None, description="URL to the node's icon.") + id: Optional[str] = Field(None, description='The unique identifier of the node.') + latest_version: Optional[NodeVersion] = None + license: Optional[str] = Field( + None, description="The path to the LICENSE file in the node's repository." + ) + name: Optional[str] = Field(None, description='The display name of the node.') + publisher: Optional[Publisher] = None + rating: Optional[float] = Field(None, description='The average rating of the node.') + repository: Optional[str] = Field(None, description="URL to the node's repository.") + status: Optional[NodeStatus] = None + status_detail: Optional[str] = Field( + None, description='The status detail of the node.' + ) + tags: Optional[List[str]] = None + translations: Optional[Dict[str, Dict[str, Any]]] = None diff --git a/comfy_api_nodes/apis/client.py b/comfy_api_nodes/apis/client.py index 22e011f47..b376aafe6 100644 --- a/comfy_api_nodes/apis/client.py +++ b/comfy_api_nodes/apis/client.py @@ -431,6 +431,11 @@ class SynchronousOperation(Generic[T, R]): else self.request.model_dump(exclude_none=True) ) + if request_dict: + for key, value in request_dict.items(): + if isinstance(value, Enum): + request_dict[key] = value.value + # Debug log for request logging.debug( f"[DEBUG] API Request: {self.endpoint.method.value} {self.endpoint.path}" diff --git a/comfy_api_nodes/nodes_ideogram.py b/comfy_api_nodes/nodes_ideogram.py index 653e797a0..51e81286f 100644 --- a/comfy_api_nodes/nodes_ideogram.py +++ b/comfy_api_nodes/nodes_ideogram.py @@ -1,9 +1,14 @@ from comfy.comfy_types.node_typing import IO, ComfyNodeABC, InputTypeDict from inspect import cleandoc +from PIL import Image +import numpy as np +import io from comfy_api_nodes.apis import ( IdeogramGenerateRequest, IdeogramGenerateResponse, ImageRequest, + IdeogramV3Request, + IdeogramV3EditRequest, ) from comfy_api_nodes.apis.client import ( @@ -17,7 +22,7 @@ from comfy_api_nodes.apinode_utils import ( bytesio_to_image_tensor, ) -RESOLUTION_MAPPING = { +V1_V1_RES_MAP = { "Auto":"AUTO", "512 x 1536":"RESOLUTION_512_1536", "576 x 1408":"RESOLUTION_576_1408", @@ -99,7 +104,7 @@ RESOLUTION_MAPPING = { "1536 x 640":"RESOLUTION_1536_640", } -ASPECT_RATIO_MAPPING = { +V1_V2_RATIO_MAP = { "1:1":"ASPECT_1_1", "4:3":"ASPECT_4_3", "3:4":"ASPECT_3_4", @@ -113,6 +118,97 @@ ASPECT_RATIO_MAPPING = { "5:4":"ASPECT_5_4", } +V3_RATIO_MAP = { + "1:3":"1x3", + "3:1":"3x1", + "1:2":"1x2", + "2:1":"2x1", + "9:16":"9x16", + "16:9":"16x9", + "10:16":"10x16", + "16:10":"16x10", + "2:3":"2x3", + "3:2":"3x2", + "3:4":"3x4", + "4:3":"4x3", + "4:5":"4x5", + "5:4":"5x4", + "1:1":"1x1", +} + +V3_RESOLUTIONS= [ + "Auto", + "512x1536", + "576x1408", + "576x1472", + "576x1536", + "640x1344", + "640x1408", + "640x1472", + "640x1536", + "704x1152", + "704x1216", + "704x1280", + "704x1344", + "704x1408", + "704x1472", + "736x1312", + "768x1088", + "768x1216", + "768x1280", + "768x1344", + "800x1280", + "832x960", + "832x1024", + "832x1088", + "832x1152", + "832x1216", + "832x1248", + "864x1152", + "896x960", + "896x1024", + "896x1088", + "896x1120", + "896x1152", + "960x832", + "960x896", + "960x1024", + "960x1088", + "1024x832", + "1024x896", + "1024x960", + "1024x1024", + "1088x768", + "1088x832", + "1088x896", + "1088x960", + "1120x896", + "1152x704", + "1152x832", + "1152x864", + "1152x896", + "1216x704", + "1216x768", + "1216x832", + "1248x832", + "1280x704", + "1280x768", + "1280x800", + "1312x736", + "1344x640", + "1344x704", + "1344x768", + "1408x576", + "1408x640", + "1408x704", + "1472x576", + "1472x640", + "1472x704", + "1536x512", + "1536x576", + "1536x640" +] + def download_and_process_image(image_url): """Helper function to download and process image from URL""" @@ -156,7 +252,7 @@ class IdeogramV1(ComfyNodeABC): "aspect_ratio": ( IO.COMBO, { - "options": list(ASPECT_RATIO_MAPPING.keys()), + "options": list(V1_V2_RATIO_MAP.keys()), "default": "1:1", "tooltip": "The aspect ratio for image generation.", }, @@ -214,7 +310,7 @@ class IdeogramV1(ComfyNodeABC): auth_token=None, ): # Determine the model based on turbo setting - aspect_ratio = ASPECT_RATIO_MAPPING.get(aspect_ratio, None) + aspect_ratio = V1_V2_RATIO_MAP.get(aspect_ratio, None) model = "V_1_TURBO" if turbo else "V_1" operation = SynchronousOperation( @@ -286,7 +382,7 @@ class IdeogramV2(ComfyNodeABC): "aspect_ratio": ( IO.COMBO, { - "options": list(ASPECT_RATIO_MAPPING.keys()), + "options": list(V1_V2_RATIO_MAP.keys()), "default": "1:1", "tooltip": "The aspect ratio for image generation. Ignored if resolution is not set to AUTO.", }, @@ -294,7 +390,7 @@ class IdeogramV2(ComfyNodeABC): "resolution": ( IO.COMBO, { - "options": list(RESOLUTION_MAPPING.keys()), + "options": list(V1_V1_RES_MAP.keys()), "default": "Auto", "tooltip": "The resolution for image generation. If not set to AUTO, this overrides the aspect_ratio setting.", }, @@ -370,8 +466,8 @@ class IdeogramV2(ComfyNodeABC): color_palette="", auth_token=None, ): - aspect_ratio = ASPECT_RATIO_MAPPING.get(aspect_ratio, None) - resolution = RESOLUTION_MAPPING.get(resolution, None) + aspect_ratio = V1_V2_RATIO_MAP.get(aspect_ratio, None) + resolution = V1_V1_RES_MAP.get(resolution, None) # Determine the model based on turbo setting model = "V_2_TURBO" if turbo else "V_2" @@ -422,11 +518,11 @@ class IdeogramV2(ComfyNodeABC): return (download_and_process_image(image_url),) - class IdeogramV3(ComfyNodeABC): """ Generates images synchronously using the Ideogram V3 model. + Supports both regular image generation from text prompts and image editing with mask. Images links are available for a limited period of time; if you would like to keep the image, you must download it. """ @@ -442,17 +538,39 @@ class IdeogramV3(ComfyNodeABC): { "multiline": True, "default": "", - "tooltip": "Prompt for the image generation", + "tooltip": "Prompt for the image generation or editing", }, ), }, "optional": { + "image": ( + IO.IMAGE, + { + "default": None, + "tooltip": "Optional reference image for image editing.", + }, + ), + "mask": ( + IO.MASK, + { + "default": None, + "tooltip": "Optional mask for inpainting (white areas will be replaced)", + }, + ), "aspect_ratio": ( IO.COMBO, { - "options": list(ASPECT_RATIO_MAPPING.keys()), + "options": list(V3_RATIO_MAP.keys()), "default": "1:1", - "tooltip": "The aspect ratio for image generation.", + "tooltip": "The aspect ratio for image generation. Ignored if resolution is not set to Auto.", + }, + ), + "resolution": ( + IO.COMBO, + { + "options": V3_RESOLUTIONS, + "default": "Auto", + "tooltip": "The resolution for image generation. If not set to Auto, this overrides the aspect_ratio setting.", }, ), "magic_prompt_option": ( @@ -478,6 +596,14 @@ class IdeogramV3(ComfyNodeABC): IO.INT, {"default": 1, "min": 1, "max": 8, "step": 1, "display": "number"}, ), + "rendering_speed": ( + IO.COMBO, + { + "options": ["BALANCED", "TURBO", "QUALITY"], + "default": "BALANCED", + "tooltip": "Controls the trade-off between generation speed and quality", + }, + ), }, "hidden": {"auth_token": "AUTH_TOKEN_COMFY_ORG"}, } @@ -491,38 +617,119 @@ class IdeogramV3(ComfyNodeABC): def api_call( self, prompt, - aspect_ratio="ASPECT_1_1", + image=None, + mask=None, + resolution="Auto", + aspect_ratio="1:1", magic_prompt_option="AUTO", seed=0, num_images=1, + rendering_speed="BALANCED", auth_token=None, ): - aspect_ratio = ASPECT_RATIO_MAPPING.get(aspect_ratio, None) - # V3 model - no turbo option - model = "V_3" + # Check if both image and mask are provided for editing mode + if image is not None and mask is not None: + # Edit mode + path = "/proxy/ideogram/ideogram-v3/edit" - operation = SynchronousOperation( - endpoint=ApiEndpoint( - path="/proxy/ideogram/generate", - method=HttpMethod.POST, - request_model=IdeogramGenerateRequest, - response_model=IdeogramGenerateResponse, - ), - request=IdeogramGenerateRequest( - image_request=ImageRequest( - prompt=prompt, - model=model, - num_images=num_images, - seed=seed, - aspect_ratio=aspect_ratio if aspect_ratio != "ASPECT_1_1" else None, - magic_prompt_option=( - magic_prompt_option if magic_prompt_option != "AUTO" else None - ), - ) - ), - auth_token=auth_token, - ) + # Process image and mask + input_tensor = image.squeeze().cpu() + # Validate mask dimensions match image + if mask.shape[1:] != image.shape[1:-1]: + raise Exception("Mask and Image must be the same size") + + # Process image + img_np = (input_tensor.numpy() * 255).astype(np.uint8) + img = Image.fromarray(img_np) + img_byte_arr = io.BytesIO() + img.save(img_byte_arr, format="PNG") + img_byte_arr.seek(0) + img_binary = img_byte_arr + img_binary.name = "image.png" + + # Process mask - white areas will be replaced + mask_np = (mask.squeeze().cpu().numpy() * 255).astype(np.uint8) + mask_img = Image.fromarray(mask_np) + mask_byte_arr = io.BytesIO() + mask_img.save(mask_byte_arr, format="PNG") + mask_byte_arr.seek(0) + mask_binary = mask_byte_arr + mask_binary.name = "mask.png" + + # Create edit request + edit_request = IdeogramV3EditRequest( + prompt=prompt, + rendering_speed=rendering_speed, + ) + + # Add optional parameters + if magic_prompt_option != "AUTO": + edit_request.magic_prompt = magic_prompt_option + if seed != 0: + edit_request.seed = seed + if num_images > 1: + edit_request.num_images = num_images + + # Execute the operation for edit mode + operation = SynchronousOperation( + endpoint=ApiEndpoint( + path=path, + method=HttpMethod.POST, + request_model=IdeogramV3EditRequest, + response_model=IdeogramGenerateResponse, + ), + request=edit_request, + files={ + "image": img_binary, + "mask": mask_binary, + }, + content_type="multipart/form-data", + auth_token=auth_token, + ) + + elif image is not None or mask is not None: + # If only one of image or mask is provided, raise an error + raise Exception("Ideogram V3 image editing requires both an image AND a mask") + else: + # Generation mode + path = "/proxy/ideogram/ideogram-v3/generate" + + # Create generation request + gen_request = IdeogramV3Request( + prompt=prompt, + rendering_speed=rendering_speed, + ) + + # Handle resolution vs aspect ratio + if resolution != "Auto": + gen_request.resolution = resolution + elif aspect_ratio != "1:1": + v3_aspect = V3_RATIO_MAP.get(aspect_ratio) + if v3_aspect: + gen_request.aspect_ratio = v3_aspect + + # Add optional parameters + if magic_prompt_option != "AUTO": + gen_request.magic_prompt = magic_prompt_option + if seed != 0: + gen_request.seed = seed + if num_images > 1: + gen_request.num_images = num_images + + # Execute the operation for generation mode + operation = SynchronousOperation( + endpoint=ApiEndpoint( + path=path, + method=HttpMethod.POST, + request_model=IdeogramV3Request, + response_model=IdeogramGenerateResponse, + ), + request=gen_request, + auth_token=auth_token, + ) + + # Execute the operation and process response response = operation.execute() if not response.data or len(response.data) == 0: @@ -534,15 +741,14 @@ class IdeogramV3(ComfyNodeABC): return (download_and_process_image(image_url),) - NODE_CLASS_MAPPINGS = { "IdeogramV1": IdeogramV1, "IdeogramV2": IdeogramV2, - #"IdeogramV3": IdeogramV3, + "IdeogramV3": IdeogramV3, } NODE_DISPLAY_NAME_MAPPINGS = { "IdeogramV1": "Ideogram V1", "IdeogramV2": "Ideogram V2", - #"IdeogramV3": "Ideogram V3", + "IdeogramV3": "Ideogram V3", } From 0382220132d9514027bac27fa6d81366e12c6e74 Mon Sep 17 00:00:00 2001 From: Christian Byrne Date: Wed, 30 Apr 2025 17:50:53 -0700 Subject: [PATCH 061/121] [Kling] Split Camera Control config to its own node (#81) --- comfy_api_nodes/nodes_kling.py | 238 +++++++++++++++++---------------- 1 file changed, 122 insertions(+), 116 deletions(-) diff --git a/comfy_api_nodes/nodes_kling.py b/comfy_api_nodes/nodes_kling.py index ac2fa6e28..59dc5485a 100644 --- a/comfy_api_nodes/nodes_kling.py +++ b/comfy_api_nodes/nodes_kling.py @@ -1,3 +1,14 @@ +""" +`camera_control` supported: + +- pro | 5s duration | kling-v1-5 + +`camera_control` not supported: + +- std | 10s duration | kling-v1-6 + +""" + from typing import Union, Optional import math import logging @@ -72,11 +83,6 @@ def is_valid_video_response(response: KlingText2VideoResponse) -> bool: ) -def is_camera_control_supported(model_name: str, duration: str, mode: str) -> bool: - """`camera_control` is only supported in `pro` mode with `5s` duration and `kling-v1-5`""" - return model_name == "kling-v1-5" and duration == "5" and mode == "pro" - - def get_camera_control_input_config( tooltip: str, default: float = 0.0 ) -> tuple[IO, InputTypeOptions]: @@ -92,39 +98,86 @@ def get_camera_control_input_config( return IO.FLOAT, input_config -def _get_camera_control_inputs() -> dict[str, tuple[IO, InputTypeOptions]]: - """Returns a dictionary of camera control inputs common to Kling video generation nodes.""" - return { - "camera_control_type": ( - IO.COMBO, - { - "options": [ - camera_control_type.value for camera_control_type in CameraType - ], - "default": "simple", - "tooltip": "Predefined camera movements type. simple: Customizable camera movement. down_back: Camera descends and moves backward. forward_up: Camera moves forward and tilts up. right_turn_forward: Rotate right and move forward. left_turn_forward: Rotate left and move forward.", - }, - ), - "camera_control_horizontal": get_camera_control_input_config( - "Controls camera's movement along horizontal axis (x-axis). Negative indicates left, positive indicates right" - ), - "camera_control_vertical": get_camera_control_input_config( - "Controls camera's movement along vertical axis (y-axis). Negative indicates downward, positive indicates upward." - ), - "camera_control_pan": get_camera_control_input_config( - "Controls camera's rotation in vertical plane (x-axis). Negative indicates downward rotation, positive indicates upward rotation.", - default=0.5, - ), - "camera_control_roll": get_camera_control_input_config( - "Controls camera's rotation in horizontal plane (y-axis). Negative indicates left rotation, positive indicates right rotation.", - ), - "camera_control_tilt": get_camera_control_input_config( - "Controls camera's rolling amount (z-axis). Negative indicates counterclockwise, positive indicates clockwise.", - ), - "camera_control_zoom": get_camera_control_input_config( - "Controls change in camera's focal length. Negative indicates narrower field of view, positive indicates wider field of view.", - ), - } +class KlingCameraControls(ComfyNodeABC): + """Kling Camera Controls Node""" + + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "camera_control_type": ( + IO.COMBO, + { + "options": [ + camera_control_type.value + for camera_control_type in CameraType + ], + "default": "simple", + "tooltip": "Predefined camera movements type. simple: Customizable camera movement. down_back: Camera descends and moves backward. forward_up: Camera moves forward and tilts up. right_turn_forward: Rotate right and move forward. left_turn_forward: Rotate left and move forward.", + }, + ), + "horizontal_movement": get_camera_control_input_config( + "Controls camera's movement along horizontal axis (x-axis). Negative indicates left, positive indicates right" + ), + "vertical_movement": get_camera_control_input_config( + "Controls camera's movement along vertical axis (y-axis). Negative indicates downward, positive indicates upward." + ), + "pan": get_camera_control_input_config( + "Controls camera's rotation in vertical plane (x-axis). Negative indicates downward rotation, positive indicates upward rotation.", + default=0.5, + ), + "tilt": get_camera_control_input_config( + "Controls camera's rotation in horizontal plane (y-axis). Negative indicates left rotation, positive indicates right rotation.", + ), + "roll": get_camera_control_input_config( + "Controls camera's rolling amount (z-axis). Negative indicates counterclockwise, positive indicates clockwise.", + ), + "zoom": get_camera_control_input_config( + "Controls change in camera's focal length. Negative indicates narrower field of view, positive indicates wider field of view.", + ), + } + } + + DESCRIPTION = "Kling Camera Controls Node. Not all model and mode combinations support camera control. Please refer to the Kling API documentation for more information." + RETURN_TYPES = ("CAMERA_CONTROL",) + RETURN_NAMES = ("camera_control",) + FUNCTION = "main" + + def main( + self, + camera_control_type: str, + horizontal_movement: float, + vertical_movement: float, + pan: float, + tilt: float, + roll: float, + zoom: float, + ): + if not is_valid_camera_control_configs( + [ + horizontal_movement, + vertical_movement, + pan, + tilt, + roll, + zoom, + ] + ): + return "Invalid camera control configs: at least one of the values must be non-zero" + + return ( + CameraControl( + type=CameraType(camera_control_type), + config=CameraConfig( + horizontal=horizontal_movement, + vertical=vertical_movement, + pan=pan, + roll=roll, + tilt=tilt, + zoom=zoom, + ), + ), + ) class KlingNodeBase(ComfyNodeABC): @@ -135,12 +188,6 @@ class KlingNodeBase(ComfyNodeABC): cls, prompt, negative_prompt, - camera_control_horizontal, - camera_control_vertical, - camera_control_pan, - camera_control_roll, - camera_control_tilt, - camera_control_zoom, ) -> Union[str, bool]: if not is_valid_prompt(prompt): return "Prompt is required" @@ -148,17 +195,6 @@ class KlingNodeBase(ComfyNodeABC): return "Prompt must be less than 2500 characters" if negative_prompt and len(negative_prompt) >= 2500: return "Negative prompt must be less than 2500 characters" - if not is_valid_camera_control_configs( - [ - camera_control_horizontal, - camera_control_vertical, - camera_control_pan, - camera_control_roll, - camera_control_tilt, - camera_control_zoom, - ] - ): - return "Invalid camera control configs" return True FUNCTION = "api_call" @@ -204,6 +240,13 @@ class KlingTextToVideoNode(KlingNodeBase): "negative_prompt": model_field_to_node_input( IO.STRING, KlingText2VideoRequest, "negative_prompt", multiline=True ), + "model_name": model_field_to_node_input( + IO.COMBO, + KlingText2VideoRequest, + "model_name", + enum_type=ModelName, + default="kling-v2-master", + ), "cfg_scale": model_field_to_node_input( IO.FLOAT, KlingText2VideoRequest, "cfg_scale" ), @@ -219,7 +262,9 @@ class KlingTextToVideoNode(KlingNodeBase): "aspect_ratio", enum_type=AspectRatio, ), - **_get_camera_control_inputs(), + }, + "optional": { + "camera_control": ("CAMERA_CONTROL", {}), }, "hidden": {"auth_token": "AUTH_TOKEN_COMFY_ORG"}, } @@ -231,33 +276,14 @@ class KlingTextToVideoNode(KlingNodeBase): self, prompt: str, negative_prompt: str, - duration: int, - mode: str, + model_name: str, cfg_scale: float, + mode: str, + duration: int, aspect_ratio: str, - camera_control_type: str, - camera_control_horizontal: float, - camera_control_vertical: float, - camera_control_pan: float, - camera_control_roll: float, - camera_control_tilt: float, - camera_control_zoom: float, + camera_control: Optional[CameraControl] = None, auth_token: Optional[str] = None, ) -> tuple[VideoFromFile]: - camera_control = None - if is_camera_control_supported("kling-v1-6", duration, mode): - camera_control = CameraControl( - type=CameraType(camera_control_type), - config=CameraConfig( - horizontal=camera_control_horizontal, - vertical=camera_control_vertical, - pan=camera_control_pan, - roll=camera_control_roll, - tilt=camera_control_tilt, - zoom=camera_control_zoom, - ).model_dump(exclude_none=True), - ) - initial_operation = SynchronousOperation( endpoint=ApiEndpoint( path=PATH_TEXT_TO_VIDEO, @@ -270,6 +296,7 @@ class KlingTextToVideoNode(KlingNodeBase): negative_prompt=negative_prompt if negative_prompt else None, duration=Duration(duration), mode=Mode(mode), + model_name=ModelName(model_name), cfg_scale=cfg_scale, aspect_ratio=AspectRatio(aspect_ratio), camera_control=camera_control, @@ -284,8 +311,6 @@ class KlingTextToVideoNode(KlingNodeBase): raise KlingApiError(error_msg) task_id = initial_response.data.task_id - logging.debug("Kling task submitted. Task ID: %s", task_id) - final_response = self.poll_for_task_status(task_id, auth_token) if not is_valid_video_response(final_response): error_msg = ( @@ -330,12 +355,6 @@ class KlingImage2VideoNode(KlingNodeBase): def INPUT_TYPES(s): return { "required": { - "model_name": model_field_to_node_input( - IO.COMBO, KlingImage2VideoRequest, "model_name", enum_type=ModelName - ), - "start_frame": model_field_to_node_input( - IO.IMAGE, KlingImage2VideoRequest, "image" - ), "prompt": model_field_to_node_input( IO.STRING, KlingImage2VideoRequest, "prompt", multiline=True ), @@ -345,6 +364,16 @@ class KlingImage2VideoNode(KlingNodeBase): "negative_prompt", multiline=True, ), + "model_name": model_field_to_node_input( + IO.COMBO, + KlingImage2VideoRequest, + "model_name", + enum_type=ModelName, + default="kling-v2-master", + ), + "start_frame": model_field_to_node_input( + IO.IMAGE, KlingImage2VideoRequest, "image" + ), "cfg_scale": model_field_to_node_input( IO.FLOAT, KlingImage2VideoRequest, "cfg_scale" ), @@ -360,9 +389,9 @@ class KlingImage2VideoNode(KlingNodeBase): "duration": model_field_to_node_input( IO.COMBO, KlingImage2VideoRequest, "duration", enum_type=Duration ), - **_get_camera_control_inputs(), }, "optional": { + "camera_control": ("CAMERA_CONTROL", {}), "end_frame": model_field_to_node_input( IO.IMAGE, KlingImage2VideoRequest, "image_tail" ), @@ -375,41 +404,18 @@ class KlingImage2VideoNode(KlingNodeBase): def api_call( self, - model_name: str, - start_frame: torch.Tensor, prompt: str, negative_prompt: str, + model_name: str, + start_frame: torch.Tensor, cfg_scale: float, mode: str, aspect_ratio: str, duration: str, - camera_control_type: str, - camera_control_horizontal: float, - camera_control_vertical: float, - camera_control_pan: float, - camera_control_roll: float, - camera_control_tilt: float, - camera_control_zoom: float, + camera_control: Optional[CameraControl] = None, end_frame: Optional[torch.Tensor] = None, auth_token: Optional[str] = None, ) -> tuple[VideoFromFile]: - camera_control = None - if is_camera_control_supported(model_name, duration, mode): - config = None - if camera_control_type != "right_turn_forward": - config = CameraConfig( - horizontal=camera_control_horizontal, - vertical=camera_control_vertical, - pan=camera_control_pan, - roll=camera_control_roll, - tilt=camera_control_tilt, - zoom=camera_control_zoom, - ) - camera_control = CameraControl( - type=CameraType(camera_control_type), - config=config if config else None, - ) - initial_operation = SynchronousOperation( endpoint=ApiEndpoint( path=PATH_IMAGE_TO_VIDEO, @@ -442,8 +448,6 @@ class KlingImage2VideoNode(KlingNodeBase): raise KlingApiError(error_msg) task_id = initial_response.data.task_id - logging.debug("Kling task submitted. Task ID: %s", task_id) - final_response = KlingImage2VideoNode.poll_for_task_status(task_id, auth_token) if not is_valid_video_response(final_response): error_msg = ( @@ -459,11 +463,13 @@ class KlingImage2VideoNode(KlingNodeBase): NODE_CLASS_MAPPINGS = { + "KlingCameraControls": KlingCameraControls, "KlingTextToVideoNode": KlingTextToVideoNode, "KlingImage2VideoNode": KlingImage2VideoNode, } NODE_DISPLAY_NAME_MAPPINGS = { + "KlingCameraControls": "Kling Camera Controls", "KlingTextToVideoNode": "Kling Text to Video", "KlingImage2VideoNode": "Kling Image to Video", } From 34ed9adbc852c6d5a12687c800400a9d947d4886 Mon Sep 17 00:00:00 2001 From: Christian Byrne Date: Wed, 30 Apr 2025 18:00:43 -0700 Subject: [PATCH 062/121] Add Pika i2v and t2v nodes (#52) --- comfy_api_nodes/apis/__init__.py | 3173 +++++++++++++++--------------- comfy_api_nodes/apis/client.py | 5 +- comfy_api_nodes/nodes_pika.py | 397 ++++ nodes.py | 1 + 4 files changed, 2026 insertions(+), 1550 deletions(-) create mode 100644 comfy_api_nodes/nodes_pika.py diff --git a/comfy_api_nodes/apis/__init__.py b/comfy_api_nodes/apis/__init__.py index 274489c23..393753048 100644 --- a/comfy_api_nodes/apis/__init__.py +++ b/comfy_api_nodes/apis/__init__.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: -# filename: https://stagingapi.comfy.org/openapi -# timestamp: 2025-05-01T00:26:01+00:00 +# filename: filtered-openapi.yaml +# timestamp: 2025-05-01T00:49:31+00:00 from __future__ import annotations @@ -9,115 +9,61 @@ from enum import Enum from typing import Any, Dict, List, Literal, Optional, Union from uuid import UUID -from pydantic import AnyUrl, BaseModel, Field, RootModel, confloat, conint, constr +from pydantic import AnyUrl, BaseModel, Field, RootModel -bytes_aliased=bytes -class BFLFluxProGenerateRequest(BaseModel): - guidance_scale: Optional[confloat(ge=1.0, le=20.0)] = Field( - None, description='The guidance scale for generation.' - ) - height: conint(ge=64, le=2048) = Field( - ..., description='The height of the image to generate.' - ) - negative_prompt: Optional[str] = Field( - None, description='The negative prompt for image generation.' - ) - num_images: Optional[conint(ge=1, le=4)] = Field( - None, description='The number of images to generate.' - ) - num_inference_steps: Optional[conint(ge=1, le=100)] = Field( - None, description='The number of inference steps.' - ) - prompt: str = Field(..., description='The text prompt for image generation.') - seed: Optional[int] = Field(None, description='The seed value for reproducibility.') - width: conint(ge=64, le=2048) = Field( - ..., description='The width of the image to generate.' - ) +bytes_aliased = bytes -class BFLFluxProGenerateResponse(BaseModel): - id: str = Field(..., description='The unique identifier for the generation task.') - polling_url: str = Field(..., description='URL to poll for the generation result.') - - -class ComfyNode(BaseModel): - category: Optional[str] = Field( +class PersonalAccessToken(BaseModel): + id: Optional[UUID] = Field(None, description="Unique identifier for the GitCommit") + name: Optional[str] = Field( None, - description='UI category where the node is listed, used for grouping nodes.', - ) - comfy_node_name: Optional[str] = Field( - None, description='Unique identifier for the node' - ) - deprecated: Optional[bool] = Field( - None, - description='Indicates if the node is deprecated. Deprecated nodes are hidden in the UI.', + description="Required. The name of the token. Can be a simple description.", ) description: Optional[str] = Field( - None, description="Brief description of the node's functionality or purpose." - ) - experimental: Optional[bool] = Field( None, - description='Indicates if the node is experimental, subject to changes or removal.', + description="Optional. A more detailed description of the token's intended use.", ) - function: Optional[str] = Field( - None, description='Name of the entry-point function to execute the node.' - ) - input_types: Optional[str] = Field(None, description='Defines input parameters') - output_is_list: Optional[List[bool]] = Field( - None, description='Boolean values indicating if each output is a list.' - ) - return_names: Optional[str] = Field( - None, description='Names of the outputs for clarity in workflows.' - ) - return_types: Optional[str] = Field( - None, description='Specifies the types of outputs produced by the node.' - ) - - -class ComfyNodeCloudBuildInfo(BaseModel): - build_id: Optional[str] = None - location: Optional[str] = None - project_id: Optional[str] = None - project_number: Optional[str] = None - - -class Customer(BaseModel): createdAt: Optional[datetime] = Field( - None, description='The date and time the user was created' + None, description="[Output Only]The date and time the token was created." ) - email: Optional[str] = Field(None, description='The email address for this user') - id: str = Field(..., description='The firebase UID of the user') - name: Optional[str] = Field(None, description='The name for this user') - updatedAt: Optional[datetime] = Field( - None, description='The date and time the user was last updated' - ) - - -class CustomerStorageResourceResponse(BaseModel): - download_url: Optional[str] = Field( + token: Optional[str] = Field( None, - description='The signed URL to use for downloading the file from the specified path', - ) - existing_file: Optional[bool] = Field( - None, description='Whether an existing file with the same hash was found' - ) - expires_at: Optional[datetime] = Field( - None, description='When the signed URL will expire' - ) - upload_url: Optional[str] = Field( - None, - description='The signed URL to use for uploading the file to the specified path', + description="[Output Only]. The personal access token. Only returned during creation.", ) -class Error(BaseModel): - details: Optional[List[str]] = Field( - None, - description='Optional detailed information about the error or hints for resolving it.', +class GitCommitSummary(BaseModel): + commit_hash: Optional[str] = Field(None, description="The hash of the commit") + commit_name: Optional[str] = Field(None, description="The name of the commit") + branch_name: Optional[str] = Field( + None, description="The branch where the commit was made" ) - message: Optional[str] = Field( - None, description='A clear and concise description of the error.' + author: Optional[str] = Field(None, description="The author of the commit") + timestamp: Optional[datetime] = Field( + None, description="The timestamp when the commit was made" ) + status_summary: Optional[Dict[str, str]] = Field( + None, description="A map of operating system to status pairs" + ) + + +class User(BaseModel): + id: Optional[str] = Field(None, description="The unique id for this user.") + email: Optional[str] = Field(None, description="The email address for this user.") + name: Optional[str] = Field(None, description="The name for this user.") + isApproved: Optional[bool] = Field( + None, description="Indicates if the user is approved." + ) + isAdmin: Optional[bool] = Field( + None, description="Indicates if the user has admin privileges." + ) + + +class PublisherUser(BaseModel): + id: Optional[str] = Field(None, description="The unique id for this user.") + email: Optional[str] = Field(None, description="The email address for this user.") + name: Optional[str] = Field(None, description="The name for this user.") class ErrorResponse(BaseModel): @@ -125,37 +71,186 @@ class ErrorResponse(BaseModel): message: str -class GitCommitSummary(BaseModel): - author: Optional[str] = Field(None, description='The author of the commit') - branch_name: Optional[str] = Field( - None, description='The branch where the commit was made' +class StorageFile(BaseModel): + id: Optional[UUID] = Field( + None, description="Unique identifier for the storage file" ) - commit_hash: Optional[str] = Field(None, description='The hash of the commit') - commit_name: Optional[str] = Field(None, description='The name of the commit') - status_summary: Optional[Dict[str, str]] = Field( - None, description='A map of operating system to status pairs' + file_path: Optional[str] = Field(None, description="Path to the file in storage") + public_url: Optional[str] = Field(None, description="Public URL") + + +class PublisherMember(BaseModel): + id: Optional[str] = Field( + None, description="The unique identifier for the publisher member." ) - timestamp: Optional[datetime] = Field( - None, description='The timestamp when the commit was made' + user: Optional[PublisherUser] = Field( + None, description="The user associated with this publisher member." ) + role: Optional[str] = Field( + None, description="The role of the user in the publisher." + ) + + +class ComfyNode(BaseModel): + comfy_node_name: Optional[str] = Field( + None, description="Unique identifier for the node" + ) + category: Optional[str] = Field( + None, + description="UI category where the node is listed, used for grouping nodes.", + ) + description: Optional[str] = Field( + None, description="Brief description of the node's functionality or purpose." + ) + input_types: Optional[str] = Field(None, description="Defines input parameters") + deprecated: Optional[bool] = Field( + None, + description="Indicates if the node is deprecated. Deprecated nodes are hidden in the UI.", + ) + experimental: Optional[bool] = Field( + None, + description="Indicates if the node is experimental, subject to changes or removal.", + ) + output_is_list: Optional[List[bool]] = Field( + None, description="Boolean values indicating if each output is a list." + ) + return_names: Optional[str] = Field( + None, description="Names of the outputs for clarity in workflows." + ) + return_types: Optional[str] = Field( + None, description="Specifies the types of outputs produced by the node." + ) + function: Optional[str] = Field( + None, description="Name of the entry-point function to execute the node." + ) + + +class ComfyNodeCloudBuildInfo(BaseModel): + project_id: Optional[str] = None + project_number: Optional[str] = None + location: Optional[str] = None + build_id: Optional[str] = None + + +class Error(BaseModel): + message: Optional[str] = Field( + None, description="A clear and concise description of the error." + ) + details: Optional[List[str]] = Field( + None, + description="Optional detailed information about the error or hints for resolving it.", + ) + + +class NodeVersionUpdateRequest(BaseModel): + changelog: Optional[str] = Field( + None, description="The changelog describing the version changes." + ) + deprecated: Optional[bool] = Field( + None, description="Whether the version is deprecated." + ) + + +class NodeStatus(str, Enum): + NodeStatusActive = "NodeStatusActive" + NodeStatusDeleted = "NodeStatusDeleted" + NodeStatusBanned = "NodeStatusBanned" + + +class NodeVersionStatus(str, Enum): + NodeVersionStatusActive = "NodeVersionStatusActive" + NodeVersionStatusDeleted = "NodeVersionStatusDeleted" + NodeVersionStatusBanned = "NodeVersionStatusBanned" + NodeVersionStatusPending = "NodeVersionStatusPending" + NodeVersionStatusFlagged = "NodeVersionStatusFlagged" + + +class PublisherStatus(str, Enum): + PublisherStatusActive = "PublisherStatusActive" + PublisherStatusBanned = "PublisherStatusBanned" + + +class WorkflowRunStatus(str, Enum): + WorkflowRunStatusStarted = "WorkflowRunStatusStarted" + WorkflowRunStatusFailed = "WorkflowRunStatusFailed" + WorkflowRunStatusCompleted = "WorkflowRunStatusCompleted" + + +class MachineStats(BaseModel): + machine_name: Optional[str] = Field(None, description="Name of the machine.") + os_version: Optional[str] = Field( + None, description="The operating system version. eg. Ubuntu Linux 20.04" + ) + gpu_type: Optional[str] = Field( + None, description="The GPU type. eg. NVIDIA Tesla K80" + ) + cpu_capacity: Optional[str] = Field(None, description="Total CPU on the machine.") + initial_cpu: Optional[str] = Field( + None, description="Initial CPU available before the job starts." + ) + memory_capacity: Optional[str] = Field( + None, description="Total memory on the machine." + ) + initial_ram: Optional[str] = Field( + None, description="Initial RAM available before the job starts." + ) + vram_time_series: Optional[Dict[str, Any]] = Field( + None, description="Time series of VRAM usage." + ) + disk_capacity: Optional[str] = Field( + None, description="Total disk capacity on the machine." + ) + initial_disk: Optional[str] = Field( + None, description="Initial disk available before the job starts." + ) + pip_freeze: Optional[str] = Field(None, description="The pip freeze output") + + +class Customer(BaseModel): + id: str = Field(..., description="The firebase UID of the user") + email: Optional[str] = Field(None, description="The email address for this user") + name: Optional[str] = Field(None, description="The name for this user") + createdAt: Optional[datetime] = Field( + None, description="The date and time the user was created" + ) + updatedAt: Optional[datetime] = Field( + None, description="The date and time the user was last updated" + ) + + +class MagicPrompt(str, Enum): + ON = "ON" + OFF = "OFF" + + +class ColorPalette(BaseModel): + name: str = Field(..., description="Name of the color palette", examples=["PASTEL"]) + + +class StyleCode(RootModel[str]): + root: str = Field(..., pattern="^[0-9A-Fa-f]{8}$") + + +class StyleType(str, Enum): + GENERAL = "GENERAL" class IdeogramColorPalette1(BaseModel): - name: str = Field(..., description='Name of the preset color palette') + name: str = Field(..., description="Name of the preset color palette") class Member(BaseModel): - color: Optional[constr(pattern=r'^#[0-9A-Fa-f]{6}$')] = Field( - None, description='Hexadecimal color code' + color: Optional[str] = Field( + None, description="Hexadecimal color code", pattern="^#[0-9A-Fa-f]{6}$" ) - weight: Optional[confloat(ge=0.0, le=1.0)] = Field( - None, description='Optional weight for the color (0-1)' + weight: Optional[float] = Field( + None, description="Optional weight for the color (0-1)", ge=0.0, le=1.0 ) class IdeogramColorPalette2(BaseModel): members: List[Member] = Field( - ..., description='Array of color definitions with optional weights' + ..., description="Array of color definitions with optional weights" ) @@ -164,232 +259,156 @@ class IdeogramColorPalette( ): root: Union[IdeogramColorPalette1, IdeogramColorPalette2] = Field( ..., - description='A color palette specification that can either use a preset name or explicit color definitions with weights', + description="A color palette specification that can either use a preset name or explicit color definitions with weights", ) class ImageRequest(BaseModel): + prompt: str = Field( + ..., description="Required. The prompt to use to generate the image." + ) aspect_ratio: Optional[str] = Field( None, description="Optional. The aspect ratio (e.g., 'ASPECT_16_9', 'ASPECT_1_1'). Cannot be used with resolution. Defaults to 'ASPECT_1_1' if unspecified.", ) - color_palette: Optional[Dict[str, Any]] = Field( - None, description='Optional. Color palette object. Only for V_2, V_2_TURBO.' - ) + model: str = Field(..., description="The model used (e.g., 'V_2', 'V_2A_TURBO')") magic_prompt_option: Optional[str] = Field( None, description="Optional. MagicPrompt usage ('AUTO', 'ON', 'OFF')." ) - model: str = Field(..., description="The model used (e.g., 'V_2', 'V_2A_TURBO')") - negative_prompt: Optional[str] = Field( + seed: Optional[int] = Field( None, - description='Optional. Description of what to exclude. Only for V_1, V_1_TURBO, V_2, V_2_TURBO.', - ) - num_images: Optional[conint(ge=1, le=8)] = Field( - 1, description='Optional. Number of images to generate (1-8). Defaults to 1.' - ) - prompt: str = Field( - ..., description='Required. The prompt to use to generate the image.' - ) - resolution: Optional[str] = Field( - None, - description="Optional. Resolution (e.g., 'RESOLUTION_1024_1024'). Only for model V_2. Cannot be used with aspect_ratio.", - ) - seed: Optional[conint(ge=0, le=2147483647)] = Field( - None, description='Optional. A number between 0 and 2147483647.' + description="Optional. A number between 0 and 2147483647.", + ge=0, + le=2147483647, ) style_type: Optional[str] = Field( None, description="Optional. Style type ('AUTO', 'GENERAL', 'REALISTIC', 'DESIGN', 'RENDER_3D', 'ANIME'). Only for models V_2 and above.", ) + negative_prompt: Optional[str] = Field( + None, + description="Optional. Description of what to exclude. Only for V_1, V_1_TURBO, V_2, V_2_TURBO.", + ) + num_images: Optional[int] = Field( + 1, + description="Optional. Number of images to generate (1-8). Defaults to 1.", + ge=1, + le=8, + ) + resolution: Optional[str] = Field( + None, + description="Optional. Resolution (e.g., 'RESOLUTION_1024_1024'). Only for model V_2. Cannot be used with aspect_ratio.", + ) + color_palette: Optional[Dict[str, Any]] = Field( + None, description="Optional. Color palette object. Only for V_2, V_2_TURBO." + ) class IdeogramGenerateRequest(BaseModel): image_request: ImageRequest = Field( - ..., description='The image generation request parameters.' + ..., description="The image generation request parameters." ) class Datum(BaseModel): - is_image_safe: Optional[bool] = Field( - None, description='Indicates whether the image is considered safe.' - ) prompt: Optional[str] = Field( - None, description='The prompt used to generate this image.' + None, description="The prompt used to generate this image." ) resolution: Optional[str] = Field( None, description="The resolution of the generated image (e.g., '1024x1024')." ) - seed: Optional[int] = Field( - None, description='The seed value used for this generation.' + is_image_safe: Optional[bool] = Field( + None, description="Indicates whether the image is considered safe." ) + seed: Optional[int] = Field( + None, description="The seed value used for this generation." + ) + url: Optional[str] = Field(None, description="URL to the generated image.") style_type: Optional[str] = Field( None, description="The style type used for generation (e.g., 'REALISTIC', 'ANIME').", ) - url: Optional[str] = Field(None, description='URL to the generated image.') class IdeogramGenerateResponse(BaseModel): created: Optional[datetime] = Field( - None, description='Timestamp when the generation was created.' + None, description="Timestamp when the generation was created." ) data: Optional[List[Datum]] = Field( - None, description='Array of generated image information.' + None, description="Array of generated image information." ) -class ColorPalette(BaseModel): - name: str = Field(..., description='Name of the color palette', examples=['PASTEL']) - - -class MagicPrompt(str, Enum): - ON = 'ON' - OFF = 'OFF' - - -class StyleType(str, Enum): - GENERAL = 'GENERAL' - - -class KlingErrorResponse(BaseModel): - code: int = Field( - ..., - description='- 1000: Authentication failed\n- 1001: Authorization is empty\n- 1002: Authorization is invalid\n- 1003: Authorization is not yet valid\n- 1004: Authorization has expired\n- 1100: Account exception\n- 1101: Account in arrears (postpaid scenario)\n- 1102: Resource pack depleted or expired (prepaid scenario)\n- 1103: Unauthorized access to requested resource\n- 1200: Invalid request parameters\n- 1201: Invalid parameters\n- 1202: Invalid request method\n- 1203: Requested resource does not exist\n- 1300: Trigger platform strategy\n- 1301: Trigger content security policy\n- 1302: API request too frequent\n- 1303: Concurrency/QPS exceeds limit\n- 1304: Trigger IP whitelist policy\n- 5000: Internal server error\n- 5001: Service temporarily unavailable\n- 5002: Server internal timeout\n', - ) - message: str = Field(..., description='Human-readable error message') - request_id: str = Field( - ..., description='Request ID for tracking and troubleshooting' - ) - - -class AspectRatio(str, Enum): - field_16_9 = '16:9' - field_9_16 = '9:16' - field_1_1 = '1:1' - - -class Config(BaseModel): - horizontal: Optional[confloat(ge=-10.0, le=10.0)] = Field( - None, - description="Controls camera's movement along horizontal axis (x-axis). Negative indicates left, positive indicates right.", - ) - pan: Optional[confloat(ge=-10.0, le=10.0)] = Field( - None, - description="Controls camera's rotation in vertical plane (x-axis). Negative indicates downward rotation, positive indicates upward rotation.", - ) - roll: Optional[confloat(ge=-10.0, le=10.0)] = Field( - None, - description="Controls camera's rolling amount (z-axis). Negative indicates counterclockwise, positive indicates clockwise.", - ) - tilt: Optional[confloat(ge=-10.0, le=10.0)] = Field( - None, - description="Controls camera's rotation in horizontal plane (y-axis). Negative indicates left rotation, positive indicates right rotation.", - ) - vertical: Optional[confloat(ge=-10.0, le=10.0)] = Field( - None, - description="Controls camera's movement along vertical axis (y-axis). Negative indicates downward, positive indicates upward.", - ) - zoom: Optional[confloat(ge=-10.0, le=10.0)] = Field( - None, - description="Controls change in camera's focal length. Negative indicates narrower field of view, positive indicates wider field of view.", - ) - - -class Type(str, Enum): - simple = 'simple' - down_back = 'down_back' - forward_up = 'forward_up' - right_turn_forward = 'right_turn_forward' - left_turn_forward = 'left_turn_forward' - - -class CameraControl(BaseModel): - config: Optional[Config] = None - type: Optional[Type] = Field( - None, - description='Predefined camera movements type. simple: Customizable camera movement. down_back: Camera descends and moves backward. forward_up: Camera moves forward and tilts up. right_turn_forward: Rotate right and move forward. left_turn_forward: Rotate left and move forward.', - ) - - -class Duration(str, Enum): - field_5 = '5' - field_10 = '10' - - -class Trajectory(BaseModel): - x: Optional[int] = Field( - None, - description='The horizontal coordinate of trajectory point. Based on bottom-left corner of image as origin (0,0).', - ) - y: Optional[int] = Field( - None, - description='The vertical coordinate of trajectory point. Based on bottom-left corner of image as origin (0,0).', - ) - - -class DynamicMask(BaseModel): - mask: Optional[AnyUrl] = Field( - None, - description='Dynamic Brush Application Area (Mask image created by users using the motion brush). The aspect ratio must match the input image.', - ) - trajectories: Optional[List[Trajectory]] = None - - -class Mode(str, Enum): - std = 'std' - pro = 'pro' - - class ModelName(str, Enum): - kling_v1 = 'kling-v1' - kling_v1_5 = 'kling-v1-5' - kling_v1_6 = 'kling-v1-6' - kling_v2_master = 'kling-v2-master' + kling_v1 = "kling-v1" + kling_v1_6 = "kling-v1-6" + kling_v2_master = "kling-v2-master" -class KlingImage2VideoRequest(BaseModel): - aspect_ratio: Optional[AspectRatio] = '16:9' - callback_url: Optional[AnyUrl] = Field( - None, - description='The callback notification address. Server will notify when the task status changes.', +class Mode(str, Enum): + std = "std" + pro = "pro" + + +class Type(str, Enum): + simple = "simple" + down_back = "down_back" + forward_up = "forward_up" + right_turn_forward = "right_turn_forward" + left_turn_forward = "left_turn_forward" + + +class Config(BaseModel): + horizontal: Optional[float] = Field(None, ge=-10.0, le=10.0) + vertical: Optional[float] = Field(None, ge=-10.0, le=10.0) + pan: Optional[float] = Field(None, ge=-10.0, le=10.0) + tilt: Optional[float] = Field(None, ge=-10.0, le=10.0) + roll: Optional[float] = Field(None, ge=-10.0, le=10.0) + zoom: Optional[float] = Field(None, ge=-10.0, le=10.0) + + +class CameraControl(BaseModel): + type: Optional[Type] = Field(None, description="Predefined camera movements type") + config: Optional[Config] = None + + +class AspectRatio(str, Enum): + field_16_9 = "16:9" + field_9_16 = "9:16" + field_1_1 = "1:1" + + +class Duration(str, Enum): + field_5 = "5" + field_10 = "10" + + +class KlingText2VideoRequest(BaseModel): + model_name: Optional[ModelName] = Field("kling-v1", description="Model Name") + prompt: Optional[str] = Field( + None, description="Positive text prompt", max_length=2500 ) + negative_prompt: Optional[str] = Field( + None, description="Negative text prompt", max_length=2500 + ) + cfg_scale: Optional[float] = Field( + 0.5, description="Flexibility in video generation", ge=0.0, le=1.0 + ) + mode: Optional[Mode] = Field("std", description="Video generation mode") camera_control: Optional[CameraControl] = None - cfg_scale: Optional[confloat(ge=0.0, le=1.0)] = Field( - 0.5, - description="Flexibility in video generation. The higher the value, the lower the model's degree of flexibility, and the stronger the relevance to the user's prompt.", - ) - duration: Optional[Duration] = Field('5', description='Video length in seconds') - dynamic_masks: Optional[List[DynamicMask]] = Field( - None, - description='Dynamic Brush Configuration List (up to 6 groups). For 5-second videos, trajectory length must not exceed 77 coordinates.', - ) - external_task_id: Optional[str] = Field( - None, - description='Customized Task ID. Must be unique within a single user account.', - ) - image: Optional[str] = Field( - None, - description='Reference Image - URL or Base64 encoded string, cannot exceed 10MB, resolution not less than 300*300px, aspect ratio between 1:2.5 ~ 2.5:1. Base64 should not include data:image prefix.', - ) - image_tail: Optional[str] = Field( - None, - description='Reference Image - End frame control. URL or Base64 encoded string, cannot exceed 10MB, resolution not less than 300*300px. Base64 should not include data:image prefix.', - ) - mode: Optional[Mode] = Field( - 'std', - description='Video generation mode. std: Standard Mode, which is cost-effective. pro: Professional Mode, generates videos with longer duration but higher quality output.', - ) - model_name: Optional[ModelName] = Field('kling-v1', description='Model Name') - negative_prompt: Optional[constr(max_length=2500)] = Field( - None, description='Negative text prompt' - ) - prompt: Optional[constr(max_length=2500)] = Field( - None, description='Positive text prompt' - ) - static_mask: Optional[AnyUrl] = Field( - None, - description='Static Brush Application Area (Mask image created by users using the motion brush). The aspect ratio must match the input image.', + aspect_ratio: Optional[AspectRatio] = "16:9" + duration: Optional[Duration] = "5" + callback_url: Optional[AnyUrl] = Field( + None, description="The callback notification address" ) + external_task_id: Optional[str] = Field(None, description="Customized Task ID") + + +class TaskStatus(str, Enum): + submitted = "submitted" + processing = "processing" + succeed = "succeed" + failed = "failed" class TaskInfo(BaseModel): @@ -397,76 +416,148 @@ class TaskInfo(BaseModel): class Video(BaseModel): - duration: Optional[str] = Field(None, description='Total video duration') - id: Optional[str] = Field(None, description='Generated video ID') - url: Optional[AnyUrl] = Field(None, description='URL for generated video') + id: Optional[str] = Field(None, description="Generated video ID") + url: Optional[AnyUrl] = Field(None, description="URL for generated video") + duration: Optional[str] = Field(None, description="Total video duration") class TaskResult(BaseModel): videos: Optional[List[Video]] = None -class TaskStatus(str, Enum): - submitted = 'submitted' - processing = 'processing' - succeed = 'succeed' - failed = 'failed' - - class Data(BaseModel): - created_at: Optional[int] = Field(None, description='Task creation time') - task_id: Optional[str] = Field(None, description='Task ID') - task_info: Optional[TaskInfo] = None - task_result: Optional[TaskResult] = None + task_id: Optional[str] = Field(None, description="Task ID") task_status: Optional[TaskStatus] = None - updated_at: Optional[int] = Field(None, description='Task update time') + task_info: Optional[TaskInfo] = None + created_at: Optional[int] = Field(None, description="Task creation time") + updated_at: Optional[int] = Field(None, description="Task update time") + task_result: Optional[TaskResult] = None -class KlingImage2VideoResponse(BaseModel): - code: Optional[int] = Field(None, description='Error code') +class KlingText2VideoResponse(BaseModel): + code: Optional[int] = Field(None, description="Error code") + message: Optional[str] = Field(None, description="Error message") + request_id: Optional[str] = Field(None, description="Request ID") data: Optional[Data] = None - message: Optional[str] = Field(None, description='Error message') - request_id: Optional[str] = Field(None, description='Request ID') - - -class Config1(BaseModel): - horizontal: Optional[confloat(ge=-10.0, le=10.0)] = None - pan: Optional[confloat(ge=-10.0, le=10.0)] = None - roll: Optional[confloat(ge=-10.0, le=10.0)] = None - tilt: Optional[confloat(ge=-10.0, le=10.0)] = None - vertical: Optional[confloat(ge=-10.0, le=10.0)] = None - zoom: Optional[confloat(ge=-10.0, le=10.0)] = None - - -class CameraControl1(BaseModel): - config: Optional[Config1] = None - type: Optional[Type] = Field(None, description='Predefined camera movements type') class ModelName1(str, Enum): - kling_v1 = 'kling-v1' - kling_v1_6 = 'kling-v1-6' - kling_v2_master = 'kling-v2-master' + kling_v1 = "kling-v1" + kling_v1_5 = "kling-v1-5" + kling_v1_6 = "kling-v1-6" + kling_v2_master = "kling-v2-master" -class KlingText2VideoRequest(BaseModel): - aspect_ratio: Optional[AspectRatio] = '16:9' - callback_url: Optional[AnyUrl] = Field( - None, description='The callback notification address' +class Trajectory(BaseModel): + x: Optional[int] = Field( + None, + description="The horizontal coordinate of trajectory point. Based on bottom-left corner of image as origin (0,0).", + ) + y: Optional[int] = Field( + None, + description="The vertical coordinate of trajectory point. Based on bottom-left corner of image as origin (0,0).", + ) + + +class DynamicMask(BaseModel): + mask: Optional[AnyUrl] = Field( + None, + description="Dynamic Brush Application Area (Mask image created by users using the motion brush). The aspect ratio must match the input image.", + ) + trajectories: Optional[List[Trajectory]] = None + + +class Config1(BaseModel): + horizontal: Optional[float] = Field( + None, + description="Controls camera's movement along horizontal axis (x-axis). Negative indicates left, positive indicates right.", + ge=-10.0, + le=10.0, + ) + vertical: Optional[float] = Field( + None, + description="Controls camera's movement along vertical axis (y-axis). Negative indicates downward, positive indicates upward.", + ge=-10.0, + le=10.0, + ) + pan: Optional[float] = Field( + None, + description="Controls camera's rotation in vertical plane (x-axis). Negative indicates downward rotation, positive indicates upward rotation.", + ge=-10.0, + le=10.0, + ) + tilt: Optional[float] = Field( + None, + description="Controls camera's rotation in horizontal plane (y-axis). Negative indicates left rotation, positive indicates right rotation.", + ge=-10.0, + le=10.0, + ) + roll: Optional[float] = Field( + None, + description="Controls camera's rolling amount (z-axis). Negative indicates counterclockwise, positive indicates clockwise.", + ge=-10.0, + le=10.0, + ) + zoom: Optional[float] = Field( + None, + description="Controls change in camera's focal length. Negative indicates narrower field of view, positive indicates wider field of view.", + ge=-10.0, + le=10.0, + ) + + +class CameraControl1(BaseModel): + type: Optional[Type] = Field( + None, + description="Predefined camera movements type. simple: Customizable camera movement. down_back: Camera descends and moves backward. forward_up: Camera moves forward and tilts up. right_turn_forward: Rotate right and move forward. left_turn_forward: Rotate left and move forward.", + ) + config: Optional[Config1] = None + + +class KlingImage2VideoRequest(BaseModel): + model_name: Optional[ModelName1] = Field("kling-v1", description="Model Name") + image: Optional[str] = Field( + None, + description="Reference Image - URL or Base64 encoded string, cannot exceed 10MB, resolution not less than 300*300px, aspect ratio between 1:2.5 ~ 2.5:1. Base64 should not include data:image prefix.", + ) + image_tail: Optional[str] = Field( + None, + description="Reference Image - End frame control. URL or Base64 encoded string, cannot exceed 10MB, resolution not less than 300*300px. Base64 should not include data:image prefix.", + ) + prompt: Optional[str] = Field( + None, description="Positive text prompt", max_length=2500 + ) + negative_prompt: Optional[str] = Field( + None, description="Negative text prompt", max_length=2500 + ) + cfg_scale: Optional[float] = Field( + 0.5, + description="Flexibility in video generation. The higher the value, the lower the model's degree of flexibility, and the stronger the relevance to the user's prompt.", + ge=0.0, + le=1.0, + ) + mode: Optional[Mode] = Field( + "std", + description="Video generation mode. std: Standard Mode, which is cost-effective. pro: Professional Mode, generates videos with longer duration but higher quality output.", + ) + static_mask: Optional[AnyUrl] = Field( + None, + description="Static Brush Application Area (Mask image created by users using the motion brush). The aspect ratio must match the input image.", + ) + dynamic_masks: Optional[List[DynamicMask]] = Field( + None, + description="Dynamic Brush Configuration List (up to 6 groups). For 5-second videos, trajectory length must not exceed 77 coordinates.", ) camera_control: Optional[CameraControl1] = None - cfg_scale: Optional[confloat(ge=0.0, le=1.0)] = Field( - 0.5, description='Flexibility in video generation' + aspect_ratio: Optional[AspectRatio] = "16:9" + duration: Optional[Duration] = Field("5", description="Video length in seconds") + callback_url: Optional[AnyUrl] = Field( + None, + description="The callback notification address. Server will notify when the task status changes.", ) - duration: Optional[Duration] = '5' - external_task_id: Optional[str] = Field(None, description='Customized Task ID') - mode: Optional[Mode] = Field('std', description='Video generation mode') - model_name: Optional[ModelName1] = Field('kling-v1', description='Model Name') - negative_prompt: Optional[constr(max_length=2500)] = Field( - None, description='Negative text prompt' - ) - prompt: Optional[constr(max_length=2500)] = Field( - None, description='Positive text prompt' + external_task_id: Optional[str] = Field( + None, + description="Customized Task ID. Must be unique within a single user account.", ) @@ -475,875 +566,44 @@ class TaskResult1(BaseModel): class Data1(BaseModel): - created_at: Optional[int] = Field(None, description='Task creation time') - task_id: Optional[str] = Field(None, description='Task ID') - task_info: Optional[TaskInfo] = None - task_result: Optional[TaskResult1] = None + task_id: Optional[str] = Field(None, description="Task ID") task_status: Optional[TaskStatus] = None - updated_at: Optional[int] = Field(None, description='Task update time') + task_info: Optional[TaskInfo] = None + created_at: Optional[int] = Field(None, description="Task creation time") + updated_at: Optional[int] = Field(None, description="Task update time") + task_result: Optional[TaskResult1] = None -class KlingText2VideoResponse(BaseModel): - code: Optional[int] = Field(None, description='Error code') +class KlingImage2VideoResponse(BaseModel): + code: Optional[int] = Field(None, description="Error code") + message: Optional[str] = Field(None, description="Error message") + request_id: Optional[str] = Field(None, description="Request ID") data: Optional[Data1] = None - message: Optional[str] = Field(None, description='Error message') - request_id: Optional[str] = Field(None, description='Request ID') -class LumaAspectRatio(str, Enum): - field_1_1 = '1:1' - field_16_9 = '16:9' - field_9_16 = '9:16' - field_4_3 = '4:3' - field_3_4 = '3:4' - field_21_9 = '21:9' - field_9_21 = '9:21' - - -class LumaAssets(BaseModel): - image: Optional[AnyUrl] = Field(None, description='The URL of the image') - progress_video: Optional[AnyUrl] = Field( - None, description='The URL of the progress video' - ) - video: Optional[AnyUrl] = Field(None, description='The URL of the video') - - -class GenerationType(str, Enum): - add_audio = 'add_audio' - - -class LumaAudioGenerationRequest(BaseModel): - callback_url: Optional[AnyUrl] = Field( - None, description='The callback URL for the audio' - ) - generation_type: Optional[GenerationType] = 'add_audio' - negative_prompt: Optional[str] = Field( - None, description='The negative prompt of the audio' - ) - prompt: Optional[str] = Field(None, description='The prompt of the audio') - - -class LumaError(BaseModel): - detail: Optional[str] = Field(None, description='The error message') +class Object(str, Enum): + event = "event" class Type2(str, Enum): - generation = 'generation' + payment_intent_succeeded = "payment_intent.succeeded" -class LumaGenerationReference(BaseModel): - id: UUID = Field(..., description='The ID of the generation') - type: Literal['generation'] +class StripeRequestInfo(BaseModel): + id: Optional[str] = None + idempotency_key: Optional[str] = None -class GenerationType1(str, Enum): - video = 'video' +class Object1(str, Enum): + payment_intent = "payment_intent" -class LumaGenerationType(str, Enum): - video = 'video' - image = 'image' +class StripeAmountDetails(BaseModel): + tip: Optional[Dict[str, Any]] = None -class GenerationType2(str, Enum): - image = 'image' - - -class LumaImageIdentity(BaseModel): - images: Optional[List[AnyUrl]] = Field( - None, description='The URLs of the image identity' - ) - - -class LumaImageModel(str, Enum): - photon_1 = 'photon-1' - photon_flash_1 = 'photon-flash-1' - - -class LumaImageRef(BaseModel): - url: Optional[AnyUrl] = Field(None, description='The URL of the image reference') - weight: Optional[float] = Field( - None, description='The weight of the image reference' - ) - - -class Type3(str, Enum): - image = 'image' - - -class LumaImageReference(BaseModel): - type: Literal['image'] - url: AnyUrl = Field(..., description='The URL of the image') - - -class LumaKeyframe(RootModel[Union[LumaGenerationReference, LumaImageReference]]): - root: Union[LumaGenerationReference, LumaImageReference] = Field( - ..., - description='A keyframe can be either a Generation reference, an Image, or a Video', - discriminator='type', - ) - - -class LumaKeyframes(BaseModel): - frame0: Optional[LumaKeyframe] = None - frame1: Optional[LumaKeyframe] = None - - -class LumaModifyImageRef(BaseModel): - url: Optional[AnyUrl] = Field(None, description='The URL of the image reference') - weight: Optional[float] = Field( - None, description='The weight of the modify image reference' - ) - - -class LumaState(str, Enum): - queued = 'queued' - dreaming = 'dreaming' - completed = 'completed' - failed = 'failed' - - -class GenerationType3(str, Enum): - upscale_video = 'upscale_video' - - -class LumaVideoModel(str, Enum): - ray_2 = 'ray-2' - ray_flash_2 = 'ray-flash-2' - ray_1_6 = 'ray-1-6' - - -class LumaVideoModelOutputDuration1(str, Enum): - field_5s = '5s' - field_9s = '9s' - - -class LumaVideoModelOutputDuration( - RootModel[Union[LumaVideoModelOutputDuration1, str]] -): - root: Union[LumaVideoModelOutputDuration1, str] - - -class LumaVideoModelOutputResolution1(str, Enum): - field_540p = '540p' - field_720p = '720p' - field_1080p = '1080p' - field_4k = '4k' - - -class LumaVideoModelOutputResolution( - RootModel[Union[LumaVideoModelOutputResolution1, str]] -): - root: Union[LumaVideoModelOutputResolution1, str] - - -class MachineStats(BaseModel): - cpu_capacity: Optional[str] = Field(None, description='Total CPU on the machine.') - disk_capacity: Optional[str] = Field( - None, description='Total disk capacity on the machine.' - ) - gpu_type: Optional[str] = Field( - None, description='The GPU type. eg. NVIDIA Tesla K80' - ) - initial_cpu: Optional[str] = Field( - None, description='Initial CPU available before the job starts.' - ) - initial_disk: Optional[str] = Field( - None, description='Initial disk available before the job starts.' - ) - initial_ram: Optional[str] = Field( - None, description='Initial RAM available before the job starts.' - ) - machine_name: Optional[str] = Field(None, description='Name of the machine.') - memory_capacity: Optional[str] = Field( - None, description='Total memory on the machine.' - ) - os_version: Optional[str] = Field( - None, description='The operating system version. eg. Ubuntu Linux 20.04' - ) - pip_freeze: Optional[str] = Field(None, description='The pip freeze output') - vram_time_series: Optional[Dict[str, Any]] = Field( - None, description='Time series of VRAM usage.' - ) - - -class MinimaxBaseResponse(BaseModel): - status_code: int = Field( - ..., - description='Status code. 0 indicates success, other values indicate errors.', - ) - status_msg: str = Field( - ..., description='Specific error details or success message.' - ) - - -class File(BaseModel): - bytes: Optional[int] = Field(None, description='File size in bytes') - created_at: Optional[int] = Field( - None, description='Unix timestamp when the file was created, in seconds' - ) - download_url: Optional[str] = Field( - None, description='The URL to download the video' - ) - file_id: Optional[int] = Field(None, description='Unique identifier for the file') - filename: Optional[str] = Field(None, description='The name of the file') - purpose: Optional[str] = Field(None, description='The purpose of using the file') - - -class MinimaxFileRetrieveResponse(BaseModel): - base_resp: MinimaxBaseResponse - file: File - - -class Status(str, Enum): - Queueing = 'Queueing' - Preparing = 'Preparing' - Processing = 'Processing' - Success = 'Success' - Fail = 'Fail' - - -class MinimaxTaskResultResponse(BaseModel): - base_resp: MinimaxBaseResponse - file_id: Optional[str] = Field( - None, - description='After the task status changes to Success, this field returns the file ID corresponding to the generated video.', - ) - status: Status = Field( - ..., - description="Task status: 'Queueing' (in queue), 'Preparing' (task is preparing), 'Processing' (generating), 'Success' (task completed successfully), or 'Fail' (task failed).", - ) - task_id: str = Field(..., description='The task ID being queried.') - - -class Model(str, Enum): - T2V_01_Director = 'T2V-01-Director' - I2V_01_Director = 'I2V-01-Director' - S2V_01 = 'S2V-01' - I2V_01 = 'I2V-01' - I2V_01_live = 'I2V-01-live' - T2V_01 = 'T2V-01' - - -class SubjectReferenceItem(BaseModel): - image: Optional[str] = Field( - None, description='URL or base64 encoding of the subject reference image.' - ) - mask: Optional[str] = Field( - None, - description='URL or base64 encoding of the mask for the subject reference image.', - ) - - -class MinimaxVideoGenerationRequest(BaseModel): - callback_url: Optional[str] = Field( - None, - description='Optional. URL to receive real-time status updates about the video generation task.', - ) - first_frame_image: Optional[str] = Field( - None, - description='URL or base64 encoding of the first frame image. Required when model is I2V-01, I2V-01-Director, or I2V-01-live.', - ) - model: Model = Field( - ..., - description='Required. ID of model. Options: T2V-01-Director, I2V-01-Director, S2V-01, I2V-01, I2V-01-live, T2V-01', - ) - prompt: Optional[constr(max_length=2000)] = Field( - None, - description='Description of the video. Should be less than 2000 characters. Supports camera movement instructions in [brackets].', - ) - prompt_optimizer: Optional[bool] = Field( - True, - description='If true (default), the model will automatically optimize the prompt. Set to false for more precise control.', - ) - subject_reference: Optional[List[SubjectReferenceItem]] = Field( - None, - description='Only available when model is S2V-01. The model will generate a video based on the subject uploaded through this parameter.', - ) - - -class MinimaxVideoGenerationResponse(BaseModel): - base_resp: MinimaxBaseResponse - task_id: str = Field( - ..., description='The task ID for the asynchronous video generation task.' - ) - - -class NodeStatus(str, Enum): - NodeStatusActive = 'NodeStatusActive' - NodeStatusDeleted = 'NodeStatusDeleted' - NodeStatusBanned = 'NodeStatusBanned' - - -class NodeVersionStatus(str, Enum): - NodeVersionStatusActive = 'NodeVersionStatusActive' - NodeVersionStatusDeleted = 'NodeVersionStatusDeleted' - NodeVersionStatusBanned = 'NodeVersionStatusBanned' - NodeVersionStatusPending = 'NodeVersionStatusPending' - NodeVersionStatusFlagged = 'NodeVersionStatusFlagged' - - -class NodeVersionUpdateRequest(BaseModel): - changelog: Optional[str] = Field( - None, description='The changelog describing the version changes.' - ) - deprecated: Optional[bool] = Field( - None, description='Whether the version is deprecated.' - ) - - -class Moderation(str, Enum): - low = 'low' - auto = 'auto' - - -class OutputFormat(str, Enum): - png = 'png' - webp = 'webp' - jpeg = 'jpeg' - - -class OpenAIImageEditRequest(BaseModel): - background: Optional[str] = Field( - None, description='Background transparency', examples=['opaque'] - ) - model: str = Field( - ..., description='The model to use for image editing', examples=['gpt-image-1'] - ) - moderation: Optional[Moderation] = Field( - None, description='Content moderation setting', examples=['auto'] - ) - n: Optional[int] = Field( - None, description='The number of images to generate', examples=[1] - ) - output_compression: Optional[int] = Field( - None, description='Compression level for JPEG or WebP (0-100)', examples=[100] - ) - output_format: Optional[OutputFormat] = Field( - None, description='Format of the output image', examples=['png'] - ) - prompt: str = Field( - ..., - description='A text description of the desired edit', - examples=['Give the rocketship rainbow coloring'], - ) - quality: Optional[str] = Field( - None, description='The quality of the edited image', examples=['low'] - ) - size: Optional[str] = Field( - None, description='Size of the output image', examples=['1024x1024'] - ) - user: Optional[str] = Field( - None, - description='A unique identifier for end-user monitoring', - examples=['user-1234'], - ) - - -class Background(str, Enum): - transparent = 'transparent' - opaque = 'opaque' - - -class Quality(str, Enum): - low = 'low' - medium = 'medium' - high = 'high' - standard = 'standard' - hd = 'hd' - - -class ResponseFormat(str, Enum): - url = 'url' - b64_json = 'b64_json' - - -class Style(str, Enum): - vivid = 'vivid' - natural = 'natural' - - -class OpenAIImageGenerationRequest(BaseModel): - background: Optional[Background] = Field( - None, description='Background transparency', examples=['opaque'] - ) - model: Optional[str] = Field( - None, description='The model to use for image generation', examples=['dall-e-3'] - ) - moderation: Optional[Moderation] = Field( - None, description='Content moderation setting', examples=['auto'] - ) - n: Optional[int] = Field( - None, - description='The number of images to generate (1-10). Only 1 supported for dall-e-3.', - examples=[1], - ) - output_compression: Optional[int] = Field( - None, description='Compression level for JPEG or WebP (0-100)', examples=[100] - ) - output_format: Optional[OutputFormat] = Field( - None, description='Format of the output image', examples=['png'] - ) - prompt: str = Field( - ..., - description='A text description of the desired image', - examples=['Draw a rocket in front of a blackhole in deep space'], - ) - quality: Optional[Quality] = Field( - None, description='The quality of the generated image', examples=['high'] - ) - response_format: Optional[ResponseFormat] = Field( - None, description='Response format of image data', examples=['b64_json'] - ) - size: Optional[str] = Field( - None, - description='Size of the image (e.g., 1024x1024, 1536x1024, auto)', - examples=['1024x1536'], - ) - style: Optional[Style] = Field( - None, description='Style of the image (only for dall-e-3)', examples=['vivid'] - ) - user: Optional[str] = Field( - None, - description='A unique identifier for end-user monitoring', - examples=['user-1234'], - ) - - -class Datum1(BaseModel): - b64_json: Optional[str] = Field(None, description='Base64 encoded image data') - revised_prompt: Optional[str] = Field(None, description='Revised prompt') - url: Optional[str] = Field(None, description='URL of the image') - - -class InputTokensDetails(BaseModel): - image_tokens: Optional[int] = None - text_tokens: Optional[int] = None - - -class Usage(BaseModel): - input_tokens: Optional[int] = None - input_tokens_details: Optional[InputTokensDetails] = None - output_tokens: Optional[int] = None - total_tokens: Optional[int] = None - - -class OpenAIImageGenerationResponse(BaseModel): - data: Optional[List[Datum1]] = None - usage: Optional[Usage] = None - - -class PersonalAccessToken(BaseModel): - createdAt: Optional[datetime] = Field( - None, description='[Output Only]The date and time the token was created.' - ) - description: Optional[str] = Field( - None, - description="Optional. A more detailed description of the token's intended use.", - ) - id: Optional[UUID] = Field(None, description='Unique identifier for the GitCommit') - name: Optional[str] = Field( - None, - description='Required. The name of the token. Can be a simple description.', - ) - token: Optional[str] = Field( - None, - description='[Output Only]. The personal access token. Only returned during creation.', - ) - - -class IngredientsMode(str, Enum): - creative = 'creative' - precise = 'precise' - - -class PikaBodyGenerate22C2vGenerate22PikascenesPost(BaseModel): - aspectRatio: Optional[confloat(ge=0.4, le=2.5)] = Field( - None, description='Aspect ratio (width / height)', title='Aspectratio' - ) - duration: Optional[int] = Field(5, title='Duration') - images: List[bytes_aliased] = Field( - ..., description='Array of images to process', title='Images' - ) - ingredientsMode: IngredientsMode = Field(..., title='Ingredientsmode') - negativePrompt: Optional[str] = Field(None, title='Negativeprompt') - promptText: Optional[str] = Field(None, title='Prompttext') - resolution: Optional[str] = Field('1080p', title='Resolution') - seed: Optional[int] = Field(None, title='Seed') - - -class PikaBodyGenerate22I2vGenerate22I2vPost(BaseModel): - duration: Optional[int] = Field(5, title='Duration') - image: bytes_aliased = Field(..., title='Image') - negativePrompt: Optional[str] = Field(None, title='Negativeprompt') - promptText: Optional[str] = Field(None, title='Prompttext') - resolution: Optional[str] = Field('1080p', title='Resolution') - seed: Optional[int] = Field(None, title='Seed') - - -class PikaBodyGenerate22KeyframeGenerate22PikaframesPost(BaseModel): - duration: Optional[int] = Field(5, title='Duration') - keyFrames: List[bytes_aliased] = Field( - ..., description='Array of keyframe images', title='Keyframes' - ) - negativePrompt: Optional[str] = Field(None, title='Negativeprompt') - promptText: str = Field(..., title='Prompttext') - resolution: Optional[str] = Field('1080p', title='Resolution') - seed: Optional[int] = Field(None, title='Seed') - - -class PikaBodyGenerate22T2vGenerate22T2vPost(BaseModel): - aspectRatio: Optional[confloat(ge=0.4, le=2.5)] = Field( - None, description='Aspect ratio (width / height)', title='Aspectratio' - ) - duration: Optional[int] = Field(5, title='Duration') - negativePrompt: Optional[str] = Field(None, title='Negativeprompt') - promptText: str = Field(..., title='Prompttext') - resolution: Optional[str] = Field('1080p', title='Resolution') - seed: Optional[int] = Field(None, title='Seed') - - -class PikaGenerateResponse(BaseModel): - video_id: str = Field(..., title='Video Id') - - -class PikaValidationError(BaseModel): - loc: List[Union[str, int]] = Field(..., title='Location') - msg: str = Field(..., title='Message') - type: str = Field(..., title='Error Type') - - -class PikaVideoResponse(BaseModel): - id: str = Field(..., title='Id') - progress: int = Field(..., title='Progress') - status: str = Field(..., title='Status') - url: str = Field(..., title='Url') - - -class Resp(BaseModel): - img_id: Optional[int] = None - - -class PixverseImageUploadResponse(BaseModel): - ErrCode: Optional[int] = None - ErrMsg: Optional[str] = None - Resp_1: Optional[Resp] = Field(None, alias='Resp') - - -class Duration2(int, Enum): - integer_5 = 5 - integer_8 = 8 - - -class Model1(str, Enum): - v3_5 = 'v3.5' - - -class MotionMode(str, Enum): - normal = 'normal' - fast = 'fast' - - -class Quality1(str, Enum): - field_360p = '360p' - field_540p = '540p' - field_720p = '720p' - field_1080p = '1080p' - - -class Style1(str, Enum): - anime = 'anime' - field_3d_animation = '3d_animation' - clay = 'clay' - comic = 'comic' - cyberpunk = 'cyberpunk' - - -class PixverseImageVideoRequest(BaseModel): - duration: Duration2 - img_id: int - model: Model1 - motion_mode: Optional[MotionMode] = None - prompt: str - quality: Quality1 - seed: Optional[int] = None - style: Optional[Style1] = None - template_id: Optional[int] = None - water_mark: Optional[bool] = None - - -class AspectRatio2(str, Enum): - field_16_9 = '16:9' - field_4_3 = '4:3' - field_1_1 = '1:1' - field_3_4 = '3:4' - field_9_16 = '9:16' - - -class PixverseTextVideoRequest(BaseModel): - aspect_ratio: AspectRatio2 - duration: Duration2 - model: Model1 - motion_mode: Optional[MotionMode] = None - negative_prompt: Optional[str] = None - prompt: str - quality: Quality1 - seed: Optional[int] = None - style: Optional[Style1] = None - template_id: Optional[int] = None - water_mark: Optional[bool] = None - - -class PixverseTransitionVideoRequest(BaseModel): - duration: Duration2 - first_frame_img: int - last_frame_img: int - model: Model1 - motion_mode: MotionMode - prompt: str - quality: Quality1 - seed: int - style: Optional[Style1] = None - template_id: Optional[int] = None - water_mark: Optional[bool] = None - - -class Resp1(BaseModel): - video_id: Optional[int] = None - - -class PixverseVideoResponse(BaseModel): - ErrCode: Optional[int] = None - ErrMsg: Optional[str] = None - Resp: Optional[Resp1] = None - - -class Status1(int, Enum): - integer_1 = 1 - integer_5 = 5 - integer_6 = 6 - integer_7 = 7 - integer_8 = 8 - - -class Resp2(BaseModel): - create_time: Optional[str] = None - id: Optional[int] = None - modify_time: Optional[str] = None - negative_prompt: Optional[str] = None - outputHeight: Optional[int] = None - outputWidth: Optional[int] = None - prompt: Optional[str] = None - resolution_ratio: Optional[int] = None - seed: Optional[int] = None - size: Optional[int] = None - status: Optional[Status1] = Field( - None, - description='Video generation status codes:\n* 1 - Generation successful\n* 5 - Generating\n* 6 - Deleted\n* 7 - Contents moderation failed\n* 8 - Generation failed\n', - ) - style: Optional[str] = None - url: Optional[str] = None - - -class PixverseVideoResultResponse(BaseModel): - ErrCode: Optional[int] = None - ErrMsg: Optional[str] = None - Resp: Optional[Resp2] = None - - -class PublisherStatus(str, Enum): - PublisherStatusActive = 'PublisherStatusActive' - PublisherStatusBanned = 'PublisherStatusBanned' - - -class PublisherUser(BaseModel): - email: Optional[str] = Field(None, description='The email address for this user.') - id: Optional[str] = Field(None, description='The unique id for this user.') - name: Optional[str] = Field(None, description='The name for this user.') - - -class RgbItem(RootModel[conint(ge=0, le=255)]): - root: conint(ge=0, le=255) - - -class RGBColor(BaseModel): - rgb: List[RgbItem] = Field(..., max_length=3, min_length=3) - - -class Controls(BaseModel): - artistic_level: Optional[conint(ge=0, le=5)] = Field( - None, - description='Defines artistic tone of your image. At a simple level, the person looks straight at the camera in a static and clean style. Dynamic and eccentric levels introduce movement and creativity.', - ) - background_color: Optional[RGBColor] = None - colors: Optional[List[RGBColor]] = Field( - None, description='An array of preferable colors' - ) - no_text: Optional[bool] = Field(None, description='Do not embed text layouts') - - -class RecraftImageGenerationRequest(BaseModel): - controls: Optional[Controls] = Field( - None, description='The controls for the generated image' - ) - model: str = Field( - ..., description='The model to use for generation (e.g., "recraftv3")' - ) - n: conint(ge=1, le=4) = Field(..., description='The number of images to generate') - prompt: str = Field( - ..., description='The text prompt describing the image to generate' - ) - size: str = Field( - ..., description='The size of the generated image (e.g., "1024x1024")' - ) - style: Optional[str] = Field( - None, - description='The style to apply to the generated image (e.g., "digital_illustration")', - ) - style_id: Optional[str] = Field( - None, - description='The style ID to apply to the generated image (e.g., "123e4567-e89b-12d3-a456-426614174000"). If style_id is provided, style should not be provided.', - ) - - -class Datum2(BaseModel): - image_id: Optional[str] = Field( - None, description='Unique identifier for the generated image' - ) - url: Optional[str] = Field(None, description='URL to access the generated image') - - -class RecraftImageGenerationResponse(BaseModel): - created: int = Field( - ..., description='Unix timestamp when the generation was created' - ) - credits: int = Field(..., description='Number of credits used for the generation') - data: List[Datum2] = Field(..., description='Array of generated image information') - - -class RenderingSpeed(str, Enum): - BALANCED = 'BALANCED' - TURBO = 'TURBO' - QUALITY = 'QUALITY' - - -class RunwayAspectRatioEnum(str, Enum): - field_1280_720 = '1280:720' - field_720_1280 = '720:1280' - field_1104_832 = '1104:832' - field_832_1104 = '832:1104' - field_960_960 = '960:960' - field_1584_672 = '1584:672' - field_1280_768 = '1280:768' - field_768_1280 = '768:1280' - - -class RunwayDurationEnum(int, Enum): - integer_5 = 5 - integer_10 = 10 - - -class RunwayImageToVideoResponse(BaseModel): - id: Optional[str] = Field(None, description='Task ID') - - -class RunwayModelEnum(str, Enum): - gen4_turbo = 'gen4_turbo' - gen3a_turbo = 'gen3a_turbo' - - -class Position(str, Enum): - first = 'first' - last = 'last' - - -class RunwayPromptImageDetailedObject(BaseModel): - position: Position = Field( - ..., - description="The position of the image in the output video. 'last' is currently supported for gen3a_turbo only.", - ) - uri: AnyUrl = Field( - ..., description='A HTTPS URL or data URI containing an encoded image.' - ) - - -class RunwayPromptImageObject( - RootModel[Union[AnyUrl, List[RunwayPromptImageDetailedObject]]] -): - root: Union[AnyUrl, List[RunwayPromptImageDetailedObject]] = Field( - ..., - description='Image(s) to use for the video generation. Can be a single URI or an array of image objects with positions.', - ) - - -class RunwayTaskStatusEnum(str, Enum): - SUCCEEDED = 'SUCCEEDED' - RUNNING = 'RUNNING' - FAILED = 'FAILED' - PENDING = 'PENDING' - CANCELLED = 'CANCELLED' - THROTTLED = 'THROTTLED' - - -class RunwayTaskStatusResponse(BaseModel): - createdAt: Optional[datetime] = Field(None, description='Task creation timestamp') - id: Optional[str] = Field(None, description='Task ID') - output: Optional[List[str]] = Field(None, description='Array of output video URLs') - status: Optional[RunwayTaskStatusEnum] = None - - -class Name(str, Enum): - content_moderation = 'content_moderation' - - -class StabilityContentModerationResponse(BaseModel): - errors: List[str] = Field( - ..., - description='One or more error messages indicating what went wrong.', - examples=[['some-field: is required']], - min_length=1, - ) - id: constr(min_length=1) = Field( - ..., - description='A unique identifier associated with this error. Please include this in any [support tickets](https://kb.stability.ai/knowledge-base/kb-tickets/new) you file, as it will greatly assist us in diagnosing the root cause of the problem.', - examples=['a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4'], - ) - name: Name = Field( - ..., - description='Our content moderation system has flagged some part of your request and subsequently denied it. You were not charged for this request. While this may at times be frustrating, it is necessary to maintain the integrity of our platform and ensure a safe experience for all users. If you would like to provide feedback, please use the [Support Form](https://kb.stability.ai/knowledge-base/kb-tickets/new).', - ) - - -class StabilityStabilityClientID(RootModel[constr(max_length=256)]): - root: constr(max_length=256) = Field( - ..., - description='The name of your application, used to help us communicate app-specific debugging or moderation issues to you.', - examples=['my-awesome-app'], - ) - - -class StabilityStabilityClientUserID(RootModel[constr(max_length=256)]): - root: constr(max_length=256) = Field( - ..., - description='A unique identifier for your end user. Used to help us communicate user-specific debugging or moderation issues to you. Feel free to obfuscate this value to protect user privacy.', - examples=['DiscordUser#9999'], - ) - - -class StabilityStabilityClientVersion(RootModel[constr(max_length=256)]): - root: constr(max_length=256) = Field( - ..., - description='The version of your application, used to help us communicate version-specific debugging or moderation issues to you.', - examples=['1.2.1'], - ) - - -class StorageFile(BaseModel): - file_path: Optional[str] = Field(None, description='Path to the file in storage') - id: Optional[UUID] = Field( - None, description='Unique identifier for the storage file' - ) - public_url: Optional[str] = Field(None, description='Public URL') +class Object2(str, Enum): + charge = "charge" class StripeAddress(BaseModel): @@ -1355,16 +615,16 @@ class StripeAddress(BaseModel): state: Optional[str] = None -class StripeAmountDetails(BaseModel): - tip: Optional[Dict[str, Any]] = None - - -class StripeBillingDetails(BaseModel): - address: Optional[StripeAddress] = None - email: Optional[str] = None - name: Optional[str] = None - phone: Optional[str] = None - tax_id: Optional[Any] = None +class StripeOutcome(BaseModel): + advice_code: Optional[Any] = None + network_advice_code: Optional[Any] = None + network_decline_code: Optional[Any] = None + network_status: Optional[str] = None + reason: Optional[Any] = None + risk_level: Optional[str] = None + risk_score: Optional[int] = None + seller_message: Optional[str] = None + type: Optional[str] = None class Checks(BaseModel): @@ -1419,37 +679,12 @@ class StripeCardDetails(BaseModel): wallet: Optional[Any] = None -class Object(str, Enum): - charge = 'charge' - - -class Object1(str, Enum): - event = 'event' - - -class Type4(str, Enum): - payment_intent_succeeded = 'payment_intent.succeeded' - - -class StripeOutcome(BaseModel): - advice_code: Optional[Any] = None - network_advice_code: Optional[Any] = None - network_decline_code: Optional[Any] = None - network_status: Optional[str] = None - reason: Optional[Any] = None - risk_level: Optional[str] = None - risk_score: Optional[int] = None - seller_message: Optional[str] = None - type: Optional[str] = None - - -class Object2(str, Enum): - payment_intent = 'payment_intent' - - -class StripePaymentMethodDetails(BaseModel): - card: Optional[StripeCardDetails] = None - type: Optional[str] = None +class StripeRefundList(BaseModel): + object: Optional[str] = None + data: Optional[List[Dict[str, Any]]] = None + has_more: Optional[bool] = None + total_count: Optional[int] = None + url: Optional[str] = None class Card(BaseModel): @@ -1463,19 +698,6 @@ class StripePaymentMethodOptions(BaseModel): card: Optional[Card] = None -class StripeRefundList(BaseModel): - data: Optional[List[Dict[str, Any]]] = None - has_more: Optional[bool] = None - object: Optional[str] = None - total_count: Optional[int] = None - url: Optional[str] = None - - -class StripeRequestInfo(BaseModel): - id: Optional[str] = None - idempotency_key: Optional[str] = None - - class StripeShipping(BaseModel): address: Optional[StripeAddress] = None carrier: Optional[str] = None @@ -1484,67 +706,460 @@ class StripeShipping(BaseModel): tracking_number: Optional[str] = None -class User(BaseModel): - email: Optional[str] = Field(None, description='The email address for this user.') - id: Optional[str] = Field(None, description='The unique id for this user.') - isAdmin: Optional[bool] = Field( - None, description='Indicates if the user has admin privileges.' +class Model(str, Enum): + T2V_01_Director = "T2V-01-Director" + I2V_01_Director = "I2V-01-Director" + S2V_01 = "S2V-01" + I2V_01 = "I2V-01" + I2V_01_live = "I2V-01-live" + T2V_01 = "T2V-01" + + +class SubjectReferenceItem(BaseModel): + image: Optional[str] = Field( + None, description="URL or base64 encoding of the subject reference image." ) - isApproved: Optional[bool] = Field( - None, description='Indicates if the user is approved.' - ) - name: Optional[str] = Field(None, description='The name for this user.') - - -class Veo2GenVidPollRequest(BaseModel): - operationName: str = Field( - ..., - description='Full operation name (from predict response)', - examples=[ - 'projects/PROJECT_ID/locations/us-central1/publishers/google/models/MODEL_ID/operations/OPERATION_ID' - ], - ) - - -class Error1(BaseModel): - code: Optional[int] = Field(None, description='Error code') - message: Optional[str] = Field(None, description='Error message') - - -class Video2(BaseModel): - bytesBase64Encoded: Optional[str] = Field( - None, description='Base64-encoded video content' - ) - gcsUri: Optional[str] = Field(None, description='Cloud Storage URI of the video') - mimeType: Optional[str] = Field(None, description='Video MIME type') - - -class Response(BaseModel): - field_type: Optional[str] = Field( + mask: Optional[str] = Field( None, - alias='@type', - examples=[ - 'type.googleapis.com/cloud.ai.large_models.vision.GenerateVideoResponse' - ], + description="URL or base64 encoding of the mask for the subject reference image.", ) - raiMediaFilteredCount: Optional[int] = Field( - None, description='Count of media filtered by responsible AI policies' - ) - raiMediaFilteredReasons: Optional[List[str]] = Field( - None, description='Reasons why media was filtered by responsible AI policies' - ) - videos: Optional[List[Video2]] = None -class Veo2GenVidPollResponse(BaseModel): - done: Optional[bool] = None - error: Optional[Error1] = Field( - None, description='Error details if operation failed' +class MinimaxVideoGenerationRequest(BaseModel): + model: Model = Field( + ..., + description="Required. ID of model. Options: T2V-01-Director, I2V-01-Director, S2V-01, I2V-01, I2V-01-live, T2V-01", ) - name: Optional[str] = None - response: Optional[Response] = Field( - None, description='The actual prediction response if done is true' + prompt: Optional[str] = Field( + None, + description="Description of the video. Should be less than 2000 characters. Supports camera movement instructions in [brackets].", + max_length=2000, ) + prompt_optimizer: Optional[bool] = Field( + True, + description="If true (default), the model will automatically optimize the prompt. Set to false for more precise control.", + ) + first_frame_image: Optional[str] = Field( + None, + description="URL or base64 encoding of the first frame image. Required when model is I2V-01, I2V-01-Director, or I2V-01-live.", + ) + subject_reference: Optional[List[SubjectReferenceItem]] = Field( + None, + description="Only available when model is S2V-01. The model will generate a video based on the subject uploaded through this parameter.", + ) + callback_url: Optional[str] = Field( + None, + description="Optional. URL to receive real-time status updates about the video generation task.", + ) + + +class MinimaxBaseResponse(BaseModel): + status_code: int = Field( + ..., + description="Status code. 0 indicates success, other values indicate errors.", + ) + status_msg: str = Field( + ..., description="Specific error details or success message." + ) + + +class MinimaxVideoGenerationResponse(BaseModel): + task_id: str = Field( + ..., description="The task ID for the asynchronous video generation task." + ) + base_resp: MinimaxBaseResponse + + +class File(BaseModel): + file_id: Optional[int] = Field(None, description="Unique identifier for the file") + bytes: Optional[int] = Field(None, description="File size in bytes") + created_at: Optional[int] = Field( + None, description="Unix timestamp when the file was created, in seconds" + ) + filename: Optional[str] = Field(None, description="The name of the file") + purpose: Optional[str] = Field(None, description="The purpose of using the file") + download_url: Optional[str] = Field( + None, description="The URL to download the video" + ) + + +class MinimaxFileRetrieveResponse(BaseModel): + file: File + base_resp: MinimaxBaseResponse + + +class Status(str, Enum): + Queueing = "Queueing" + Preparing = "Preparing" + Processing = "Processing" + Success = "Success" + Fail = "Fail" + + +class MinimaxTaskResultResponse(BaseModel): + task_id: str = Field(..., description="The task ID being queried.") + status: Status = Field( + ..., + description="Task status: 'Queueing' (in queue), 'Preparing' (task is preparing), 'Processing' (generating), 'Success' (task completed successfully), or 'Fail' (task failed).", + ) + file_id: Optional[str] = Field( + None, + description="After the task status changes to Success, this field returns the file ID corresponding to the generated video.", + ) + base_resp: MinimaxBaseResponse + + +class BFLFluxProGenerateRequest(BaseModel): + prompt: str = Field(..., description="The text prompt for image generation.") + negative_prompt: Optional[str] = Field( + None, description="The negative prompt for image generation." + ) + width: int = Field( + ..., description="The width of the image to generate.", ge=64, le=2048 + ) + height: int = Field( + ..., description="The height of the image to generate.", ge=64, le=2048 + ) + num_inference_steps: Optional[int] = Field( + None, description="The number of inference steps.", ge=1, le=100 + ) + guidance_scale: Optional[float] = Field( + None, description="The guidance scale for generation.", ge=1.0, le=20.0 + ) + seed: Optional[int] = Field(None, description="The seed value for reproducibility.") + num_images: Optional[int] = Field( + None, description="The number of images to generate.", ge=1, le=4 + ) + + +class BFLFluxProGenerateResponse(BaseModel): + id: str = Field(..., description="The unique identifier for the generation task.") + polling_url: str = Field(..., description="URL to poll for the generation result.") + + +class Datum1(BaseModel): + image_id: Optional[str] = Field( + None, description="Unique identifier for the generated image" + ) + url: Optional[str] = Field(None, description="URL to access the generated image") + + +class RecraftImageGenerationResponse(BaseModel): + created: int = Field( + ..., description="Unix timestamp when the generation was created" + ) + credits: int = Field(..., description="Number of credits used for the generation") + data: List[Datum1] = Field(..., description="Array of generated image information") + + +class KlingErrorResponse(BaseModel): + code: int = Field( + ..., + description="- 1000: Authentication failed\n- 1001: Authorization is empty\n- 1002: Authorization is invalid\n- 1003: Authorization is not yet valid\n- 1004: Authorization has expired\n- 1100: Account exception\n- 1101: Account in arrears (postpaid scenario)\n- 1102: Resource pack depleted or expired (prepaid scenario)\n- 1103: Unauthorized access to requested resource\n- 1200: Invalid request parameters\n- 1201: Invalid parameters\n- 1202: Invalid request method\n- 1203: Requested resource does not exist\n- 1300: Trigger platform strategy\n- 1301: Trigger content security policy\n- 1302: API request too frequent\n- 1303: Concurrency/QPS exceeds limit\n- 1304: Trigger IP whitelist policy\n- 5000: Internal server error\n- 5001: Service temporarily unavailable\n- 5002: Server internal timeout\n", + ) + message: str = Field(..., description="Human-readable error message") + request_id: str = Field( + ..., description="Request ID for tracking and troubleshooting" + ) + + +class LumaAspectRatio(str, Enum): + field_1_1 = "1:1" + field_16_9 = "16:9" + field_9_16 = "9:16" + field_4_3 = "4:3" + field_3_4 = "3:4" + field_21_9 = "21:9" + field_9_21 = "9:21" + + +class LumaVideoModel(str, Enum): + ray_2 = "ray-2" + ray_flash_2 = "ray-flash-2" + ray_1_6 = "ray-1-6" + + +class LumaVideoModelOutputResolution1(str, Enum): + field_540p = "540p" + field_720p = "720p" + field_1080p = "1080p" + field_4k = "4k" + + +class LumaVideoModelOutputResolution( + RootModel[Union[LumaVideoModelOutputResolution1, str]] +): + root: Union[LumaVideoModelOutputResolution1, str] + + +class LumaVideoModelOutputDuration1(str, Enum): + field_5s = "5s" + field_9s = "9s" + + +class LumaVideoModelOutputDuration( + RootModel[Union[LumaVideoModelOutputDuration1, str]] +): + root: Union[LumaVideoModelOutputDuration1, str] + + +class LumaImageModel(str, Enum): + photon_1 = "photon-1" + photon_flash_1 = "photon-flash-1" + + +class LumaImageRef(BaseModel): + url: Optional[AnyUrl] = Field(None, description="The URL of the image reference") + weight: Optional[float] = Field( + None, description="The weight of the image reference" + ) + + +class LumaImageIdentity(BaseModel): + images: Optional[List[AnyUrl]] = Field( + None, description="The URLs of the image identity" + ) + + +class LumaModifyImageRef(BaseModel): + url: Optional[AnyUrl] = Field(None, description="The URL of the image reference") + weight: Optional[float] = Field( + None, description="The weight of the modify image reference" + ) + + +class Type3(str, Enum): + generation = "generation" + + +class LumaGenerationReference(BaseModel): + type: Literal["generation"] + id: UUID = Field(..., description="The ID of the generation") + + +class Type4(str, Enum): + image = "image" + + +class LumaImageReference(BaseModel): + type: Literal["image"] + url: AnyUrl = Field(..., description="The URL of the image") + + +class LumaKeyframe(RootModel[Union[LumaGenerationReference, LumaImageReference]]): + root: Union[LumaGenerationReference, LumaImageReference] = Field( + ..., + description="A keyframe can be either a Generation reference, an Image, or a Video", + discriminator="type", + ) + + +class LumaGenerationType(str, Enum): + video = "video" + image = "image" + + +class LumaState(str, Enum): + queued = "queued" + dreaming = "dreaming" + completed = "completed" + failed = "failed" + + +class LumaAssets(BaseModel): + video: Optional[AnyUrl] = Field(None, description="The URL of the video") + image: Optional[AnyUrl] = Field(None, description="The URL of the image") + progress_video: Optional[AnyUrl] = Field( + None, description="The URL of the progress video" + ) + + +class GenerationType(str, Enum): + video = "video" + + +class GenerationType1(str, Enum): + image = "image" + + +class CharacterRef(BaseModel): + identity0: Optional[LumaImageIdentity] = None + + +class LumaImageGenerationRequest(BaseModel): + generation_type: Optional[GenerationType1] = "image" + model: Optional[LumaImageModel] = "photon-1" + prompt: Optional[str] = Field(None, description="The prompt of the generation") + aspect_ratio: Optional[LumaAspectRatio] = "16:9" + callback_url: Optional[AnyUrl] = Field( + None, description="The callback URL for the generation" + ) + image_ref: Optional[List[LumaImageRef]] = None + style_ref: Optional[List[LumaImageRef]] = None + character_ref: Optional[CharacterRef] = None + modify_image_ref: Optional[LumaModifyImageRef] = None + + +class GenerationType2(str, Enum): + upscale_video = "upscale_video" + + +class LumaUpscaleVideoGenerationRequest(BaseModel): + generation_type: Optional[GenerationType2] = "upscale_video" + resolution: Optional[LumaVideoModelOutputResolution] = None + callback_url: Optional[AnyUrl] = Field( + None, description="The callback URL for the upscale" + ) + + +class GenerationType3(str, Enum): + add_audio = "add_audio" + + +class LumaAudioGenerationRequest(BaseModel): + generation_type: Optional[GenerationType3] = "add_audio" + prompt: Optional[str] = Field(None, description="The prompt of the audio") + negative_prompt: Optional[str] = Field( + None, description="The negative prompt of the audio" + ) + callback_url: Optional[AnyUrl] = Field( + None, description="The callback URL for the audio" + ) + + +class LumaError(BaseModel): + detail: Optional[str] = Field(None, description="The error message") + + +class AspectRatio2(str, Enum): + field_16_9 = "16:9" + field_4_3 = "4:3" + field_1_1 = "1:1" + field_3_4 = "3:4" + field_9_16 = "9:16" + + +class Duration2(int, Enum): + integer_5 = 5 + integer_8 = 8 + + +class Model1(str, Enum): + v3_5 = "v3.5" + + +class MotionMode(str, Enum): + normal = "normal" + fast = "fast" + + +class Quality(str, Enum): + field_360p = "360p" + field_540p = "540p" + field_720p = "720p" + field_1080p = "1080p" + + +class Style(str, Enum): + anime = "anime" + field_3d_animation = "3d_animation" + clay = "clay" + comic = "comic" + cyberpunk = "cyberpunk" + + +class PixverseTextVideoRequest(BaseModel): + aspect_ratio: AspectRatio2 + duration: Duration2 + model: Model1 + motion_mode: Optional[MotionMode] = None + negative_prompt: Optional[str] = None + prompt: str + quality: Quality + seed: Optional[int] = None + style: Optional[Style] = None + template_id: Optional[int] = None + water_mark: Optional[bool] = None + + +class Resp(BaseModel): + video_id: Optional[int] = None + + +class PixverseVideoResponse(BaseModel): + ErrCode: Optional[int] = None + ErrMsg: Optional[str] = None + Resp_1: Optional[Resp] = Field(None, alias="Resp") + + +class Resp1(BaseModel): + img_id: Optional[int] = None + + +class PixverseImageUploadResponse(BaseModel): + ErrCode: Optional[int] = None + ErrMsg: Optional[str] = None + Resp: Optional[Resp1] = None + + +class PixverseImageVideoRequest(BaseModel): + img_id: int + model: Model1 + prompt: str + duration: Duration2 + quality: Quality + motion_mode: Optional[MotionMode] = None + seed: Optional[int] = None + style: Optional[Style] = None + template_id: Optional[int] = None + water_mark: Optional[bool] = None + + +class PixverseTransitionVideoRequest(BaseModel): + first_frame_img: int + last_frame_img: int + model: Model1 + duration: Duration2 + quality: Quality + motion_mode: MotionMode + seed: int + prompt: str + style: Optional[Style] = None + template_id: Optional[int] = None + water_mark: Optional[bool] = None + + +class Status1(int, Enum): + integer_1 = 1 + integer_5 = 5 + integer_6 = 6 + integer_7 = 7 + integer_8 = 8 + + +class Resp2(BaseModel): + create_time: Optional[str] = None + id: Optional[int] = None + modify_time: Optional[str] = None + negative_prompt: Optional[str] = None + outputHeight: Optional[int] = None + outputWidth: Optional[int] = None + prompt: Optional[str] = None + resolution_ratio: Optional[int] = None + seed: Optional[int] = None + size: Optional[int] = None + status: Optional[Status1] = Field( + None, + description="Video generation status codes:\n* 1 - Generation successful\n* 5 - Generating\n* 6 - Deleted\n* 7 - Contents moderation failed\n* 8 - Generation failed\n", + ) + style: Optional[str] = None + url: Optional[str] = None + + +class PixverseVideoResultResponse(BaseModel): + ErrCode: Optional[int] = None + ErrMsg: Optional[str] = None + Resp: Optional[Resp2] = None class Image(BaseModel): @@ -1560,28 +1175,28 @@ class Image1(BaseModel): class Instance(BaseModel): + prompt: str = Field(..., description="Text description of the video") image: Optional[Union[Image, Image1]] = Field( - None, description='Optional image to guide video generation' + None, description="Optional image to guide video generation" ) - prompt: str = Field(..., description='Text description of the video') class PersonGeneration(str, Enum): - ALLOW = 'ALLOW' - BLOCK = 'BLOCK' + ALLOW = "ALLOW" + BLOCK = "BLOCK" class Parameters(BaseModel): - aspectRatio: Optional[str] = Field(None, examples=['16:9']) - durationSeconds: Optional[int] = None - enhancePrompt: Optional[bool] = None + aspectRatio: Optional[str] = Field(None, examples=["16:9"]) negativePrompt: Optional[str] = None personGeneration: Optional[PersonGeneration] = None sampleCount: Optional[int] = None seed: Optional[int] = None storageUri: Optional[str] = Field( - None, description='Optional Cloud Storage URI to upload the video' + None, description="Optional Cloud Storage URI to upload the video" ) + durationSeconds: Optional[int] = None + enhancePrompt: Optional[bool] = None class Veo2GenVidRequest(BaseModel): @@ -1592,234 +1207,771 @@ class Veo2GenVidRequest(BaseModel): class Veo2GenVidResponse(BaseModel): name: str = Field( ..., - description='Operation resource name', + description="Operation resource name", examples=[ - 'projects/PROJECT_ID/locations/us-central1/publishers/google/models/MODEL_ID/operations/a1b07c8e-7b5a-4aba-bb34-3e1ccb8afcc8' + "projects/PROJECT_ID/locations/us-central1/publishers/google/models/MODEL_ID/operations/a1b07c8e-7b5a-4aba-bb34-3e1ccb8afcc8" ], ) -class WorkflowRunStatus(str, Enum): - WorkflowRunStatusStarted = 'WorkflowRunStatusStarted' - WorkflowRunStatusFailed = 'WorkflowRunStatusFailed' - WorkflowRunStatusCompleted = 'WorkflowRunStatusCompleted' +class Veo2GenVidPollRequest(BaseModel): + operationName: str = Field( + ..., + description="Full operation name (from predict response)", + examples=[ + "projects/PROJECT_ID/locations/us-central1/publishers/google/models/MODEL_ID/operations/OPERATION_ID" + ], + ) + + +class Video2(BaseModel): + gcsUri: Optional[str] = Field(None, description="Cloud Storage URI of the video") + bytesBase64Encoded: Optional[str] = Field( + None, description="Base64-encoded video content" + ) + mimeType: Optional[str] = Field(None, description="Video MIME type") + + +class Response(BaseModel): + field_type: Optional[str] = Field( + None, + alias="@type", + examples=[ + "type.googleapis.com/cloud.ai.large_models.vision.GenerateVideoResponse" + ], + ) + raiMediaFilteredCount: Optional[int] = Field( + None, description="Count of media filtered by responsible AI policies" + ) + raiMediaFilteredReasons: Optional[List[str]] = Field( + None, description="Reasons why media was filtered by responsible AI policies" + ) + videos: Optional[List[Video2]] = None + + +class Error1(BaseModel): + code: Optional[int] = Field(None, description="Error code") + message: Optional[str] = Field(None, description="Error message") + + +class Veo2GenVidPollResponse(BaseModel): + name: Optional[str] = None + done: Optional[bool] = None + response: Optional[Response] = Field( + None, description="The actual prediction response if done is true" + ) + error: Optional[Error1] = Field( + None, description="Error details if operation failed" + ) + + +class RunwayImageToVideoResponse(BaseModel): + id: Optional[str] = Field(None, description="Task ID") + + +class RunwayTaskStatusEnum(str, Enum): + SUCCEEDED = "SUCCEEDED" + RUNNING = "RUNNING" + FAILED = "FAILED" + PENDING = "PENDING" + CANCELLED = "CANCELLED" + THROTTLED = "THROTTLED" + + +class RunwayModelEnum(str, Enum): + gen4_turbo = "gen4_turbo" + gen3a_turbo = "gen3a_turbo" + + +class Position(str, Enum): + first = "first" + last = "last" + + +class RunwayPromptImageDetailedObject(BaseModel): + uri: AnyUrl = Field( + ..., description="A HTTPS URL or data URI containing an encoded image." + ) + position: Position = Field( + ..., + description="The position of the image in the output video. 'last' is currently supported for gen3a_turbo only.", + ) + + +class RunwayDurationEnum(int, Enum): + integer_5 = 5 + integer_10 = 10 + + +class RunwayAspectRatioEnum(str, Enum): + field_1280_720 = "1280:720" + field_720_1280 = "720:1280" + field_1104_832 = "1104:832" + field_832_1104 = "832:1104" + field_960_960 = "960:960" + field_1584_672 = "1584:672" + field_1280_768 = "1280:768" + field_768_1280 = "768:1280" + + +class RunwayPromptImageObject( + RootModel[Union[AnyUrl, List[RunwayPromptImageDetailedObject]]] +): + root: Union[AnyUrl, List[RunwayPromptImageDetailedObject]] = Field( + ..., + description="Image(s) to use for the video generation. Can be a single URI or an array of image objects with positions.", + ) + + +class Datum2(BaseModel): + b64_json: Optional[str] = Field(None, description="Base64 encoded image data") + url: Optional[str] = Field(None, description="URL of the image") + revised_prompt: Optional[str] = Field(None, description="Revised prompt") + + +class InputTokensDetails(BaseModel): + text_tokens: Optional[int] = None + image_tokens: Optional[int] = None + + +class Usage(BaseModel): + input_tokens: Optional[int] = None + input_tokens_details: Optional[InputTokensDetails] = None + output_tokens: Optional[int] = None + total_tokens: Optional[int] = None + + +class OpenAIImageGenerationResponse(BaseModel): + data: Optional[List[Datum2]] = None + usage: Optional[Usage] = None + + +class Quality3(str, Enum): + low = "low" + medium = "medium" + high = "high" + standard = "standard" + hd = "hd" + + +class OutputFormat(str, Enum): + png = "png" + webp = "webp" + jpeg = "jpeg" + + +class Moderation(str, Enum): + low = "low" + auto = "auto" + + +class Background(str, Enum): + transparent = "transparent" + opaque = "opaque" + + +class ResponseFormat(str, Enum): + url = "url" + b64_json = "b64_json" + + +class Style3(str, Enum): + vivid = "vivid" + natural = "natural" + + +class OpenAIImageGenerationRequest(BaseModel): + model: Optional[str] = Field( + None, description="The model to use for image generation", examples=["dall-e-3"] + ) + prompt: str = Field( + ..., + description="A text description of the desired image", + examples=["Draw a rocket in front of a blackhole in deep space"], + ) + n: Optional[int] = Field( + None, + description="The number of images to generate (1-10). Only 1 supported for dall-e-3.", + examples=[1], + ) + quality: Optional[Quality3] = Field( + None, description="The quality of the generated image", examples=["high"] + ) + size: Optional[str] = Field( + None, + description="Size of the image (e.g., 1024x1024, 1536x1024, auto)", + examples=["1024x1536"], + ) + output_format: Optional[OutputFormat] = Field( + None, description="Format of the output image", examples=["png"] + ) + output_compression: Optional[int] = Field( + None, description="Compression level for JPEG or WebP (0-100)", examples=[100] + ) + moderation: Optional[Moderation] = Field( + None, description="Content moderation setting", examples=["auto"] + ) + background: Optional[Background] = Field( + None, description="Background transparency", examples=["opaque"] + ) + response_format: Optional[ResponseFormat] = Field( + None, description="Response format of image data", examples=["b64_json"] + ) + style: Optional[Style3] = Field( + None, description="Style of the image (only for dall-e-3)", examples=["vivid"] + ) + user: Optional[str] = Field( + None, + description="A unique identifier for end-user monitoring", + examples=["user-1234"], + ) + + +class OpenAIImageEditRequest(BaseModel): + model: str = Field( + ..., description="The model to use for image editing", examples=["gpt-image-1"] + ) + prompt: str = Field( + ..., + description="A text description of the desired edit", + examples=["Give the rocketship rainbow coloring"], + ) + n: Optional[int] = Field( + None, description="The number of images to generate", examples=[1] + ) + quality: Optional[str] = Field( + None, description="The quality of the edited image", examples=["low"] + ) + size: Optional[str] = Field( + None, description="Size of the output image", examples=["1024x1024"] + ) + output_format: Optional[OutputFormat] = Field( + None, description="Format of the output image", examples=["png"] + ) + output_compression: Optional[int] = Field( + None, description="Compression level for JPEG or WebP (0-100)", examples=[100] + ) + moderation: Optional[Moderation] = Field( + None, description="Content moderation setting", examples=["auto"] + ) + background: Optional[str] = Field( + None, description="Background transparency", examples=["opaque"] + ) + user: Optional[str] = Field( + None, + description="A unique identifier for end-user monitoring", + examples=["user-1234"], + ) + + +class CustomerStorageResourceResponse(BaseModel): + download_url: Optional[str] = Field( + None, + description="The signed URL to use for downloading the file from the specified path", + ) + upload_url: Optional[str] = Field( + None, + description="The signed URL to use for uploading the file to the specified path", + ) + expires_at: Optional[datetime] = Field( + None, description="When the signed URL will expire" + ) + existing_file: Optional[bool] = Field( + None, description="Whether an existing file with the same hash was found" + ) + + +class PikaBodyGenerate22T2vGenerate22T2vPost(BaseModel): + promptText: str = Field(..., title="Prompttext") + negativePrompt: Optional[str] = Field(None, title="Negativeprompt") + seed: Optional[int] = Field(None, title="Seed") + resolution: Optional[str] = Field("1080p", title="Resolution") + duration: Optional[int] = Field(5, title="Duration") + aspectRatio: Optional[float] = Field( + 1.7777777777777777, + description="Aspect ratio (width / height)", + ge=0.4, + le=2.5, + title="Aspectratio", + ) + + +class PikaGenerateResponse(BaseModel): + video_id: str = Field(..., title="Video Id") + + +class PikaBodyGenerate22I2vGenerate22I2vPost(BaseModel): + image: Optional[str] = Field(None, title="Image") + promptText: Optional[str] = Field(None, title="Prompttext") + negativePrompt: Optional[str] = Field(None, title="Negativeprompt") + seed: Optional[int] = Field(None, title="Seed") + resolution: Optional[str] = Field("1080p", title="Resolution") + duration: Optional[int] = Field(5, title="Duration") + + +class IngredientsMode(str, Enum): + creative = "creative" + precise = "precise" + + +class AspectRatio3(RootModel[float]): + root: float = Field( + ..., + description="Aspect ratio (width / height)", + ge=0.4, + le=2.5, + title="Aspectratio", + ) + + +class PikaBodyGenerate22C2vGenerate22PikascenesPost(BaseModel): + images: Optional[List[bytes_aliased]] = Field( + None, description="Array of images to process", title="Images" + ) + ingredientsMode: IngredientsMode = Field(..., title="Ingredientsmode") + promptText: Optional[str] = Field(None, title="Prompttext") + negativePrompt: Optional[str] = Field(None, title="Negativeprompt") + seed: Optional[int] = Field(None, title="Seed") + resolution: Optional[str] = Field("1080p", title="Resolution") + duration: Optional[int] = Field(5, title="Duration") + aspectRatio: Optional[AspectRatio3] = Field( + None, description="Aspect ratio (width / height)", title="Aspectratio" + ) + + +class PikaBodyGenerate22KeyframeGenerate22PikaframesPost(BaseModel): + keyFrames: List[bytes_aliased] = Field( + ..., description="Array of keyframe images", title="Keyframes" + ) + promptText: str = Field(..., title="Prompttext") + negativePrompt: Optional[str] = Field(None, title="Negativeprompt") + seed: Optional[int] = Field(None, title="Seed") + resolution: Optional[str] = Field("1080p", title="Resolution") + duration: Optional[int] = Field(5, title="Duration") + + +class PikaStatusEnum(str, Enum): + queued = "queued" + started = "started" + finished = "finished" + + +class PikaValidationError(BaseModel): + loc: List[Union[str, int]] = Field(..., title="Location") + msg: str = Field(..., title="Message") + type: str = Field(..., title="Error Type") + + +class RgbItem(RootModel[int]): + root: int = Field(..., ge=0, le=255) + + +class RGBColor(BaseModel): + rgb: List[RgbItem] = Field(..., max_length=3, min_length=3) + + +class StabilityStabilityClientID(RootModel[str]): + root: str = Field( + ..., + description="The name of your application, used to help us communicate app-specific debugging or moderation issues to you.", + examples=["my-awesome-app"], + max_length=256, + ) + + +class StabilityStabilityClientUserID(RootModel[str]): + root: str = Field( + ..., + description="A unique identifier for your end user. Used to help us communicate user-specific debugging or moderation issues to you. Feel free to obfuscate this value to protect user privacy.", + examples=["DiscordUser#9999"], + max_length=256, + ) + + +class StabilityStabilityClientVersion(RootModel[str]): + root: str = Field( + ..., + description="The version of your application, used to help us communicate version-specific debugging or moderation issues to you.", + examples=["1.2.1"], + max_length=256, + ) + + +class Name(str, Enum): + content_moderation = "content_moderation" + + +class StabilityContentModerationResponse(BaseModel): + id: str = Field( + ..., + description="A unique identifier associated with this error. Please include this in any [support tickets](https://kb.stability.ai/knowledge-base/kb-tickets/new) you file, as it will greatly assist us in diagnosing the root cause of the problem.", + examples=["a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4"], + min_length=1, + ) + name: Name = Field( + ..., + description="Our content moderation system has flagged some part of your request and subsequently denied it. You were not charged for this request. While this may at times be frustrating, it is necessary to maintain the integrity of our platform and ensure a safe experience for all users. If you would like to provide feedback, please use the [Support Form](https://kb.stability.ai/knowledge-base/kb-tickets/new).", + ) + errors: List[str] = Field( + ..., + description="One or more error messages indicating what went wrong.", + examples=[["some-field: is required"]], + min_length=1, + ) + + +class RenderingSpeed(str, Enum): + BALANCED = "BALANCED" + TURBO = "TURBO" + QUALITY = "QUALITY" class ActionJobResult(BaseModel): - action_job_id: Optional[str] = Field( - None, description='Identifier of the job this result belongs to' - ) + id: Optional[UUID] = Field(None, description="Unique identifier for the job result") + workflow_name: Optional[str] = Field(None, description="Name of the workflow") + operating_system: Optional[str] = Field(None, description="Operating system used") + python_version: Optional[str] = Field(None, description="PyTorch version used") + pytorch_version: Optional[str] = Field(None, description="PyTorch version used") action_run_id: Optional[str] = Field( - None, description='Identifier of the run this result belongs to' + None, description="Identifier of the run this result belongs to" ) - author: Optional[str] = Field(None, description='The author of the commit') - avg_vram: Optional[int] = Field( - None, description='The average VRAM used by the job' + action_job_id: Optional[str] = Field( + None, description="Identifier of the job this result belongs to" ) + cuda_version: Optional[str] = Field(None, description="CUDA version used") branch_name: Optional[str] = Field( - None, description='Name of the relevant git branch' + None, description="Name of the relevant git branch" ) - comfy_run_flags: Optional[str] = Field( - None, description='The comfy run flags. E.g. `--low-vram`' - ) - commit_hash: Optional[str] = Field(None, description='The hash of the commit') - commit_id: Optional[str] = Field(None, description='The ID of the commit') - commit_message: Optional[str] = Field(None, description='The message of the commit') + commit_hash: Optional[str] = Field(None, description="The hash of the commit") + commit_id: Optional[str] = Field(None, description="The ID of the commit") commit_time: Optional[int] = Field( - None, description='The Unix timestamp when the commit was made' + None, description="The Unix timestamp when the commit was made" ) - cuda_version: Optional[str] = Field(None, description='CUDA version used') - end_time: Optional[int] = Field( - None, description='The end time of the job as a Unix timestamp.' + commit_message: Optional[str] = Field(None, description="The message of the commit") + comfy_run_flags: Optional[str] = Field( + None, description="The comfy run flags. E.g. `--low-vram`" ) - git_repo: Optional[str] = Field(None, description='The repository name') - id: Optional[UUID] = Field(None, description='Unique identifier for the job result') - job_trigger_user: Optional[str] = Field( - None, description='The user who triggered the job.' - ) - machine_stats: Optional[MachineStats] = None - operating_system: Optional[str] = Field(None, description='Operating system used') - peak_vram: Optional[int] = Field(None, description='The peak VRAM used by the job') - pr_number: Optional[str] = Field(None, description='The pull request number') - python_version: Optional[str] = Field(None, description='PyTorch version used') - pytorch_version: Optional[str] = Field(None, description='PyTorch version used') + git_repo: Optional[str] = Field(None, description="The repository name") + pr_number: Optional[str] = Field(None, description="The pull request number") start_time: Optional[int] = Field( - None, description='The start time of the job as a Unix timestamp.' + None, description="The start time of the job as a Unix timestamp." ) + end_time: Optional[int] = Field( + None, description="The end time of the job as a Unix timestamp." + ) + avg_vram: Optional[int] = Field( + None, description="The average VRAM used by the job" + ) + peak_vram: Optional[int] = Field(None, description="The peak VRAM used by the job") + job_trigger_user: Optional[str] = Field( + None, description="The user who triggered the job." + ) + author: Optional[str] = Field(None, description="The author of the commit") + machine_stats: Optional[MachineStats] = None status: Optional[WorkflowRunStatus] = None storage_file: Optional[StorageFile] = None - workflow_name: Optional[str] = Field(None, description='Name of the workflow') -class IdeogramV3EditRequest(BaseModel): - color_palette: Optional[IdeogramColorPalette] = None - image: Optional[bytes_aliased] = Field( +class Publisher(BaseModel): + name: Optional[str] = None + id: Optional[str] = Field( None, - description='The image being edited (max size 10MB); only JPEG, WebP and PNG formats are supported at this time.', + description="The unique identifier for the publisher. It's akin to a username. Should be lowercase.", ) - magic_prompt: Optional[str] = Field( + description: Optional[str] = None + website: Optional[str] = None + support: Optional[str] = None + source_code_repo: Optional[str] = None + logo: Optional[str] = Field(None, description="URL to the publisher's logo.") + createdAt: Optional[datetime] = Field( + None, description="The date and time the publisher was created." + ) + members: Optional[List[PublisherMember]] = Field( + None, description="A list of members in the publisher." + ) + status: Optional[PublisherStatus] = Field( + None, description="The status of the publisher." + ) + + +class NodeVersion(BaseModel): + id: Optional[str] = None + version: Optional[str] = Field( None, - description='Determine if MagicPrompt should be used in generating the request or not.', + description="The version identifier, following semantic versioning. Must be unique for the node.", ) - mask: Optional[bytes_aliased] = Field( - None, - description='A black and white image of the same size as the image being edited (max size 10MB). Black regions in the mask should match up with the regions of the image that you would like to edit; only JPEG, WebP and PNG formats are supported at this time.', + createdAt: Optional[datetime] = Field( + None, description="The date and time the version was created." ) - num_images: Optional[int] = Field( - None, description='The number of images to generate.' + changelog: Optional[str] = Field( + None, description="Summary of changes made in this version" ) - prompt: str = Field( - ..., description='The prompt used to describe the edited result.' + dependencies: Optional[List[str]] = Field( + None, description="A list of pip dependencies required by the node." ) - rendering_speed: RenderingSpeed - seed: Optional[int] = Field( - None, description='Random seed. Set for reproducible generation.' + downloadUrl: Optional[str] = Field( + None, description="[Output Only] URL to download this version of the node" ) - style_codes: Optional[List[constr(pattern=r'^[0-9A-Fa-f]{8}$')]] = Field( - None, - description='A list of 8 character hexadecimal codes representing the style of the image. Cannot be used in conjunction with style_reference_images or style_type.', + deprecated: Optional[bool] = Field( + None, description="Indicates if this version is deprecated." ) - style_reference_images: Optional[List[bytes_aliased]] = Field( - None, - description='A set of images to use as style references (maximum total size 10MB across all style references). The images should be in JPEG, PNG or WebP format.', + status: Optional[NodeVersionStatus] = Field( + None, description="The status of the node version." + ) + status_reason: Optional[str] = Field( + None, description="The reason for the status change." + ) + node_id: Optional[str] = Field( + None, description="The unique identifier of the node." + ) + comfy_node_extract_status: Optional[str] = Field( + None, description="The status of comfy node extraction process." ) class IdeogramV3Request(BaseModel): - aspect_ratio: Optional[str] = Field( - None, description='Aspect ratio in format WxH', examples=['1x3'] + prompt: str = Field(..., description="The text prompt for image generation") + seed: Optional[int] = Field( + None, description="Seed value for reproducible generation" ) - color_palette: Optional[ColorPalette] = None + resolution: Optional[str] = Field( + None, description="Image resolution in format WxH", examples=["1280x800"] + ) + aspect_ratio: Optional[str] = Field( + None, description="Aspect ratio in format WxH", examples=["1x3"] + ) + rendering_speed: RenderingSpeed magic_prompt: Optional[MagicPrompt] = Field( - None, description='Whether to enable magic prompt enhancement' + None, description="Whether to enable magic prompt enhancement" ) negative_prompt: Optional[str] = Field( - None, description='Text prompt specifying what to avoid in the generation' + None, description="Text prompt specifying what to avoid in the generation" ) - num_images: Optional[conint(ge=1)] = Field( - None, description='Number of images to generate' + num_images: Optional[int] = Field( + None, description="Number of images to generate", ge=1 ) - prompt: str = Field(..., description='The text prompt for image generation') - rendering_speed: RenderingSpeed - resolution: Optional[str] = Field( - None, description='Image resolution in format WxH', examples=['1280x800'] - ) - seed: Optional[int] = Field( - None, description='Seed value for reproducible generation' - ) - style_codes: Optional[List[constr(pattern=r'^[0-9A-Fa-f]{8}$')]] = Field( - None, description='Array of style codes in hexadecimal format' - ) - style_reference_images: Optional[List[str]] = Field( - None, description='Array of reference image URLs or identifiers' + color_palette: Optional[ColorPalette] = None + style_codes: Optional[List[StyleCode]] = Field( + None, description="Array of style codes in hexadecimal format" ) style_type: Optional[StyleType] = Field( - None, description='The type of style to apply' + None, description="The type of style to apply" ) + style_reference_images: Optional[List[str]] = Field( + None, description="Array of reference image URLs or identifiers" + ) + + +class IdeogramV3EditRequest(BaseModel): + image: Optional[bytes_aliased] = Field( + None, + description="The image being edited (max size 10MB); only JPEG, WebP and PNG formats are supported at this time.", + ) + mask: Optional[bytes_aliased] = Field( + None, + description="A black and white image of the same size as the image being edited (max size 10MB). Black regions in the mask should match up with the regions of the image that you would like to edit; only JPEG, WebP and PNG formats are supported at this time.", + ) + prompt: str = Field( + ..., description="The prompt used to describe the edited result." + ) + magic_prompt: Optional[str] = Field( + None, + description="Determine if MagicPrompt should be used in generating the request or not.", + ) + num_images: Optional[int] = Field( + None, description="The number of images to generate." + ) + seed: Optional[int] = Field( + None, description="Random seed. Set for reproducible generation." + ) + rendering_speed: RenderingSpeed + color_palette: Optional[IdeogramColorPalette] = Field( + None, + description="A color palette for generation, must EITHER be specified via one of the presets (name) or explicitly via hexadecimal representations of the color with optional weights (members). Not supported by V_1, V_1_TURBO, V_2A and V_2A_TURBO models.", + ) + style_codes: Optional[List[StyleCode]] = Field( + None, + description="A list of 8 character hexadecimal codes representing the style of the image. Cannot be used in conjunction with style_reference_images or style_type.", + ) + style_reference_images: Optional[List[bytes_aliased]] = Field( + None, + description="A set of images to use as style references (maximum total size 10MB across all style references). The images should be in JPEG, PNG or WebP format.", + ) + + +class StripeBillingDetails(BaseModel): + address: Optional[StripeAddress] = None + email: Optional[str] = None + name: Optional[str] = None + phone: Optional[str] = None + tax_id: Optional[Any] = None + + +class StripePaymentMethodDetails(BaseModel): + card: Optional[StripeCardDetails] = None + type: Optional[str] = None + + +class Controls(BaseModel): + artistic_level: Optional[int] = Field( + None, + description="Defines artistic tone of your image. At a simple level, the person looks straight at the camera in a static and clean style. Dynamic and eccentric levels introduce movement and creativity.", + ge=0, + le=5, + ) + colors: Optional[List[RGBColor]] = Field( + None, description="An array of preferable colors" + ) + background_color: Optional[RGBColor] = Field( + None, description="Use given color as a desired background color" + ) + no_text: Optional[bool] = Field(None, description="Do not embed text layouts") + + +class RecraftImageGenerationRequest(BaseModel): + prompt: str = Field( + ..., description="The text prompt describing the image to generate" + ) + model: str = Field( + ..., description='The model to use for generation (e.g., "recraftv3")' + ) + style: Optional[str] = Field( + None, + description='The style to apply to the generated image (e.g., "digital_illustration")', + ) + style_id: Optional[str] = Field( + None, + description='The style ID to apply to the generated image (e.g., "123e4567-e89b-12d3-a456-426614174000"). If style_id is provided, style should not be provided.', + ) + size: str = Field( + ..., description='The size of the generated image (e.g., "1024x1024")' + ) + controls: Optional[Controls] = Field( + None, description="The controls for the generated image" + ) + n: int = Field(..., description="The number of images to generate", ge=1, le=4) + + +class LumaKeyframes(BaseModel): + frame0: Optional[LumaKeyframe] = None + frame1: Optional[LumaKeyframe] = None class LumaGenerationRequest(BaseModel): + generation_type: Optional[GenerationType] = "video" + prompt: str = Field(..., description="The prompt of the generation") aspect_ratio: LumaAspectRatio - callback_url: Optional[AnyUrl] = Field( - None, - description='The callback URL of the generation, a POST request with Generation object will be sent to the callback URL when the generation is dreaming, completed, or failed', - ) - duration: LumaVideoModelOutputDuration - generation_type: Optional[GenerationType1] = 'video' + loop: Optional[bool] = Field(None, description="Whether to loop the video") keyframes: Optional[LumaKeyframes] = None - loop: Optional[bool] = Field(None, description='Whether to loop the video') - model: LumaVideoModel - prompt: str = Field(..., description='The prompt of the generation') - resolution: LumaVideoModelOutputResolution - - -class CharacterRef(BaseModel): - identity0: Optional[LumaImageIdentity] = None - - -class LumaImageGenerationRequest(BaseModel): - aspect_ratio: Optional[LumaAspectRatio] = '16:9' callback_url: Optional[AnyUrl] = Field( - None, description='The callback URL for the generation' - ) - character_ref: Optional[CharacterRef] = None - generation_type: Optional[GenerationType2] = 'image' - image_ref: Optional[List[LumaImageRef]] = None - model: Optional[LumaImageModel] = 'photon-1' - modify_image_ref: Optional[LumaModifyImageRef] = None - prompt: Optional[str] = Field(None, description='The prompt of the generation') - style_ref: Optional[List[LumaImageRef]] = None - - -class LumaUpscaleVideoGenerationRequest(BaseModel): - callback_url: Optional[AnyUrl] = Field( - None, description='The callback URL for the upscale' - ) - generation_type: Optional[GenerationType3] = 'upscale_video' - resolution: Optional[LumaVideoModelOutputResolution] = None - - -class NodeVersion(BaseModel): - changelog: Optional[str] = Field( - None, description='Summary of changes made in this version' - ) - comfy_node_extract_status: Optional[str] = Field( - None, description='The status of comfy node extraction process.' - ) - createdAt: Optional[datetime] = Field( - None, description='The date and time the version was created.' - ) - dependencies: Optional[List[str]] = Field( - None, description='A list of pip dependencies required by the node.' - ) - deprecated: Optional[bool] = Field( - None, description='Indicates if this version is deprecated.' - ) - downloadUrl: Optional[str] = Field( - None, description='[Output Only] URL to download this version of the node' - ) - id: Optional[str] = None - node_id: Optional[str] = Field( - None, description='The unique identifier of the node.' - ) - status: Optional[NodeVersionStatus] = None - status_reason: Optional[str] = Field( - None, description='The reason for the status change.' - ) - version: Optional[str] = Field( None, - description='The version identifier, following semantic versioning. Must be unique for the node.', + description="The callback URL of the generation, a POST request with Generation object will be sent to the callback URL when the generation is dreaming, completed, or failed", ) + model: LumaVideoModel + resolution: LumaVideoModelOutputResolution + duration: LumaVideoModelOutputDuration -class PikaHTTPValidationError(BaseModel): - detail: Optional[List[PikaValidationError]] = Field(None, title='Detail') +class StripeChargeList(BaseModel): + data: Optional[List[StripeCharge]] = None + has_more: Optional[bool] = None + object: Optional[str] = None + total_count: Optional[int] = None + url: Optional[str] = None -class PublisherMember(BaseModel): - id: Optional[str] = Field( - None, description='The unique identifier for the publisher member.' +class LumaGeneration(BaseModel): + id: Optional[UUID] = Field(None, description="The ID of the generation") + generation_type: Optional[LumaGenerationType] = None + state: Optional[LumaState] = None + failure_reason: Optional[str] = Field( + None, description="The reason for the state of the generation" ) - role: Optional[str] = Field( - None, description='The role of the user in the publisher.' + created_at: Optional[datetime] = Field( + None, description="The date and time when the generation was created" ) - user: Optional[PublisherUser] = None + assets: Optional[LumaAssets] = None + model: Optional[str] = Field(None, description="The model used for the generation") + request: Optional[ + Union[ + LumaGenerationRequest, + LumaImageGenerationRequest, + LumaUpscaleVideoGenerationRequest, + LumaAudioGenerationRequest, + ] + ] = Field(None, description="The request of the generation") class RunwayImageToVideoRequest(BaseModel): - duration: RunwayDurationEnum - model: RunwayModelEnum promptImage: RunwayPromptImageObject - promptText: Optional[constr(max_length=1000)] = Field( - None, description='Text prompt for the generation' + seed: int = Field( + ..., description="Random seed for generation", ge=0, le=4294967295 ) - ratio: RunwayAspectRatioEnum - seed: conint(ge=0, le=4294967295) = Field( - ..., description='Random seed for generation' + model: RunwayModelEnum = Field(..., description="Model to use for generation") + promptText: Optional[str] = Field( + None, description="Text prompt for the generation", max_length=1000 ) + duration: RunwayDurationEnum = Field( + ..., description="The number of seconds of duration for the output video." + ) + ratio: RunwayAspectRatioEnum = Field( + ..., + description="The resolution (aspect ratio) of the output video. Allowable values depend on the selected model. 1280:768 and 768:1280 are only supported for gen3a_turbo.", + ) + + +class RunwayTaskStatusResponse(BaseModel): + id: Optional[str] = Field(None, description="Task ID") + status: Optional[RunwayTaskStatusEnum] = Field(None, description="Task status") + createdAt: Optional[datetime] = Field(None, description="Task creation timestamp") + output: Optional[List[str]] = Field(None, description="Array of output video URLs") + + +class PikaHTTPValidationError(BaseModel): + detail: Optional[List[PikaValidationError]] = Field(None, title="Detail") + + +class PikaVideoResponse(BaseModel): + id: str = Field(..., title="Id") + status: PikaStatusEnum = Field( + ..., description="The status of the video", title="Status" + ) + url: Optional[str] = Field(None, title="Url") + progress: Optional[int] = Field(None, title="Progress") + + +class Node(BaseModel): + id: Optional[str] = Field(None, description="The unique identifier of the node.") + name: Optional[str] = Field(None, description="The display name of the node.") + category: Optional[str] = Field(None, description="The category of the node.") + description: Optional[str] = None + author: Optional[str] = None + license: Optional[str] = Field( + None, description="The path to the LICENSE file in the node's repository." + ) + icon: Optional[str] = Field(None, description="URL to the node's icon.") + repository: Optional[str] = Field(None, description="URL to the node's repository.") + tags: Optional[List[str]] = None + latest_version: Optional[NodeVersion] = Field( + None, description="The latest version of the node." + ) + rating: Optional[float] = Field(None, description="The average rating of the node.") + downloads: Optional[int] = Field( + None, description="The number of downloads of the node." + ) + publisher: Optional[Publisher] = Field( + None, description="The publisher of the node." + ) + status: Optional[NodeStatus] = Field(None, description="The status of the node.") + status_detail: Optional[str] = Field( + None, description="The status detail of the node." + ) + translations: Optional[Dict[str, Dict[str, Any]]] = None class StripeCharge(BaseModel): + id: Optional[str] = None + object: Optional[Object2] = None amount: Optional[int] = None amount_captured: Optional[int] = None amount_refunded: Optional[int] = None @@ -1841,11 +1993,9 @@ class StripeCharge(BaseModel): failure_code: Optional[Any] = None failure_message: Optional[Any] = None fraud_details: Optional[Dict[str, Any]] = None - id: Optional[str] = None invoice: Optional[Any] = None livemode: Optional[bool] = None metadata: Optional[Dict[str, Any]] = None - object: Optional[Object] = None on_behalf_of: Optional[Any] = None order: Optional[Any] = None outcome: Optional[StripeOutcome] = None @@ -1870,15 +2020,9 @@ class StripeCharge(BaseModel): transfer_group: Optional[Any] = None -class StripeChargeList(BaseModel): - data: Optional[List[StripeCharge]] = None - has_more: Optional[bool] = None - object: Optional[str] = None - total_count: Optional[int] = None - url: Optional[str] = None - - class StripePaymentIntent(BaseModel): + id: Optional[str] = None + object: Optional[Object1] = None amount: Optional[int] = None amount_capturable: Optional[int] = None amount_details: Optional[StripeAmountDetails] = None @@ -1896,14 +2040,12 @@ class StripePaymentIntent(BaseModel): currency: Optional[str] = None customer: Optional[str] = None description: Optional[str] = None - id: Optional[str] = None invoice: Optional[str] = None last_payment_error: Optional[Any] = None latest_charge: Optional[str] = None livemode: Optional[bool] = None metadata: Optional[Dict[str, Any]] = None next_action: Optional[Any] = None - object: Optional[Object2] = None on_behalf_of: Optional[Any] = None payment_method: Optional[str] = None payment_method_configuration_details: Optional[Any] = None @@ -1922,84 +2064,17 @@ class StripePaymentIntent(BaseModel): transfer_group: Optional[Any] = None -class LumaGeneration(BaseModel): - assets: Optional[LumaAssets] = None - created_at: Optional[datetime] = Field( - None, description='The date and time when the generation was created' - ) - failure_reason: Optional[str] = Field( - None, description='The reason for the state of the generation' - ) - generation_type: Optional[LumaGenerationType] = None - id: Optional[UUID] = Field(None, description='The ID of the generation') - model: Optional[str] = Field(None, description='The model used for the generation') - request: Optional[ - Union[ - LumaGenerationRequest, - LumaImageGenerationRequest, - LumaUpscaleVideoGenerationRequest, - LumaAudioGenerationRequest, - ] - ] = Field(None, description='The request of the generation') - state: Optional[LumaState] = None - - -class Publisher(BaseModel): - createdAt: Optional[datetime] = Field( - None, description='The date and time the publisher was created.' - ) - description: Optional[str] = None - id: Optional[str] = Field( - None, - description="The unique identifier for the publisher. It's akin to a username. Should be lowercase.", - ) - logo: Optional[str] = Field(None, description="URL to the publisher's logo.") - members: Optional[List[PublisherMember]] = Field( - None, description='A list of members in the publisher.' - ) - name: Optional[str] = None - source_code_repo: Optional[str] = None - status: Optional[PublisherStatus] = None - support: Optional[str] = None - website: Optional[str] = None - - class Data2(BaseModel): object: Optional[StripePaymentIntent] = None class StripeEvent(BaseModel): + id: str + object: Object api_version: Optional[str] = None created: Optional[int] = None data: Data2 - id: str livemode: Optional[bool] = None - object: Object1 pending_webhooks: Optional[int] = None request: Optional[StripeRequestInfo] = None - type: Type4 - - -class Node(BaseModel): - author: Optional[str] = None - category: Optional[str] = Field(None, description='The category of the node.') - description: Optional[str] = None - downloads: Optional[int] = Field( - None, description='The number of downloads of the node.' - ) - icon: Optional[str] = Field(None, description="URL to the node's icon.") - id: Optional[str] = Field(None, description='The unique identifier of the node.') - latest_version: Optional[NodeVersion] = None - license: Optional[str] = Field( - None, description="The path to the LICENSE file in the node's repository." - ) - name: Optional[str] = Field(None, description='The display name of the node.') - publisher: Optional[Publisher] = None - rating: Optional[float] = Field(None, description='The average rating of the node.') - repository: Optional[str] = Field(None, description="URL to the node's repository.") - status: Optional[NodeStatus] = None - status_detail: Optional[str] = Field( - None, description='The status detail of the node.' - ) - tags: Optional[List[str]] = None - translations: Optional[Dict[str, Dict[str, Any]]] = None + type: Type2 diff --git a/comfy_api_nodes/apis/client.py b/comfy_api_nodes/apis/client.py index b376aafe6..e9c68bf5b 100644 --- a/comfy_api_nodes/apis/client.py +++ b/comfy_api_nodes/apis/client.py @@ -411,7 +411,6 @@ class SynchronousOperation(Generic[T, R]): self.verify_ssl = verify_ssl self.files = files self.content_type = content_type - def execute(self, client: Optional[ApiClient] = None) -> R: """Execute the API operation using the provided client or create one""" try: @@ -430,6 +429,10 @@ class SynchronousOperation(Generic[T, R]): if isinstance(self.request, EmptyRequest) else self.request.model_dump(exclude_none=True) ) + if request_dict: + for key, value in request_dict.items(): + if isinstance(value, Enum): + request_dict[key] = value.value if request_dict: for key, value in request_dict.items(): diff --git a/comfy_api_nodes/nodes_pika.py b/comfy_api_nodes/nodes_pika.py new file mode 100644 index 000000000..230af9669 --- /dev/null +++ b/comfy_api_nodes/nodes_pika.py @@ -0,0 +1,397 @@ +"""Pika API docs: https://pika-827374fb.mintlify.app/api-reference""" + +from typing import Optional, TypeVar +import logging +import torch +from comfy_api_nodes.apis import ( + PikaBodyGenerate22T2vGenerate22T2vPost, + PikaGenerateResponse, + PikaBodyGenerate22I2vGenerate22I2vPost, + PikaVideoResponse, + PikaBodyGenerate22C2vGenerate22PikascenesPost, + IngredientsMode, +) +from comfy_api_nodes.apis.client import ( + ApiEndpoint, + HttpMethod, + SynchronousOperation, + PollingOperation, + EmptyRequest, +) +from comfy_api_nodes.apinode_utils import ( + tensor_to_bytesio, + download_url_to_video_output, +) +from comfy_api_nodes.mapper_utils import model_field_to_node_input +from comfy.comfy_types.node_typing import IO, ComfyNodeABC, InputTypeOptions +from comfy_api.input_impl import VideoFromFile + +R = TypeVar("R") + +PIKA_API_VERSION = "2.2" +PATH_TEXT_TO_VIDEO = f"/proxy/pika/generate/{PIKA_API_VERSION}/t2v" +PATH_IMAGE_TO_VIDEO = f"/proxy/pika/generate/{PIKA_API_VERSION}/i2v" +PATH_PIKAFRAMES = f"/proxy/pika/generate/{PIKA_API_VERSION}/pikaframes" +PATH_PIKASCENES = f"/proxy/pika/generate/{PIKA_API_VERSION}/pikascenes" +PATH_VIDEO_GET = "/proxy/pika/videos" + + +class PikaApiError(Exception): + """Exception for Pika API errors.""" + + pass + + +def is_valid_video_response(response: PikaVideoResponse) -> bool: + """Check if the video response is valid.""" + return hasattr(response, "url") and response.url is not None + + +def is_valid_initial_response(response: PikaGenerateResponse) -> bool: + """Check if the initial response is valid.""" + return hasattr(response, "video_id") and response.video_id is not None + + +class PikaNodeBase(ComfyNodeABC): + """Base class for Pika nodes.""" + + @classmethod + def get_base_inputs_types( + cls, request_model + ) -> dict[str, tuple[IO, InputTypeOptions]]: + """Get the base required inputs types common to all Pika nodes.""" + return { + "prompt_text": model_field_to_node_input( + IO.STRING, + request_model, + "promptText", + multiline=True, + ), + "negative_prompt": model_field_to_node_input( + IO.STRING, + request_model, + "negativePrompt", + multiline=True, + ), + "seed": model_field_to_node_input( + IO.INT, + request_model, + "seed", + min=0, + max=0xFFFFFFFF, + control_after_generate=True, + ), + "resolution": model_field_to_node_input( + IO.STRING, + request_model, + "resolution", + ), + "duration": model_field_to_node_input( + IO.INT, + request_model, + "duration", + ), + } + + CATEGORY = "api node/video/Pika" + API_NODE = True + FUNCTION = "api_call" + + def poll_for_task_status( + self, task_id: str, auth_token: str + ) -> PikaGenerateResponse: + """Polls the Pika API endpoint until the task reaches a terminal state.""" + polling_operation = PollingOperation( + poll_endpoint=ApiEndpoint( + path=f"{PATH_VIDEO_GET}/{task_id}", + method=HttpMethod.GET, + request_model=EmptyRequest, + response_model=PikaVideoResponse, + ), + completed_statuses=[ + "finished", + ], + failed_statuses=["failed", "cancelled"], + status_extractor=lambda response: ( + response.status.value if response.status else None + ), + progress_extractor=lambda response: ( + response.progress if hasattr(response, "progress") else None + ), + auth_token=auth_token, + ) + return polling_operation.execute() + + def execute_task( + self, + initial_operation: SynchronousOperation[R, PikaGenerateResponse], + auth_token: Optional[str] = None, + ) -> tuple[VideoFromFile]: + """Executes the initial operation then polls for the task status until it is completed. + + Args: + initial_operation: The initial operation to execute. + auth_token: The authentication token to use for the API call. + + Returns: + A tuple containing the video file as a VIDEO output. + """ + initial_response = initial_operation.execute() + if not is_valid_initial_response(initial_response): + error_msg = f"Pika initial request failed. Code: {initial_response.code}, Message: {initial_response.message}, Data: {initial_response.data}" + logging.error(error_msg) + raise PikaApiError(error_msg) + + task_id = initial_response.video_id + final_response = self.poll_for_task_status(task_id, auth_token) + if not is_valid_video_response(final_response): + error_msg = ( + f"Pika task {task_id} succeeded but no video data found in response." + ) + logging.error(error_msg) + raise PikaApiError(error_msg) + + video_url = str(final_response.url) + logging.debug("Pika task %s succeeded. Video URL: %s", task_id, video_url) + + return download_url_to_video_output(video_url) + + +class PikaImageToVideoV2_2(PikaNodeBase): + """Pika 2.2 Image to Video Node.""" + + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "image": ( + IO.IMAGE, + {"tooltip": "The image to convert to video"}, + ), + **cls.get_base_inputs_types(PikaBodyGenerate22I2vGenerate22I2vPost), + }, + "hidden": { + "auth_token": "AUTH_TOKEN_COMFY_ORG", + }, + } + + DESCRIPTION = "Sends an image and prompt to the Pika API v2.2 to generate a video." + RETURN_TYPES = ("VIDEO",) + + def api_call( + self, + image: torch.Tensor, + prompt_text: str, + negative_prompt: str, + seed: int, + resolution: str, + duration: int, + auth_token: Optional[str] = None, + ) -> tuple[VideoFromFile]: + """API call for Pika 2.2 Image to Video.""" + # Convert image to BytesIO + image_bytes_io = tensor_to_bytesio(image) + image_bytes_io.seek(0) # Reset stream position + + # Prepare file data for multipart upload + pika_files = {"image": ("image.png", image_bytes_io, "image/png")} + + # Prepare non-file data using the Pydantic model + pika_request_data = PikaBodyGenerate22I2vGenerate22I2vPost( + promptText=prompt_text, + negativePrompt=negative_prompt, + seed=seed, + resolution=resolution, + duration=duration, + ) + + initial_operation = SynchronousOperation( + endpoint=ApiEndpoint( + path=PATH_IMAGE_TO_VIDEO, + method=HttpMethod.POST, + request_model=PikaBodyGenerate22I2vGenerate22I2vPost, + response_model=PikaGenerateResponse, + ), + request=pika_request_data, + files=pika_files, + content_type="multipart/form-data", + auth_token=auth_token, + ) + + return self.execute_task(initial_operation, auth_token) + + +class PikaTextToVideoNodeV2_2(PikaNodeBase): + """Pika 2.2 Text to Video Node.""" + + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + **cls.get_base_inputs_types(PikaBodyGenerate22T2vGenerate22T2vPost), + "aspect_ratio": model_field_to_node_input( + IO.FLOAT, + PikaBodyGenerate22T2vGenerate22T2vPost, + "aspectRatio", + step=0.001, + ), + }, + "hidden": { + "auth_token": "AUTH_TOKEN_COMFY_ORG", + }, + } + + RETURN_TYPES = ("VIDEO",) + DESCRIPTION = "Sends a text prompt to the Pika API v2.2 to generate a video." + + def api_call( + self, + prompt_text: str, + negative_prompt: str, + seed: int, + resolution: str, + duration: int, + aspect_ratio: float, + auth_token: Optional[str] = None, + ) -> tuple[VideoFromFile]: + """API call for Pika 2.2 Text to Video.""" + initial_operation = SynchronousOperation( + endpoint=ApiEndpoint( + path=PATH_TEXT_TO_VIDEO, + method=HttpMethod.POST, + request_model=PikaBodyGenerate22T2vGenerate22T2vPost, + response_model=PikaGenerateResponse, + ), + request=PikaBodyGenerate22T2vGenerate22T2vPost( + promptText=prompt_text, + negativePrompt=negative_prompt, + seed=seed, + resolution=resolution, + duration=duration, + aspectRatio=aspect_ratio, + ), + auth_token=auth_token, + content_type="application/x-www-form-urlencoded", + ) + + return self.execute_task(initial_operation, auth_token) + + +class PikaScenesV2_2(PikaNodeBase): + """Pika 2.2 Scenes Node.""" + + @classmethod + def INPUT_TYPES(cls): + image_ingredient_input = ( + IO.IMAGE, + {"tooltip": "Image that will be used as ingredient to create a video."}, + ) + return { + "required": { + **cls.get_base_inputs_types( + PikaBodyGenerate22C2vGenerate22PikascenesPost, + ), + "ingredients_mode": model_field_to_node_input( + IO.COMBO, + PikaBodyGenerate22C2vGenerate22PikascenesPost, + "ingredientsMode", + enum_type=IngredientsMode, + default="creative", + ), + "aspect_ratio": model_field_to_node_input( + IO.FLOAT, + PikaBodyGenerate22C2vGenerate22PikascenesPost, + "aspectRatio", + step=0.001, + default=1.7777777777777777, + ), + }, + "optional": { + "image_ingredient_1": image_ingredient_input, + "image_ingredient_2": image_ingredient_input, + "image_ingredient_3": image_ingredient_input, + "image_ingredient_4": image_ingredient_input, + "image_ingredient_5": image_ingredient_input, + }, + "hidden": { + "auth_token": "AUTH_TOKEN_COMFY_ORG", + }, + } + + DESCRIPTION = "Combine your images to create a video with the objects in them. Upload multiple images as ingredients and generate a high-quality video that incorporates all of them." + RETURN_TYPES = ("VIDEO",) + + def api_call( + self, + prompt_text: str, + negative_prompt: str, + seed: int, + resolution: str, + duration: int, + ingredients_mode: str, + aspect_ratio: float, + image_ingredient_1: Optional[torch.Tensor] = None, + image_ingredient_2: Optional[torch.Tensor] = None, + image_ingredient_3: Optional[torch.Tensor] = None, + image_ingredient_4: Optional[torch.Tensor] = None, + image_ingredient_5: Optional[torch.Tensor] = None, + auth_token: Optional[str] = None, + ) -> tuple[VideoFromFile]: + """API call for Pika Scenes 2.2.""" + all_image_bytes_io = [] + for image in [ + image_ingredient_1, + image_ingredient_2, + image_ingredient_3, + image_ingredient_4, + image_ingredient_5, + ]: + if image is not None: + image_bytes_io = tensor_to_bytesio(image) + image_bytes_io.seek(0) + all_image_bytes_io.append(image_bytes_io) + + # Prepare files data for multipart upload + pika_files = [ + ("images", (f"image_{i}.png", image_bytes_io, "image/png")) + for i, image_bytes_io in enumerate(all_image_bytes_io) + ] + + # Prepare non-file data using the Pydantic model + pika_request_data = PikaBodyGenerate22C2vGenerate22PikascenesPost( + ingredientsMode=ingredients_mode, + promptText=prompt_text, + negativePrompt=negative_prompt, + seed=seed, + resolution=resolution, + duration=duration, + aspectRatio=aspect_ratio, + ) + + initial_operation = SynchronousOperation( + endpoint=ApiEndpoint( + path=PATH_PIKASCENES, + method=HttpMethod.POST, + request_model=PikaBodyGenerate22C2vGenerate22PikascenesPost, + response_model=PikaGenerateResponse, + ), + request=pika_request_data, + files=pika_files, + content_type="multipart/form-data", + auth_token=auth_token, + ) + + return self.execute_task(initial_operation, auth_token) + + +NODE_CLASS_MAPPINGS = { + "PikaImageToVideoNode2_2": PikaImageToVideoV2_2, + "PikaTextToVideoNode2_2": PikaTextToVideoNodeV2_2, + "PikaScenesV2_2": PikaScenesV2_2, +} + +NODE_DISPLAY_NAME_MAPPINGS = { + "PikaImageToVideoNode2_2": "Pika 2.2 Image to Video", + "PikaTextToVideoNode2_2": "Pika 2.2 Text to Video", + "PikaScenesV2_2": "Pika 2.2 Scenes", +} diff --git a/nodes.py b/nodes.py index 1fc782cd4..9ac9be365 100644 --- a/nodes.py +++ b/nodes.py @@ -2272,6 +2272,7 @@ def init_builtin_extra_nodes(): "nodes_luma.py", "nodes_recraft.py", "nodes_pixverse.py", + "nodes_pika.py", ] import_failed = [] From 2c0df4dc5b44d97221b9eb340af3d5be73fa49a0 Mon Sep 17 00:00:00 2001 From: Christian Byrne Date: Wed, 30 Apr 2025 18:28:22 -0700 Subject: [PATCH 063/121] Temporary Fix for Runway (#87) --- comfy_api_nodes/apis/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/comfy_api_nodes/apis/__init__.py b/comfy_api_nodes/apis/__init__.py index 393753048..d90d1decd 100644 --- a/comfy_api_nodes/apis/__init__.py +++ b/comfy_api_nodes/apis/__init__.py @@ -1289,7 +1289,7 @@ class Position(str, Enum): class RunwayPromptImageDetailedObject(BaseModel): - uri: AnyUrl = Field( + uri: str = Field( ..., description="A HTTPS URL or data URI containing an encoded image." ) position: Position = Field( @@ -1315,9 +1315,9 @@ class RunwayAspectRatioEnum(str, Enum): class RunwayPromptImageObject( - RootModel[Union[AnyUrl, List[RunwayPromptImageDetailedObject]]] + RootModel[Union[str, List[RunwayPromptImageDetailedObject]]] ): - root: Union[AnyUrl, List[RunwayPromptImageDetailedObject]] = Field( + root: Union[str, List[RunwayPromptImageDetailedObject]] = Field( ..., description="Image(s) to use for the video generation. Can be a single URI or an array of image objects with positions.", ) From 5891b576e9841efba5d3f44f6bb4e695bec63c67 Mon Sep 17 00:00:00 2001 From: Jedrzej Kosinski Date: Wed, 30 Apr 2025 21:50:37 -0500 Subject: [PATCH 064/121] Added Stability Stable Image Ultra node (#86) --- comfy_api_nodes/apis/stability_api.py | 69 +++++++++++ comfy_api_nodes/nodes_stability.py | 160 ++++++++++++++++++++++++++ nodes.py | 1 + 3 files changed, 230 insertions(+) create mode 100644 comfy_api_nodes/apis/stability_api.py create mode 100644 comfy_api_nodes/nodes_stability.py diff --git a/comfy_api_nodes/apis/stability_api.py b/comfy_api_nodes/apis/stability_api.py new file mode 100644 index 000000000..410884c6e --- /dev/null +++ b/comfy_api_nodes/apis/stability_api.py @@ -0,0 +1,69 @@ +from __future__ import annotations + +from enum import Enum +from typing import Optional + +from pydantic import BaseModel, Field, confloat + + +class StabilityFormat(str, Enum): + png = 'png' + jpeg = 'jpeg' + webp = 'webp' + + +class StabilityAspectRatio(str, Enum): + ratio_1_1 = "1:1" + ratio_16_9 = "16:9" + ratio_9_16 = "9:16" + ratio_3_2 = "3:2" + ratio_2_3 = "2:3" + ratio_5_4 = "5:4" + ratio_4_5 = "4:5" + ratio_21_9 = "21:9" + ratio_9_21 = "9:21" + + +def get_stability_style_presets(include_none=True): + presets = [] + if include_none: + presets.append("None") + return presets + [x.value for x in StabilityStylePreset] + + +class StabilityStylePreset(str, Enum): + _3d_model = "3d-model" + analog_film = "analog-film" + anime = "anime" + cinematic = "cinematic" + comic_book = "comic-book" + digital_art = "digital-art" + enhance = "enhance" + fantasy_art = "fantasy-art" + isometric = "isometric" + line_art = "line-art" + low_poly = "low-poly" + modeling_compound = "modeling-compound" + neon_punk = "neon-punk" + origami = "origami" + photographic = "photographic" + pixel_art = "pixel-art" + tile_texture = "tile-texture" + + +class StabilityStableUltraRequest(BaseModel): + prompt: str = Field(...) + negative_prompt: Optional[str] = Field(None) + aspect_ratio: Optional[str] = Field(None) + seed: Optional[int] = Field(None) + output_format: Optional[str] = Field(StabilityFormat.png.value) + image: Optional[str] = Field(None) + style_preset: Optional[str] = Field(None) + strength: Optional[confloat(ge=0.0, le=1.0)] = Field(None) + + +class StabilityStableUltraResponse(BaseModel): + image: Optional[str] = Field(None) + finish_reason: Optional[str] = Field(None) + seed: Optional[int] = Field(None) + diff --git a/comfy_api_nodes/nodes_stability.py b/comfy_api_nodes/nodes_stability.py new file mode 100644 index 000000000..44d8aa17a --- /dev/null +++ b/comfy_api_nodes/nodes_stability.py @@ -0,0 +1,160 @@ +from inspect import cleandoc +from comfy.comfy_types.node_typing import IO +from comfy_api_nodes.apis.stability_api import ( + StabilityStableUltraRequest, + StabilityStableUltraResponse, + StabilityAspectRatio, + get_stability_style_presets, +) +from comfy_api_nodes.apis.client import ( + ApiEndpoint, + HttpMethod, + SynchronousOperation, +) +from comfy_api_nodes.apinode_utils import ( + bytesio_to_image_tensor, + tensor_to_bytesio, +) + +import torch +import base64 +from io import BytesIO + + + +class StabilityStableImageUltraNode: + """ + Generates images synchronously based on prompt and resolution. + """ + + RETURN_TYPES = (IO.IMAGE,) + DESCRIPTION = cleandoc(__doc__ or "") # Handle potential None value + FUNCTION = "api_call" + API_NODE = True + CATEGORY = "api node/image/stability" + + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "prompt": ( + IO.STRING, + { + "multiline": True, + "default": "", + "tooltip": "What you wish to see in the output image. A strong, descriptive prompt that clearly defines" + + "What you wish to see in the output image. A strong, descriptive prompt that clearly defines" + + "elements, colors, and subjects will lead to better results. " + + "To control the weight of a given word use the format `(word:weight)`," + + "where `word` is the word you'd like to control the weight of and `weight`" + + "is a value between 0 and 1. For example: `The sky was a crisp (blue:0.3) and (green:0.8)`" + + "would convey a sky that was blue and green, but more green than blue." + }, + ), + "aspect_ratio": ([x.value for x in StabilityAspectRatio], + { + "default": StabilityAspectRatio.ratio_1_1, + "tooltip": "Aspect ratio of generated image.", + }, + ), + "style_preset": (get_stability_style_presets(), + { + "tooltip": "Optional desired style of generated image.", + }, + ), + "seed": ( + IO.INT, + { + "default": 0, + "min": 0, + "max": 4294967294, + "control_after_generate": True, + "tooltip": "The random seed used for creating the noise.", + }, + ), + }, + "optional": { + "image": (IO.IMAGE,), + "negative_prompt": ( + IO.STRING, + { + "default": "", + "forceInput": True, + "tooltip": "A blurb of text describing what you do not wish to see in the output image. This is an advanced feature." + }, + ), + "image_denoise": ( + IO.FLOAT, + { + "default": 0.5, + "min": 0.0, + "max": 1.0, + "step": 0.01, + "tooltip": "Denoise of input image; 0.0 yields image identical to input, 1.0 is as if no image was provided at all.", + }, + ), + }, + "hidden": { + "auth_token": "AUTH_TOKEN_COMFY_ORG", + }, + } + + def api_call(self, prompt: str, aspect_ratio: str, style_preset: str, seed: int, + negative_prompt: str=None, image: torch.Tensor = None, image_denoise: float=None, + auth_token=None): + # prepare image binary if image present + image_binary = None + if image is not None: + image_binary = tensor_to_bytesio(image, 1504 * 1504).read() + else: + image_denoise = None + + if not negative_prompt: + negative_prompt = None + if style_preset == "None": + style_preset = None + + files = { + "image": image_binary + } + + operation = SynchronousOperation( + endpoint=ApiEndpoint( + path="/proxy/stability/v2beta/stable-image/generate/ultra", + method=HttpMethod.POST, + request_model=StabilityStableUltraRequest, + response_model=StabilityStableUltraResponse, + ), + request=StabilityStableUltraRequest( + prompt=prompt, + negative_prompt=negative_prompt, + aspect_ratio=aspect_ratio, + seed=seed, + strength=image_denoise, + style_preset=style_preset, + ), + files=files, + content_type="multipart/form-data", + auth_token=auth_token, + ) + response_api = operation.execute() + + if response_api.finish_reason != "SUCCESS": + raise Exception(f"Stable Image Ultra generation failed: {response_api.finish_reason}.") + + image_data = base64.b64decode(response_api.image) + returned_image = bytesio_to_image_tensor(BytesIO(image_data)) + + return (returned_image,) + + +# A dictionary that contains all nodes you want to export with their names +# NOTE: names should be globally unique +NODE_CLASS_MAPPINGS = { + "StabilityStableImageUltraNode": StabilityStableImageUltraNode, +} + +# A dictionary that contains the friendly/humanly readable titles for the nodes +NODE_DISPLAY_NAME_MAPPINGS = { + "StabilityStableImageUltraNode": "Stabiliy Stable Image Ultra", +} diff --git a/nodes.py b/nodes.py index 9ac9be365..9d5ebd5c0 100644 --- a/nodes.py +++ b/nodes.py @@ -2272,6 +2272,7 @@ def init_builtin_extra_nodes(): "nodes_luma.py", "nodes_recraft.py", "nodes_pixverse.py", + "nodes_stability.py", "nodes_pika.py", ] From 8636e3274beb198e0b1e4b47af084b71f5ee1e8b Mon Sep 17 00:00:00 2001 From: Christian Byrne Date: Wed, 30 Apr 2025 20:23:26 -0700 Subject: [PATCH 065/121] Remove Runway nodes (#88) --- comfy_api_nodes/apis/__init__.py | 24 +-- comfy_api_nodes/nodes_runway.py | 275 ------------------------------- nodes.py | 1 - 3 files changed, 12 insertions(+), 288 deletions(-) delete mode 100644 comfy_api_nodes/nodes_runway.py diff --git a/comfy_api_nodes/apis/__init__.py b/comfy_api_nodes/apis/__init__.py index d90d1decd..b948027e0 100644 --- a/comfy_api_nodes/apis/__init__.py +++ b/comfy_api_nodes/apis/__init__.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: filtered-openapi.yaml -# timestamp: 2025-05-01T00:49:31+00:00 +# timestamp: 2025-05-01T03:12:50+00:00 from __future__ import annotations @@ -1289,7 +1289,7 @@ class Position(str, Enum): class RunwayPromptImageDetailedObject(BaseModel): - uri: str = Field( + uri: AnyUrl = Field( ..., description="A HTTPS URL or data URI containing an encoded image." ) position: Position = Field( @@ -1315,9 +1315,9 @@ class RunwayAspectRatioEnum(str, Enum): class RunwayPromptImageObject( - RootModel[Union[str, List[RunwayPromptImageDetailedObject]]] + RootModel[Union[AnyUrl, List[RunwayPromptImageDetailedObject]]] ): - root: Union[str, List[RunwayPromptImageDetailedObject]] = Field( + root: Union[AnyUrl, List[RunwayPromptImageDetailedObject]] = Field( ..., description="Image(s) to use for the video generation. Can be a single URI or an array of image objects with positions.", ) @@ -1872,14 +1872,6 @@ class LumaGenerationRequest(BaseModel): duration: LumaVideoModelOutputDuration -class StripeChargeList(BaseModel): - data: Optional[List[StripeCharge]] = None - has_more: Optional[bool] = None - object: Optional[str] = None - total_count: Optional[int] = None - url: Optional[str] = None - - class LumaGeneration(BaseModel): id: Optional[UUID] = Field(None, description="The ID of the generation") generation_type: Optional[LumaGenerationType] = None @@ -2020,6 +2012,14 @@ class StripeCharge(BaseModel): transfer_group: Optional[Any] = None +class StripeChargeList(BaseModel): + object: Optional[str] = None + data: Optional[List[StripeCharge]] = None + has_more: Optional[bool] = None + total_count: Optional[int] = None + url: Optional[str] = None + + class StripePaymentIntent(BaseModel): id: Optional[str] = None object: Optional[Object1] = None diff --git a/comfy_api_nodes/nodes_runway.py b/comfy_api_nodes/nodes_runway.py deleted file mode 100644 index e472debd0..000000000 --- a/comfy_api_nodes/nodes_runway.py +++ /dev/null @@ -1,275 +0,0 @@ -from inspect import cleandoc -from typing import Union, Optional -import logging - -import torch -from comfy_api_nodes.apis import ( - RunwayImageToVideoRequest, - RunwayImageToVideoResponse, - RunwayTaskStatusResponse as TaskStatusResponse, - RunwayTaskStatusEnum as TaskStatus, - RunwayModelEnum as Model, - RunwayDurationEnum as Duration, - RunwayAspectRatioEnum as AspectRatio, - RunwayPromptImageObject, - RunwayPromptImageDetailedObject, -) -from comfy_api_nodes.apis.client import ( - ApiEndpoint, - HttpMethod, - SynchronousOperation, - PollingOperation, - EmptyRequest, -) -from comfy_api_nodes.apinode_utils import ( - download_url_to_bytesio, - upload_images_to_comfyapi, -) -from comfy.comfy_types.node_typing import IO, ComfyNodeABC -from comfy_api.input_impl import VideoFromFile -from comfy_api_nodes.mapper_utils import model_field_to_node_input - -PATH_IMAGE_TO_VIDEO = "/proxy/runway/image-to-video" -PATH_GET_TASK_STATUS = "/proxy/runway/tasks" - - -class RunwayApiError(Exception): - """Base exception for Runway API errors.""" - - pass - - -def extract_progress_from_task_status(response: TaskStatusResponse) -> float: - if hasattr(response, "progress") and response.progress is not None: - return response.progress * 100 - return None - - -class RunwayImageToVideoNode(ComfyNodeABC): - """ - Runway Image to Video Node. - """ - - @staticmethod - def is_ratio_supported(model: str, ratio: str) -> bool: - """ - Checks if the chosen aspect ratio is supported by the chosen model. - """ - if model != "gen3a_turbo" and ratio in [ - "1280:768", - "768:1280", - ]: - return False - return True - - @staticmethod - def is_end_frame_supported(model: str) -> bool: - """ - Checks if the chosen model supports the end frame input. - """ - return model == "gen3a_turbo" - - @staticmethod - def is_valid_prompt(prompt: str) -> bool: - return bool(prompt) - - @staticmethod - def is_valid_initial_response(response: RunwayImageToVideoResponse) -> bool: - return bool(response.id) - - @staticmethod - def is_valid_image(image: torch.Tensor) -> bool: - """https://docs.dev.runwayml.com/assets/inputs/#common-error-reasons""" - return image.shape[2] < 8000 and image.shape[1] < 8000 - - @staticmethod - def is_valid_video_response(response: RunwayImageToVideoResponse) -> bool: - return response.output and len(response.output) > 0 - - @staticmethod - def poll_for_task_status(task_id: str, auth_token: str) -> TaskStatusResponse: - """ - Polls the Runway API endpoint until the task reaches a terminal state. - """ - polling_operation = PollingOperation( - poll_endpoint=ApiEndpoint( - path=f"{PATH_GET_TASK_STATUS}/{task_id}", - method=HttpMethod.GET, - request_model=EmptyRequest, - response_model=TaskStatusResponse, - ), - completed_statuses=[ - TaskStatus.SUCCEEDED.value, - ], - failed_statuses=[ - TaskStatus.FAILED.value, - TaskStatus.CANCELLED.value, - ], - progress_extractor=extract_progress_from_task_status, - status_extractor=lambda response: (response.status.value), - auth_token=auth_token, - ) - return polling_operation.execute() - - @classmethod - def INPUT_TYPES(s): - return { - "required": { - "model": model_field_to_node_input( - IO.COMBO, RunwayImageToVideoRequest, "model", enum_type=Model - ), - "prompt": model_field_to_node_input( - IO.STRING, RunwayImageToVideoRequest, "promptText", multiline=True - ), - "duration": model_field_to_node_input( - IO.COMBO, RunwayImageToVideoRequest, "duration", enum_type=Duration - ), - "ratio": model_field_to_node_input( - IO.COMBO, RunwayImageToVideoRequest, "ratio", enum_type=AspectRatio - ), - "seed": model_field_to_node_input( - IO.INT, RunwayImageToVideoRequest, "seed", control_after_generate=True - ), - }, - "optional": { - "start_frame": ( - IO.IMAGE, - {"tooltip": "Start frame to be used for the video"}, - ), - "end_frame": ( - IO.IMAGE, - { - "tooltip": "End frame to be used for the video. Supported for gen3a_turbo only." - }, - ), - }, - "hidden": {"auth_token": "AUTH_TOKEN_COMFY_ORG"}, - } - - RETURN_TYPES = ("VIDEO",) - FUNCTION = "api_call" - CATEGORY = "api node/video/Runway" - DESCRIPTION = cleandoc(__doc__ or "") - API_NODE = True - - @classmethod - def VALIDATE_INPUTS( - cls, - model: str, - ratio: str, - ) -> Union[str, bool]: - if not RunwayImageToVideoNode.is_ratio_supported(model, ratio): - return "Invalid aspect ratio for the chosen model. 1280:768 and 768:1280 are only supported for gen3a_turbo." - return True - - def api_call( - self, - model: str, - prompt: str, - duration: str, - ratio: str, - seed: int, - start_frame: Optional[torch.Tensor] = None, - end_frame: Optional[torch.Tensor] = None, - auth_token: Optional[str] = None, - ) -> tuple[VideoFromFile]: - # Validate manually because optional inputs are not passed to VALIDATE_INPUTS. - if start_frame is None and end_frame is None: - message = "Start frame and end frame cannot both be empty." - raise RunwayApiError(message) - if end_frame is not None and not RunwayImageToVideoNode.is_end_frame_supported( - model - ): - message = "End frame is only supported for gen3a_turbo model." - raise RunwayApiError(message) - - prompt_images_tensors: list[torch.Tensor] = [] - if start_frame is not None: - if not RunwayImageToVideoNode.is_valid_image(start_frame): - message = "Start frame is not a valid image." - raise RunwayApiError(message) - prompt_images_tensors.append(start_frame) - - if end_frame != None: - if not RunwayImageToVideoNode.is_valid_image(end_frame): - message = "End frame is not a valid image." - raise RunwayApiError(message) - prompt_images_tensors.append(end_frame) - - # stack tensors - prompt_images_tensor = torch.cat(prompt_images_tensors, dim=0) - - download_urls = upload_images_to_comfyapi( - prompt_images_tensor, - max_images=2, - auth_token=auth_token, - mime_type="image/png", - ) - - # Create a list of detailed image objects - prompt_image_details: list[RunwayPromptImageDetailedObject] = [ - RunwayPromptImageDetailedObject(uri=str(download_urls[0]), position="first") - ] - if len(download_urls) > 1: - prompt_image_details.append( - RunwayPromptImageDetailedObject( - uri=str(download_urls[1]), position="last" - ) - ) - - # Wrap the list in the main object if details exist - prompt_image_object: Optional[RunwayPromptImageObject] = None - if prompt_image_details: - prompt_image_object = RunwayPromptImageObject(root=prompt_image_details) - - initial_operation = SynchronousOperation( - endpoint=ApiEndpoint( - path=PATH_IMAGE_TO_VIDEO, - method=HttpMethod.POST, - request_model=RunwayImageToVideoRequest, - response_model=RunwayImageToVideoResponse, - ), - request=RunwayImageToVideoRequest( - promptText=prompt, - seed=seed, - model=Model(model), - duration=Duration(duration), - ratio=AspectRatio(ratio), - promptImage=prompt_image_object, - ), - auth_token=auth_token, - ) - - initial_response = initial_operation.execute() - if not RunwayImageToVideoNode.is_valid_initial_response(initial_response): - error_message = "Invalid initial response from Runway API." - logging.error(error_message) - raise RunwayApiError(error_message) - - task_id = initial_response.id - logging.debug("Runway task submitted. Task ID: %s", task_id) - - final_response = self.poll_for_task_status(task_id, auth_token) - if not RunwayImageToVideoNode.is_valid_video_response(final_response): - error_message = "Runway task succeeded but no video data found in response." - logging.error(error_message) - raise RunwayApiError(error_message) - - video_url = final_response.output[0] - logging.debug("Attempting to download video from URL: %s", video_url) - - video_io = download_url_to_bytesio(video_url) - if video_io is None: - error_msg = f"Failed to download video from {video_url}" - logging.error(error_msg) - raise RunwayApiError(error_msg) - return (VideoFromFile(video_io),) - - -NODE_CLASS_MAPPINGS = { - "RunwayImageToVideoNode": RunwayImageToVideoNode, -} - -NODE_DISPLAY_NAME_MAPPINGS = { - "RunwayImageToVideoNode": "Runway Image to Video", -} diff --git a/nodes.py b/nodes.py index 9d5ebd5c0..b79ef730c 100644 --- a/nodes.py +++ b/nodes.py @@ -2267,7 +2267,6 @@ def init_builtin_extra_nodes(): "nodes_minimax.py", "nodes_veo2.py", "nodes_kling.py", - "nodes_runway.py", "nodes_bfl.py", "nodes_luma.py", "nodes_recraft.py", From 552bc92f7cd8a086f12ceb5f8e757e6d70358fac Mon Sep 17 00:00:00 2001 From: Christian Byrne Date: Wed, 30 Apr 2025 21:48:07 -0700 Subject: [PATCH 066/121] Fix: Prompt text can't be validated in Kling nodes when using primitive nodes (#90) --- comfy_api_nodes/nodes_kling.py | 47 +++++++++++++++------------------- 1 file changed, 20 insertions(+), 27 deletions(-) diff --git a/comfy_api_nodes/nodes_kling.py b/comfy_api_nodes/nodes_kling.py index 59dc5485a..ce29aa78b 100644 --- a/comfy_api_nodes/nodes_kling.py +++ b/comfy_api_nodes/nodes_kling.py @@ -1,15 +1,4 @@ -""" -`camera_control` supported: - -- pro | 5s duration | kling-v1-5 - -`camera_control` not supported: - -- std | 10s duration | kling-v1-6 - -""" - -from typing import Union, Optional +from typing import Optional import math import logging import torch @@ -39,9 +28,9 @@ from comfy_api_nodes.apinode_utils import ( tensor_to_base64_string, download_url_to_video_output, ) +from comfy_api_nodes.mapper_utils import model_field_to_node_input from comfy.comfy_types.node_typing import IO, InputTypeOptions, ComfyNodeABC from comfy_api.input_impl import VideoFromFile -from comfy_api_nodes.mapper_utils import model_field_to_node_input KLING_API_VERSION = "v1" PATH_TEXT_TO_VIDEO = f"/proxy/kling/{KLING_API_VERSION}/videos/text2video" @@ -52,6 +41,9 @@ PATH_VIDEO_EFFECTS = f"/proxy/kling/{KLING_API_VERSION}/videos/effects" PATH_CHARACTER_IMAGE = f"/proxy/kling/{KLING_API_VERSION}/images/generations" PATH_VIRTUAL_TRY_ON = f"/proxy/kling/{KLING_API_VERSION}/images/kolors-virtual-try-on" +MAX_PROMPT_LENGTH_T2V = 2500 +MAX_PROMPT_LENGTH_I2V = 500 + class KlingApiError(Exception): """Base exception for Kling API errors.""" @@ -83,6 +75,19 @@ def is_valid_video_response(response: KlingText2VideoResponse) -> bool: ) +def validate_prompts(prompt: str, negative_prompt: str, max_length: int) -> bool: + """Verifies that the positive prompt is not empty and that neither promt is too long.""" + if not prompt: + raise ValueError("Positive prompt is empty") + if len(prompt) > max_length: + raise ValueError(f"Positive prompt is too long: {len(prompt)} characters") + if negative_prompt and len(negative_prompt) > max_length: + raise ValueError( + f"Negative prompt is too long: {len(negative_prompt)} characters" + ) + return True + + def get_camera_control_input_config( tooltip: str, default: float = 0.0 ) -> tuple[IO, InputTypeOptions]: @@ -183,20 +188,6 @@ class KlingCameraControls(ComfyNodeABC): class KlingNodeBase(ComfyNodeABC): """Base class for Kling nodes.""" - @classmethod - def VALIDATE_INPUTS( - cls, - prompt, - negative_prompt, - ) -> Union[str, bool]: - if not is_valid_prompt(prompt): - return "Prompt is required" - if len(prompt) >= 2500: - return "Prompt must be less than 2500 characters" - if negative_prompt and len(negative_prompt) >= 2500: - return "Negative prompt must be less than 2500 characters" - return True - FUNCTION = "api_call" CATEGORY = "api node/video/Kling" API_NODE = True @@ -284,6 +275,7 @@ class KlingTextToVideoNode(KlingNodeBase): camera_control: Optional[CameraControl] = None, auth_token: Optional[str] = None, ) -> tuple[VideoFromFile]: + validate_prompts(prompt, negative_prompt, MAX_PROMPT_LENGTH_T2V) initial_operation = SynchronousOperation( endpoint=ApiEndpoint( path=PATH_TEXT_TO_VIDEO, @@ -416,6 +408,7 @@ class KlingImage2VideoNode(KlingNodeBase): end_frame: Optional[torch.Tensor] = None, auth_token: Optional[str] = None, ) -> tuple[VideoFromFile]: + validate_prompts(prompt, negative_prompt, MAX_PROMPT_LENGTH_I2V) initial_operation = SynchronousOperation( endpoint=ApiEndpoint( path=PATH_IMAGE_TO_VIDEO, From 1f5818aa00b7082921484bed8f59f5a37d8d9de8 Mon Sep 17 00:00:00 2001 From: Christian Byrne Date: Wed, 30 Apr 2025 22:04:33 -0700 Subject: [PATCH 067/121] Fix: typo in node name "Stabiliy" => "Stability" (#91) --- comfy_api_nodes/nodes_stability.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/comfy_api_nodes/nodes_stability.py b/comfy_api_nodes/nodes_stability.py index 44d8aa17a..25ab6e0b6 100644 --- a/comfy_api_nodes/nodes_stability.py +++ b/comfy_api_nodes/nodes_stability.py @@ -156,5 +156,5 @@ NODE_CLASS_MAPPINGS = { # A dictionary that contains the friendly/humanly readable titles for the nodes NODE_DISPLAY_NAME_MAPPINGS = { - "StabilityStableImageUltraNode": "Stabiliy Stable Image Ultra", + "StabilityStableImageUltraNode": "Stability Stable Image Ultra", } From aad07cb34c03132779ba5c4831dbaf4f696a3f60 Mon Sep 17 00:00:00 2001 From: Jedrzej Kosinski Date: Thu, 1 May 2025 00:54:33 -0500 Subject: [PATCH 068/121] Add String (Multiline) node (#93) --- comfy_extras/nodes_primitive.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/comfy_extras/nodes_primitive.py b/comfy_extras/nodes_primitive.py index 184b990c3..1f93f87a7 100644 --- a/comfy_extras/nodes_primitive.py +++ b/comfy_extras/nodes_primitive.py @@ -21,6 +21,21 @@ class String(ComfyNodeABC): return (value,) +class StringMultiline(ComfyNodeABC): + @classmethod + def INPUT_TYPES(cls) -> InputTypeDict: + return { + "required": {"value": (IO.STRING, {"multiline": True,},)}, + } + + RETURN_TYPES = (IO.STRING,) + FUNCTION = "execute" + CATEGORY = "utils/primitive" + + def execute(self, value: str) -> tuple[str]: + return (value,) + + class Int(ComfyNodeABC): @classmethod def INPUT_TYPES(cls) -> InputTypeDict: @@ -68,6 +83,7 @@ class Boolean(ComfyNodeABC): NODE_CLASS_MAPPINGS = { "PrimitiveString": String, + "PrimitiveStringMultiline": StringMultiline, "PrimitiveInt": Int, "PrimitiveFloat": Float, "PrimitiveBoolean": Boolean, @@ -75,6 +91,7 @@ NODE_CLASS_MAPPINGS = { NODE_DISPLAY_NAME_MAPPINGS = { "PrimitiveString": "String", + "PrimitiveStringMultiline": "String (Multiline)", "PrimitiveInt": "Int", "PrimitiveFloat": "Float", "PrimitiveBoolean": "Boolean", From 47babda75e4e4dec97ababa16d8ad11e7f41b9eb Mon Sep 17 00:00:00 2001 From: Christian Byrne Date: Wed, 30 Apr 2025 23:27:10 -0700 Subject: [PATCH 069/121] Update Pika Duration and Resolution options (#94) --- comfy_api_nodes/apis/__init__.py | 2409 +++++++++++------------------- comfy_api_nodes/nodes_pika.py | 13 +- 2 files changed, 848 insertions(+), 1574 deletions(-) diff --git a/comfy_api_nodes/apis/__init__.py b/comfy_api_nodes/apis/__init__.py index b948027e0..9031b39f7 100644 --- a/comfy_api_nodes/apis/__init__.py +++ b/comfy_api_nodes/apis/__init__.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: filtered-openapi.yaml -# timestamp: 2025-05-01T03:12:50+00:00 +# timestamp: 2025-05-01T05:55:00+00:00 from __future__ import annotations @@ -14,56 +14,32 @@ from pydantic import AnyUrl, BaseModel, Field, RootModel bytes_aliased = bytes -class PersonalAccessToken(BaseModel): - id: Optional[UUID] = Field(None, description="Unique identifier for the GitCommit") - name: Optional[str] = Field( - None, - description="Required. The name of the token. Can be a simple description.", +class BFLFluxProGenerateRequest(BaseModel): + guidance_scale: Optional[float] = Field( + None, description="The guidance scale for generation.", ge=1.0, le=20.0 ) - description: Optional[str] = Field( - None, - description="Optional. A more detailed description of the token's intended use.", + height: int = Field( + ..., description="The height of the image to generate.", ge=64, le=2048 ) - createdAt: Optional[datetime] = Field( - None, description="[Output Only]The date and time the token was created." + negative_prompt: Optional[str] = Field( + None, description="The negative prompt for image generation." ) - token: Optional[str] = Field( - None, - description="[Output Only]. The personal access token. Only returned during creation.", + num_images: Optional[int] = Field( + None, description="The number of images to generate.", ge=1, le=4 + ) + num_inference_steps: Optional[int] = Field( + None, description="The number of inference steps.", ge=1, le=100 + ) + prompt: str = Field(..., description="The text prompt for image generation.") + seed: Optional[int] = Field(None, description="The seed value for reproducibility.") + width: int = Field( + ..., description="The width of the image to generate.", ge=64, le=2048 ) -class GitCommitSummary(BaseModel): - commit_hash: Optional[str] = Field(None, description="The hash of the commit") - commit_name: Optional[str] = Field(None, description="The name of the commit") - branch_name: Optional[str] = Field( - None, description="The branch where the commit was made" - ) - author: Optional[str] = Field(None, description="The author of the commit") - timestamp: Optional[datetime] = Field( - None, description="The timestamp when the commit was made" - ) - status_summary: Optional[Dict[str, str]] = Field( - None, description="A map of operating system to status pairs" - ) - - -class User(BaseModel): - id: Optional[str] = Field(None, description="The unique id for this user.") - email: Optional[str] = Field(None, description="The email address for this user.") - name: Optional[str] = Field(None, description="The name for this user.") - isApproved: Optional[bool] = Field( - None, description="Indicates if the user is approved." - ) - isAdmin: Optional[bool] = Field( - None, description="Indicates if the user has admin privileges." - ) - - -class PublisherUser(BaseModel): - id: Optional[str] = Field(None, description="The unique id for this user.") - email: Optional[str] = Field(None, description="The email address for this user.") - name: Optional[str] = Field(None, description="The name for this user.") +class BFLFluxProGenerateResponse(BaseModel): + id: str = Field(..., description="The unique identifier for the generation task.") + polling_url: str = Field(..., description="URL to poll for the generation result.") class ErrorResponse(BaseModel): @@ -71,170 +47,6 @@ class ErrorResponse(BaseModel): message: str -class StorageFile(BaseModel): - id: Optional[UUID] = Field( - None, description="Unique identifier for the storage file" - ) - file_path: Optional[str] = Field(None, description="Path to the file in storage") - public_url: Optional[str] = Field(None, description="Public URL") - - -class PublisherMember(BaseModel): - id: Optional[str] = Field( - None, description="The unique identifier for the publisher member." - ) - user: Optional[PublisherUser] = Field( - None, description="The user associated with this publisher member." - ) - role: Optional[str] = Field( - None, description="The role of the user in the publisher." - ) - - -class ComfyNode(BaseModel): - comfy_node_name: Optional[str] = Field( - None, description="Unique identifier for the node" - ) - category: Optional[str] = Field( - None, - description="UI category where the node is listed, used for grouping nodes.", - ) - description: Optional[str] = Field( - None, description="Brief description of the node's functionality or purpose." - ) - input_types: Optional[str] = Field(None, description="Defines input parameters") - deprecated: Optional[bool] = Field( - None, - description="Indicates if the node is deprecated. Deprecated nodes are hidden in the UI.", - ) - experimental: Optional[bool] = Field( - None, - description="Indicates if the node is experimental, subject to changes or removal.", - ) - output_is_list: Optional[List[bool]] = Field( - None, description="Boolean values indicating if each output is a list." - ) - return_names: Optional[str] = Field( - None, description="Names of the outputs for clarity in workflows." - ) - return_types: Optional[str] = Field( - None, description="Specifies the types of outputs produced by the node." - ) - function: Optional[str] = Field( - None, description="Name of the entry-point function to execute the node." - ) - - -class ComfyNodeCloudBuildInfo(BaseModel): - project_id: Optional[str] = None - project_number: Optional[str] = None - location: Optional[str] = None - build_id: Optional[str] = None - - -class Error(BaseModel): - message: Optional[str] = Field( - None, description="A clear and concise description of the error." - ) - details: Optional[List[str]] = Field( - None, - description="Optional detailed information about the error or hints for resolving it.", - ) - - -class NodeVersionUpdateRequest(BaseModel): - changelog: Optional[str] = Field( - None, description="The changelog describing the version changes." - ) - deprecated: Optional[bool] = Field( - None, description="Whether the version is deprecated." - ) - - -class NodeStatus(str, Enum): - NodeStatusActive = "NodeStatusActive" - NodeStatusDeleted = "NodeStatusDeleted" - NodeStatusBanned = "NodeStatusBanned" - - -class NodeVersionStatus(str, Enum): - NodeVersionStatusActive = "NodeVersionStatusActive" - NodeVersionStatusDeleted = "NodeVersionStatusDeleted" - NodeVersionStatusBanned = "NodeVersionStatusBanned" - NodeVersionStatusPending = "NodeVersionStatusPending" - NodeVersionStatusFlagged = "NodeVersionStatusFlagged" - - -class PublisherStatus(str, Enum): - PublisherStatusActive = "PublisherStatusActive" - PublisherStatusBanned = "PublisherStatusBanned" - - -class WorkflowRunStatus(str, Enum): - WorkflowRunStatusStarted = "WorkflowRunStatusStarted" - WorkflowRunStatusFailed = "WorkflowRunStatusFailed" - WorkflowRunStatusCompleted = "WorkflowRunStatusCompleted" - - -class MachineStats(BaseModel): - machine_name: Optional[str] = Field(None, description="Name of the machine.") - os_version: Optional[str] = Field( - None, description="The operating system version. eg. Ubuntu Linux 20.04" - ) - gpu_type: Optional[str] = Field( - None, description="The GPU type. eg. NVIDIA Tesla K80" - ) - cpu_capacity: Optional[str] = Field(None, description="Total CPU on the machine.") - initial_cpu: Optional[str] = Field( - None, description="Initial CPU available before the job starts." - ) - memory_capacity: Optional[str] = Field( - None, description="Total memory on the machine." - ) - initial_ram: Optional[str] = Field( - None, description="Initial RAM available before the job starts." - ) - vram_time_series: Optional[Dict[str, Any]] = Field( - None, description="Time series of VRAM usage." - ) - disk_capacity: Optional[str] = Field( - None, description="Total disk capacity on the machine." - ) - initial_disk: Optional[str] = Field( - None, description="Initial disk available before the job starts." - ) - pip_freeze: Optional[str] = Field(None, description="The pip freeze output") - - -class Customer(BaseModel): - id: str = Field(..., description="The firebase UID of the user") - email: Optional[str] = Field(None, description="The email address for this user") - name: Optional[str] = Field(None, description="The name for this user") - createdAt: Optional[datetime] = Field( - None, description="The date and time the user was created" - ) - updatedAt: Optional[datetime] = Field( - None, description="The date and time the user was last updated" - ) - - -class MagicPrompt(str, Enum): - ON = "ON" - OFF = "OFF" - - -class ColorPalette(BaseModel): - name: str = Field(..., description="Name of the color palette", examples=["PASTEL"]) - - -class StyleCode(RootModel[str]): - root: str = Field(..., pattern="^[0-9A-Fa-f]{8}$") - - -class StyleType(str, Enum): - GENERAL = "GENERAL" - - class IdeogramColorPalette1(BaseModel): name: str = Field(..., description="Name of the preset color palette") @@ -264,17 +76,34 @@ class IdeogramColorPalette( class ImageRequest(BaseModel): - prompt: str = Field( - ..., description="Required. The prompt to use to generate the image." - ) aspect_ratio: Optional[str] = Field( None, description="Optional. The aspect ratio (e.g., 'ASPECT_16_9', 'ASPECT_1_1'). Cannot be used with resolution. Defaults to 'ASPECT_1_1' if unspecified.", ) - model: str = Field(..., description="The model used (e.g., 'V_2', 'V_2A_TURBO')") + color_palette: Optional[Dict[str, Any]] = Field( + None, description="Optional. Color palette object. Only for V_2, V_2_TURBO." + ) magic_prompt_option: Optional[str] = Field( None, description="Optional. MagicPrompt usage ('AUTO', 'ON', 'OFF')." ) + model: str = Field(..., description="The model used (e.g., 'V_2', 'V_2A_TURBO')") + negative_prompt: Optional[str] = Field( + None, + description="Optional. Description of what to exclude. Only for V_1, V_1_TURBO, V_2, V_2_TURBO.", + ) + num_images: Optional[int] = Field( + 1, + description="Optional. Number of images to generate (1-8). Defaults to 1.", + ge=1, + le=8, + ) + prompt: str = Field( + ..., description="Required. The prompt to use to generate the image." + ) + resolution: Optional[str] = Field( + None, + description="Optional. Resolution (e.g., 'RESOLUTION_1024_1024'). Only for model V_2. Cannot be used with aspect_ratio.", + ) seed: Optional[int] = Field( None, description="Optional. A number between 0 and 2147483647.", @@ -285,23 +114,6 @@ class ImageRequest(BaseModel): None, description="Optional. Style type ('AUTO', 'GENERAL', 'REALISTIC', 'DESIGN', 'RENDER_3D', 'ANIME'). Only for models V_2 and above.", ) - negative_prompt: Optional[str] = Field( - None, - description="Optional. Description of what to exclude. Only for V_1, V_1_TURBO, V_2, V_2_TURBO.", - ) - num_images: Optional[int] = Field( - 1, - description="Optional. Number of images to generate (1-8). Defaults to 1.", - ge=1, - le=8, - ) - resolution: Optional[str] = Field( - None, - description="Optional. Resolution (e.g., 'RESOLUTION_1024_1024'). Only for model V_2. Cannot be used with aspect_ratio.", - ) - color_palette: Optional[Dict[str, Any]] = Field( - None, description="Optional. Color palette object. Only for V_2, V_2_TURBO." - ) class IdeogramGenerateRequest(BaseModel): @@ -311,23 +123,23 @@ class IdeogramGenerateRequest(BaseModel): class Datum(BaseModel): + is_image_safe: Optional[bool] = Field( + None, description="Indicates whether the image is considered safe." + ) prompt: Optional[str] = Field( None, description="The prompt used to generate this image." ) resolution: Optional[str] = Field( None, description="The resolution of the generated image (e.g., '1024x1024')." ) - is_image_safe: Optional[bool] = Field( - None, description="Indicates whether the image is considered safe." - ) seed: Optional[int] = Field( None, description="The seed value used for this generation." ) - url: Optional[str] = Field(None, description="URL to the generated image.") style_type: Optional[str] = Field( None, description="The style type used for generation (e.g., 'REALISTIC', 'ANIME').", ) + url: Optional[str] = Field(None, description="URL to the generated image.") class IdeogramGenerateResponse(BaseModel): @@ -339,15 +151,77 @@ class IdeogramGenerateResponse(BaseModel): ) -class ModelName(str, Enum): - kling_v1 = "kling-v1" - kling_v1_6 = "kling-v1-6" - kling_v2_master = "kling-v2-master" +class StyleCode(RootModel[str]): + root: str = Field(..., pattern="^[0-9A-Fa-f]{8}$") -class Mode(str, Enum): - std = "std" - pro = "pro" +class ColorPalette(BaseModel): + name: str = Field(..., description="Name of the color palette", examples=["PASTEL"]) + + +class MagicPrompt(str, Enum): + ON = "ON" + OFF = "OFF" + + +class StyleType(str, Enum): + GENERAL = "GENERAL" + + +class KlingErrorResponse(BaseModel): + code: int = Field( + ..., + description="- 1000: Authentication failed\n- 1001: Authorization is empty\n- 1002: Authorization is invalid\n- 1003: Authorization is not yet valid\n- 1004: Authorization has expired\n- 1100: Account exception\n- 1101: Account in arrears (postpaid scenario)\n- 1102: Resource pack depleted or expired (prepaid scenario)\n- 1103: Unauthorized access to requested resource\n- 1200: Invalid request parameters\n- 1201: Invalid parameters\n- 1202: Invalid request method\n- 1203: Requested resource does not exist\n- 1300: Trigger platform strategy\n- 1301: Trigger content security policy\n- 1302: API request too frequent\n- 1303: Concurrency/QPS exceeds limit\n- 1304: Trigger IP whitelist policy\n- 5000: Internal server error\n- 5001: Service temporarily unavailable\n- 5002: Server internal timeout\n", + ) + message: str = Field(..., description="Human-readable error message") + request_id: str = Field( + ..., description="Request ID for tracking and troubleshooting" + ) + + +class AspectRatio(str, Enum): + field_16_9 = "16:9" + field_9_16 = "9:16" + field_1_1 = "1:1" + + +class Config(BaseModel): + horizontal: Optional[float] = Field( + None, + description="Controls camera's movement along horizontal axis (x-axis). Negative indicates left, positive indicates right.", + ge=-10.0, + le=10.0, + ) + pan: Optional[float] = Field( + None, + description="Controls camera's rotation in vertical plane (x-axis). Negative indicates downward rotation, positive indicates upward rotation.", + ge=-10.0, + le=10.0, + ) + roll: Optional[float] = Field( + None, + description="Controls camera's rolling amount (z-axis). Negative indicates counterclockwise, positive indicates clockwise.", + ge=-10.0, + le=10.0, + ) + tilt: Optional[float] = Field( + None, + description="Controls camera's rotation in horizontal plane (y-axis). Negative indicates left rotation, positive indicates right rotation.", + ge=-10.0, + le=10.0, + ) + vertical: Optional[float] = Field( + None, + description="Controls camera's movement along vertical axis (y-axis). Negative indicates downward, positive indicates upward.", + ge=-10.0, + le=10.0, + ) + zoom: Optional[float] = Field( + None, + description="Controls change in camera's focal length. Negative indicates narrower field of view, positive indicates wider field of view.", + ge=-10.0, + le=10.0, + ) class Type(str, Enum): @@ -358,24 +232,12 @@ class Type(str, Enum): left_turn_forward = "left_turn_forward" -class Config(BaseModel): - horizontal: Optional[float] = Field(None, ge=-10.0, le=10.0) - vertical: Optional[float] = Field(None, ge=-10.0, le=10.0) - pan: Optional[float] = Field(None, ge=-10.0, le=10.0) - tilt: Optional[float] = Field(None, ge=-10.0, le=10.0) - roll: Optional[float] = Field(None, ge=-10.0, le=10.0) - zoom: Optional[float] = Field(None, ge=-10.0, le=10.0) - - class CameraControl(BaseModel): - type: Optional[Type] = Field(None, description="Predefined camera movements type") config: Optional[Config] = None - - -class AspectRatio(str, Enum): - field_16_9 = "16:9" - field_9_16 = "9:16" - field_1_1 = "1:1" + type: Optional[Type] = Field( + None, + description="Predefined camera movements type. simple: Customizable camera movement. down_back: Camera descends and moves backward. forward_up: Camera moves forward and tilts up. right_turn_forward: Rotate right and move forward. left_turn_forward: Rotate left and move forward.", + ) class Duration(str, Enum): @@ -383,71 +245,6 @@ class Duration(str, Enum): field_10 = "10" -class KlingText2VideoRequest(BaseModel): - model_name: Optional[ModelName] = Field("kling-v1", description="Model Name") - prompt: Optional[str] = Field( - None, description="Positive text prompt", max_length=2500 - ) - negative_prompt: Optional[str] = Field( - None, description="Negative text prompt", max_length=2500 - ) - cfg_scale: Optional[float] = Field( - 0.5, description="Flexibility in video generation", ge=0.0, le=1.0 - ) - mode: Optional[Mode] = Field("std", description="Video generation mode") - camera_control: Optional[CameraControl] = None - aspect_ratio: Optional[AspectRatio] = "16:9" - duration: Optional[Duration] = "5" - callback_url: Optional[AnyUrl] = Field( - None, description="The callback notification address" - ) - external_task_id: Optional[str] = Field(None, description="Customized Task ID") - - -class TaskStatus(str, Enum): - submitted = "submitted" - processing = "processing" - succeed = "succeed" - failed = "failed" - - -class TaskInfo(BaseModel): - external_task_id: Optional[str] = None - - -class Video(BaseModel): - id: Optional[str] = Field(None, description="Generated video ID") - url: Optional[AnyUrl] = Field(None, description="URL for generated video") - duration: Optional[str] = Field(None, description="Total video duration") - - -class TaskResult(BaseModel): - videos: Optional[List[Video]] = None - - -class Data(BaseModel): - task_id: Optional[str] = Field(None, description="Task ID") - task_status: Optional[TaskStatus] = None - task_info: Optional[TaskInfo] = None - created_at: Optional[int] = Field(None, description="Task creation time") - updated_at: Optional[int] = Field(None, description="Task update time") - task_result: Optional[TaskResult] = None - - -class KlingText2VideoResponse(BaseModel): - code: Optional[int] = Field(None, description="Error code") - message: Optional[str] = Field(None, description="Error message") - request_id: Optional[str] = Field(None, description="Request ID") - data: Optional[Data] = None - - -class ModelName1(str, Enum): - kling_v1 = "kling-v1" - kling_v1_5 = "kling-v1-5" - kling_v1_6 = "kling-v1-6" - kling_v2_master = "kling-v2-master" - - class Trajectory(BaseModel): x: Optional[int] = Field( None, @@ -467,55 +264,40 @@ class DynamicMask(BaseModel): trajectories: Optional[List[Trajectory]] = None -class Config1(BaseModel): - horizontal: Optional[float] = Field( - None, - description="Controls camera's movement along horizontal axis (x-axis). Negative indicates left, positive indicates right.", - ge=-10.0, - le=10.0, - ) - vertical: Optional[float] = Field( - None, - description="Controls camera's movement along vertical axis (y-axis). Negative indicates downward, positive indicates upward.", - ge=-10.0, - le=10.0, - ) - pan: Optional[float] = Field( - None, - description="Controls camera's rotation in vertical plane (x-axis). Negative indicates downward rotation, positive indicates upward rotation.", - ge=-10.0, - le=10.0, - ) - tilt: Optional[float] = Field( - None, - description="Controls camera's rotation in horizontal plane (y-axis). Negative indicates left rotation, positive indicates right rotation.", - ge=-10.0, - le=10.0, - ) - roll: Optional[float] = Field( - None, - description="Controls camera's rolling amount (z-axis). Negative indicates counterclockwise, positive indicates clockwise.", - ge=-10.0, - le=10.0, - ) - zoom: Optional[float] = Field( - None, - description="Controls change in camera's focal length. Negative indicates narrower field of view, positive indicates wider field of view.", - ge=-10.0, - le=10.0, - ) +class Mode(str, Enum): + std = "std" + pro = "pro" -class CameraControl1(BaseModel): - type: Optional[Type] = Field( - None, - description="Predefined camera movements type. simple: Customizable camera movement. down_back: Camera descends and moves backward. forward_up: Camera moves forward and tilts up. right_turn_forward: Rotate right and move forward. left_turn_forward: Rotate left and move forward.", - ) - config: Optional[Config1] = None +class ModelName(str, Enum): + kling_v1 = "kling-v1" + kling_v1_5 = "kling-v1-5" + kling_v1_6 = "kling-v1-6" + kling_v2_master = "kling-v2-master" class KlingImage2VideoRequest(BaseModel): - model_name: Optional[ModelName1] = Field("kling-v1", description="Model Name") + aspect_ratio: Optional[AspectRatio] = "16:9" + callback_url: Optional[AnyUrl] = Field( + None, + description="The callback notification address. Server will notify when the task status changes.", + ) + camera_control: Optional[CameraControl] = None + cfg_scale: Optional[float] = Field( + 0.5, + description="Flexibility in video generation. The higher the value, the lower the model's degree of flexibility, and the stronger the relevance to the user's prompt.", + ge=0.0, + le=1.0, + ) + duration: Optional[Duration] = Field("5", description="Video length in seconds") + dynamic_masks: Optional[List[DynamicMask]] = Field( + None, + description="Dynamic Brush Configuration List (up to 6 groups). For 5-second videos, trajectory length must not exceed 77 coordinates.", + ) + external_task_id: Optional[str] = Field( + None, + description="Customized Task ID. Must be unique within a single user account.", + ) image: Optional[str] = Field( None, description="Reference Image - URL or Base64 encoded string, cannot exceed 10MB, resolution not less than 300*300px, aspect ratio between 1:2.5 ~ 2.5:1. Base64 should not include data:image prefix.", @@ -524,40 +306,98 @@ class KlingImage2VideoRequest(BaseModel): None, description="Reference Image - End frame control. URL or Base64 encoded string, cannot exceed 10MB, resolution not less than 300*300px. Base64 should not include data:image prefix.", ) - prompt: Optional[str] = Field( - None, description="Positive text prompt", max_length=2500 - ) - negative_prompt: Optional[str] = Field( - None, description="Negative text prompt", max_length=2500 - ) - cfg_scale: Optional[float] = Field( - 0.5, - description="Flexibility in video generation. The higher the value, the lower the model's degree of flexibility, and the stronger the relevance to the user's prompt.", - ge=0.0, - le=1.0, - ) mode: Optional[Mode] = Field( "std", description="Video generation mode. std: Standard Mode, which is cost-effective. pro: Professional Mode, generates videos with longer duration but higher quality output.", ) + model_name: Optional[ModelName] = Field("kling-v1", description="Model Name") + negative_prompt: Optional[str] = Field( + None, description="Negative text prompt", max_length=2500 + ) + prompt: Optional[str] = Field( + None, description="Positive text prompt", max_length=2500 + ) static_mask: Optional[AnyUrl] = Field( None, description="Static Brush Application Area (Mask image created by users using the motion brush). The aspect ratio must match the input image.", ) - dynamic_masks: Optional[List[DynamicMask]] = Field( - None, - description="Dynamic Brush Configuration List (up to 6 groups). For 5-second videos, trajectory length must not exceed 77 coordinates.", + + +class TaskInfo(BaseModel): + external_task_id: Optional[str] = None + + +class Video(BaseModel): + duration: Optional[str] = Field(None, description="Total video duration") + id: Optional[str] = Field(None, description="Generated video ID") + url: Optional[AnyUrl] = Field(None, description="URL for generated video") + + +class TaskResult(BaseModel): + videos: Optional[List[Video]] = None + + +class TaskStatus(str, Enum): + submitted = "submitted" + processing = "processing" + succeed = "succeed" + failed = "failed" + + +class Data(BaseModel): + created_at: Optional[int] = Field(None, description="Task creation time") + task_id: Optional[str] = Field(None, description="Task ID") + task_info: Optional[TaskInfo] = None + task_result: Optional[TaskResult] = None + task_status: Optional[TaskStatus] = None + updated_at: Optional[int] = Field(None, description="Task update time") + + +class KlingImage2VideoResponse(BaseModel): + code: Optional[int] = Field(None, description="Error code") + data: Optional[Data] = None + message: Optional[str] = Field(None, description="Error message") + request_id: Optional[str] = Field(None, description="Request ID") + + +class Config1(BaseModel): + horizontal: Optional[float] = Field(None, ge=-10.0, le=10.0) + pan: Optional[float] = Field(None, ge=-10.0, le=10.0) + roll: Optional[float] = Field(None, ge=-10.0, le=10.0) + tilt: Optional[float] = Field(None, ge=-10.0, le=10.0) + vertical: Optional[float] = Field(None, ge=-10.0, le=10.0) + zoom: Optional[float] = Field(None, ge=-10.0, le=10.0) + + +class CameraControl1(BaseModel): + config: Optional[Config1] = None + type: Optional[Type] = Field(None, description="Predefined camera movements type") + + +class ModelName1(str, Enum): + kling_v1 = "kling-v1" + kling_v1_6 = "kling-v1-6" + kling_v2_master = "kling-v2-master" + + +class KlingText2VideoRequest(BaseModel): + aspect_ratio: Optional[AspectRatio] = "16:9" + callback_url: Optional[AnyUrl] = Field( + None, description="The callback notification address" ) camera_control: Optional[CameraControl1] = None - aspect_ratio: Optional[AspectRatio] = "16:9" - duration: Optional[Duration] = Field("5", description="Video length in seconds") - callback_url: Optional[AnyUrl] = Field( - None, - description="The callback notification address. Server will notify when the task status changes.", + cfg_scale: Optional[float] = Field( + 0.5, description="Flexibility in video generation", ge=0.0, le=1.0 ) - external_task_id: Optional[str] = Field( - None, - description="Customized Task ID. Must be unique within a single user account.", + duration: Optional[Duration] = "5" + external_task_id: Optional[str] = Field(None, description="Customized Task ID") + mode: Optional[Mode] = Field("std", description="Video generation mode") + model_name: Optional[ModelName1] = Field("kling-v1", description="Model Name") + negative_prompt: Optional[str] = Field( + None, description="Negative text prompt", max_length=2500 + ) + prompt: Optional[str] = Field( + None, description="Positive text prompt", max_length=2500 ) @@ -566,144 +406,215 @@ class TaskResult1(BaseModel): class Data1(BaseModel): - task_id: Optional[str] = Field(None, description="Task ID") - task_status: Optional[TaskStatus] = None - task_info: Optional[TaskInfo] = None created_at: Optional[int] = Field(None, description="Task creation time") - updated_at: Optional[int] = Field(None, description="Task update time") + task_id: Optional[str] = Field(None, description="Task ID") + task_info: Optional[TaskInfo] = None task_result: Optional[TaskResult1] = None + task_status: Optional[TaskStatus] = None + updated_at: Optional[int] = Field(None, description="Task update time") -class KlingImage2VideoResponse(BaseModel): +class KlingText2VideoResponse(BaseModel): code: Optional[int] = Field(None, description="Error code") + data: Optional[Data1] = None message: Optional[str] = Field(None, description="Error message") request_id: Optional[str] = Field(None, description="Request ID") - data: Optional[Data1] = None -class Object(str, Enum): - event = "event" +class LumaAspectRatio(str, Enum): + field_1_1 = "1:1" + field_16_9 = "16:9" + field_9_16 = "9:16" + field_4_3 = "4:3" + field_3_4 = "3:4" + field_21_9 = "21:9" + field_9_21 = "9:21" + + +class LumaAssets(BaseModel): + image: Optional[AnyUrl] = Field(None, description="The URL of the image") + progress_video: Optional[AnyUrl] = Field( + None, description="The URL of the progress video" + ) + video: Optional[AnyUrl] = Field(None, description="The URL of the video") + + +class GenerationType(str, Enum): + add_audio = "add_audio" + + +class LumaAudioGenerationRequest(BaseModel): + callback_url: Optional[AnyUrl] = Field( + None, description="The callback URL for the audio" + ) + generation_type: Optional[GenerationType] = "add_audio" + negative_prompt: Optional[str] = Field( + None, description="The negative prompt of the audio" + ) + prompt: Optional[str] = Field(None, description="The prompt of the audio") + + +class LumaError(BaseModel): + detail: Optional[str] = Field(None, description="The error message") class Type2(str, Enum): - payment_intent_succeeded = "payment_intent.succeeded" + generation = "generation" -class StripeRequestInfo(BaseModel): - id: Optional[str] = None - idempotency_key: Optional[str] = None +class LumaGenerationReference(BaseModel): + id: UUID = Field(..., description="The ID of the generation") + type: Literal["generation"] -class Object1(str, Enum): - payment_intent = "payment_intent" +class GenerationType1(str, Enum): + video = "video" -class StripeAmountDetails(BaseModel): - tip: Optional[Dict[str, Any]] = None +class LumaGenerationType(str, Enum): + video = "video" + image = "image" -class Object2(str, Enum): - charge = "charge" +class GenerationType2(str, Enum): + image = "image" -class StripeAddress(BaseModel): - city: Optional[str] = None - country: Optional[str] = None - line1: Optional[str] = None - line2: Optional[str] = None - postal_code: Optional[str] = None - state: Optional[str] = None +class LumaImageIdentity(BaseModel): + images: Optional[List[AnyUrl]] = Field( + None, description="The URLs of the image identity" + ) -class StripeOutcome(BaseModel): - advice_code: Optional[Any] = None - network_advice_code: Optional[Any] = None - network_decline_code: Optional[Any] = None - network_status: Optional[str] = None - reason: Optional[Any] = None - risk_level: Optional[str] = None - risk_score: Optional[int] = None - seller_message: Optional[str] = None - type: Optional[str] = None +class LumaImageModel(str, Enum): + photon_1 = "photon-1" + photon_flash_1 = "photon-flash-1" -class Checks(BaseModel): - address_line1_check: Optional[Any] = None - address_postal_code_check: Optional[Any] = None - cvc_check: Optional[str] = None +class LumaImageRef(BaseModel): + url: Optional[AnyUrl] = Field(None, description="The URL of the image reference") + weight: Optional[float] = Field( + None, description="The weight of the image reference" + ) -class ExtendedAuthorization(BaseModel): - status: Optional[str] = None +class Type3(str, Enum): + image = "image" -class IncrementalAuthorization(BaseModel): - status: Optional[str] = None +class LumaImageReference(BaseModel): + type: Literal["image"] + url: AnyUrl = Field(..., description="The URL of the image") -class Multicapture(BaseModel): - status: Optional[str] = None +class LumaKeyframe(RootModel[Union[LumaGenerationReference, LumaImageReference]]): + root: Union[LumaGenerationReference, LumaImageReference] = Field( + ..., + description="A keyframe can be either a Generation reference, an Image, or a Video", + discriminator="type", + ) -class NetworkToken(BaseModel): - used: Optional[bool] = None +class LumaKeyframes(BaseModel): + frame0: Optional[LumaKeyframe] = None + frame1: Optional[LumaKeyframe] = None -class Overcapture(BaseModel): - maximum_amount_capturable: Optional[int] = None - status: Optional[str] = None +class LumaModifyImageRef(BaseModel): + url: Optional[AnyUrl] = Field(None, description="The URL of the image reference") + weight: Optional[float] = Field( + None, description="The weight of the modify image reference" + ) -class StripeCardDetails(BaseModel): - amount_authorized: Optional[int] = None - authorization_code: Optional[Any] = None - brand: Optional[str] = None - checks: Optional[Checks] = None - country: Optional[str] = None - exp_month: Optional[int] = None - exp_year: Optional[int] = None - extended_authorization: Optional[ExtendedAuthorization] = None - fingerprint: Optional[str] = None - funding: Optional[str] = None - incremental_authorization: Optional[IncrementalAuthorization] = None - installments: Optional[Any] = None - last4: Optional[str] = None - mandate: Optional[Any] = None - multicapture: Optional[Multicapture] = None - network: Optional[str] = None - network_token: Optional[NetworkToken] = None - network_transaction_id: Optional[str] = None - overcapture: Optional[Overcapture] = None - regulated_status: Optional[str] = None - three_d_secure: Optional[Any] = None - wallet: Optional[Any] = None +class LumaState(str, Enum): + queued = "queued" + dreaming = "dreaming" + completed = "completed" + failed = "failed" -class StripeRefundList(BaseModel): - object: Optional[str] = None - data: Optional[List[Dict[str, Any]]] = None - has_more: Optional[bool] = None - total_count: Optional[int] = None - url: Optional[str] = None +class GenerationType3(str, Enum): + upscale_video = "upscale_video" -class Card(BaseModel): - installments: Optional[Any] = None - mandate_options: Optional[Any] = None - network: Optional[Any] = None - request_three_d_secure: Optional[str] = None +class LumaVideoModel(str, Enum): + ray_2 = "ray-2" + ray_flash_2 = "ray-flash-2" + ray_1_6 = "ray-1-6" -class StripePaymentMethodOptions(BaseModel): - card: Optional[Card] = None +class LumaVideoModelOutputDuration1(str, Enum): + field_5s = "5s" + field_9s = "9s" -class StripeShipping(BaseModel): - address: Optional[StripeAddress] = None - carrier: Optional[str] = None - name: Optional[str] = None - phone: Optional[str] = None - tracking_number: Optional[str] = None +class LumaVideoModelOutputDuration( + RootModel[Union[LumaVideoModelOutputDuration1, str]] +): + root: Union[LumaVideoModelOutputDuration1, str] + + +class LumaVideoModelOutputResolution1(str, Enum): + field_540p = "540p" + field_720p = "720p" + field_1080p = "1080p" + field_4k = "4k" + + +class LumaVideoModelOutputResolution( + RootModel[Union[LumaVideoModelOutputResolution1, str]] +): + root: Union[LumaVideoModelOutputResolution1, str] + + +class MinimaxBaseResponse(BaseModel): + status_code: int = Field( + ..., + description="Status code. 0 indicates success, other values indicate errors.", + ) + status_msg: str = Field( + ..., description="Specific error details or success message." + ) + + +class File(BaseModel): + bytes: Optional[int] = Field(None, description="File size in bytes") + created_at: Optional[int] = Field( + None, description="Unix timestamp when the file was created, in seconds" + ) + download_url: Optional[str] = Field( + None, description="The URL to download the video" + ) + file_id: Optional[int] = Field(None, description="Unique identifier for the file") + filename: Optional[str] = Field(None, description="The name of the file") + purpose: Optional[str] = Field(None, description="The purpose of using the file") + + +class MinimaxFileRetrieveResponse(BaseModel): + base_resp: MinimaxBaseResponse + file: File + + +class Status(str, Enum): + Queueing = "Queueing" + Preparing = "Preparing" + Processing = "Processing" + Success = "Success" + Fail = "Fail" + + +class MinimaxTaskResultResponse(BaseModel): + base_resp: MinimaxBaseResponse + file_id: Optional[str] = Field( + None, + description="After the task status changes to Success, this field returns the file ID corresponding to the generated video.", + ) + status: Status = Field( + ..., + description="Task status: 'Queueing' (in queue), 'Preparing' (task is preparing), 'Processing' (generating), 'Success' (task completed successfully), or 'Fail' (task failed).", + ) + task_id: str = Field(..., description="The task ID being queried.") class Model(str, Enum): @@ -726,6 +637,14 @@ class SubjectReferenceItem(BaseModel): class MinimaxVideoGenerationRequest(BaseModel): + callback_url: Optional[str] = Field( + None, + description="Optional. URL to receive real-time status updates about the video generation task.", + ) + first_frame_image: Optional[str] = Field( + None, + description="URL or base64 encoding of the first frame image. Required when model is I2V-01, I2V-01-Director, or I2V-01-live.", + ) model: Model = Field( ..., description="Required. ID of model. Options: T2V-01-Director, I2V-01-Director, S2V-01, I2V-01, I2V-01-live, T2V-01", @@ -739,304 +658,216 @@ class MinimaxVideoGenerationRequest(BaseModel): True, description="If true (default), the model will automatically optimize the prompt. Set to false for more precise control.", ) - first_frame_image: Optional[str] = Field( - None, - description="URL or base64 encoding of the first frame image. Required when model is I2V-01, I2V-01-Director, or I2V-01-live.", - ) subject_reference: Optional[List[SubjectReferenceItem]] = Field( None, description="Only available when model is S2V-01. The model will generate a video based on the subject uploaded through this parameter.", ) - callback_url: Optional[str] = Field( - None, - description="Optional. URL to receive real-time status updates about the video generation task.", - ) - - -class MinimaxBaseResponse(BaseModel): - status_code: int = Field( - ..., - description="Status code. 0 indicates success, other values indicate errors.", - ) - status_msg: str = Field( - ..., description="Specific error details or success message." - ) class MinimaxVideoGenerationResponse(BaseModel): + base_resp: MinimaxBaseResponse task_id: str = Field( ..., description="The task ID for the asynchronous video generation task." ) - base_resp: MinimaxBaseResponse -class File(BaseModel): - file_id: Optional[int] = Field(None, description="Unique identifier for the file") - bytes: Optional[int] = Field(None, description="File size in bytes") - created_at: Optional[int] = Field( - None, description="Unix timestamp when the file was created, in seconds" +class Moderation(str, Enum): + low = "low" + auto = "auto" + + +class OutputFormat(str, Enum): + png = "png" + webp = "webp" + jpeg = "jpeg" + + +class OpenAIImageEditRequest(BaseModel): + background: Optional[str] = Field( + None, description="Background transparency", examples=["opaque"] ) - filename: Optional[str] = Field(None, description="The name of the file") - purpose: Optional[str] = Field(None, description="The purpose of using the file") - download_url: Optional[str] = Field( - None, description="The URL to download the video" + model: str = Field( + ..., description="The model to use for image editing", examples=["gpt-image-1"] ) - - -class MinimaxFileRetrieveResponse(BaseModel): - file: File - base_resp: MinimaxBaseResponse - - -class Status(str, Enum): - Queueing = "Queueing" - Preparing = "Preparing" - Processing = "Processing" - Success = "Success" - Fail = "Fail" - - -class MinimaxTaskResultResponse(BaseModel): - task_id: str = Field(..., description="The task ID being queried.") - status: Status = Field( + moderation: Optional[Moderation] = Field( + None, description="Content moderation setting", examples=["auto"] + ) + n: Optional[int] = Field( + None, description="The number of images to generate", examples=[1] + ) + output_compression: Optional[int] = Field( + None, description="Compression level for JPEG or WebP (0-100)", examples=[100] + ) + output_format: Optional[OutputFormat] = Field( + None, description="Format of the output image", examples=["png"] + ) + prompt: str = Field( ..., - description="Task status: 'Queueing' (in queue), 'Preparing' (task is preparing), 'Processing' (generating), 'Success' (task completed successfully), or 'Fail' (task failed).", + description="A text description of the desired edit", + examples=["Give the rocketship rainbow coloring"], ) - file_id: Optional[str] = Field( + quality: Optional[str] = Field( + None, description="The quality of the edited image", examples=["low"] + ) + size: Optional[str] = Field( + None, description="Size of the output image", examples=["1024x1024"] + ) + user: Optional[str] = Field( None, - description="After the task status changes to Success, this field returns the file ID corresponding to the generated video.", - ) - base_resp: MinimaxBaseResponse - - -class BFLFluxProGenerateRequest(BaseModel): - prompt: str = Field(..., description="The text prompt for image generation.") - negative_prompt: Optional[str] = Field( - None, description="The negative prompt for image generation." - ) - width: int = Field( - ..., description="The width of the image to generate.", ge=64, le=2048 - ) - height: int = Field( - ..., description="The height of the image to generate.", ge=64, le=2048 - ) - num_inference_steps: Optional[int] = Field( - None, description="The number of inference steps.", ge=1, le=100 - ) - guidance_scale: Optional[float] = Field( - None, description="The guidance scale for generation.", ge=1.0, le=20.0 - ) - seed: Optional[int] = Field(None, description="The seed value for reproducibility.") - num_images: Optional[int] = Field( - None, description="The number of images to generate.", ge=1, le=4 + description="A unique identifier for end-user monitoring", + examples=["user-1234"], ) -class BFLFluxProGenerateResponse(BaseModel): - id: str = Field(..., description="The unique identifier for the generation task.") - polling_url: str = Field(..., description="URL to poll for the generation result.") +class Background(str, Enum): + transparent = "transparent" + opaque = "opaque" + + +class Quality(str, Enum): + low = "low" + medium = "medium" + high = "high" + standard = "standard" + hd = "hd" + + +class ResponseFormat(str, Enum): + url = "url" + b64_json = "b64_json" + + +class Style(str, Enum): + vivid = "vivid" + natural = "natural" + + +class OpenAIImageGenerationRequest(BaseModel): + background: Optional[Background] = Field( + None, description="Background transparency", examples=["opaque"] + ) + model: Optional[str] = Field( + None, description="The model to use for image generation", examples=["dall-e-3"] + ) + moderation: Optional[Moderation] = Field( + None, description="Content moderation setting", examples=["auto"] + ) + n: Optional[int] = Field( + None, + description="The number of images to generate (1-10). Only 1 supported for dall-e-3.", + examples=[1], + ) + output_compression: Optional[int] = Field( + None, description="Compression level for JPEG or WebP (0-100)", examples=[100] + ) + output_format: Optional[OutputFormat] = Field( + None, description="Format of the output image", examples=["png"] + ) + prompt: str = Field( + ..., + description="A text description of the desired image", + examples=["Draw a rocket in front of a blackhole in deep space"], + ) + quality: Optional[Quality] = Field( + None, description="The quality of the generated image", examples=["high"] + ) + response_format: Optional[ResponseFormat] = Field( + None, description="Response format of image data", examples=["b64_json"] + ) + size: Optional[str] = Field( + None, + description="Size of the image (e.g., 1024x1024, 1536x1024, auto)", + examples=["1024x1536"], + ) + style: Optional[Style] = Field( + None, description="Style of the image (only for dall-e-3)", examples=["vivid"] + ) + user: Optional[str] = Field( + None, + description="A unique identifier for end-user monitoring", + examples=["user-1234"], + ) class Datum1(BaseModel): - image_id: Optional[str] = Field( - None, description="Unique identifier for the generated image" - ) - url: Optional[str] = Field(None, description="URL to access the generated image") + b64_json: Optional[str] = Field(None, description="Base64 encoded image data") + revised_prompt: Optional[str] = Field(None, description="Revised prompt") + url: Optional[str] = Field(None, description="URL of the image") -class RecraftImageGenerationResponse(BaseModel): - created: int = Field( - ..., description="Unix timestamp when the generation was created" - ) - credits: int = Field(..., description="Number of credits used for the generation") - data: List[Datum1] = Field(..., description="Array of generated image information") +class InputTokensDetails(BaseModel): + image_tokens: Optional[int] = None + text_tokens: Optional[int] = None -class KlingErrorResponse(BaseModel): - code: int = Field( +class Usage(BaseModel): + input_tokens: Optional[int] = None + input_tokens_details: Optional[InputTokensDetails] = None + output_tokens: Optional[int] = None + total_tokens: Optional[int] = None + + +class OpenAIImageGenerationResponse(BaseModel): + data: Optional[List[Datum1]] = None + usage: Optional[Usage] = None + + +class AspectRatio2(RootModel[float]): + root: float = Field( ..., - description="- 1000: Authentication failed\n- 1001: Authorization is empty\n- 1002: Authorization is invalid\n- 1003: Authorization is not yet valid\n- 1004: Authorization has expired\n- 1100: Account exception\n- 1101: Account in arrears (postpaid scenario)\n- 1102: Resource pack depleted or expired (prepaid scenario)\n- 1103: Unauthorized access to requested resource\n- 1200: Invalid request parameters\n- 1201: Invalid parameters\n- 1202: Invalid request method\n- 1203: Requested resource does not exist\n- 1300: Trigger platform strategy\n- 1301: Trigger content security policy\n- 1302: API request too frequent\n- 1303: Concurrency/QPS exceeds limit\n- 1304: Trigger IP whitelist policy\n- 5000: Internal server error\n- 5001: Service temporarily unavailable\n- 5002: Server internal timeout\n", - ) - message: str = Field(..., description="Human-readable error message") - request_id: str = Field( - ..., description="Request ID for tracking and troubleshooting" + description="Aspect ratio (width / height)", + ge=0.4, + le=2.5, + title="Aspectratio", ) -class LumaAspectRatio(str, Enum): - field_1_1 = "1:1" - field_16_9 = "16:9" - field_9_16 = "9:16" - field_4_3 = "4:3" - field_3_4 = "3:4" - field_21_9 = "21:9" - field_9_21 = "9:21" +class IngredientsMode(str, Enum): + creative = "creative" + precise = "precise" -class LumaVideoModel(str, Enum): - ray_2 = "ray-2" - ray_flash_2 = "ray-flash-2" - ray_1_6 = "ray-1-6" +class PikaDurationEnum(int, Enum): + integer_5 = 5 + integer_10 = 10 -class LumaVideoModelOutputResolution1(str, Enum): - field_540p = "540p" - field_720p = "720p" +class PikaGenerateResponse(BaseModel): + video_id: str = Field(..., title="Video Id") + + +class PikaResolutionEnum(str, Enum): field_1080p = "1080p" - field_4k = "4k" + field_720p = "720p" -class LumaVideoModelOutputResolution( - RootModel[Union[LumaVideoModelOutputResolution1, str]] -): - root: Union[LumaVideoModelOutputResolution1, str] - - -class LumaVideoModelOutputDuration1(str, Enum): - field_5s = "5s" - field_9s = "9s" - - -class LumaVideoModelOutputDuration( - RootModel[Union[LumaVideoModelOutputDuration1, str]] -): - root: Union[LumaVideoModelOutputDuration1, str] - - -class LumaImageModel(str, Enum): - photon_1 = "photon-1" - photon_flash_1 = "photon-flash-1" - - -class LumaImageRef(BaseModel): - url: Optional[AnyUrl] = Field(None, description="The URL of the image reference") - weight: Optional[float] = Field( - None, description="The weight of the image reference" - ) - - -class LumaImageIdentity(BaseModel): - images: Optional[List[AnyUrl]] = Field( - None, description="The URLs of the image identity" - ) - - -class LumaModifyImageRef(BaseModel): - url: Optional[AnyUrl] = Field(None, description="The URL of the image reference") - weight: Optional[float] = Field( - None, description="The weight of the modify image reference" - ) - - -class Type3(str, Enum): - generation = "generation" - - -class LumaGenerationReference(BaseModel): - type: Literal["generation"] - id: UUID = Field(..., description="The ID of the generation") - - -class Type4(str, Enum): - image = "image" - - -class LumaImageReference(BaseModel): - type: Literal["image"] - url: AnyUrl = Field(..., description="The URL of the image") - - -class LumaKeyframe(RootModel[Union[LumaGenerationReference, LumaImageReference]]): - root: Union[LumaGenerationReference, LumaImageReference] = Field( - ..., - description="A keyframe can be either a Generation reference, an Image, or a Video", - discriminator="type", - ) - - -class LumaGenerationType(str, Enum): - video = "video" - image = "image" - - -class LumaState(str, Enum): +class PikaStatusEnum(str, Enum): queued = "queued" - dreaming = "dreaming" - completed = "completed" - failed = "failed" + started = "started" + finished = "finished" -class LumaAssets(BaseModel): - video: Optional[AnyUrl] = Field(None, description="The URL of the video") - image: Optional[AnyUrl] = Field(None, description="The URL of the image") - progress_video: Optional[AnyUrl] = Field( - None, description="The URL of the progress video" - ) +class PikaValidationError(BaseModel): + loc: List[Union[str, int]] = Field(..., title="Location") + msg: str = Field(..., title="Message") + type: str = Field(..., title="Error Type") -class GenerationType(str, Enum): - video = "video" +class PikaVideoResponse(BaseModel): + id: str = Field(..., title="Id") + progress: Optional[int] = Field(None, title="Progress") + status: PikaStatusEnum + url: Optional[str] = Field(None, title="Url") -class GenerationType1(str, Enum): - image = "image" +class Resp(BaseModel): + img_id: Optional[int] = None -class CharacterRef(BaseModel): - identity0: Optional[LumaImageIdentity] = None - - -class LumaImageGenerationRequest(BaseModel): - generation_type: Optional[GenerationType1] = "image" - model: Optional[LumaImageModel] = "photon-1" - prompt: Optional[str] = Field(None, description="The prompt of the generation") - aspect_ratio: Optional[LumaAspectRatio] = "16:9" - callback_url: Optional[AnyUrl] = Field( - None, description="The callback URL for the generation" - ) - image_ref: Optional[List[LumaImageRef]] = None - style_ref: Optional[List[LumaImageRef]] = None - character_ref: Optional[CharacterRef] = None - modify_image_ref: Optional[LumaModifyImageRef] = None - - -class GenerationType2(str, Enum): - upscale_video = "upscale_video" - - -class LumaUpscaleVideoGenerationRequest(BaseModel): - generation_type: Optional[GenerationType2] = "upscale_video" - resolution: Optional[LumaVideoModelOutputResolution] = None - callback_url: Optional[AnyUrl] = Field( - None, description="The callback URL for the upscale" - ) - - -class GenerationType3(str, Enum): - add_audio = "add_audio" - - -class LumaAudioGenerationRequest(BaseModel): - generation_type: Optional[GenerationType3] = "add_audio" - prompt: Optional[str] = Field(None, description="The prompt of the audio") - negative_prompt: Optional[str] = Field( - None, description="The negative prompt of the audio" - ) - callback_url: Optional[AnyUrl] = Field( - None, description="The callback URL for the audio" - ) - - -class LumaError(BaseModel): - detail: Optional[str] = Field(None, description="The error message") - - -class AspectRatio2(str, Enum): - field_16_9 = "16:9" - field_4_3 = "4:3" - field_1_1 = "1:1" - field_3_4 = "3:4" - field_9_16 = "9:16" +class PixverseImageUploadResponse(BaseModel): + ErrCode: Optional[int] = None + ErrMsg: Optional[str] = None + Resp_1: Optional[Resp] = Field(None, alias="Resp") class Duration2(int, Enum): @@ -1053,14 +884,14 @@ class MotionMode(str, Enum): fast = "fast" -class Quality(str, Enum): +class Quality1(str, Enum): field_360p = "360p" field_540p = "540p" field_720p = "720p" field_1080p = "1080p" -class Style(str, Enum): +class Style1(str, Enum): anime = "anime" field_3d_animation = "3d_animation" clay = "clay" @@ -1068,67 +899,65 @@ class Style(str, Enum): cyberpunk = "cyberpunk" +class PixverseImageVideoRequest(BaseModel): + duration: Duration2 + img_id: int + model: Model1 + motion_mode: Optional[MotionMode] = None + prompt: str + quality: Quality1 + seed: Optional[int] = None + style: Optional[Style1] = None + template_id: Optional[int] = None + water_mark: Optional[bool] = None + + +class AspectRatio3(str, Enum): + field_16_9 = "16:9" + field_4_3 = "4:3" + field_1_1 = "1:1" + field_3_4 = "3:4" + field_9_16 = "9:16" + + class PixverseTextVideoRequest(BaseModel): - aspect_ratio: AspectRatio2 + aspect_ratio: AspectRatio3 duration: Duration2 model: Model1 motion_mode: Optional[MotionMode] = None negative_prompt: Optional[str] = None prompt: str - quality: Quality + quality: Quality1 seed: Optional[int] = None - style: Optional[Style] = None + style: Optional[Style1] = None template_id: Optional[int] = None water_mark: Optional[bool] = None -class Resp(BaseModel): +class PixverseTransitionVideoRequest(BaseModel): + duration: Duration2 + first_frame_img: int + last_frame_img: int + model: Model1 + motion_mode: MotionMode + prompt: str + quality: Quality1 + seed: int + style: Optional[Style1] = None + template_id: Optional[int] = None + water_mark: Optional[bool] = None + + +class Resp1(BaseModel): video_id: Optional[int] = None class PixverseVideoResponse(BaseModel): - ErrCode: Optional[int] = None - ErrMsg: Optional[str] = None - Resp_1: Optional[Resp] = Field(None, alias="Resp") - - -class Resp1(BaseModel): - img_id: Optional[int] = None - - -class PixverseImageUploadResponse(BaseModel): ErrCode: Optional[int] = None ErrMsg: Optional[str] = None Resp: Optional[Resp1] = None -class PixverseImageVideoRequest(BaseModel): - img_id: int - model: Model1 - prompt: str - duration: Duration2 - quality: Quality - motion_mode: Optional[MotionMode] = None - seed: Optional[int] = None - style: Optional[Style] = None - template_id: Optional[int] = None - water_mark: Optional[bool] = None - - -class PixverseTransitionVideoRequest(BaseModel): - first_frame_img: int - last_frame_img: int - model: Model1 - duration: Duration2 - quality: Quality - motion_mode: MotionMode - seed: int - prompt: str - style: Optional[Style] = None - template_id: Optional[int] = None - water_mark: Optional[bool] = None - - class Status1(int, Enum): integer_1 = 1 integer_5 = 5 @@ -1162,56 +991,71 @@ class PixverseVideoResultResponse(BaseModel): Resp: Optional[Resp2] = None -class Image(BaseModel): - bytesBase64Encoded: str - gcsUri: Optional[str] = None - mimeType: Optional[str] = None +class RgbItem(RootModel[int]): + root: int = Field(..., ge=0, le=255) -class Image1(BaseModel): - bytesBase64Encoded: Optional[str] = None - gcsUri: str - mimeType: Optional[str] = None +class RGBColor(BaseModel): + rgb: List[RgbItem] = Field(..., max_length=3, min_length=3) -class Instance(BaseModel): - prompt: str = Field(..., description="Text description of the video") - image: Optional[Union[Image, Image1]] = Field( - None, description="Optional image to guide video generation" +class Controls(BaseModel): + artistic_level: Optional[int] = Field( + None, + description="Defines artistic tone of your image. At a simple level, the person looks straight at the camera in a static and clean style. Dynamic and eccentric levels introduce movement and creativity.", + ge=0, + le=5, + ) + background_color: Optional[RGBColor] = None + colors: Optional[List[RGBColor]] = Field( + None, description="An array of preferable colors" + ) + no_text: Optional[bool] = Field(None, description="Do not embed text layouts") + + +class RecraftImageGenerationRequest(BaseModel): + controls: Optional[Controls] = Field( + None, description="The controls for the generated image" + ) + model: str = Field( + ..., description='The model to use for generation (e.g., "recraftv3")' + ) + n: int = Field(..., description="The number of images to generate", ge=1, le=4) + prompt: str = Field( + ..., description="The text prompt describing the image to generate" + ) + size: str = Field( + ..., description='The size of the generated image (e.g., "1024x1024")' + ) + style: Optional[str] = Field( + None, + description='The style to apply to the generated image (e.g., "digital_illustration")', + ) + style_id: Optional[str] = Field( + None, + description='The style ID to apply to the generated image (e.g., "123e4567-e89b-12d3-a456-426614174000"). If style_id is provided, style should not be provided.', ) -class PersonGeneration(str, Enum): - ALLOW = "ALLOW" - BLOCK = "BLOCK" - - -class Parameters(BaseModel): - aspectRatio: Optional[str] = Field(None, examples=["16:9"]) - negativePrompt: Optional[str] = None - personGeneration: Optional[PersonGeneration] = None - sampleCount: Optional[int] = None - seed: Optional[int] = None - storageUri: Optional[str] = Field( - None, description="Optional Cloud Storage URI to upload the video" +class Datum2(BaseModel): + image_id: Optional[str] = Field( + None, description="Unique identifier for the generated image" ) - durationSeconds: Optional[int] = None - enhancePrompt: Optional[bool] = None + url: Optional[str] = Field(None, description="URL to access the generated image") -class Veo2GenVidRequest(BaseModel): - instances: Optional[List[Instance]] = None - parameters: Optional[Parameters] = None - - -class Veo2GenVidResponse(BaseModel): - name: str = Field( - ..., - description="Operation resource name", - examples=[ - "projects/PROJECT_ID/locations/us-central1/publishers/google/models/MODEL_ID/operations/a1b07c8e-7b5a-4aba-bb34-3e1ccb8afcc8" - ], +class RecraftImageGenerationResponse(BaseModel): + created: int = Field( + ..., description="Unix timestamp when the generation was created" ) + credits: int = Field(..., description="Number of credits used for the generation") + data: List[Datum2] = Field(..., description="Array of generated image information") + + +class RenderingSpeed(str, Enum): + BALANCED = "BALANCED" + TURBO = "TURBO" + QUALITY = "QUALITY" class Veo2GenVidPollRequest(BaseModel): @@ -1224,11 +1068,16 @@ class Veo2GenVidPollRequest(BaseModel): ) +class Error(BaseModel): + code: Optional[int] = Field(None, description="Error code") + message: Optional[str] = Field(None, description="Error message") + + class Video2(BaseModel): - gcsUri: Optional[str] = Field(None, description="Cloud Storage URI of the video") bytesBase64Encoded: Optional[str] = Field( None, description="Base64-encoded video content" ) + gcsUri: Optional[str] = Field(None, description="Cloud Storage URI of the video") mimeType: Optional[str] = Field(None, description="Video MIME type") @@ -1249,545 +1098,92 @@ class Response(BaseModel): videos: Optional[List[Video2]] = None -class Error1(BaseModel): - code: Optional[int] = Field(None, description="Error code") - message: Optional[str] = Field(None, description="Error message") - - class Veo2GenVidPollResponse(BaseModel): - name: Optional[str] = None done: Optional[bool] = None + error: Optional[Error] = Field( + None, description="Error details if operation failed" + ) + name: Optional[str] = None response: Optional[Response] = Field( None, description="The actual prediction response if done is true" ) - error: Optional[Error1] = Field( - None, description="Error details if operation failed" + + +class Image(BaseModel): + bytesBase64Encoded: str + gcsUri: Optional[str] = None + mimeType: Optional[str] = None + + +class Image1(BaseModel): + bytesBase64Encoded: Optional[str] = None + gcsUri: str + mimeType: Optional[str] = None + + +class Instance(BaseModel): + image: Optional[Union[Image, Image1]] = Field( + None, description="Optional image to guide video generation" + ) + prompt: str = Field(..., description="Text description of the video") + + +class PersonGeneration(str, Enum): + ALLOW = "ALLOW" + BLOCK = "BLOCK" + + +class Parameters(BaseModel): + aspectRatio: Optional[str] = Field(None, examples=["16:9"]) + durationSeconds: Optional[int] = None + enhancePrompt: Optional[bool] = None + negativePrompt: Optional[str] = None + personGeneration: Optional[PersonGeneration] = None + sampleCount: Optional[int] = None + seed: Optional[int] = None + storageUri: Optional[str] = Field( + None, description="Optional Cloud Storage URI to upload the video" ) -class RunwayImageToVideoResponse(BaseModel): - id: Optional[str] = Field(None, description="Task ID") +class Veo2GenVidRequest(BaseModel): + instances: Optional[List[Instance]] = None + parameters: Optional[Parameters] = None -class RunwayTaskStatusEnum(str, Enum): - SUCCEEDED = "SUCCEEDED" - RUNNING = "RUNNING" - FAILED = "FAILED" - PENDING = "PENDING" - CANCELLED = "CANCELLED" - THROTTLED = "THROTTLED" - - -class RunwayModelEnum(str, Enum): - gen4_turbo = "gen4_turbo" - gen3a_turbo = "gen3a_turbo" - - -class Position(str, Enum): - first = "first" - last = "last" - - -class RunwayPromptImageDetailedObject(BaseModel): - uri: AnyUrl = Field( - ..., description="A HTTPS URL or data URI containing an encoded image." - ) - position: Position = Field( +class Veo2GenVidResponse(BaseModel): + name: str = Field( ..., - description="The position of the image in the output video. 'last' is currently supported for gen3a_turbo only.", - ) - - -class RunwayDurationEnum(int, Enum): - integer_5 = 5 - integer_10 = 10 - - -class RunwayAspectRatioEnum(str, Enum): - field_1280_720 = "1280:720" - field_720_1280 = "720:1280" - field_1104_832 = "1104:832" - field_832_1104 = "832:1104" - field_960_960 = "960:960" - field_1584_672 = "1584:672" - field_1280_768 = "1280:768" - field_768_1280 = "768:1280" - - -class RunwayPromptImageObject( - RootModel[Union[AnyUrl, List[RunwayPromptImageDetailedObject]]] -): - root: Union[AnyUrl, List[RunwayPromptImageDetailedObject]] = Field( - ..., - description="Image(s) to use for the video generation. Can be a single URI or an array of image objects with positions.", - ) - - -class Datum2(BaseModel): - b64_json: Optional[str] = Field(None, description="Base64 encoded image data") - url: Optional[str] = Field(None, description="URL of the image") - revised_prompt: Optional[str] = Field(None, description="Revised prompt") - - -class InputTokensDetails(BaseModel): - text_tokens: Optional[int] = None - image_tokens: Optional[int] = None - - -class Usage(BaseModel): - input_tokens: Optional[int] = None - input_tokens_details: Optional[InputTokensDetails] = None - output_tokens: Optional[int] = None - total_tokens: Optional[int] = None - - -class OpenAIImageGenerationResponse(BaseModel): - data: Optional[List[Datum2]] = None - usage: Optional[Usage] = None - - -class Quality3(str, Enum): - low = "low" - medium = "medium" - high = "high" - standard = "standard" - hd = "hd" - - -class OutputFormat(str, Enum): - png = "png" - webp = "webp" - jpeg = "jpeg" - - -class Moderation(str, Enum): - low = "low" - auto = "auto" - - -class Background(str, Enum): - transparent = "transparent" - opaque = "opaque" - - -class ResponseFormat(str, Enum): - url = "url" - b64_json = "b64_json" - - -class Style3(str, Enum): - vivid = "vivid" - natural = "natural" - - -class OpenAIImageGenerationRequest(BaseModel): - model: Optional[str] = Field( - None, description="The model to use for image generation", examples=["dall-e-3"] - ) - prompt: str = Field( - ..., - description="A text description of the desired image", - examples=["Draw a rocket in front of a blackhole in deep space"], - ) - n: Optional[int] = Field( - None, - description="The number of images to generate (1-10). Only 1 supported for dall-e-3.", - examples=[1], - ) - quality: Optional[Quality3] = Field( - None, description="The quality of the generated image", examples=["high"] - ) - size: Optional[str] = Field( - None, - description="Size of the image (e.g., 1024x1024, 1536x1024, auto)", - examples=["1024x1536"], - ) - output_format: Optional[OutputFormat] = Field( - None, description="Format of the output image", examples=["png"] - ) - output_compression: Optional[int] = Field( - None, description="Compression level for JPEG or WebP (0-100)", examples=[100] - ) - moderation: Optional[Moderation] = Field( - None, description="Content moderation setting", examples=["auto"] - ) - background: Optional[Background] = Field( - None, description="Background transparency", examples=["opaque"] - ) - response_format: Optional[ResponseFormat] = Field( - None, description="Response format of image data", examples=["b64_json"] - ) - style: Optional[Style3] = Field( - None, description="Style of the image (only for dall-e-3)", examples=["vivid"] - ) - user: Optional[str] = Field( - None, - description="A unique identifier for end-user monitoring", - examples=["user-1234"], - ) - - -class OpenAIImageEditRequest(BaseModel): - model: str = Field( - ..., description="The model to use for image editing", examples=["gpt-image-1"] - ) - prompt: str = Field( - ..., - description="A text description of the desired edit", - examples=["Give the rocketship rainbow coloring"], - ) - n: Optional[int] = Field( - None, description="The number of images to generate", examples=[1] - ) - quality: Optional[str] = Field( - None, description="The quality of the edited image", examples=["low"] - ) - size: Optional[str] = Field( - None, description="Size of the output image", examples=["1024x1024"] - ) - output_format: Optional[OutputFormat] = Field( - None, description="Format of the output image", examples=["png"] - ) - output_compression: Optional[int] = Field( - None, description="Compression level for JPEG or WebP (0-100)", examples=[100] - ) - moderation: Optional[Moderation] = Field( - None, description="Content moderation setting", examples=["auto"] - ) - background: Optional[str] = Field( - None, description="Background transparency", examples=["opaque"] - ) - user: Optional[str] = Field( - None, - description="A unique identifier for end-user monitoring", - examples=["user-1234"], - ) - - -class CustomerStorageResourceResponse(BaseModel): - download_url: Optional[str] = Field( - None, - description="The signed URL to use for downloading the file from the specified path", - ) - upload_url: Optional[str] = Field( - None, - description="The signed URL to use for uploading the file to the specified path", - ) - expires_at: Optional[datetime] = Field( - None, description="When the signed URL will expire" - ) - existing_file: Optional[bool] = Field( - None, description="Whether an existing file with the same hash was found" - ) - - -class PikaBodyGenerate22T2vGenerate22T2vPost(BaseModel): - promptText: str = Field(..., title="Prompttext") - negativePrompt: Optional[str] = Field(None, title="Negativeprompt") - seed: Optional[int] = Field(None, title="Seed") - resolution: Optional[str] = Field("1080p", title="Resolution") - duration: Optional[int] = Field(5, title="Duration") - aspectRatio: Optional[float] = Field( - 1.7777777777777777, - description="Aspect ratio (width / height)", - ge=0.4, - le=2.5, - title="Aspectratio", - ) - - -class PikaGenerateResponse(BaseModel): - video_id: str = Field(..., title="Video Id") - - -class PikaBodyGenerate22I2vGenerate22I2vPost(BaseModel): - image: Optional[str] = Field(None, title="Image") - promptText: Optional[str] = Field(None, title="Prompttext") - negativePrompt: Optional[str] = Field(None, title="Negativeprompt") - seed: Optional[int] = Field(None, title="Seed") - resolution: Optional[str] = Field("1080p", title="Resolution") - duration: Optional[int] = Field(5, title="Duration") - - -class IngredientsMode(str, Enum): - creative = "creative" - precise = "precise" - - -class AspectRatio3(RootModel[float]): - root: float = Field( - ..., - description="Aspect ratio (width / height)", - ge=0.4, - le=2.5, - title="Aspectratio", - ) - - -class PikaBodyGenerate22C2vGenerate22PikascenesPost(BaseModel): - images: Optional[List[bytes_aliased]] = Field( - None, description="Array of images to process", title="Images" - ) - ingredientsMode: IngredientsMode = Field(..., title="Ingredientsmode") - promptText: Optional[str] = Field(None, title="Prompttext") - negativePrompt: Optional[str] = Field(None, title="Negativeprompt") - seed: Optional[int] = Field(None, title="Seed") - resolution: Optional[str] = Field("1080p", title="Resolution") - duration: Optional[int] = Field(5, title="Duration") - aspectRatio: Optional[AspectRatio3] = Field( - None, description="Aspect ratio (width / height)", title="Aspectratio" - ) - - -class PikaBodyGenerate22KeyframeGenerate22PikaframesPost(BaseModel): - keyFrames: List[bytes_aliased] = Field( - ..., description="Array of keyframe images", title="Keyframes" - ) - promptText: str = Field(..., title="Prompttext") - negativePrompt: Optional[str] = Field(None, title="Negativeprompt") - seed: Optional[int] = Field(None, title="Seed") - resolution: Optional[str] = Field("1080p", title="Resolution") - duration: Optional[int] = Field(5, title="Duration") - - -class PikaStatusEnum(str, Enum): - queued = "queued" - started = "started" - finished = "finished" - - -class PikaValidationError(BaseModel): - loc: List[Union[str, int]] = Field(..., title="Location") - msg: str = Field(..., title="Message") - type: str = Field(..., title="Error Type") - - -class RgbItem(RootModel[int]): - root: int = Field(..., ge=0, le=255) - - -class RGBColor(BaseModel): - rgb: List[RgbItem] = Field(..., max_length=3, min_length=3) - - -class StabilityStabilityClientID(RootModel[str]): - root: str = Field( - ..., - description="The name of your application, used to help us communicate app-specific debugging or moderation issues to you.", - examples=["my-awesome-app"], - max_length=256, - ) - - -class StabilityStabilityClientUserID(RootModel[str]): - root: str = Field( - ..., - description="A unique identifier for your end user. Used to help us communicate user-specific debugging or moderation issues to you. Feel free to obfuscate this value to protect user privacy.", - examples=["DiscordUser#9999"], - max_length=256, - ) - - -class StabilityStabilityClientVersion(RootModel[str]): - root: str = Field( - ..., - description="The version of your application, used to help us communicate version-specific debugging or moderation issues to you.", - examples=["1.2.1"], - max_length=256, - ) - - -class Name(str, Enum): - content_moderation = "content_moderation" - - -class StabilityContentModerationResponse(BaseModel): - id: str = Field( - ..., - description="A unique identifier associated with this error. Please include this in any [support tickets](https://kb.stability.ai/knowledge-base/kb-tickets/new) you file, as it will greatly assist us in diagnosing the root cause of the problem.", - examples=["a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4"], - min_length=1, - ) - name: Name = Field( - ..., - description="Our content moderation system has flagged some part of your request and subsequently denied it. You were not charged for this request. While this may at times be frustrating, it is necessary to maintain the integrity of our platform and ensure a safe experience for all users. If you would like to provide feedback, please use the [Support Form](https://kb.stability.ai/knowledge-base/kb-tickets/new).", - ) - errors: List[str] = Field( - ..., - description="One or more error messages indicating what went wrong.", - examples=[["some-field: is required"]], - min_length=1, - ) - - -class RenderingSpeed(str, Enum): - BALANCED = "BALANCED" - TURBO = "TURBO" - QUALITY = "QUALITY" - - -class ActionJobResult(BaseModel): - id: Optional[UUID] = Field(None, description="Unique identifier for the job result") - workflow_name: Optional[str] = Field(None, description="Name of the workflow") - operating_system: Optional[str] = Field(None, description="Operating system used") - python_version: Optional[str] = Field(None, description="PyTorch version used") - pytorch_version: Optional[str] = Field(None, description="PyTorch version used") - action_run_id: Optional[str] = Field( - None, description="Identifier of the run this result belongs to" - ) - action_job_id: Optional[str] = Field( - None, description="Identifier of the job this result belongs to" - ) - cuda_version: Optional[str] = Field(None, description="CUDA version used") - branch_name: Optional[str] = Field( - None, description="Name of the relevant git branch" - ) - commit_hash: Optional[str] = Field(None, description="The hash of the commit") - commit_id: Optional[str] = Field(None, description="The ID of the commit") - commit_time: Optional[int] = Field( - None, description="The Unix timestamp when the commit was made" - ) - commit_message: Optional[str] = Field(None, description="The message of the commit") - comfy_run_flags: Optional[str] = Field( - None, description="The comfy run flags. E.g. `--low-vram`" - ) - git_repo: Optional[str] = Field(None, description="The repository name") - pr_number: Optional[str] = Field(None, description="The pull request number") - start_time: Optional[int] = Field( - None, description="The start time of the job as a Unix timestamp." - ) - end_time: Optional[int] = Field( - None, description="The end time of the job as a Unix timestamp." - ) - avg_vram: Optional[int] = Field( - None, description="The average VRAM used by the job" - ) - peak_vram: Optional[int] = Field(None, description="The peak VRAM used by the job") - job_trigger_user: Optional[str] = Field( - None, description="The user who triggered the job." - ) - author: Optional[str] = Field(None, description="The author of the commit") - machine_stats: Optional[MachineStats] = None - status: Optional[WorkflowRunStatus] = None - storage_file: Optional[StorageFile] = None - - -class Publisher(BaseModel): - name: Optional[str] = None - id: Optional[str] = Field( - None, - description="The unique identifier for the publisher. It's akin to a username. Should be lowercase.", - ) - description: Optional[str] = None - website: Optional[str] = None - support: Optional[str] = None - source_code_repo: Optional[str] = None - logo: Optional[str] = Field(None, description="URL to the publisher's logo.") - createdAt: Optional[datetime] = Field( - None, description="The date and time the publisher was created." - ) - members: Optional[List[PublisherMember]] = Field( - None, description="A list of members in the publisher." - ) - status: Optional[PublisherStatus] = Field( - None, description="The status of the publisher." - ) - - -class NodeVersion(BaseModel): - id: Optional[str] = None - version: Optional[str] = Field( - None, - description="The version identifier, following semantic versioning. Must be unique for the node.", - ) - createdAt: Optional[datetime] = Field( - None, description="The date and time the version was created." - ) - changelog: Optional[str] = Field( - None, description="Summary of changes made in this version" - ) - dependencies: Optional[List[str]] = Field( - None, description="A list of pip dependencies required by the node." - ) - downloadUrl: Optional[str] = Field( - None, description="[Output Only] URL to download this version of the node" - ) - deprecated: Optional[bool] = Field( - None, description="Indicates if this version is deprecated." - ) - status: Optional[NodeVersionStatus] = Field( - None, description="The status of the node version." - ) - status_reason: Optional[str] = Field( - None, description="The reason for the status change." - ) - node_id: Optional[str] = Field( - None, description="The unique identifier of the node." - ) - comfy_node_extract_status: Optional[str] = Field( - None, description="The status of comfy node extraction process." - ) - - -class IdeogramV3Request(BaseModel): - prompt: str = Field(..., description="The text prompt for image generation") - seed: Optional[int] = Field( - None, description="Seed value for reproducible generation" - ) - resolution: Optional[str] = Field( - None, description="Image resolution in format WxH", examples=["1280x800"] - ) - aspect_ratio: Optional[str] = Field( - None, description="Aspect ratio in format WxH", examples=["1x3"] - ) - rendering_speed: RenderingSpeed - magic_prompt: Optional[MagicPrompt] = Field( - None, description="Whether to enable magic prompt enhancement" - ) - negative_prompt: Optional[str] = Field( - None, description="Text prompt specifying what to avoid in the generation" - ) - num_images: Optional[int] = Field( - None, description="Number of images to generate", ge=1 - ) - color_palette: Optional[ColorPalette] = None - style_codes: Optional[List[StyleCode]] = Field( - None, description="Array of style codes in hexadecimal format" - ) - style_type: Optional[StyleType] = Field( - None, description="The type of style to apply" - ) - style_reference_images: Optional[List[str]] = Field( - None, description="Array of reference image URLs or identifiers" + description="Operation resource name", + examples=[ + "projects/PROJECT_ID/locations/us-central1/publishers/google/models/MODEL_ID/operations/a1b07c8e-7b5a-4aba-bb34-3e1ccb8afcc8" + ], ) class IdeogramV3EditRequest(BaseModel): + color_palette: Optional[IdeogramColorPalette] = None image: Optional[bytes_aliased] = Field( None, description="The image being edited (max size 10MB); only JPEG, WebP and PNG formats are supported at this time.", ) - mask: Optional[bytes_aliased] = Field( - None, - description="A black and white image of the same size as the image being edited (max size 10MB). Black regions in the mask should match up with the regions of the image that you would like to edit; only JPEG, WebP and PNG formats are supported at this time.", - ) - prompt: str = Field( - ..., description="The prompt used to describe the edited result." - ) magic_prompt: Optional[str] = Field( None, description="Determine if MagicPrompt should be used in generating the request or not.", ) + mask: Optional[bytes_aliased] = Field( + None, + description="A black and white image of the same size as the image being edited (max size 10MB). Black regions in the mask should match up with the regions of the image that you would like to edit; only JPEG, WebP and PNG formats are supported at this time.", + ) num_images: Optional[int] = Field( None, description="The number of images to generate." ) - seed: Optional[int] = Field( - None, description="Random seed. Set for reproducible generation." + prompt: str = Field( + ..., description="The prompt used to describe the edited result." ) rendering_speed: RenderingSpeed - color_palette: Optional[IdeogramColorPalette] = Field( - None, - description="A color palette for generation, must EITHER be specified via one of the presets (name) or explicitly via hexadecimal representations of the color with optional weights (members). Not supported by V_1, V_1_TURBO, V_2A and V_2A_TURBO models.", + seed: Optional[int] = Field( + None, description="Random seed. Set for reproducible generation." ) style_codes: Optional[List[StyleCode]] = Field( None, @@ -1799,90 +1195,144 @@ class IdeogramV3EditRequest(BaseModel): ) -class StripeBillingDetails(BaseModel): - address: Optional[StripeAddress] = None - email: Optional[str] = None - name: Optional[str] = None - phone: Optional[str] = None - tax_id: Optional[Any] = None - - -class StripePaymentMethodDetails(BaseModel): - card: Optional[StripeCardDetails] = None - type: Optional[str] = None - - -class Controls(BaseModel): - artistic_level: Optional[int] = Field( - None, - description="Defines artistic tone of your image. At a simple level, the person looks straight at the camera in a static and clean style. Dynamic and eccentric levels introduce movement and creativity.", - ge=0, - le=5, +class IdeogramV3Request(BaseModel): + aspect_ratio: Optional[str] = Field( + None, description="Aspect ratio in format WxH", examples=["1x3"] ) - colors: Optional[List[RGBColor]] = Field( - None, description="An array of preferable colors" + color_palette: Optional[ColorPalette] = None + magic_prompt: Optional[MagicPrompt] = Field( + None, description="Whether to enable magic prompt enhancement" ) - background_color: Optional[RGBColor] = Field( - None, description="Use given color as a desired background color" + negative_prompt: Optional[str] = Field( + None, description="Text prompt specifying what to avoid in the generation" ) - no_text: Optional[bool] = Field(None, description="Do not embed text layouts") - - -class RecraftImageGenerationRequest(BaseModel): - prompt: str = Field( - ..., description="The text prompt describing the image to generate" + num_images: Optional[int] = Field( + None, description="Number of images to generate", ge=1 ) - model: str = Field( - ..., description='The model to use for generation (e.g., "recraftv3")' + prompt: str = Field(..., description="The text prompt for image generation") + rendering_speed: RenderingSpeed + resolution: Optional[str] = Field( + None, description="Image resolution in format WxH", examples=["1280x800"] ) - style: Optional[str] = Field( - None, - description='The style to apply to the generated image (e.g., "digital_illustration")', + seed: Optional[int] = Field( + None, description="Seed value for reproducible generation" ) - style_id: Optional[str] = Field( - None, - description='The style ID to apply to the generated image (e.g., "123e4567-e89b-12d3-a456-426614174000"). If style_id is provided, style should not be provided.', + style_codes: Optional[List[StyleCode]] = Field( + None, description="Array of style codes in hexadecimal format" ) - size: str = Field( - ..., description='The size of the generated image (e.g., "1024x1024")' + style_reference_images: Optional[List[str]] = Field( + None, description="Array of reference image URLs or identifiers" ) - controls: Optional[Controls] = Field( - None, description="The controls for the generated image" + style_type: Optional[StyleType] = Field( + None, description="The type of style to apply" ) - n: int = Field(..., description="The number of images to generate", ge=1, le=4) - - -class LumaKeyframes(BaseModel): - frame0: Optional[LumaKeyframe] = None - frame1: Optional[LumaKeyframe] = None class LumaGenerationRequest(BaseModel): - generation_type: Optional[GenerationType] = "video" - prompt: str = Field(..., description="The prompt of the generation") aspect_ratio: LumaAspectRatio - loop: Optional[bool] = Field(None, description="Whether to loop the video") - keyframes: Optional[LumaKeyframes] = None callback_url: Optional[AnyUrl] = Field( None, description="The callback URL of the generation, a POST request with Generation object will be sent to the callback URL when the generation is dreaming, completed, or failed", ) - model: LumaVideoModel - resolution: LumaVideoModelOutputResolution duration: LumaVideoModelOutputDuration + generation_type: Optional[GenerationType1] = "video" + keyframes: Optional[LumaKeyframes] = None + loop: Optional[bool] = Field(None, description="Whether to loop the video") + model: LumaVideoModel + prompt: str = Field(..., description="The prompt of the generation") + resolution: LumaVideoModelOutputResolution + + +class CharacterRef(BaseModel): + identity0: Optional[LumaImageIdentity] = None + + +class LumaImageGenerationRequest(BaseModel): + aspect_ratio: Optional[LumaAspectRatio] = "16:9" + callback_url: Optional[AnyUrl] = Field( + None, description="The callback URL for the generation" + ) + character_ref: Optional[CharacterRef] = None + generation_type: Optional[GenerationType2] = "image" + image_ref: Optional[List[LumaImageRef]] = None + model: Optional[LumaImageModel] = "photon-1" + modify_image_ref: Optional[LumaModifyImageRef] = None + prompt: Optional[str] = Field(None, description="The prompt of the generation") + style_ref: Optional[List[LumaImageRef]] = None + + +class LumaUpscaleVideoGenerationRequest(BaseModel): + callback_url: Optional[AnyUrl] = Field( + None, description="The callback URL for the upscale" + ) + generation_type: Optional[GenerationType3] = "upscale_video" + resolution: Optional[LumaVideoModelOutputResolution] = None + + +class PikaBodyGenerate22C2vGenerate22PikascenesPost(BaseModel): + aspectRatio: Optional[AspectRatio2] = Field( + None, description="Aspect ratio (width / height)", title="Aspectratio" + ) + duration: Optional[PikaDurationEnum] = 5 + images: Optional[List[bytes_aliased]] = Field( + None, description="Array of images to process", title="Images" + ) + ingredientsMode: IngredientsMode = Field(..., title="Ingredientsmode") + negativePrompt: Optional[str] = Field(None, title="Negativeprompt") + promptText: Optional[str] = Field(None, title="Prompttext") + resolution: Optional[PikaResolutionEnum] = "1080p" + seed: Optional[int] = Field(None, title="Seed") + + +class PikaBodyGenerate22I2vGenerate22I2vPost(BaseModel): + duration: Optional[PikaDurationEnum] = 5 + image: Optional[str] = Field(None, title="Image") + negativePrompt: Optional[str] = Field(None, title="Negativeprompt") + promptText: Optional[str] = Field(None, title="Prompttext") + resolution: Optional[PikaResolutionEnum] = "1080p" + seed: Optional[int] = Field(None, title="Seed") + + +class PikaBodyGenerate22KeyframeGenerate22PikaframesPost(BaseModel): + duration: Optional[int] = Field(None, ge=5, le=10, title="Duration") + keyFrames: List[bytes_aliased] = Field( + ..., description="Array of keyframe images", title="Keyframes" + ) + negativePrompt: Optional[str] = Field(None, title="Negativeprompt") + promptText: str = Field(..., title="Prompttext") + resolution: Optional[PikaResolutionEnum] = "1080p" + seed: Optional[int] = Field(None, title="Seed") + + +class PikaBodyGenerate22T2vGenerate22T2vPost(BaseModel): + aspectRatio: Optional[float] = Field( + 1.7777777777777777, + description="Aspect ratio (width / height)", + ge=0.4, + le=2.5, + title="Aspectratio", + ) + duration: Optional[PikaDurationEnum] = 5 + negativePrompt: Optional[str] = Field(None, title="Negativeprompt") + promptText: str = Field(..., title="Prompttext") + resolution: Optional[PikaResolutionEnum] = "1080p" + seed: Optional[int] = Field(None, title="Seed") + + +class PikaHTTPValidationError(BaseModel): + detail: Optional[List[PikaValidationError]] = Field(None, title="Detail") class LumaGeneration(BaseModel): - id: Optional[UUID] = Field(None, description="The ID of the generation") - generation_type: Optional[LumaGenerationType] = None - state: Optional[LumaState] = None - failure_reason: Optional[str] = Field( - None, description="The reason for the state of the generation" - ) + assets: Optional[LumaAssets] = None created_at: Optional[datetime] = Field( None, description="The date and time when the generation was created" ) - assets: Optional[LumaAssets] = None + failure_reason: Optional[str] = Field( + None, description="The reason for the state of the generation" + ) + generation_type: Optional[LumaGenerationType] = None + id: Optional[UUID] = Field(None, description="The ID of the generation") model: Optional[str] = Field(None, description="The model used for the generation") request: Optional[ Union[ @@ -1892,189 +1342,4 @@ class LumaGeneration(BaseModel): LumaAudioGenerationRequest, ] ] = Field(None, description="The request of the generation") - - -class RunwayImageToVideoRequest(BaseModel): - promptImage: RunwayPromptImageObject - seed: int = Field( - ..., description="Random seed for generation", ge=0, le=4294967295 - ) - model: RunwayModelEnum = Field(..., description="Model to use for generation") - promptText: Optional[str] = Field( - None, description="Text prompt for the generation", max_length=1000 - ) - duration: RunwayDurationEnum = Field( - ..., description="The number of seconds of duration for the output video." - ) - ratio: RunwayAspectRatioEnum = Field( - ..., - description="The resolution (aspect ratio) of the output video. Allowable values depend on the selected model. 1280:768 and 768:1280 are only supported for gen3a_turbo.", - ) - - -class RunwayTaskStatusResponse(BaseModel): - id: Optional[str] = Field(None, description="Task ID") - status: Optional[RunwayTaskStatusEnum] = Field(None, description="Task status") - createdAt: Optional[datetime] = Field(None, description="Task creation timestamp") - output: Optional[List[str]] = Field(None, description="Array of output video URLs") - - -class PikaHTTPValidationError(BaseModel): - detail: Optional[List[PikaValidationError]] = Field(None, title="Detail") - - -class PikaVideoResponse(BaseModel): - id: str = Field(..., title="Id") - status: PikaStatusEnum = Field( - ..., description="The status of the video", title="Status" - ) - url: Optional[str] = Field(None, title="Url") - progress: Optional[int] = Field(None, title="Progress") - - -class Node(BaseModel): - id: Optional[str] = Field(None, description="The unique identifier of the node.") - name: Optional[str] = Field(None, description="The display name of the node.") - category: Optional[str] = Field(None, description="The category of the node.") - description: Optional[str] = None - author: Optional[str] = None - license: Optional[str] = Field( - None, description="The path to the LICENSE file in the node's repository." - ) - icon: Optional[str] = Field(None, description="URL to the node's icon.") - repository: Optional[str] = Field(None, description="URL to the node's repository.") - tags: Optional[List[str]] = None - latest_version: Optional[NodeVersion] = Field( - None, description="The latest version of the node." - ) - rating: Optional[float] = Field(None, description="The average rating of the node.") - downloads: Optional[int] = Field( - None, description="The number of downloads of the node." - ) - publisher: Optional[Publisher] = Field( - None, description="The publisher of the node." - ) - status: Optional[NodeStatus] = Field(None, description="The status of the node.") - status_detail: Optional[str] = Field( - None, description="The status detail of the node." - ) - translations: Optional[Dict[str, Dict[str, Any]]] = None - - -class StripeCharge(BaseModel): - id: Optional[str] = None - object: Optional[Object2] = None - amount: Optional[int] = None - amount_captured: Optional[int] = None - amount_refunded: Optional[int] = None - application: Optional[str] = None - application_fee: Optional[str] = None - application_fee_amount: Optional[int] = None - balance_transaction: Optional[str] = None - billing_details: Optional[StripeBillingDetails] = None - calculated_statement_descriptor: Optional[str] = None - captured: Optional[bool] = None - created: Optional[int] = None - currency: Optional[str] = None - customer: Optional[str] = None - description: Optional[str] = None - destination: Optional[Any] = None - dispute: Optional[Any] = None - disputed: Optional[bool] = None - failure_balance_transaction: Optional[Any] = None - failure_code: Optional[Any] = None - failure_message: Optional[Any] = None - fraud_details: Optional[Dict[str, Any]] = None - invoice: Optional[Any] = None - livemode: Optional[bool] = None - metadata: Optional[Dict[str, Any]] = None - on_behalf_of: Optional[Any] = None - order: Optional[Any] = None - outcome: Optional[StripeOutcome] = None - paid: Optional[bool] = None - payment_intent: Optional[str] = None - payment_method: Optional[str] = None - payment_method_details: Optional[StripePaymentMethodDetails] = None - radar_options: Optional[Dict[str, Any]] = None - receipt_email: Optional[str] = None - receipt_number: Optional[str] = None - receipt_url: Optional[str] = None - refunded: Optional[bool] = None - refunds: Optional[StripeRefundList] = None - review: Optional[Any] = None - shipping: Optional[StripeShipping] = None - source: Optional[Any] = None - source_transfer: Optional[Any] = None - statement_descriptor: Optional[Any] = None - statement_descriptor_suffix: Optional[Any] = None - status: Optional[str] = None - transfer_data: Optional[Any] = None - transfer_group: Optional[Any] = None - - -class StripeChargeList(BaseModel): - object: Optional[str] = None - data: Optional[List[StripeCharge]] = None - has_more: Optional[bool] = None - total_count: Optional[int] = None - url: Optional[str] = None - - -class StripePaymentIntent(BaseModel): - id: Optional[str] = None - object: Optional[Object1] = None - amount: Optional[int] = None - amount_capturable: Optional[int] = None - amount_details: Optional[StripeAmountDetails] = None - amount_received: Optional[int] = None - application: Optional[str] = None - application_fee_amount: Optional[int] = None - automatic_payment_methods: Optional[Any] = None - canceled_at: Optional[int] = None - cancellation_reason: Optional[str] = None - capture_method: Optional[str] = None - charges: Optional[StripeChargeList] = None - client_secret: Optional[str] = None - confirmation_method: Optional[str] = None - created: Optional[int] = None - currency: Optional[str] = None - customer: Optional[str] = None - description: Optional[str] = None - invoice: Optional[str] = None - last_payment_error: Optional[Any] = None - latest_charge: Optional[str] = None - livemode: Optional[bool] = None - metadata: Optional[Dict[str, Any]] = None - next_action: Optional[Any] = None - on_behalf_of: Optional[Any] = None - payment_method: Optional[str] = None - payment_method_configuration_details: Optional[Any] = None - payment_method_options: Optional[StripePaymentMethodOptions] = None - payment_method_types: Optional[List[str]] = None - processing: Optional[Any] = None - receipt_email: Optional[str] = None - review: Optional[Any] = None - setup_future_usage: Optional[Any] = None - shipping: Optional[StripeShipping] = None - source: Optional[Any] = None - statement_descriptor: Optional[Any] = None - statement_descriptor_suffix: Optional[Any] = None - status: Optional[str] = None - transfer_data: Optional[Any] = None - transfer_group: Optional[Any] = None - - -class Data2(BaseModel): - object: Optional[StripePaymentIntent] = None - - -class StripeEvent(BaseModel): - id: str - object: Object - api_version: Optional[str] = None - created: Optional[int] = None - data: Data2 - livemode: Optional[bool] = None - pending_webhooks: Optional[int] = None - request: Optional[StripeRequestInfo] = None - type: Type2 + state: Optional[LumaState] = None diff --git a/comfy_api_nodes/nodes_pika.py b/comfy_api_nodes/nodes_pika.py index 230af9669..ad44a6e60 100644 --- a/comfy_api_nodes/nodes_pika.py +++ b/comfy_api_nodes/nodes_pika.py @@ -10,6 +10,8 @@ from comfy_api_nodes.apis import ( PikaVideoResponse, PikaBodyGenerate22C2vGenerate22PikascenesPost, IngredientsMode, + PikaDurationEnum, + PikaResolutionEnum, ) from comfy_api_nodes.apis.client import ( ApiEndpoint, @@ -82,14 +84,16 @@ class PikaNodeBase(ComfyNodeABC): control_after_generate=True, ), "resolution": model_field_to_node_input( - IO.STRING, + IO.COMBO, request_model, "resolution", + enum_type=PikaResolutionEnum, ), "duration": model_field_to_node_input( - IO.INT, + IO.COMBO, request_model, "duration", + enum_type=PikaDurationEnum, ), } @@ -234,6 +238,9 @@ class PikaTextToVideoNodeV2_2(PikaNodeBase): PikaBodyGenerate22T2vGenerate22T2vPost, "aspectRatio", step=0.001, + min=0.4, + max=2.5, + default=1.7777777777777777, ), }, "hidden": { @@ -303,6 +310,8 @@ class PikaScenesV2_2(PikaNodeBase): PikaBodyGenerate22C2vGenerate22PikascenesPost, "aspectRatio", step=0.001, + min=0.4, + max=2.5, default=1.7777777777777777, ), }, From 8fd2f0e71b659a385828122442b45b2a573dbf3f Mon Sep 17 00:00:00 2001 From: Robin Huang Date: Thu, 1 May 2025 00:28:50 -0700 Subject: [PATCH 070/121] Change base branch to master. Not main. (#95) --- .github/workflows/update-api-stubs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/update-api-stubs.yml b/.github/workflows/update-api-stubs.yml index fe9e4cd6a..c99ec9fc1 100644 --- a/.github/workflows/update-api-stubs.yml +++ b/.github/workflows/update-api-stubs.yml @@ -53,4 +53,4 @@ jobs: Generated automatically by the a Github workflow. branch: update-api-stubs delete-branch: true - base: main + base: master From 24d0e9001fc882a79f6ddfb3a455f0ceceb35e88 Mon Sep 17 00:00:00 2001 From: Jedrzej Kosinski Date: Thu, 1 May 2025 02:59:39 -0500 Subject: [PATCH 071/121] Fix UploadRequest file_name param (#98) --- comfy_api_nodes/apinode_utils.py | 4 ++-- comfy_api_nodes/apis/client.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/comfy_api_nodes/apinode_utils.py b/comfy_api_nodes/apinode_utils.py index 5a507a6ac..e5d0e29a9 100644 --- a/comfy_api_nodes/apinode_utils.py +++ b/comfy_api_nodes/apinode_utils.py @@ -331,10 +331,10 @@ def upload_images_to_comfyapi( img_binary = tensor_to_bytesio(curr_image, mime_type=mime_type) # first, request upload/download urls from comfy API if not mime_type: - request_object = UploadRequest(filename=img_binary.name) + request_object = UploadRequest(file_name=img_binary.name) else: request_object = UploadRequest( - filename=img_binary.name, content_type=mime_type + file_name=img_binary.name, content_type=mime_type ) operation = SynchronousOperation( endpoint=ApiEndpoint( diff --git a/comfy_api_nodes/apis/client.py b/comfy_api_nodes/apis/client.py index e9c68bf5b..032c02b2e 100644 --- a/comfy_api_nodes/apis/client.py +++ b/comfy_api_nodes/apis/client.py @@ -128,7 +128,7 @@ class EmptyRequest(BaseModel): class UploadRequest(BaseModel): - filename: str = Field(..., description="Filename to upload") + file_name: str = Field(..., description="Filename to upload") content_type: str | None = Field( None, description="Mime type of the file. For example: image/png, image/jpeg, video/mp4, etc.", From 5bc1aeaf457ee633e2100877ed9acd409541793c Mon Sep 17 00:00:00 2001 From: Jedrzej Kosinski Date: Thu, 1 May 2025 03:39:34 -0500 Subject: [PATCH 072/121] Removed Infinite Style Library until later (#99) --- comfy_api_nodes/nodes_recraft.py | 30 ------------------------------ 1 file changed, 30 deletions(-) diff --git a/comfy_api_nodes/nodes_recraft.py b/comfy_api_nodes/nodes_recraft.py index 80cccd969..0b98f8eb7 100644 --- a/comfy_api_nodes/nodes_recraft.py +++ b/comfy_api_nodes/nodes_recraft.py @@ -225,34 +225,6 @@ class RecraftStyleV3LogoRasterNode(RecraftStyleV3RealisticImageNode): RECRAFT_STYLE = RecraftStyleV3.logo_raster -class RecraftStyleInfiniteStyleLibrary: - """ - Select style based on preexisting UUID from the Infinite Style Library. - """ - - RETURN_TYPES = (RecraftIO.STYLEV3,) - RETURN_NAMES = ("recraft_style",) - DESCRIPTION = cleandoc(__doc__ or "") # Handle potential None value - FUNCTION = "create_style" - CATEGORY = "api node/image/Recraft" - - @classmethod - def INPUT_TYPES(s): - return { - "required": { - "style_id": (IO.STRING, { - "default": "", - "tooltip": "UUID of style from Infinite Style Library.", - }) - } - } - - def create_style(self, style_id: str): - if not style_id: - raise Exception("The style_id input cannot be empty.") - return (RecraftStyle(style_id=style_id),) - - class RecraftTextToImageNode: """ Generates images synchronously based on prompt and resolution. @@ -511,7 +483,6 @@ NODE_CLASS_MAPPINGS = { "RecraftStyleV3RealisticImage": RecraftStyleV3RealisticImageNode, "RecraftStyleV3DigitalIllustration": RecraftStyleV3DigitalIllustrationNode, "RecraftStyleV3LogoRaster": RecraftStyleV3LogoRasterNode, - "RecraftStyleV3InfiniteStyleLibrary": RecraftStyleInfiniteStyleLibrary, "RecraftColorRGB": RecraftColorRGBNode, "RecraftControls": RecraftControlsNode, "SaveSVG": SaveSVGNode, @@ -524,7 +495,6 @@ NODE_DISPLAY_NAME_MAPPINGS = { "RecraftStyleV3RealisticImage": "Recraft Style - Realistic Image", "RecraftStyleV3DigitalIllustration": "Recraft Style - Digital Illustration", "RecraftStyleV3LogoRaster": "Recraft Style - Logo Raster", - "RecraftStyleV3InfiniteStyleLibrary": "Recraft Style - Infinite Style Library", "RecraftColorRGB": "Recraft Color RGB", "RecraftControls": "Recraft Controls", "SaveSVG": "Save SVG", From dec4dd9858af155dc89d5754f14e02484b5a7b45 Mon Sep 17 00:00:00 2001 From: thot experiment <94414189+thot-experiment@users.noreply.github.com> Date: Thu, 1 May 2025 11:34:03 -0700 Subject: [PATCH 073/121] fix ideogram style types (#100) --- comfy_api_nodes/nodes_ideogram.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/comfy_api_nodes/nodes_ideogram.py b/comfy_api_nodes/nodes_ideogram.py index 51e81286f..4ac871624 100644 --- a/comfy_api_nodes/nodes_ideogram.py +++ b/comfy_api_nodes/nodes_ideogram.py @@ -417,7 +417,7 @@ class IdeogramV2(ComfyNodeABC): "style_type": ( IO.COMBO, { - "options": ["NONE", "ANIME", "CINEMATIC", "CREATIVE", "DIGITAL_ART", "PHOTOGRAPHIC"], + "options": ["AUTO", "GENERAL", "REALISTIC", "DESIGN", "RENDER_3D", "ANIME"], "default": "NONE", "tooltip": "Style type for generation (V2 only)", }, From e710a19f9bc49446b8f0a003956410bb9dec08c2 Mon Sep 17 00:00:00 2001 From: thot experiment <94414189+thot-experiment@users.noreply.github.com> Date: Thu, 1 May 2025 13:06:27 -0700 Subject: [PATCH 074/121] fix multi image return (#101) close #96 --- comfy_api_nodes/nodes_ideogram.py | 54 ++++++++++++++++++++----------- 1 file changed, 36 insertions(+), 18 deletions(-) diff --git a/comfy_api_nodes/nodes_ideogram.py b/comfy_api_nodes/nodes_ideogram.py index 4ac871624..1142bd5e5 100644 --- a/comfy_api_nodes/nodes_ideogram.py +++ b/comfy_api_nodes/nodes_ideogram.py @@ -3,6 +3,7 @@ from inspect import cleandoc from PIL import Image import numpy as np import io +import torch from comfy_api_nodes.apis import ( IdeogramGenerateRequest, IdeogramGenerateResponse, @@ -209,14 +210,26 @@ V3_RESOLUTIONS= [ "1536x640" ] -def download_and_process_image(image_url): - """Helper function to download and process image from URL""" +def download_and_process_images(image_urls): + """Helper function to download and process multiple images from URLs""" - # Using functions from apinode_utils.py to handle downloading and processing - image_bytesio = download_url_to_bytesio(image_url) # Download image content to BytesIO - img_tensor = bytesio_to_image_tensor(image_bytesio, mode="RGB") # Convert to torch.Tensor with RGB mode + # Initialize list to store image tensors + image_tensors = [] + + for image_url in image_urls: + # Using functions from apinode_utils.py to handle downloading and processing + image_bytesio = download_url_to_bytesio(image_url) # Download image content to BytesIO + img_tensor = bytesio_to_image_tensor(image_bytesio, mode="RGB") # Convert to torch.Tensor with RGB mode + image_tensors.append(img_tensor) + + # Stack tensors to match (N, width, height, channels) + if image_tensors: + stacked_tensors = torch.cat(image_tensors, dim=0) + else: + raise Exception("No valid images were processed") + + return stacked_tensors - return img_tensor class IdeogramV1(ComfyNodeABC): """ @@ -340,12 +353,13 @@ class IdeogramV1(ComfyNodeABC): if not response.data or len(response.data) == 0: raise Exception("No images were generated in the response") - image_url = response.data[0].url - if not image_url: - raise Exception("No image URL was generated in the response") + image_urls = [image_data.url for image_data in response.data if image_data.url] - return (download_and_process_image(image_url),) + if not image_urls: + raise Exception("No image URLs were generated in the response") + + return (download_and_process_images(image_urls),) class IdeogramV2(ComfyNodeABC): @@ -511,12 +525,13 @@ class IdeogramV2(ComfyNodeABC): if not response.data or len(response.data) == 0: raise Exception("No images were generated in the response") - image_url = response.data[0].url - if not image_url: - raise Exception("No image URL was generated in the response") + image_urls = [image_data.url for image_data in response.data if image_data.url] - return (download_and_process_image(image_url),) + if not image_urls: + raise Exception("No image URLs were generated in the response") + + return (download_and_process_images(image_urls),) class IdeogramV3(ComfyNodeABC): """ @@ -734,12 +749,14 @@ class IdeogramV3(ComfyNodeABC): if not response.data or len(response.data) == 0: raise Exception("No images were generated in the response") - image_url = response.data[0].url - if not image_url: - raise Exception("No image URL was generated in the response") + image_urls = [image_data.url for image_data in response.data if image_data.url] + + if not image_urls: + raise Exception("No image URLs were generated in the response") + + return (download_and_process_images(image_urls),) - return (download_and_process_image(image_url),) NODE_CLASS_MAPPINGS = { "IdeogramV1": IdeogramV1, @@ -752,3 +769,4 @@ NODE_DISPLAY_NAME_MAPPINGS = { "IdeogramV2": "Ideogram V2", "IdeogramV3": "Ideogram V3", } + From ae27988af8b5d723f0342d1ae48c1062c634f6b0 Mon Sep 17 00:00:00 2001 From: thot experiment <94414189+thot-experiment@users.noreply.github.com> Date: Thu, 1 May 2025 14:22:41 -0700 Subject: [PATCH 075/121] add metadata saving to SVG (#102) --- comfy_api_nodes/nodes_recraft.py | 46 +++++++++++++++++++++++--------- 1 file changed, 33 insertions(+), 13 deletions(-) diff --git a/comfy_api_nodes/nodes_recraft.py b/comfy_api_nodes/nodes_recraft.py index 0b98f8eb7..c769923ed 100644 --- a/comfy_api_nodes/nodes_recraft.py +++ b/comfy_api_nodes/nodes_recraft.py @@ -23,7 +23,7 @@ from comfy_api_nodes.apinode_utils import ( download_url_to_bytesio, ) import folder_paths - +import json import os import torch from io import BytesIO @@ -70,22 +70,42 @@ class SaveSVGNode: filename_prefix += self.prefix_append full_output_folder, filename, counter, subfolder, filename_prefix = folder_paths.get_save_image_path(filename_prefix, self.output_dir) results = list() - for batch_number, svg_bytes in enumerate(svg.data): - # NOTE: no way to do metadata for SVG right now, maybe figure this out later - # metadata = None - # if not args.disable_metadata: - # metadata = PngInfo() - # if prompt is not None: - # metadata.add_text("prompt", json.dumps(prompt)) - # if extra_pnginfo is not None: - # for x in extra_pnginfo: - # metadata.add_text(x, json.dumps(extra_pnginfo[x])) + # Prepare metadata JSON + metadata_dict = {} + if prompt is not None: + metadata_dict["prompt"] = prompt + if extra_pnginfo is not None: + metadata_dict.update(extra_pnginfo) + + # Convert metadata to JSON string + metadata_json = json.dumps(metadata_dict, indent=2) if metadata_dict else None + + for batch_number, svg_bytes in enumerate(svg.data): filename_with_batch_num = filename.replace("%batch_num%", str(batch_number)) file = f"{filename_with_batch_num}_{counter:05}_.svg" + + # Read SVG content + svg_bytes.seek(0) + svg_content = svg_bytes.read().decode('utf-8') + + # Inject metadata if available + if metadata_json: + # Create metadata element with CDATA section + metadata_element = f""" + + +""" + # Insert metadata after opening svg tag using regex + import re + svg_content = re.sub(r'(]*>)', r'\1\n' + metadata_element, svg_content) + + # Write the modified SVG to file with open(os.path.join(full_output_folder, file), 'wb') as svg_file: - svg_bytes.seek(0) - svg_file.write(svg_bytes.read()) + svg_file.write(svg_content.encode('utf-8')) + results.append({ "filename": file, "subfolder": subfolder, From 7967154b13c7d3814401e108b0178da62561ac95 Mon Sep 17 00:00:00 2001 From: Christian Byrne Date: Thu, 1 May 2025 17:31:49 -0700 Subject: [PATCH 076/121] Bump templates version to include API node template workflows (#104) --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index f64a05947..bc79168c1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ comfyui-frontend-package==1.17.11 -comfyui-workflow-templates==0.1.3 +comfyui-workflow-templates==0.1.8 torch torchsde torchvision From 67e9395017c0a5c56772f8b23184c03d246617b4 Mon Sep 17 00:00:00 2001 From: Christian Byrne Date: Thu, 1 May 2025 17:40:05 -0700 Subject: [PATCH 077/121] Fix: `download_url_to_video_output` return type (#103) --- comfy_api_nodes/apinode_utils.py | 6 ++---- comfy_api_nodes/nodes_kling.py | 4 ++-- comfy_api_nodes/nodes_pika.py | 2 +- 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/comfy_api_nodes/apinode_utils.py b/comfy_api_nodes/apinode_utils.py index e5d0e29a9..3ff04c2a3 100644 --- a/comfy_api_nodes/apinode_utils.py +++ b/comfy_api_nodes/apinode_utils.py @@ -23,9 +23,7 @@ import uuid from io import BytesIO -def download_url_to_video_output( - video_url: str, timeout: int = None -) -> tuple[VideoFromFile]: +def download_url_to_video_output(video_url: str, timeout: int = None) -> VideoFromFile: """Downloads a video from a URL and returns a `VIDEO` output. Args: @@ -39,7 +37,7 @@ def download_url_to_video_output( error_msg = f"Failed to download video from {video_url}" logging.error(error_msg) raise ValueError(error_msg) - return (VideoFromFile(video_io),) + return VideoFromFile(video_io) def downscale_image_tensor(image, total_pixels=1536 * 1024) -> torch.Tensor: diff --git a/comfy_api_nodes/nodes_kling.py b/comfy_api_nodes/nodes_kling.py index ce29aa78b..ca64f8a8e 100644 --- a/comfy_api_nodes/nodes_kling.py +++ b/comfy_api_nodes/nodes_kling.py @@ -314,7 +314,7 @@ class KlingTextToVideoNode(KlingNodeBase): video_url = str(final_response.data.task_result.videos[0].url) logging.debug("Kling task %s succeeded. Video URL: %s", task_id, video_url) - return download_url_to_video_output(video_url) + return (download_url_to_video_output(video_url),) class KlingImage2VideoNode(KlingNodeBase): @@ -452,7 +452,7 @@ class KlingImage2VideoNode(KlingNodeBase): video_url = str(final_response.data.task_result.videos[0].url) logging.info("Attempting to download video from URL: %s", video_url) - return download_url_to_video_output(video_url) + return (download_url_to_video_output(video_url),) NODE_CLASS_MAPPINGS = { diff --git a/comfy_api_nodes/nodes_pika.py b/comfy_api_nodes/nodes_pika.py index ad44a6e60..825d570d7 100644 --- a/comfy_api_nodes/nodes_pika.py +++ b/comfy_api_nodes/nodes_pika.py @@ -158,7 +158,7 @@ class PikaNodeBase(ComfyNodeABC): video_url = str(final_response.url) logging.debug("Pika task %s succeeded. Video URL: %s", task_id, video_url) - return download_url_to_video_output(video_url) + return (download_url_to_video_output(video_url),) class PikaImageToVideoV2_2(PikaNodeBase): From f4f2e031e35ee9e94c3b9a167c1b274b42e9e112 Mon Sep 17 00:00:00 2001 From: thot experiment <94414189+thot-experiment@users.noreply.github.com> Date: Thu, 1 May 2025 18:23:41 -0700 Subject: [PATCH 078/121] fix 4o generation bug (#106) --- comfy_api_nodes/nodes_openai.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/comfy_api_nodes/nodes_openai.py b/comfy_api_nodes/nodes_openai.py index a734e8a95..612653a55 100644 --- a/comfy_api_nodes/nodes_openai.py +++ b/comfy_api_nodes/nodes_openai.py @@ -392,6 +392,7 @@ class OpenAIGPTImage1(ComfyNodeABC): ): model = "gpt-image-1" path = "/proxy/openai/images/generations" + content_type="application/json" request_class = OpenAIImageGenerationRequest img_binaries = [] mask_binary = None @@ -400,6 +401,7 @@ class OpenAIGPTImage1(ComfyNodeABC): if image is not None: path = "/proxy/openai/images/edits" request_class = OpenAIImageEditRequest + content_type="multipart/form-data", batch_size = image.shape[0] @@ -461,7 +463,7 @@ class OpenAIGPTImage1(ComfyNodeABC): size=size, ), files=files if files else None, - content_type="multipart/form-data", + content_type=content_type, auth_token=auth_token, ) From 3957839ff145fe226cc132b618f1954301c09797 Mon Sep 17 00:00:00 2001 From: Christian Byrne Date: Thu, 1 May 2025 20:01:41 -0700 Subject: [PATCH 079/121] Serve SVG files directly (#107) --- server.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/server.py b/server.py index f64ec27d4..33112c227 100644 --- a/server.py +++ b/server.py @@ -416,6 +416,17 @@ class PromptServer(): if os.path.isfile(file): if 'preview' in request.rel_url.query: + extension = os.path.splitext(filename)[-1] + if extension and extension.lower() == ".svg": + with open(file, "r") as f: + return web.Response( + body=f.read(), + content_type="image/svg+xml", + headers={ + "Content-Disposition": f'filename="{filename}"' + }, + ) + with Image.open(file) as img: preview_info = request.rel_url.query['preview'].split(';') image_format = preview_info[0] From 3850c47fe1c4ba1c1135ede88acd8dac2a8d9041 Mon Sep 17 00:00:00 2001 From: Jedrzej Kosinski Date: Fri, 2 May 2025 05:49:05 -0500 Subject: [PATCH 080/121] Add a bunch of nodes, 3 ready to use, the rest waiting for endpoint support (#108) --- comfy_api_nodes/apis/bfl_api.py | 97 ++++ comfy_api_nodes/apis/recraft_api.py | 10 +- comfy_api_nodes/apis/stability_api.py | 25 + comfy_api_nodes/nodes_bfl.py | 782 +++++++++++++++++++++++--- comfy_api_nodes/nodes_recraft.py | 575 ++++++++++++++++++- comfy_api_nodes/nodes_stability.py | 141 +++++ 6 files changed, 1557 insertions(+), 73 deletions(-) diff --git a/comfy_api_nodes/apis/bfl_api.py b/comfy_api_nodes/apis/bfl_api.py index 722f75eb3..c189038fb 100644 --- a/comfy_api_nodes/apis/bfl_api.py +++ b/comfy_api_nodes/apis/bfl_api.py @@ -11,7 +11,104 @@ class BFLOutputFormat(str, Enum): jpeg = 'jpeg' +class BFLFluxExpandImageRequest(BaseModel): + prompt: str = Field(..., description='The description of the changes you want to make. This text guides the expansion process, allowing you to specify features, styles, or modifications for the expanded areas.') + prompt_upsampling: Optional[bool] = Field( + None, description='Whether to perform upsampling on the prompt. If active, automatically modifies the prompt for more creative generation.' + ) + seed: Optional[int] = Field(None, description='The seed value for reproducibility.') + top: conint(ge=0, le=2048) = Field(..., description='Number of pixels to expand at the top of the image') + bottom: conint(ge=0, le=2048) = Field(..., description='Number of pixels to expand at the bottom of the image') + left: conint(ge=0, le=2048) = Field(..., description='Number of pixels to expand at the left side of the image') + right: conint(ge=0, le=2048) = Field(..., description='Number of pixels to expand at the right side of the image') + steps: conint(ge=15, le=50) = Field(..., description='Number of steps for the image generation process') + guidance: confloat(ge=1.5, le=100) = Field(..., description='Guidance strength for the image generation process') + safety_tolerance: Optional[conint(ge=0, le=6)] = Field( + 6, description='Tolerance level for input and output moderation. Between 0 and 6, 0 being most strict, 6 being least strict. Defaults to 2.' + ) + output_format: Optional[BFLOutputFormat] = Field( + BFLOutputFormat.png, description="Output format for the generated image. Can be 'jpeg' or 'png'.", examples=['png'] + ) + image: str = Field(None, description='A Base64-encoded string representing the image you wish to expand') + + +class BFLFluxFillImageRequest(BaseModel): + prompt: str = Field(..., description='The description of the changes you want to make. This text guides the expansion process, allowing you to specify features, styles, or modifications for the expanded areas.') + prompt_upsampling: Optional[bool] = Field( + None, description='Whether to perform upsampling on the prompt. If active, automatically modifies the prompt for more creative generation.' + ) + seed: Optional[int] = Field(None, description='The seed value for reproducibility.') + steps: conint(ge=15, le=50) = Field(..., description='Number of steps for the image generation process') + guidance: confloat(ge=1.5, le=100) = Field(..., description='Guidance strength for the image generation process') + safety_tolerance: Optional[conint(ge=0, le=6)] = Field( + 6, description='Tolerance level for input and output moderation. Between 0 and 6, 0 being most strict, 6 being least strict. Defaults to 2.' + ) + output_format: Optional[BFLOutputFormat] = Field( + BFLOutputFormat.png, description="Output format for the generated image. Can be 'jpeg' or 'png'.", examples=['png'] + ) + image: str = Field(None, description='A Base64-encoded string representing the image you wish to modify. Can contain alpha mask if desired.') + mask: str = Field(None, description='A Base64-encoded string representing the mask of the areas you with to modify.') + + +class BFLFluxCannyImageRequest(BaseModel): + prompt: str = Field(..., description='Text prompt for image generation') + prompt_upsampling: Optional[bool] = Field( + None, description='Whether to perform upsampling on the prompt. If active, automatically modifies the prompt for more creative generation.' + ) + canny_low_threshold: Optional[int] = Field(None, description='Low threshold for Canny edge detection') + canny_high_threshold: Optional[int] = Field(None, description='High threshold for Canny edge detection') + seed: Optional[int] = Field(None, description='The seed value for reproducibility.') + steps: conint(ge=15, le=50) = Field(..., description='Number of steps for the image generation process') + guidance: confloat(ge=1, le=100) = Field(..., description='Guidance strength for the image generation process') + safety_tolerance: Optional[conint(ge=0, le=6)] = Field( + 6, description='Tolerance level for input and output moderation. Between 0 and 6, 0 being most strict, 6 being least strict. Defaults to 2.' + ) + output_format: Optional[BFLOutputFormat] = Field( + BFLOutputFormat.png, description="Output format for the generated image. Can be 'jpeg' or 'png'.", examples=['png'] + ) + control_image: Optional[str] = Field(None, description='Base64 encoded image to use as control input if no preprocessed image is provided') + preprocessed_image: Optional[str] = Field(None, description='Optional pre-processed image that will bypass the control preprocessing step') + + +class BFLFluxDepthImageRequest(BaseModel): + prompt: str = Field(..., description='Text prompt for image generation') + prompt_upsampling: Optional[bool] = Field( + None, description='Whether to perform upsampling on the prompt. If active, automatically modifies the prompt for more creative generation.' + ) + seed: Optional[int] = Field(None, description='The seed value for reproducibility.') + steps: conint(ge=15, le=50) = Field(..., description='Number of steps for the image generation process') + guidance: confloat(ge=1, le=100) = Field(..., description='Guidance strength for the image generation process') + safety_tolerance: Optional[conint(ge=0, le=6)] = Field( + 6, description='Tolerance level for input and output moderation. Between 0 and 6, 0 being most strict, 6 being least strict. Defaults to 2.' + ) + output_format: Optional[BFLOutputFormat] = Field( + BFLOutputFormat.png, description="Output format for the generated image. Can be 'jpeg' or 'png'.", examples=['png'] + ) + control_image: Optional[str] = Field(None, description='Base64 encoded image to use as control input if no preprocessed image is provided') + preprocessed_image: Optional[str] = Field(None, description='Optional pre-processed image that will bypass the control preprocessing step') + + class BFLFluxProGenerateRequest(BaseModel): + prompt: str = Field(..., description='The text prompt for image generation.') + prompt_upsampling: Optional[bool] = Field( + None, description='Whether to perform upsampling on the prompt. If active, automatically modifies the prompt for more creative generation.' + ) + seed: Optional[int] = Field(None, description='The seed value for reproducibility.') + width: conint(ge=256, le=1440) = Field(1024, description='Width of the generated image in pixels. Must be a multiple of 32.') + height: conint(ge=256, le=1440) = Field(768, description='Height of the generated image in pixels. Must be a multiple of 32.') + safety_tolerance: Optional[conint(ge=0, le=6)] = Field( + 6, description='Tolerance level for input and output moderation. Between 0 and 6, 0 being most strict, 6 being least strict. Defaults to 2.' + ) + output_format: Optional[BFLOutputFormat] = Field( + BFLOutputFormat.png, description="Output format for the generated image. Can be 'jpeg' or 'png'.", examples=['png'] + ) + image_prompt: Optional[str] = Field(None, description='Optional image to remix in base64 format') + # image_prompt_strength: Optional[confloat(ge=0.0, le=1.0)] = Field( + # None, description='Blend between the prompt and the image prompt.' + # ) + + +class BFLFluxProUltraGenerateRequest(BaseModel): prompt: str = Field(..., description='The text prompt for image generation.') prompt_upsampling: Optional[bool] = Field( None, description='Whether to perform upsampling on the prompt. If active, automatically modifies the prompt for more creative generation.' diff --git a/comfy_api_nodes/apis/recraft_api.py b/comfy_api_nodes/apis/recraft_api.py index da6232992..e5060bcd9 100644 --- a/comfy_api_nodes/apis/recraft_api.py +++ b/comfy_api_nodes/apis/recraft_api.py @@ -5,7 +5,7 @@ from __future__ import annotations from enum import Enum from typing import Optional -from pydantic import BaseModel, Field, conint +from pydantic import BaseModel, Field, conint, confloat class RecraftColor: @@ -238,14 +238,15 @@ class RecraftControlsObject(BaseModel): class RecraftImageGenerationRequest(BaseModel): prompt: str = Field(..., description='The text prompt describing the image to generate') - size: RecraftImageSize = Field(..., description='The size of the generated image (e.g., "1024x1024")') + size: Optional[RecraftImageSize] = Field(None, description='The size of the generated image (e.g., "1024x1024")') n: conint(ge=1, le=6) = Field(..., description='The number of images to generate') - negative_prompts: Optional[str] = Field(None, description='A text description of undesired elements on an image') + negative_prompt: Optional[str] = Field(None, description='A text description of undesired elements on an image') model: Optional[RecraftModel] = Field(RecraftModel.recraftv3, description='The model to use for generation (e.g., "recraftv3")') style: Optional[str] = Field(None, description='The style to apply to the generated image (e.g., "digital_illustration")') substyle: Optional[str] = Field(None, description='The substyle to apply to the generated image, depending on the style input') controls: Optional[RecraftControlsObject] = Field(None, description='A set of custom parameters to tweak generation process') style_id: Optional[str] = Field(None, description='Use a previously uploaded style as a reference; UUID') + strength: Optional[confloat(ge=0.0, le=1.0)] = Field(None, description='Defines the difference with the original image, should lie in [0, 1], where 0 means almost identical, and 1 means miserable similarity') # text_layout @@ -257,4 +258,5 @@ class RecraftReturnedObject(BaseModel): class RecraftImageGenerationResponse(BaseModel): created: int = Field(..., description='Unix timestamp when the generation was created') credits: int = Field(..., description='Number of credits used for the generation') - data: list[RecraftReturnedObject] = Field(..., description=' Array of generated image information') + data: Optional[list[RecraftReturnedObject]] = Field(None, description='Array of generated image information') + image: Optional[RecraftReturnedObject] = Field(None, description='Single generated image') diff --git a/comfy_api_nodes/apis/stability_api.py b/comfy_api_nodes/apis/stability_api.py index 410884c6e..d8d4f4e6a 100644 --- a/comfy_api_nodes/apis/stability_api.py +++ b/comfy_api_nodes/apis/stability_api.py @@ -51,6 +51,31 @@ class StabilityStylePreset(str, Enum): tile_texture = "tile-texture" +class Stability_SD3_5_Model(str, Enum): + sd3_5_large = "sd3.5-large" + sd3_5_large_turbo = "sd3.5-large-turbo" + #sd3_5_medium = "sd3.5-medium" + + +class Stability_SD3_5_GenerationMode(str, Enum): + text_to_image = "text-to-image" + image_to_image = "image-to-image" + + +class StabilityStable3_5Request(BaseModel): + model: str = Field(...) + mode: str = Field(...) + prompt: str = Field(...) + negative_prompt: Optional[str] = Field(None) + aspect_ratio: Optional[str] = Field(None) + seed: Optional[int] = Field(None) + output_format: Optional[str] = Field(StabilityFormat.png.value) + image: Optional[str] = Field(None) + style_preset: Optional[str] = Field(None) + cfg_scale: float = Field(...) + strength: Optional[confloat(ge=0.0, le=1.0)] = Field(None) + + class StabilityStableUltraRequest(BaseModel): prompt: str = Field(...) negative_prompt: Optional[str] = Field(None) diff --git a/comfy_api_nodes/nodes_bfl.py b/comfy_api_nodes/nodes_bfl.py index 84ff267f2..fd58249c8 100644 --- a/comfy_api_nodes/nodes_bfl.py +++ b/comfy_api_nodes/nodes_bfl.py @@ -3,7 +3,12 @@ from inspect import cleandoc from comfy.comfy_types.node_typing import IO, ComfyNodeABC from comfy_api_nodes.apis.bfl_api import ( BFLStatus, + BFLFluxExpandImageRequest, + BFLFluxFillImageRequest, + BFLFluxCannyImageRequest, + BFLFluxDepthImageRequest, BFLFluxProGenerateRequest, + BFLFluxProUltraGenerateRequest, BFLFluxProGenerateResponse, ) from comfy_api_nodes.apis.client import ( @@ -25,6 +30,84 @@ import base64 import time +def convert_mask_to_image(mask: torch.Tensor): + """ + Make mask have the expected amount of dims (4) and channels (3) to be recognized as an image. + """ + mask = mask.unsqueeze(-1) + mask = torch.cat([mask]*3, dim=-1) + return mask + + +def handle_bfl_synchronous_operation( + operation: SynchronousOperation, timeout_bfl_calls=360 +): + response_api: BFLFluxProGenerateResponse = operation.execute() + return _poll_until_generated( + response_api.polling_url, timeout=timeout_bfl_calls + ) + +def _poll_until_generated(polling_url: str, timeout=360): + # used bfl-comfy-nodes to verify code implementation: + # https://github.com/black-forest-labs/bfl-comfy-nodes/tree/main + start_time = time.time() + retries_404 = 0 + max_retries_404 = 5 + retry_404_seconds = 2 + retry_202_seconds = 2 + retry_pending_seconds = 1 + request = requests.Request(method=HttpMethod.GET, url=polling_url) + # NOTE: should True loop be replaced with checking if workflow has been interrupted? + while True: + response = requests.Session().send(request.prepare()) + if response.status_code == 200: + result = response.json() + if result["status"] == BFLStatus.ready: + img_url = result["result"]["sample"] + img_response = requests.get(img_url) + return process_image_response(img_response) + elif result["status"] in [ + BFLStatus.request_moderated, + BFLStatus.content_moderated, + ]: + status = result["status"] + raise Exception( + f"BFL API did not return an image due to: {status}." + ) + elif result["status"] == BFLStatus.error: + raise Exception(f"BFL API encountered an error: {result}.") + elif result["status"] == BFLStatus.pending: + time.sleep(retry_pending_seconds) + continue + elif response.status_code == 404: + if retries_404 < max_retries_404: + retries_404 += 1 + time.sleep(retry_404_seconds) + continue + raise Exception( + f"BFL API could not find task after {max_retries_404} tries." + ) + elif response.status_code == 202: + time.sleep(retry_202_seconds) + elif time.time() - start_time > timeout: + raise Exception( + f"BFL API experienced a timeout; could not return request under {timeout} seconds." + ) + else: + raise Exception(f"BFL API encountered an error: {response.json()}") + +def convert_image_to_base64(image: torch.Tensor): + scaled_image = downscale_image_tensor(image, total_pixels=2048 * 2048) + # remove batch dimension if present + if len(scaled_image.shape) > 3: + scaled_image = scaled_image[0] + image_np = (scaled_image.numpy() * 255).astype(np.uint8) + img = Image.fromarray(image_np) + img_byte_arr = io.BytesIO() + img.save(img_byte_arr, format="PNG") + return base64.b64encode(img_byte_arr.getvalue()).decode() + + class FluxProUltraImageNode(ComfyNodeABC): """ Generates images synchronously based on prompt and resolution. @@ -133,10 +216,10 @@ class FluxProUltraImageNode(ComfyNodeABC): endpoint=ApiEndpoint( path="/proxy/bfl/flux-pro-1.1-ultra/generate", method=HttpMethod.POST, - request_model=BFLFluxProGenerateRequest, + request_model=BFLFluxProUltraGenerateRequest, response_model=BFLFluxProGenerateResponse, ), - request=BFLFluxProGenerateRequest( + request=BFLFluxProUltraGenerateRequest( prompt=prompt, prompt_upsampling=prompt_upsampling, seed=seed, @@ -151,7 +234,7 @@ class FluxProUltraImageNode(ComfyNodeABC): image_prompt=( image_prompt if image_prompt is None - else self._convert_image_to_base64(image_prompt) + else convert_image_to_base64(image_prompt) ), image_prompt_strength=( None if image_prompt is None else round(image_prompt_strength, 2) @@ -159,85 +242,650 @@ class FluxProUltraImageNode(ComfyNodeABC): ), auth_token=auth_token, ) - output_image = self._handle_bfl_synchronous_operation(operation) + output_image = handle_bfl_synchronous_operation(operation) return (output_image,) - def _handle_bfl_synchronous_operation( - self, operation: SynchronousOperation, timeout_bfl_calls=360 + + +class FluxProImageNode(ComfyNodeABC): + """ + Generates images synchronously based on prompt and resolution. + """ + + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "prompt": ( + IO.STRING, + { + "multiline": True, + "default": "", + "tooltip": "Prompt for the image generation", + }, + ), + "prompt_upsampling": ( + IO.BOOLEAN, + { + "default": False, + "tooltip": "Whether to perform upsampling on the prompt. If active, automatically modifies the prompt for more creative generation, but results are nondeterministic (same seed will not produce exactly the same result).", + }, + ), + "width": ( + IO.INT, + { + "default": 1024, + "min": 256, + "max": 1440, + "step": 32, + }, + ), + "height": ( + IO.INT, + { + "default": 768, + "min": 256, + "max": 1440, + "step": 32, + }, + ), + "seed": ( + IO.INT, + { + "default": 0, + "min": 0, + "max": 0xFFFFFFFFFFFFFFFF, + "control_after_generate": True, + "tooltip": "The random seed used for creating the noise.", + }, + ), + }, + "optional": { + "image_prompt": (IO.IMAGE,), + # "image_prompt_strength": ( + # IO.FLOAT, + # { + # "default": 0.1, + # "min": 0.0, + # "max": 1.0, + # "step": 0.01, + # "tooltip": "Blend between the prompt and the image prompt.", + # }, + # ), + }, + "hidden": { + "auth_token": "AUTH_TOKEN_COMFY_ORG", + }, + } + + RETURN_TYPES = (IO.IMAGE,) + DESCRIPTION = cleandoc(__doc__ or "") # Handle potential None value + FUNCTION = "api_call" + API_NODE = True + CATEGORY = "api node/image/bfl" + + def api_call( + self, + prompt: str, + prompt_upsampling, + width: int, + height: int, + seed=0, + image_prompt=None, + # image_prompt_strength=0.1, + auth_token=None, + **kwargs, ): - response_api: BFLFluxProGenerateResponse = operation.execute() - return self._poll_until_generated( - response_api.polling_url, timeout=timeout_bfl_calls + image_prompt = ( + image_prompt + if image_prompt is None + else convert_image_to_base64(image_prompt) + ) + + operation = SynchronousOperation( + endpoint=ApiEndpoint( + path="/proxy/bfl/flux-pro-1.1/generate", + method=HttpMethod.POST, + request_model=BFLFluxProGenerateRequest, + response_model=BFLFluxProGenerateResponse, + ), + request=BFLFluxProGenerateRequest( + prompt=prompt, + prompt_upsampling=prompt_upsampling, + width=width, + height=height, + seed=seed, + image_prompt=image_prompt, + ), + auth_token=auth_token, ) + output_image = handle_bfl_synchronous_operation(operation) + return (output_image,) - def _poll_until_generated(self, polling_url: str, timeout=360): - # used bfl-comfy-nodes to verify code implementation: - # https://github.com/black-forest-labs/bfl-comfy-nodes/tree/main - start_time = time.time() - retries_404 = 0 - max_retries_404 = 5 - retry_404_seconds = 2 - retry_202_seconds = 2 - retry_pending_seconds = 1 - request = requests.Request(method=HttpMethod.GET, url=polling_url) - # NOTE: should True loop be replaced with checking if workflow has been interrupted? - while True: - response = requests.Session().send(request.prepare()) - if response.status_code == 200: - result = response.json() - if result["status"] == BFLStatus.ready: - img_url = result["result"]["sample"] - img_response = requests.get(img_url) - return process_image_response(img_response) - elif result["status"] in [ - BFLStatus.request_moderated, - BFLStatus.content_moderated, - ]: - status = result["status"] - raise Exception( - f"BFL API did not return an image due to: {status}." - ) - elif result["status"] == BFLStatus.error: - raise Exception(f"BFL API encountered an error: {result}.") - elif result["status"] == BFLStatus.pending: - time.sleep(retry_pending_seconds) - continue - elif response.status_code == 404: - if retries_404 < max_retries_404: - retries_404 += 1 - time.sleep(retry_404_seconds) - continue - raise Exception( - f"BFL API could not find task after {max_retries_404} tries." - ) - elif response.status_code == 202: - time.sleep(retry_202_seconds) - elif time.time() - start_time > timeout: - raise Exception( - f"BFL API experienced a timeout; could not return request under {timeout} seconds." - ) - else: - raise Exception(f"BFL API encountered an error: {response.json()}") - def _convert_image_to_base64(self, image: torch.Tensor): - scaled_image = downscale_image_tensor(image, total_pixels=2048 * 2048) - # remove batch dimension if present - if len(scaled_image.shape) > 3: - scaled_image = scaled_image[0] - image_np = (scaled_image.numpy() * 255).astype(np.uint8) - img = Image.fromarray(image_np) - img_byte_arr = io.BytesIO() - img.save(img_byte_arr, format="PNG") - return base64.b64encode(img_byte_arr.getvalue()).decode() +class FluxProExpandNode(ComfyNodeABC): + """ + Outpaints image based on prompt. + """ + + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "image": (IO.IMAGE,), + "prompt": ( + IO.STRING, + { + "multiline": True, + "default": "", + "tooltip": "Prompt for the image generation", + }, + ), + "prompt_upsampling": ( + IO.BOOLEAN, + { + "default": False, + "tooltip": "Whether to perform upsampling on the prompt. If active, automatically modifies the prompt for more creative generation, but results are nondeterministic (same seed will not produce exactly the same result).", + }, + ), + "top": ( + IO.INT, + { + "default": 0, + "min": 0, + "max": 2048, + "tooltip": "Number of pixels to expand at the top of the image" + }, + ), + "bottom": ( + IO.INT, + { + "default": 0, + "min": 0, + "max": 2048, + "tooltip": "Number of pixels to expand at the bottom of the image" + }, + ), + "left": ( + IO.INT, + { + "default": 0, + "min": 0, + "max": 2048, + "tooltip": "Number of pixels to expand at the left side of the image" + }, + ), + "right": ( + IO.INT, + { + "default": 0, + "min": 0, + "max": 2048, + "tooltip": "Number of pixels to expand at the right side of the image" + }, + ), + "guidance": ( + IO.FLOAT, + { + "default": 60, + "min": 1.5, + "max": 100, + "tooltip": "Guidance strength for the image generation process" + }, + ), + "steps": ( + IO.INT, + { + "default": 50, + "min": 15, + "max": 50, + "tooltip": "Number of steps for the image generation process" + }, + ), + "seed": ( + IO.INT, + { + "default": 0, + "min": 0, + "max": 0xFFFFFFFFFFFFFFFF, + "control_after_generate": True, + "tooltip": "The random seed used for creating the noise.", + }, + ), + }, + "optional": { + }, + "hidden": { + "auth_token": "AUTH_TOKEN_COMFY_ORG", + }, + } + + RETURN_TYPES = (IO.IMAGE,) + DESCRIPTION = cleandoc(__doc__ or "") # Handle potential None value + FUNCTION = "api_call" + API_NODE = True + CATEGORY = "api node/image/bfl" + + def api_call( + self, + image: torch.Tensor, + prompt: str, + prompt_upsampling: bool, + top: int, + bottom: int, + left: int, + right: int, + steps: int, + guidance: float, + seed=0, + auth_token=None, + **kwargs, + ): + image = convert_image_to_base64(image) + + operation = SynchronousOperation( + endpoint=ApiEndpoint( + path="/proxy/bfl/flux-pro-1.0-expand/generate", + method=HttpMethod.POST, + request_model=BFLFluxExpandImageRequest, + response_model=BFLFluxProGenerateResponse, + ), + request=BFLFluxExpandImageRequest( + prompt=prompt, + prompt_upsampling=prompt_upsampling, + top=top, + bottom=bottom, + left=left, + right=right, + steps=steps, + guidance=guidance, + seed=seed, + image=image, + ), + auth_token=auth_token, + ) + output_image = handle_bfl_synchronous_operation(operation) + return (output_image,) + + + +class FluxProFillNode(ComfyNodeABC): + """ + Inpaints image based on mask and prompt. + """ + + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "image": (IO.IMAGE,), + "mask": (IO.MASK,), + "prompt": ( + IO.STRING, + { + "multiline": True, + "default": "", + "tooltip": "Prompt for the image generation", + }, + ), + "prompt_upsampling": ( + IO.BOOLEAN, + { + "default": False, + "tooltip": "Whether to perform upsampling on the prompt. If active, automatically modifies the prompt for more creative generation, but results are nondeterministic (same seed will not produce exactly the same result).", + }, + ), + "guidance": ( + IO.FLOAT, + { + "default": 60, + "min": 1.5, + "max": 100, + "tooltip": "Guidance strength for the image generation process" + }, + ), + "steps": ( + IO.INT, + { + "default": 50, + "min": 15, + "max": 50, + "tooltip": "Number of steps for the image generation process" + }, + ), + "seed": ( + IO.INT, + { + "default": 0, + "min": 0, + "max": 0xFFFFFFFFFFFFFFFF, + "control_after_generate": True, + "tooltip": "The random seed used for creating the noise.", + }, + ), + }, + "optional": { + }, + "hidden": { + "auth_token": "AUTH_TOKEN_COMFY_ORG", + }, + } + + RETURN_TYPES = (IO.IMAGE,) + DESCRIPTION = cleandoc(__doc__ or "") # Handle potential None value + FUNCTION = "api_call" + API_NODE = True + CATEGORY = "api node/image/bfl" + + def api_call( + self, + image: torch.Tensor, + mask: torch.Tensor, + prompt: str, + prompt_upsampling: bool, + steps: int, + guidance: float, + seed=0, + auth_token=None, + **kwargs, + ): + # make sure image will have alpha channel removed + image = convert_image_to_base64(image[:,:,:,:3]) + mask = convert_image_to_base64(convert_mask_to_image(mask)) + + operation = SynchronousOperation( + endpoint=ApiEndpoint( + path="/proxy/bfl/flux-pro-1.0-fill/generate", + method=HttpMethod.POST, + request_model=BFLFluxFillImageRequest, + response_model=BFLFluxProGenerateResponse, + ), + request=BFLFluxFillImageRequest( + prompt=prompt, + prompt_upsampling=prompt_upsampling, + steps=steps, + guidance=guidance, + seed=seed, + image=image, + mask=mask, + ), + auth_token=auth_token, + ) + output_image = handle_bfl_synchronous_operation(operation) + return (output_image,) + + +class FluxProCannyNode(ComfyNodeABC): + """ + Generate image using a control image (canny). + """ + + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "control_image": (IO.IMAGE,), + "prompt": ( + IO.STRING, + { + "multiline": True, + "default": "", + "tooltip": "Prompt for the image generation", + }, + ), + "prompt_upsampling": ( + IO.BOOLEAN, + { + "default": False, + "tooltip": "Whether to perform upsampling on the prompt. If active, automatically modifies the prompt for more creative generation, but results are nondeterministic (same seed will not produce exactly the same result).", + }, + ), + "canny_low_threshold": ( + IO.INT, + { + "default": 0, + "min": 0, + "max": 500, + "tooltip": "Low threshold for Canny edge detection; ignored if skip_processing is True" + }, + ), + "canny_high_threshold": ( + IO.INT, + { + "default": 0, + "min": 0, + "max": 500, + "tooltip": "High threshold for Canny edge detection; ignored if skip_processing is True" + }, + ), + "skip_preprocessing": ( + IO.BOOLEAN, + { + "default": False, + "tooltip": "Whether to skip preprocessing; set to True if control_image already is canny-fied, False if it is a raw image.", + }, + ), + "guidance": ( + IO.FLOAT, + { + "default": 30, + "min": 1, + "max": 100, + "tooltip": "Guidance strength for the image generation process" + }, + ), + "steps": ( + IO.INT, + { + "default": 50, + "min": 15, + "max": 50, + "tooltip": "Number of steps for the image generation process" + }, + ), + "seed": ( + IO.INT, + { + "default": 0, + "min": 0, + "max": 0xFFFFFFFFFFFFFFFF, + "control_after_generate": True, + "tooltip": "The random seed used for creating the noise.", + }, + ), + }, + "optional": { + }, + "hidden": { + "auth_token": "AUTH_TOKEN_COMFY_ORG", + }, + } + + RETURN_TYPES = (IO.IMAGE,) + DESCRIPTION = cleandoc(__doc__ or "") # Handle potential None value + FUNCTION = "api_call" + API_NODE = True + CATEGORY = "api node/image/bfl" + + def api_call( + self, + control_image: torch.Tensor, + prompt: str, + prompt_upsampling: bool, + canny_low_threshold: int, + canny_high_threshold: int, + skip_preprocessing: bool, + steps: int, + guidance: float, + seed=0, + auth_token=None, + **kwargs, + ): + control_image = convert_image_to_base64(control_image[:,:,:,:3]) + preprocessed_image = None + + if skip_preprocessing: + preprocessed_image = control_image + control_image = None + canny_low_threshold = None + canny_high_threshold = None + + operation = SynchronousOperation( + endpoint=ApiEndpoint( + path="/proxy/bfl/flux-pro-1.0-canny/generate", + method=HttpMethod.POST, + request_model=BFLFluxCannyImageRequest, + response_model=BFLFluxProGenerateResponse, + ), + request=BFLFluxCannyImageRequest( + prompt=prompt, + prompt_upsampling=prompt_upsampling, + steps=steps, + guidance=guidance, + seed=seed, + control_image=control_image, + canny_low_threshold=canny_low_threshold, + canny_high_threshold=canny_high_threshold, + preprocessed_image=preprocessed_image, + ), + auth_token=auth_token, + ) + output_image = handle_bfl_synchronous_operation(operation) + return (output_image,) + + +class FluxProDepthNode(ComfyNodeABC): + """ + Generate image using a control image (depth). + """ + + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "control_image": (IO.IMAGE,), + "prompt": ( + IO.STRING, + { + "multiline": True, + "default": "", + "tooltip": "Prompt for the image generation", + }, + ), + "prompt_upsampling": ( + IO.BOOLEAN, + { + "default": False, + "tooltip": "Whether to perform upsampling on the prompt. If active, automatically modifies the prompt for more creative generation, but results are nondeterministic (same seed will not produce exactly the same result).", + }, + ), + "skip_preprocessing": ( + IO.BOOLEAN, + { + "default": False, + "tooltip": "Whether to skip preprocessing; set to True if control_image already is depth-ified, False if it is a raw image.", + }, + ), + "guidance": ( + IO.FLOAT, + { + "default": 15, + "min": 1, + "max": 100, + "tooltip": "Guidance strength for the image generation process" + }, + ), + "steps": ( + IO.INT, + { + "default": 50, + "min": 15, + "max": 50, + "tooltip": "Number of steps for the image generation process" + }, + ), + "seed": ( + IO.INT, + { + "default": 0, + "min": 0, + "max": 0xFFFFFFFFFFFFFFFF, + "control_after_generate": True, + "tooltip": "The random seed used for creating the noise.", + }, + ), + }, + "optional": { + }, + "hidden": { + "auth_token": "AUTH_TOKEN_COMFY_ORG", + }, + } + + RETURN_TYPES = (IO.IMAGE,) + DESCRIPTION = cleandoc(__doc__ or "") # Handle potential None value + FUNCTION = "api_call" + API_NODE = True + CATEGORY = "api node/image/bfl" + + def api_call( + self, + control_image: torch.Tensor, + prompt: str, + prompt_upsampling: bool, + skip_preprocessing: bool, + steps: int, + guidance: float, + seed=0, + auth_token=None, + **kwargs, + ): + control_image = convert_image_to_base64(control_image[:,:,:,:3]) + preprocessed_image = None + + if skip_preprocessing: + preprocessed_image = control_image + control_image = None + + operation = SynchronousOperation( + endpoint=ApiEndpoint( + path="/proxy/bfl/flux-pro-1.0-canny/generate", + method=HttpMethod.POST, + request_model=BFLFluxDepthImageRequest, + response_model=BFLFluxProGenerateResponse, + ), + request=BFLFluxDepthImageRequest( + prompt=prompt, + prompt_upsampling=prompt_upsampling, + steps=steps, + guidance=guidance, + seed=seed, + control_image=control_image, + preprocessed_image=preprocessed_image, + ), + auth_token=auth_token, + ) + output_image = handle_bfl_synchronous_operation(operation) + return (output_image,) # A dictionary that contains all nodes you want to export with their names # NOTE: names should be globally unique NODE_CLASS_MAPPINGS = { "FluxProUltraImageNode": FluxProUltraImageNode, + # "FluxProImageNode": FluxProImageNode, + # "FluxProExpandNode": FluxProExpandNode, + # "FluxProFillNode": FluxProFillNode, + # "FluxProCannyNode": FluxProCannyNode, + # "FluxProDepthNode": FluxProDepthNode, } # A dictionary that contains the friendly/humanly readable titles for the nodes NODE_DISPLAY_NAME_MAPPINGS = { "FluxProUltraImageNode": "Flux 1.1 [pro] Ultra Image", + # "FluxProImageNode": "Flux 1.1 [pro] Image", + # "FluxProExpandNode": "Flux.1 Expand Image", + # "FluxProFillNode": "Flux.1 Fill Image", + # "FluxProCannyNode": "Flux.1 Canny Control Image", + # "FluxProDepthNode": "Flux.1 Depth Control Image", } diff --git a/comfy_api_nodes/nodes_recraft.py b/comfy_api_nodes/nodes_recraft.py index c769923ed..86c26c92c 100644 --- a/comfy_api_nodes/nodes_recraft.py +++ b/comfy_api_nodes/nodes_recraft.py @@ -1,4 +1,6 @@ +from __future__ import annotations from inspect import cleandoc +from comfy.utils import ProgressBar from comfy.comfy_types.node_typing import IO from comfy_api_nodes.apis.recraft_api import ( RecraftImageGenerationRequest, @@ -17,10 +19,12 @@ from comfy_api_nodes.apis.client import ( ApiEndpoint, HttpMethod, SynchronousOperation, + EmptyRequest, ) from comfy_api_nodes.apinode_utils import ( bytesio_to_image_tensor, download_url_to_bytesio, + tensor_to_bytesio, ) import folder_paths import json @@ -29,6 +33,50 @@ import torch from io import BytesIO +def handle_recraft_file_request( + image: torch.Tensor, + path: str, + mask: torch.Tensor=None, + total_pixels=4096*4096, + timeout=1024, + request=None, + auth_token=None + ) -> list[BytesIO]: + """ + Handle sending common Recraft file-only request to get back file bytes. + """ + if request is None: + request = EmptyRequest() + + files = { + 'image': tensor_to_bytesio(image, total_pixels=total_pixels).read() + } + if mask is not None: + files['mask'] = tensor_to_bytesio(mask, total_pixels=total_pixels).read() + + operation = SynchronousOperation( + endpoint=ApiEndpoint( + path=path, + method=HttpMethod.POST, + request_model=type(request), + response_model=RecraftImageGenerationResponse, + ), + request=request, + files=files, + content_type="multipart/form-data", + auth_token=auth_token, + ) + response: RecraftImageGenerationResponse = operation.execute() + all_bytesio = [] + if response.image is not None: + all_bytesio.append(download_url_to_bytesio(response.image.url, timeout=timeout)) + else: + for data in response.data: + all_bytesio.append(download_url_to_bytesio(data.url, timeout=timeout)) + + return all_bytesio + + class SVG: """ Stores SVG representations via a list of BytesIO objects. @@ -36,6 +84,16 @@ class SVG: def __init__(self, data: list[BytesIO]): self.data = data + def combine(self, other: SVG): + return SVG(self.data + other.data) + + @staticmethod + def combine_all(svgs: list[SVG]): + all_svgs = [] + for svg in svgs: + all_svgs.extend(svg.data) + return SVG(all_svgs) + class SaveSVGNode: """ @@ -349,7 +407,7 @@ class RecraftTextToImageNode: ), request=RecraftImageGenerationRequest( prompt=prompt, - negative_prompts=negative_prompt, + negative_prompt=negative_prompt, model=RecraftModel.recraftv3, size=size, n=n, @@ -374,6 +432,243 @@ class RecraftTextToImageNode: return (output_image,) +class RecraftImageToImageNode: + """ + Modify image based on prompt and strength. + """ + + RETURN_TYPES = (IO.IMAGE,) + DESCRIPTION = cleandoc(__doc__ or "") # Handle potential None value + FUNCTION = "api_call" + API_NODE = True + CATEGORY = "api node/image/Recraft" + + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "image": (IO.IMAGE, ), + "prompt": ( + IO.STRING, + { + "multiline": True, + "default": "", + "tooltip": "Prompt for the image generation.", + }, + ), + "n": ( + IO.INT, + { + "default": 1, + "min": 1, + "max": 6, + "tooltip": "The number of images to generate.", + }, + ), + "strength": ( + IO.FLOAT, + { + "default": 0.5, + "min": 0.0, + "max": 1.0, + "step": 0.01, + "tooltip": "Defines the difference with the original image, should lie in [0, 1], where 0 means almost identical, and 1 means miserable similarity." + } + ), + "seed": ( + IO.INT, + { + "default": 0, + "min": 0, + "max": 0xFFFFFFFFFFFFFFFF, + "control_after_generate": True, + "tooltip": "Seed to determine if node should re-run; actual results are nondeterministic regardless of seed.", + }, + ), + }, + "optional": { + "recraft_style": (RecraftIO.STYLEV3,), + "negative_prompt": ( + IO.STRING, + { + "default": "", + "forceInput": True, + "tooltip": "An optional text description of undesired elements on an image.", + }, + ), + "recraft_controls": ( + RecraftIO.CONTROLS, + { + "tooltip": "Optional additional controls over the generation via the Recraft Controls node." + }, + ), + }, + "hidden": { + "auth_token": "AUTH_TOKEN_COMFY_ORG", + }, + } + + def api_call( + self, + image: torch.Tensor, + prompt: str, + n: int, + strength: float, + seed, + auth_token=None, + recraft_style: RecraftStyle = None, + negative_prompt: str = None, + recraft_controls: RecraftControls = None, + **kwargs, + ): + default_style = RecraftStyle(RecraftStyleV3.realistic_image) + if recraft_style is None: + recraft_style = default_style + + controls_api = None + if recraft_controls: + controls_api = recraft_controls.create_api_model() + + if not negative_prompt: + negative_prompt = None + + request = RecraftImageGenerationRequest( + prompt=prompt, + negative_prompt=negative_prompt, + model=RecraftModel.recraftv3, + n=n, + strength=round(strength, 2), + style=recraft_style.style, + substyle=recraft_style.substyle, + style_id=recraft_style.style_id, + controls=controls_api, + ) + + images = [] + total = image.shape[0] + pbar = ProgressBar(total) + for i in range(total): + sub_bytes = handle_recraft_file_request( + image=image[i], + path="/proxy/recraft/images/imageToImage", + request=request, + auth_token=auth_token, + ) + images.append(torch.cat([bytesio_to_image_tensor(x) for x in sub_bytes], dim=0)) + pbar.update(1) + + images_tensor = torch.cat(images, dim=0) + return (images_tensor, ) + + +class RecraftImageInpaintingNode: + """ + Modify image based on prompt and mask. + """ + + RETURN_TYPES = (IO.IMAGE,) + DESCRIPTION = cleandoc(__doc__ or "") # Handle potential None value + FUNCTION = "api_call" + API_NODE = True + CATEGORY = "api node/image/Recraft" + + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "image": (IO.IMAGE, ), + "mask": (IO.MASK, ), + "prompt": ( + IO.STRING, + { + "multiline": True, + "default": "", + "tooltip": "Prompt for the image generation.", + }, + ), + "n": ( + IO.INT, + { + "default": 1, + "min": 1, + "max": 6, + "tooltip": "The number of images to generate.", + }, + ), + "seed": ( + IO.INT, + { + "default": 0, + "min": 0, + "max": 0xFFFFFFFFFFFFFFFF, + "control_after_generate": True, + "tooltip": "Seed to determine if node should re-run; actual results are nondeterministic regardless of seed.", + }, + ), + }, + "optional": { + "recraft_style": (RecraftIO.STYLEV3,), + "negative_prompt": ( + IO.STRING, + { + "default": "", + "forceInput": True, + "tooltip": "An optional text description of undesired elements on an image.", + }, + ), + }, + "hidden": { + "auth_token": "AUTH_TOKEN_COMFY_ORG", + }, + } + + def api_call( + self, + image: torch.Tensor, + mask: torch.Tensor, + prompt: str, + n: int, + seed, + auth_token=None, + recraft_style: RecraftStyle = None, + negative_prompt: str = None, + **kwargs, + ): + default_style = RecraftStyle(RecraftStyleV3.realistic_image) + if recraft_style is None: + recraft_style = default_style + + if not negative_prompt: + negative_prompt = None + + request = RecraftImageGenerationRequest( + prompt=prompt, + negative_prompt=negative_prompt, + model=RecraftModel.recraftv3, + n=n, + style=recraft_style.style, + substyle=recraft_style.substyle, + style_id=recraft_style.style_id, + ) + + images = [] + total = image.shape[0] + pbar = ProgressBar(total) + for i in range(total): + sub_bytes = handle_recraft_file_request( + image=image[i], + mask=mask[i:i+1], + path="/proxy/recraft/images/imageInpainting", + request=request, + auth_token=auth_token, + ) + images.append(torch.cat([bytesio_to_image_tensor(x) for x in sub_bytes], dim=0)) + pbar.update(1) + + images_tensor = torch.cat(images, dim=0) + return (images_tensor, ) + + class RecraftTextToVectorNode: """ Generates SVG synchronously based on prompt and resolution. @@ -477,7 +772,7 @@ class RecraftTextToVectorNode: ), request=RecraftImageGenerationRequest( prompt=prompt, - negative_prompts=negative_prompt, + negative_prompt=negative_prompt, model=RecraftModel.recraftv3, size=size, n=n, @@ -495,11 +790,280 @@ class RecraftTextToVectorNode: return (SVG(svg_data),) +class RecraftVectorizeImageNode: + """ + Generates SVG synchronously from an input image. + """ + + RETURN_TYPES = (RecraftIO.SVG,) + DESCRIPTION = cleandoc(__doc__ or "") # Handle potential None value + FUNCTION = "api_call" + API_NODE = True + CATEGORY = "api node/image/Recraft" + + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "image": (IO.IMAGE, ), + }, + "optional": { + }, + "hidden": { + "auth_token": "AUTH_TOKEN_COMFY_ORG", + }, + } + + def api_call( + self, + image: torch.Tensor, + auth_token=None, + **kwargs, + ): + svgs = [] + total = image.shape[0] + pbar = ProgressBar(total) + for i in range(total): + sub_bytes = handle_recraft_file_request( + image=image[i], + path="/proxy/recraft/images/vectorize", + auth_token=auth_token, + ) + svgs.append(SVG(sub_bytes)) + pbar.update(1) + + return (SVG.combine_all(svgs), ) + + +class RecraftReplaceBackgroundNode: + """ + Replace background on image, based on provided prompt. + """ + + RETURN_TYPES = (IO.IMAGE,) + DESCRIPTION = cleandoc(__doc__ or "") # Handle potential None value + FUNCTION = "api_call" + API_NODE = True + CATEGORY = "api node/image/Recraft" + + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "image": (IO.IMAGE, ), + "prompt": ( + IO.STRING, + { + "multiline": True, + "default": "", + "tooltip": "Prompt for the image generation.", + }, + ), + "n": ( + IO.INT, + { + "default": 1, + "min": 1, + "max": 6, + "tooltip": "The number of images to generate.", + }, + ), + "seed": ( + IO.INT, + { + "default": 0, + "min": 0, + "max": 0xFFFFFFFFFFFFFFFF, + "control_after_generate": True, + "tooltip": "Seed to determine if node should re-run; actual results are nondeterministic regardless of seed.", + }, + ), + }, + "optional": { + "recraft_style": (RecraftIO.STYLEV3,), + "negative_prompt": ( + IO.STRING, + { + "default": "", + "forceInput": True, + "tooltip": "An optional text description of undesired elements on an image.", + }, + ), + }, + "hidden": { + "auth_token": "AUTH_TOKEN_COMFY_ORG", + }, + } + + def api_call( + self, + image: torch.Tensor, + prompt: str, + n: int, + seed, + auth_token=None, + recraft_style: RecraftStyle = None, + negative_prompt: str = None, + **kwargs, + ): + default_style = RecraftStyle(RecraftStyleV3.realistic_image) + if recraft_style is None: + recraft_style = default_style + + if not negative_prompt: + negative_prompt = None + + request = RecraftImageGenerationRequest( + prompt=prompt, + negative_prompt=negative_prompt, + model=RecraftModel.recraftv3, + n=n, + style=recraft_style.style, + substyle=recraft_style.substyle, + style_id=recraft_style.style_id, + ) + + images = [] + total = image.shape[0] + pbar = ProgressBar(total) + for i in range(total): + sub_bytes = handle_recraft_file_request( + image=image[i], + path="/proxy/recraft/images/replaceBackground", + request=request, + auth_token=auth_token, + ) + images.append(torch.cat([bytesio_to_image_tensor(x) for x in sub_bytes], dim=0)) + pbar.update(1) + + images_tensor = torch.cat(images, dim=0) + return (images_tensor, ) + + +class RecraftRemoveBackgroundNode: + """ + Remove background from image, and return processed image and mask. + """ + + RETURN_TYPES = (IO.IMAGE, IO.MASK) + DESCRIPTION = cleandoc(__doc__ or "") # Handle potential None value + FUNCTION = "api_call" + API_NODE = True + CATEGORY = "api node/image/Recraft" + + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "image": (IO.IMAGE, ), + }, + "optional": { + }, + "hidden": { + "auth_token": "AUTH_TOKEN_COMFY_ORG", + }, + } + + def api_call( + self, + image: torch.Tensor, + auth_token=None, + **kwargs, + ): + images = [] + total = image.shape[0] + pbar = ProgressBar(total) + for i in range(total): + sub_bytes = handle_recraft_file_request( + image=image[i], + path="/proxy/recraft/images/removeBackground", + auth_token=auth_token, + ) + images.append(torch.cat([bytesio_to_image_tensor(x) for x in sub_bytes], dim=0)) + pbar.update(1) + + images_tensor = torch.cat(images, dim=0) + # use alpha channel as masks, in B,H,W format + masks_tensor = images_tensor[:,:,:,-1:].squeeze(-1) + return (images_tensor, masks_tensor) + + +class RecraftCrispUpscaleNode: + """ + Upscale image synchronously. + Enhances a given raster image using ‘crisp upscale’ tool, increasing image resolution, making the image sharper and cleaner. + """ + + RETURN_TYPES = (IO.IMAGE,) + DESCRIPTION = cleandoc(__doc__ or "") # Handle potential None value + FUNCTION = "api_call" + API_NODE = True + CATEGORY = "api node/image/Recraft" + + RECRAFT_PATH = "/proxy/recraft/images/crispUpscale" + + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "image": (IO.IMAGE, ), + }, + "optional": { + }, + "hidden": { + "auth_token": "AUTH_TOKEN_COMFY_ORG", + }, + } + + def api_call( + self, + image: torch.Tensor, + auth_token=None, + **kwargs, + ): + images = [] + total = image.shape[0] + pbar = ProgressBar(total) + for i in range(total): + sub_bytes = handle_recraft_file_request( + image=image[i], + path=self.RECRAFT_PATH, + auth_token=auth_token, + ) + images.append(torch.cat([bytesio_to_image_tensor(x) for x in sub_bytes], dim=0)) + pbar.update(1) + + images_tensor = torch.cat(images, dim=0) + return (images_tensor,) + + +class RecraftCreativeUpscaleNode(RecraftCrispUpscaleNode): + """ + Upscale image synchronously. + Enhances a given raster image using ‘creative upscale’ tool, boosting resolution with a focus on refining small details and faces. + """ + + RETURN_TYPES = (IO.IMAGE,) + DESCRIPTION = cleandoc(__doc__ or "") # Handle potential None value + FUNCTION = "api_call" + API_NODE = True + CATEGORY = "api node/image/Recraft" + + RECRAFT_PATH = "/proxy/recraft/images/creativeUpscale" + + # A dictionary that contains all nodes you want to export with their names # NOTE: names should be globally unique NODE_CLASS_MAPPINGS = { "RecraftTextToImageNode": RecraftTextToImageNode, + # "RecraftImageToImageNode": RecraftImageToImageNode, + # "RecraftImageInpaintingNode": RecraftImageInpaintingNode, "RecraftTextToVectorNode": RecraftTextToVectorNode, + "RecraftVectorizeImageNode": RecraftVectorizeImageNode, + "RecraftRemoveBackgroundNode": RecraftRemoveBackgroundNode, + # "RecraftReplaceBackgroundNode": RecraftReplaceBackgroundNode, + "RecraftCrispUpscaleNode": RecraftCrispUpscaleNode, + # "RecraftCreativeUpscaleNode": RecraftCreativeUpscaleNode, "RecraftStyleV3RealisticImage": RecraftStyleV3RealisticImageNode, "RecraftStyleV3DigitalIllustration": RecraftStyleV3DigitalIllustrationNode, "RecraftStyleV3LogoRaster": RecraftStyleV3LogoRasterNode, @@ -511,7 +1075,14 @@ NODE_CLASS_MAPPINGS = { # A dictionary that contains the friendly/humanly readable titles for the nodes NODE_DISPLAY_NAME_MAPPINGS = { "RecraftTextToImageNode": "Recraft Text to Image", + # "RecraftImageToImageNode": "Recraft Image to Image", + # "RecraftImageInpaintingNode": "Recraft Image Inpainting", "RecraftTextToVectorNode": "Recraft Text to Vector", + "RecraftVectorizeImageNode": "Recraft Vectorize Image", + "RecraftRemoveBackgroundNode": "Recraft Remove Background", + # "RecraftReplaceBackgroundNode": "Recraft Replace Background", + "RecraftCrispUpscaleNode": "Recraft Crisp Upscale Image", + # "RecraftCreativeUpscaleNode": "Recraft Creative Upscale Image", "RecraftStyleV3RealisticImage": "Recraft Style - Realistic Image", "RecraftStyleV3DigitalIllustration": "Recraft Style - Digital Illustration", "RecraftStyleV3LogoRaster": "Recraft Style - Logo Raster", diff --git a/comfy_api_nodes/nodes_stability.py b/comfy_api_nodes/nodes_stability.py index 25ab6e0b6..2ac6b30a2 100644 --- a/comfy_api_nodes/nodes_stability.py +++ b/comfy_api_nodes/nodes_stability.py @@ -1,9 +1,12 @@ from inspect import cleandoc from comfy.comfy_types.node_typing import IO from comfy_api_nodes.apis.stability_api import ( + StabilityStable3_5Request, StabilityStableUltraRequest, StabilityStableUltraResponse, StabilityAspectRatio, + Stability_SD3_5_Model, + Stability_SD3_5_GenerationMode, get_stability_style_presets, ) from comfy_api_nodes.apis.client import ( @@ -148,13 +151,151 @@ class StabilityStableImageUltraNode: return (returned_image,) +class StabilityStableImageSD_3_5Node: + """ + Generates images synchronously based on prompt and resolution. + """ + + RETURN_TYPES = (IO.IMAGE,) + DESCRIPTION = cleandoc(__doc__ or "") # Handle potential None value + FUNCTION = "api_call" + API_NODE = True + CATEGORY = "api node/image/stability" + + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "prompt": ( + IO.STRING, + { + "multiline": True, + "default": "", + "tooltip": "What you wish to see in the output image. A strong, descriptive prompt that clearly defines elements, colors, and subjects will lead to better results." + }, + ), + "aspect_ratio": ([x.value for x in StabilityAspectRatio], + { + "default": StabilityAspectRatio.ratio_1_1, + "tooltip": "Aspect ratio of generated image.", + }, + ), + "style_preset": (get_stability_style_presets(), + { + "tooltip": "Optional desired style of generated image.", + }, + ), + "cfg_scale": ( + IO.FLOAT, + { + "default": 4.0, + "min": 1.0, + "max": 10.0, + "step": 0.1, + "tooltip": "How strictly the diffusion process adheres to the prompt text (higher values keep your image closer to your prompt)", + }, + ), + "seed": ( + IO.INT, + { + "default": 0, + "min": 0, + "max": 4294967294, + "control_after_generate": True, + "tooltip": "The random seed used for creating the noise.", + }, + ), + }, + "optional": { + "image": (IO.IMAGE,), + "negative_prompt": ( + IO.STRING, + { + "default": "", + "forceInput": True, + "tooltip": "Keywords of what you do not wish to see in the output image. This is an advanced feature." + }, + ), + "image_denoise": ( + IO.FLOAT, + { + "default": 0.5, + "min": 0.0, + "max": 1.0, + "step": 0.01, + "tooltip": "Denoise of input image; 0.0 yields image identical to input, 1.0 is as if no image was provided at all.", + }, + ), + }, + "hidden": { + "auth_token": "AUTH_TOKEN_COMFY_ORG", + }, + } + + def api_call(self, prompt: str, aspect_ratio: str, style_preset: str, seed: int, cfg_scale: float, + negative_prompt: str=None, image: torch.Tensor = None, image_denoise: float=None, + auth_token=None): + model = Stability_SD3_5_Model.sd3_5_large.value + # prepare image binary if image present + image_binary = None + mode = Stability_SD3_5_GenerationMode.text_to_image.value + if image is not None: + image_binary = tensor_to_bytesio(image, 1504 * 1504).read() + mode = Stability_SD3_5_GenerationMode.image_to_image.value + else: + image_denoise = None + + if not negative_prompt: + negative_prompt = None + if style_preset == "None": + style_preset = None + + files = { + "image": image_binary + } + + operation = SynchronousOperation( + endpoint=ApiEndpoint( + path="/proxy/stability/v2beta/stable-image/generate/sd3", + method=HttpMethod.POST, + request_model=StabilityStable3_5Request, + response_model=StabilityStableUltraResponse, + ), + request=StabilityStable3_5Request( + prompt=prompt, + negative_prompt=negative_prompt, + aspect_ratio=aspect_ratio, + seed=seed, + strength=image_denoise, + style_preset=style_preset, + cfg_scale=cfg_scale, + model=model, + mode=mode, + ), + files=files, + content_type="multipart/form-data", + auth_token=auth_token, + ) + response_api = operation.execute() + + if response_api.finish_reason != "SUCCESS": + raise Exception(f"Stable Diffusion 3.5 Image generation failed: {response_api.finish_reason}.") + + image_data = base64.b64decode(response_api.image) + returned_image = bytesio_to_image_tensor(BytesIO(image_data)) + + return (returned_image,) + + # A dictionary that contains all nodes you want to export with their names # NOTE: names should be globally unique NODE_CLASS_MAPPINGS = { "StabilityStableImageUltraNode": StabilityStableImageUltraNode, + # "StabilityStableImageSD_3_5Node": StabilityStableImageSD_3_5Node, } # A dictionary that contains the friendly/humanly readable titles for the nodes NODE_DISPLAY_NAME_MAPPINGS = { "StabilityStableImageUltraNode": "Stability Stable Image Ultra", + # "StabilityStableImageSD_3_5Node": "Stability Stable Diffusion 3.5 Image", } From e7f241071b68585f60955453d1bf0c016875070e Mon Sep 17 00:00:00 2001 From: Christian Byrne Date: Fri, 2 May 2025 09:47:47 -0700 Subject: [PATCH 081/121] Revert "Serve SVG files directly" (#111) --- server.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/server.py b/server.py index 33112c227..f64ec27d4 100644 --- a/server.py +++ b/server.py @@ -416,17 +416,6 @@ class PromptServer(): if os.path.isfile(file): if 'preview' in request.rel_url.query: - extension = os.path.splitext(filename)[-1] - if extension and extension.lower() == ".svg": - with open(file, "r") as f: - return web.Response( - body=f.read(), - content_type="image/svg+xml", - headers={ - "Content-Disposition": f'filename="{filename}"' - }, - ) - with Image.open(file) as img: preview_info = request.rel_url.query['preview'].split(';') image_format = preview_info[0] From 1178022a7aa64810631a8541b3a752435531d88d Mon Sep 17 00:00:00 2001 From: Jedrzej Kosinski Date: Fri, 2 May 2025 14:10:46 -0500 Subject: [PATCH 082/121] Expose 4 remaining Recraft nodes (#112) --- comfy_api_nodes/apis/recraft_api.py | 1 + comfy_api_nodes/nodes_recraft.py | 30 +++++++++++++++++++---------- 2 files changed, 21 insertions(+), 10 deletions(-) diff --git a/comfy_api_nodes/apis/recraft_api.py b/comfy_api_nodes/apis/recraft_api.py index e5060bcd9..c0ec9d0c8 100644 --- a/comfy_api_nodes/apis/recraft_api.py +++ b/comfy_api_nodes/apis/recraft_api.py @@ -247,6 +247,7 @@ class RecraftImageGenerationRequest(BaseModel): controls: Optional[RecraftControlsObject] = Field(None, description='A set of custom parameters to tweak generation process') style_id: Optional[str] = Field(None, description='Use a previously uploaded style as a reference; UUID') strength: Optional[confloat(ge=0.0, le=1.0)] = Field(None, description='Defines the difference with the original image, should lie in [0, 1], where 0 means almost identical, and 1 means miserable similarity') + random_seed: Optional[int] = Field(None, description="Seed for video generation") # text_layout diff --git a/comfy_api_nodes/nodes_recraft.py b/comfy_api_nodes/nodes_recraft.py index 86c26c92c..62cc503e2 100644 --- a/comfy_api_nodes/nodes_recraft.py +++ b/comfy_api_nodes/nodes_recraft.py @@ -1,6 +1,6 @@ from __future__ import annotations from inspect import cleandoc -from comfy.utils import ProgressBar +from comfy.utils import ProgressBar, common_upscale from comfy.comfy_types.node_typing import IO from comfy_api_nodes.apis.recraft_api import ( RecraftImageGenerationRequest, @@ -542,6 +542,7 @@ class RecraftImageToImageNode: substyle=recraft_style.substyle, style_id=recraft_style.style_id, controls=controls_api, + random_seed=seed, ) images = [] @@ -649,8 +650,17 @@ class RecraftImageInpaintingNode: style=recraft_style.style, substyle=recraft_style.substyle, style_id=recraft_style.style_id, + random_seed=seed, ) + # prepare mask tensor + _, H, W, _ = image.shape + mask = mask.unsqueeze(-1) + mask = mask.movedim(-1,1) + mask = common_upscale(mask, width=W, height=H, upscale_method="nearest-exact", crop="disabled") + mask = mask.movedim(1,-1) + mask = (mask > 0.5).float() + images = [] total = image.shape[0] pbar = ProgressBar(total) @@ -658,7 +668,7 @@ class RecraftImageInpaintingNode: sub_bytes = handle_recraft_file_request( image=image[i], mask=mask[i:i+1], - path="/proxy/recraft/images/imageInpainting", + path="/proxy/recraft/images/inpaint", request=request, auth_token=auth_token, ) @@ -1056,14 +1066,14 @@ class RecraftCreativeUpscaleNode(RecraftCrispUpscaleNode): # NOTE: names should be globally unique NODE_CLASS_MAPPINGS = { "RecraftTextToImageNode": RecraftTextToImageNode, - # "RecraftImageToImageNode": RecraftImageToImageNode, - # "RecraftImageInpaintingNode": RecraftImageInpaintingNode, + "RecraftImageToImageNode": RecraftImageToImageNode, + "RecraftImageInpaintingNode": RecraftImageInpaintingNode, "RecraftTextToVectorNode": RecraftTextToVectorNode, "RecraftVectorizeImageNode": RecraftVectorizeImageNode, "RecraftRemoveBackgroundNode": RecraftRemoveBackgroundNode, - # "RecraftReplaceBackgroundNode": RecraftReplaceBackgroundNode, + "RecraftReplaceBackgroundNode": RecraftReplaceBackgroundNode, "RecraftCrispUpscaleNode": RecraftCrispUpscaleNode, - # "RecraftCreativeUpscaleNode": RecraftCreativeUpscaleNode, + "RecraftCreativeUpscaleNode": RecraftCreativeUpscaleNode, "RecraftStyleV3RealisticImage": RecraftStyleV3RealisticImageNode, "RecraftStyleV3DigitalIllustration": RecraftStyleV3DigitalIllustrationNode, "RecraftStyleV3LogoRaster": RecraftStyleV3LogoRasterNode, @@ -1075,14 +1085,14 @@ NODE_CLASS_MAPPINGS = { # A dictionary that contains the friendly/humanly readable titles for the nodes NODE_DISPLAY_NAME_MAPPINGS = { "RecraftTextToImageNode": "Recraft Text to Image", - # "RecraftImageToImageNode": "Recraft Image to Image", - # "RecraftImageInpaintingNode": "Recraft Image Inpainting", + "RecraftImageToImageNode": "Recraft Image to Image", + "RecraftImageInpaintingNode": "Recraft Image Inpainting", "RecraftTextToVectorNode": "Recraft Text to Vector", "RecraftVectorizeImageNode": "Recraft Vectorize Image", "RecraftRemoveBackgroundNode": "Recraft Remove Background", - # "RecraftReplaceBackgroundNode": "Recraft Replace Background", + "RecraftReplaceBackgroundNode": "Recraft Replace Background", "RecraftCrispUpscaleNode": "Recraft Crisp Upscale Image", - # "RecraftCreativeUpscaleNode": "Recraft Creative Upscale Image", + "RecraftCreativeUpscaleNode": "Recraft Creative Upscale Image", "RecraftStyleV3RealisticImage": "Recraft Style - Realistic Image", "RecraftStyleV3DigitalIllustration": "Recraft Style - Digital Illustration", "RecraftStyleV3LogoRaster": "Recraft Style - Logo Raster", From 99a6b59cbbfccd4592ad6b75d07e5f9965caa851 Mon Sep 17 00:00:00 2001 From: Christian Byrne Date: Fri, 2 May 2025 14:23:58 -0700 Subject: [PATCH 083/121] [Kling] Add `Duration` and `Video ID` outputs (#105) --- comfy_api_nodes/nodes_kling.py | 344 +++++++++++++++++++++++++++++---- 1 file changed, 303 insertions(+), 41 deletions(-) diff --git a/comfy_api_nodes/nodes_kling.py b/comfy_api_nodes/nodes_kling.py index ca64f8a8e..4b2492908 100644 --- a/comfy_api_nodes/nodes_kling.py +++ b/comfy_api_nodes/nodes_kling.py @@ -103,7 +103,42 @@ def get_camera_control_input_config( return IO.FLOAT, input_config -class KlingCameraControls(ComfyNodeABC): +class KlingNodeBase(ComfyNodeABC): + """ + Base class for Kling nodes. + + Compatibility Table + =================== + | Mode | Duration | Model Name | Camera Control | Image Tail | + |------|----------|------------------|----------------|------------| + | std | 5 | kling-v1 | No | Yes | + | std | 5 | kling-v1-5 | No | Yes | + | std | 5 | kling-v1-6 | No | No | + | std | 5 | kling-v2-master | No | No | + | std | 10 | kling-v1 | No | No | + | std | 10 | kling-v1-5 | No | No | + | std | 10 | kling-v1-6 | No | No | + | std | 10 | kling-v2-master | No | No | + | pro | 5 | kling-v1 | No | Yes | + | pro | 5 | kling-v1-5 | Yes | Yes | + | pro | 5 | kling-v1-6 | No | Yes | + | pro | 5 | kling-v2-master | No | No | + | pro | 10 | kling-v1 | No | No | + | pro | 10 | kling-v1-5 | No | Yes | + | pro | 10 | kling-v1-6 | No | Yes | + | pro | 10 | kling-v2-master | No | No | + + **Note**: Although the combo of pro mode, kling-v1-5 model, and 5s duration + supports both camera_control and image_tail, you can only use one feature + at a time. + """ + + FUNCTION = "api_call" + CATEGORY = "api node/video/Kling" + API_NODE = True + + +class KlingCameraControls(KlingNodeBase): """Kling Camera Controls Node""" @classmethod @@ -148,16 +183,16 @@ class KlingCameraControls(ComfyNodeABC): RETURN_NAMES = ("camera_control",) FUNCTION = "main" - def main( - self, - camera_control_type: str, + @classmethod + def VALIDATE_INPUTS( + cls, horizontal_movement: float, vertical_movement: float, pan: float, tilt: float, roll: float, zoom: float, - ): + ) -> bool | str: if not is_valid_camera_control_configs( [ horizontal_movement, @@ -169,7 +204,18 @@ class KlingCameraControls(ComfyNodeABC): ] ): return "Invalid camera control configs: at least one of the values must be non-zero" + return True + def main( + self, + camera_control_type: str, + horizontal_movement: float, + vertical_movement: float, + pan: float, + tilt: float, + roll: float, + zoom: float, + ) -> tuple[CameraControl]: return ( CameraControl( type=CameraType(camera_control_type), @@ -185,18 +231,8 @@ class KlingCameraControls(ComfyNodeABC): ) -class KlingNodeBase(ComfyNodeABC): - """Base class for Kling nodes.""" - - FUNCTION = "api_call" - CATEGORY = "api node/video/Kling" - API_NODE = True - - class KlingTextToVideoNode(KlingNodeBase): - """ - Kling Text to Video Node. - """ + """Kling Text to Video Node""" @staticmethod def poll_for_task_status(task_id: str, auth_token: str) -> KlingText2VideoResponse: @@ -254,13 +290,11 @@ class KlingTextToVideoNode(KlingNodeBase): enum_type=AspectRatio, ), }, - "optional": { - "camera_control": ("CAMERA_CONTROL", {}), - }, "hidden": {"auth_token": "AUTH_TOKEN_COMFY_ORG"}, } - RETURN_TYPES = ("VIDEO",) + RETURN_TYPES = ("VIDEO", "STRING", "STRING") + RETURN_NAMES = ("VIDEO", "Kling ID", "Duration (sec)") DESCRIPTION = "Kling Text to Video Node" def api_call( @@ -274,7 +308,7 @@ class KlingTextToVideoNode(KlingNodeBase): aspect_ratio: str, camera_control: Optional[CameraControl] = None, auth_token: Optional[str] = None, - ) -> tuple[VideoFromFile]: + ) -> tuple[VideoFromFile, str, str]: validate_prompts(prompt, negative_prompt, MAX_PROMPT_LENGTH_T2V) initial_operation = SynchronousOperation( endpoint=ApiEndpoint( @@ -311,16 +345,79 @@ class KlingTextToVideoNode(KlingNodeBase): logging.error(error_msg) raise KlingApiError(error_msg) - video_url = str(final_response.data.task_result.videos[0].url) - logging.debug("Kling task %s succeeded. Video URL: %s", task_id, video_url) + video = final_response.data.task_result.videos[0] + logging.debug("Kling task %s succeeded. Video URL: %s", task_id, video.url) + return ( + download_url_to_video_output(video.url), + str(video.id), + str(video.duration), + ) - return (download_url_to_video_output(video_url),) + +class KlingCameraControlT2VNode(KlingTextToVideoNode): + """ + Kling Text to Video Camera Control Node. This node is a text to video node, but it supports controlling the camera. + Duration, mode, and model_name request fields are hard-coded because camera control is only supported in pro mode with the kling-v1-5 model at 5s duration as of 2025-05-02. + """ + + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "prompt": model_field_to_node_input( + IO.STRING, KlingText2VideoRequest, "prompt", multiline=True + ), + "negative_prompt": model_field_to_node_input( + IO.STRING, + KlingText2VideoRequest, + "negative_prompt", + multiline=True, + ), + "cfg_scale": model_field_to_node_input( + IO.FLOAT, KlingText2VideoRequest, "cfg_scale" + ), + "aspect_ratio": model_field_to_node_input( + IO.COMBO, + KlingText2VideoRequest, + "aspect_ratio", + enum_type=AspectRatio, + ), + "camera_control": ( + "CAMERA_CONTROL", + { + "tooltip": "Can be created using the Kling Camera Controls node. Controls the camera movement and motion during the video generation.", + }, + ), + }, + "hidden": {"auth_token": "AUTH_TOKEN_COMFY_ORG"}, + } + + DESCRIPTION = "Transform text into cinematic videos with professional camera movements that simulate real-world cinematography. Control virtual camera actions including zoom, rotation, pan, tilt, and first-person view, while maintaining focus on your original text." + + def api_call( + self, + prompt: str, + negative_prompt: str, + cfg_scale: float, + aspect_ratio: str, + camera_control: Optional[CameraControl] = None, + auth_token: Optional[str] = None, + ): + return super().api_call( + model_name="kling-v1-5", + cfg_scale=cfg_scale, + mode="pro", + aspect_ratio=aspect_ratio, + duration="5", + prompt=prompt, + negative_prompt=negative_prompt, + camera_control=camera_control, + auth_token=auth_token, + ) class KlingImage2VideoNode(KlingNodeBase): - """ - Kling Image to Video Node. - """ + """Kling Image to Video Node""" @staticmethod def poll_for_task_status(task_id: str, auth_token: str) -> KlingImage2VideoResponse: @@ -347,6 +444,9 @@ class KlingImage2VideoNode(KlingNodeBase): def INPUT_TYPES(s): return { "required": { + "start_frame": model_field_to_node_input( + IO.IMAGE, KlingImage2VideoRequest, "image" + ), "prompt": model_field_to_node_input( IO.STRING, KlingImage2VideoRequest, "prompt", multiline=True ), @@ -363,9 +463,6 @@ class KlingImage2VideoNode(KlingNodeBase): enum_type=ModelName, default="kling-v2-master", ), - "start_frame": model_field_to_node_input( - IO.IMAGE, KlingImage2VideoRequest, "image" - ), "cfg_scale": model_field_to_node_input( IO.FLOAT, KlingImage2VideoRequest, "cfg_scale" ), @@ -382,24 +479,19 @@ class KlingImage2VideoNode(KlingNodeBase): IO.COMBO, KlingImage2VideoRequest, "duration", enum_type=Duration ), }, - "optional": { - "camera_control": ("CAMERA_CONTROL", {}), - "end_frame": model_field_to_node_input( - IO.IMAGE, KlingImage2VideoRequest, "image_tail" - ), - }, "hidden": {"auth_token": "AUTH_TOKEN_COMFY_ORG"}, } - RETURN_TYPES = ("VIDEO",) + RETURN_TYPES = ("VIDEO", "STRING", "STRING") + RETURN_NAMES = ("VIDEO", "Kling ID", "Duration (sec)") DESCRIPTION = "Kling Image to Video Node" def api_call( self, + start_frame: torch.Tensor, prompt: str, negative_prompt: str, model_name: str, - start_frame: torch.Tensor, cfg_scale: float, mode: str, aspect_ratio: str, @@ -449,20 +541,190 @@ class KlingImage2VideoNode(KlingNodeBase): logging.error(error_msg) raise KlingApiError(error_msg) - video_url = str(final_response.data.task_result.videos[0].url) - logging.info("Attempting to download video from URL: %s", video_url) + video = final_response.data.task_result.videos[0] + logging.info("Kling task %s succeeded. Video URL: %s", task_id, video.url) - return (download_url_to_video_output(video_url),) + return ( + download_url_to_video_output(video.url), + str(video.id), + str(video.duration), + ) + + +class KlingCameraControlI2VNode(KlingImage2VideoNode): + """ + Kling Image to Video Camera Control Node. This node is a image to video node, but it supports controlling the camera. + Duration, mode, and model_name request fields are hard-coded because camera control is only supported in pro mode with the kling-v1-5 model at 5s duration as of 2025-05-02. + """ + + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "start_frame": model_field_to_node_input( + IO.IMAGE, KlingImage2VideoRequest, "image" + ), + "prompt": model_field_to_node_input( + IO.STRING, KlingImage2VideoRequest, "prompt", multiline=True + ), + "negative_prompt": model_field_to_node_input( + IO.STRING, + KlingImage2VideoRequest, + "negative_prompt", + multiline=True, + ), + "cfg_scale": model_field_to_node_input( + IO.FLOAT, KlingImage2VideoRequest, "cfg_scale" + ), + "aspect_ratio": model_field_to_node_input( + IO.COMBO, + KlingImage2VideoRequest, + "aspect_ratio", + enum_type=AspectRatio, + ), + "camera_control": ( + "CAMERA_CONTROL", + { + "tooltip": "Can be created using the Kling Camera Controls node. Controls the camera movement and motion during the video generation.", + }, + ), + }, + "hidden": {"auth_token": "AUTH_TOKEN_COMFY_ORG"}, + } + + DESCRIPTION = "Transform still images into cinematic videos with professional camera movements that simulate real-world cinematography. Control virtual camera actions including zoom, rotation, pan, tilt, and first-person view, while maintaining focus on your original image." + + def api_call( + self, + start_frame: torch.Tensor, + prompt: str, + negative_prompt: str, + cfg_scale: float, + aspect_ratio: str, + camera_control: CameraControl, + auth_token: Optional[str] = None, + ): + return super().api_call( + model_name="kling-v1-5", + start_frame=start_frame, + cfg_scale=cfg_scale, + mode="pro", + aspect_ratio=aspect_ratio, + duration="5", + prompt=prompt, + negative_prompt=negative_prompt, + camera_control=camera_control, + auth_token=auth_token, + ) + + +class KlingStartEndFrameNode(KlingImage2VideoNode): + """ + Kling First Last Frame Node. This node allows creation of a video from a first and last frame. It calls the normal image to video endpoint, but only allows the subset of input options that support the `image_tail` request field. + """ + + @staticmethod + def get_mode_string_mapping() -> dict[str, tuple[str, str, str]]: + """ + Returns a mapping of mode strings to their corresponding (mode, duration, model_name) tuples. + Only includes config combos that support the `image_tail` request field. + """ + return { + "standard mode / 5s duration / kling-v1": ("std", "5", "kling-v1"), + "standard mode / 5s duration / kling-v1-5": ("std", "5", "kling-v1-5"), + "pro mode / 5s duration / kling-v1": ("pro", "5", "kling-v1"), + "pro mode / 5s duration / kling-v1-5": ("pro", "5", "kling-v1-5"), + "pro mode / 5s duration / kling-v1-6": ("pro", "5", "kling-v1-6"), + "pro mode / 10s duration / kling-v1-5": ("pro", "10", "kling-v1-5"), + "pro mode / 10s duration / kling-v1-6": ("pro", "10", "kling-v1-6"), + } + + @classmethod + def INPUT_TYPES(s): + modes = list(KlingStartEndFrameNode.get_mode_string_mapping().keys()) + return { + "required": { + "start_frame": model_field_to_node_input( + IO.IMAGE, KlingImage2VideoRequest, "image" + ), + "end_frame": model_field_to_node_input( + IO.IMAGE, KlingImage2VideoRequest, "image_tail" + ), + "prompt": model_field_to_node_input( + IO.STRING, KlingImage2VideoRequest, "prompt", multiline=True + ), + "negative_prompt": model_field_to_node_input( + IO.STRING, + KlingImage2VideoRequest, + "negative_prompt", + multiline=True, + ), + "cfg_scale": model_field_to_node_input( + IO.FLOAT, KlingImage2VideoRequest, "cfg_scale" + ), + "aspect_ratio": model_field_to_node_input( + IO.COMBO, + KlingImage2VideoRequest, + "aspect_ratio", + enum_type=AspectRatio, + ), + "mode": ( + modes, + { + "default": modes[2], + "tooltip": "The configuration to use for the video generation following the format: mode / duration / model_name.", + }, + ), + }, + "hidden": {"auth_token": "AUTH_TOKEN_COMFY_ORG"}, + } + + DESCRIPTION = "Generate a video sequence that transitions between your provided start and end images. The node creates all frames in between, producing a smooth transformation from the first frame to the last." + + def parse_inputs_from_mode(self, mode: str) -> tuple[str, str, str]: + """Parses the mode input into a tuple of (model_name, duration, mode).""" + return KlingStartEndFrameNode.get_mode_string_mapping()[mode] + + def api_call( + self, + start_frame: torch.Tensor, + end_frame: torch.Tensor, + prompt: str, + negative_prompt: str, + cfg_scale: float, + aspect_ratio: str, + mode: str, + auth_token: Optional[str] = None, + ): + mode, duration, model_name = self.parse_inputs_from_mode(mode) + return super().api_call( + prompt=prompt, + negative_prompt=negative_prompt, + model_name=model_name, + start_frame=start_frame, + cfg_scale=cfg_scale, + mode=mode, + aspect_ratio=aspect_ratio, + duration=duration, + end_frame=end_frame, + auth_token=auth_token, + ) NODE_CLASS_MAPPINGS = { "KlingCameraControls": KlingCameraControls, "KlingTextToVideoNode": KlingTextToVideoNode, "KlingImage2VideoNode": KlingImage2VideoNode, + "KlingCameraControlI2VNode": KlingCameraControlI2VNode, + "KlingCameraControlT2VNode": KlingCameraControlT2VNode, + "KlingStartEndFrameNode": KlingStartEndFrameNode, } NODE_DISPLAY_NAME_MAPPINGS = { "KlingCameraControls": "Kling Camera Controls", "KlingTextToVideoNode": "Kling Text to Video", "KlingImage2VideoNode": "Kling Image to Video", + "KlingCameraControlI2VNode": "Kling Image to Video (Camera Control)", + "KlingCameraControlT2VNode": "Kling Text to Video (Camera Control)", + "KlingStartEndFrameNode": "Kling Start-End Frame to Video", } From 7a10383f902cfe360da4b6028b667f5da05a0650 Mon Sep 17 00:00:00 2001 From: Christian Byrne Date: Fri, 2 May 2025 15:25:29 -0700 Subject: [PATCH 084/121] Fix: datamodel-codegen sets string#binary type to non-existent `bytes_aliased` variable (#114) --- comfy_api_nodes/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/comfy_api_nodes/README.md b/comfy_api_nodes/README.md index 0ebecae63..e2633a769 100644 --- a/comfy_api_nodes/README.md +++ b/comfy_api_nodes/README.md @@ -36,6 +36,6 @@ npm install -g @redocly/cli redocly bundle openapi.yaml --output filtered-openapi.yaml --config comfy_api_nodes/redocly-dev.yaml --remove-unused-components # Generate the pydantic datamodels for validation. -datamodel-codegen --use-subclass-enum --field-constraints --input filtered-openapi.yaml --output comfy_api_nodes/apis/__init__.py --output-model-type pydantic_v2.BaseModel +datamodel-codegen --use-subclass-enum --field-constraints --strict-types bytes --input filtered-openapi.yaml --output comfy_api_nodes/apis/__init__.py --output-model-type pydantic_v2.BaseModel ``` From b9c03f6d4f761b27f3ea27fae76f338b4a30546f Mon Sep 17 00:00:00 2001 From: Christian Byrne Date: Fri, 2 May 2025 16:31:04 -0700 Subject: [PATCH 085/121] Fix: Dall-e 2 not setting request content-type dynamically (#113) --- comfy_api_nodes/nodes_openai.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/comfy_api_nodes/nodes_openai.py b/comfy_api_nodes/nodes_openai.py index 612653a55..e3fbbb868 100644 --- a/comfy_api_nodes/nodes_openai.py +++ b/comfy_api_nodes/nodes_openai.py @@ -116,11 +116,13 @@ class OpenAIDalle2(ComfyNodeABC): ): model = "dall-e-2" path = "/proxy/openai/images/generations" + content_type = "application/json" request_class = OpenAIImageGenerationRequest img_binary = None if image is not None and mask is not None: path = "/proxy/openai/images/edits" + content_type = "multipart/form-data" request_class = OpenAIImageEditRequest input_tensor = image.squeeze().cpu() @@ -166,6 +168,7 @@ class OpenAIDalle2(ComfyNodeABC): if img_binary else None ), + content_type=content_type, auth_token=auth_token, ) @@ -401,7 +404,7 @@ class OpenAIGPTImage1(ComfyNodeABC): if image is not None: path = "/proxy/openai/images/edits" request_class = OpenAIImageEditRequest - content_type="multipart/form-data", + content_type ="multipart/form-data" batch_size = image.shape[0] From 8295d758a3222d41cfc218c017155d71def89350 Mon Sep 17 00:00:00 2001 From: Robin Huang Date: Sat, 3 May 2025 14:59:49 -0700 Subject: [PATCH 086/121] Default request timeout: one hour. (#116) --- comfy_api_nodes/apis/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/comfy_api_nodes/apis/client.py b/comfy_api_nodes/apis/client.py index 032c02b2e..1d9f40881 100644 --- a/comfy_api_nodes/apis/client.py +++ b/comfy_api_nodes/apis/client.py @@ -157,7 +157,7 @@ class ApiClient: self, base_url: str, api_key: Optional[str] = None, - timeout: float = 30.0, + timeout: float = 3600.0, verify_ssl: bool = True, ): self.base_url = base_url From 1560c9bc8cb2e6dae617cd4c934af89ca38526b3 Mon Sep 17 00:00:00 2001 From: Christian Byrne Date: Sat, 3 May 2025 15:19:37 -0700 Subject: [PATCH 087/121] Add Kling nodes: camera control, start-end frame, lip-sync, video extend (#115) --- comfy_api/input_impl/video_types.py | 20 +- comfy_api_nodes/apinode_utils.py | 159 + comfy_api_nodes/apis/__init__.py | 4444 +++++++++++++++++++++------ comfy_api_nodes/nodes_kling.py | 1151 +++++-- 4 files changed, 4607 insertions(+), 1167 deletions(-) diff --git a/comfy_api/input_impl/video_types.py b/comfy_api/input_impl/video_types.py index 12e5783db..146d6daf8 100644 --- a/comfy_api/input_impl/video_types.py +++ b/comfy_api/input_impl/video_types.py @@ -89,7 +89,7 @@ class VideoFromFile(VideoInput): def save_to( self, - path: str, + path: str | io.BytesIO, format: VideoContainer = VideoContainer.AUTO, codec: VideoCodec = VideoCodec.AUTO, metadata: Optional[dict] = None @@ -116,7 +116,23 @@ class VideoFromFile(VideoInput): ) streams = container.streams - with av.open(path, mode='w', options={"movflags": "use_metadata_tags"}) as output_container: + open_kwargs = { + "mode": "w", + "options": {"movflags": "use_metadata_tags"} + } + + if not isinstance(path, str): + # Explicit format is needed for non-path destinations (like BytesIO) + output_format_str = ( + format.value.lower() + if format != VideoContainer.AUTO + else container.format.name + ) + if "," in output_format_str: + output_format_str = output_format_str.split(",")[0] + open_kwargs["format"] = output_format_str + + with av.open(path, **open_kwargs) as output_container: # Copy over the original metadata for key, value in container.metadata.items(): if metadata is None or key not in metadata: diff --git a/comfy_api_nodes/apinode_utils.py b/comfy_api_nodes/apinode_utils.py index 3ff04c2a3..baeaa0ceb 100644 --- a/comfy_api_nodes/apinode_utils.py +++ b/comfy_api_nodes/apinode_utils.py @@ -3,6 +3,9 @@ import logging from typing import Optional from comfy.utils import common_upscale from comfy_api.input_impl import VideoFromFile +from comfy_api.util import VideoContainer, VideoCodec +from comfy_api.input.video_types import VideoInput +from comfy_api.input.basic_types import AudioInput from comfy_api_nodes.apis.client import ( ApiClient, ApiEndpoint, @@ -21,6 +24,7 @@ import math import base64 import uuid from io import BytesIO +import av def download_url_to_video_output(video_url: str, timeout: int = None) -> VideoFromFile: @@ -197,6 +201,11 @@ def bytesio_to_image_tensor(image_bytesio: BytesIO, mode: str = "RGBA") -> torch return torch.from_numpy(image_array).unsqueeze(0) +def download_url_to_image_tensor(url: str, timeout: int = None) -> torch.Tensor: + """Downloads an image from a URL and returns a [B, H, W, C] tensor.""" + image_bytesio = download_url_to_bytesio(url, timeout) + return bytesio_to_image_tensor(image_bytesio) + def process_image_response(response: requests.Response) -> torch.Tensor: """Uses content from a Response object and converts it to a torch.Tensor""" return bytesio_to_image_tensor(BytesIO(response.content)) @@ -301,6 +310,156 @@ def tensor_to_data_uri( return f"data:{mime_type};base64,{base64_string}" +def upload_file_to_comfyapi( + file_bytes_io: BytesIO, + filename: str, + upload_mime_type: str, + auth_token: Optional[str] = None, +) -> str: + """ + Uploads a single file to ComfyUI API and returns its download URL. + + Args: + file_bytes_io: BytesIO object containing the file data. + filename: The filename of the file. + upload_mime_type: MIME type of the file. + auth_token: Optional authentication token. + + Returns: + The download URL for the uploaded file. + """ + request_object = UploadRequest(file_name=filename, content_type=upload_mime_type) + operation = SynchronousOperation( + endpoint=ApiEndpoint( + path="/customers/storage", + method=HttpMethod.POST, + request_model=UploadRequest, + response_model=UploadResponse, + ), + request=request_object, + auth_token=auth_token, + ) + + response: UploadResponse = operation.execute() + upload_response = ApiClient.upload_file( + response.upload_url, file_bytes_io, content_type=upload_mime_type + ) + upload_response.raise_for_status() + + return response.download_url + + +def upload_video_to_comfyapi( + video: VideoInput, + auth_token: Optional[str] = None, + container: VideoContainer = VideoContainer.MP4, + codec: VideoCodec = VideoCodec.H264, + max_duration: Optional[int] = None, +) -> str: + """ + Uploads a single video to ComfyUI API and returns its download URL. + Uses the specified container and codec for saving the video before upload. + + Args: + video: Input VideoInput object. + auth_token: Optional authentication token. + container: The video container format to use (default: MP4). + codec: The video codec to use (default: H264). + max_duration: Optional maximum duration of the video in seconds. If the video is longer than this, an error will be raised. + + Returns: + The download URL for the uploaded video file. + """ + if max_duration is not None: + try: + actual_duration = video.duration_seconds + if actual_duration is not None and actual_duration > max_duration: + raise ValueError( + f"Video duration ({actual_duration:.2f}s) exceeds the maximum allowed ({max_duration}s)." + ) + except Exception as e: + logging.error(f"Error getting video duration: {e}") + raise ValueError(f"Could not verify video duration from source: {e}") from e + + upload_mime_type = f"video/{container.value.lower()}" + filename = f"uploaded_video.{container.value.lower()}" + + # Convert VideoInput to BytesIO using specified container/codec + video_bytes_io = io.BytesIO() + video.save_to(video_bytes_io, format=container, codec=codec) + video_bytes_io.seek(0) + + return upload_file_to_comfyapi( + video_bytes_io, filename, upload_mime_type, auth_token + ) + + +def upload_audio_to_comfyapi( + audio: AudioInput, + auth_token: Optional[str] = None, +) -> str: + """ + Uploads a single audio input to ComfyUI API and returns its download URL. + Encodes the raw waveform into MP4/AAC format before uploading. + + Args: + audio: Input AudioInput object (containing waveform tensor and sample_rate). + auth_token: Optional authentication token. + + Returns: + The download URL for the uploaded audio file. + """ + waveform: torch.Tensor = audio["waveform"] + sample_rate: int = audio["sample_rate"] + + # If batch is > 1, take first item + if waveform.shape[0] > 1: + waveform = waveform[0] + + # Check waveform tensor shape + if waveform.ndim != 3 or waveform.shape[0] != 1: + raise ValueError("Expected waveform tensor shape (1, channels, samples)") + + # Prepare data for av library + audio_data_np = waveform.squeeze(0).cpu().numpy() + if audio_data_np.dtype != np.float32: + audio_data_np = audio_data_np.astype(np.float32) + + # Ensure the array is C-contiguous + if not audio_data_np.flags["C_CONTIGUOUS"]: + audio_data_np = np.ascontiguousarray(audio_data_np) + + # Default to MP4/AAC + container_format = "mp4" + codec_name = "aac" + upload_mime_type = "audio/mp4" + filename = "uploaded_audio.mp4" + + audio_bytes_io = io.BytesIO() + with av.open(audio_bytes_io, mode="w", format=container_format) as output_container: + audio_stream = output_container.add_stream(codec_name, rate=sample_rate) + frame = av.AudioFrame.from_ndarray( + audio_data_np, + format="fltp", + layout="stereo" if audio_data_np.shape[0] > 1 else "mono", + ) + frame.sample_rate = sample_rate + frame.pts = 0 + + for packet in audio_stream.encode(frame): + output_container.mux(packet) + + # Flush stream + for packet in audio_stream.encode(None): + output_container.mux(packet) + + audio_bytes_io.seek(0) # Reset buffer position for reading + + return upload_file_to_comfyapi( + audio_bytes_io, filename, upload_mime_type, auth_token + ) + + def upload_images_to_comfyapi( image: torch.Tensor, max_images=8, auth_token=None, mime_type: Optional[str] = None ) -> list[str]: diff --git a/comfy_api_nodes/apis/__init__.py b/comfy_api_nodes/apis/__init__.py index 9031b39f7..9998bb07a 100644 --- a/comfy_api_nodes/apis/__init__.py +++ b/comfy_api_nodes/apis/__init__.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: filtered-openapi.yaml -# timestamp: 2025-05-01T05:55:00+00:00 +# timestamp: 2025-05-03T21:40:30+00:00 from __future__ import annotations @@ -9,37 +9,59 @@ from enum import Enum from typing import Any, Dict, List, Literal, Optional, Union from uuid import UUID -from pydantic import AnyUrl, BaseModel, Field, RootModel - -bytes_aliased = bytes +from pydantic import AnyUrl, BaseModel, Field, RootModel, StrictBytes -class BFLFluxProGenerateRequest(BaseModel): - guidance_scale: Optional[float] = Field( - None, description="The guidance scale for generation.", ge=1.0, le=20.0 +class PersonalAccessToken(BaseModel): + id: Optional[UUID] = Field(None, description='Unique identifier for the GitCommit') + name: Optional[str] = Field( + None, + description='Required. The name of the token. Can be a simple description.', ) - height: int = Field( - ..., description="The height of the image to generate.", ge=64, le=2048 + description: Optional[str] = Field( + None, + description="Optional. A more detailed description of the token's intended use.", ) - negative_prompt: Optional[str] = Field( - None, description="The negative prompt for image generation." + createdAt: Optional[datetime] = Field( + None, description='[Output Only]The date and time the token was created.' ) - num_images: Optional[int] = Field( - None, description="The number of images to generate.", ge=1, le=4 - ) - num_inference_steps: Optional[int] = Field( - None, description="The number of inference steps.", ge=1, le=100 - ) - prompt: str = Field(..., description="The text prompt for image generation.") - seed: Optional[int] = Field(None, description="The seed value for reproducibility.") - width: int = Field( - ..., description="The width of the image to generate.", ge=64, le=2048 + token: Optional[str] = Field( + None, + description='[Output Only]. The personal access token. Only returned during creation.', ) -class BFLFluxProGenerateResponse(BaseModel): - id: str = Field(..., description="The unique identifier for the generation task.") - polling_url: str = Field(..., description="URL to poll for the generation result.") +class GitCommitSummary(BaseModel): + commit_hash: Optional[str] = Field(None, description='The hash of the commit') + commit_name: Optional[str] = Field(None, description='The name of the commit') + branch_name: Optional[str] = Field( + None, description='The branch where the commit was made' + ) + author: Optional[str] = Field(None, description='The author of the commit') + timestamp: Optional[datetime] = Field( + None, description='The timestamp when the commit was made' + ) + status_summary: Optional[Dict[str, str]] = Field( + None, description='A map of operating system to status pairs' + ) + + +class User(BaseModel): + id: Optional[str] = Field(None, description='The unique id for this user.') + email: Optional[str] = Field(None, description='The email address for this user.') + name: Optional[str] = Field(None, description='The name for this user.') + isApproved: Optional[bool] = Field( + None, description='Indicates if the user is approved.' + ) + isAdmin: Optional[bool] = Field( + None, description='Indicates if the user has admin privileges.' + ) + + +class PublisherUser(BaseModel): + id: Optional[str] = Field(None, description='The unique id for this user.') + email: Optional[str] = Field(None, description='The email address for this user.') + name: Optional[str] = Field(None, description='The name for this user.') class ErrorResponse(BaseModel): @@ -47,22 +69,186 @@ class ErrorResponse(BaseModel): message: str +class StorageFile(BaseModel): + id: Optional[UUID] = Field( + None, description='Unique identifier for the storage file' + ) + file_path: Optional[str] = Field(None, description='Path to the file in storage') + public_url: Optional[str] = Field(None, description='Public URL') + + +class PublisherMember(BaseModel): + id: Optional[str] = Field( + None, description='The unique identifier for the publisher member.' + ) + user: Optional[PublisherUser] = Field( + None, description='The user associated with this publisher member.' + ) + role: Optional[str] = Field( + None, description='The role of the user in the publisher.' + ) + + +class ComfyNode(BaseModel): + comfy_node_name: Optional[str] = Field( + None, description='Unique identifier for the node' + ) + category: Optional[str] = Field( + None, + description='UI category where the node is listed, used for grouping nodes.', + ) + description: Optional[str] = Field( + None, description="Brief description of the node's functionality or purpose." + ) + input_types: Optional[str] = Field(None, description='Defines input parameters') + deprecated: Optional[bool] = Field( + None, + description='Indicates if the node is deprecated. Deprecated nodes are hidden in the UI.', + ) + experimental: Optional[bool] = Field( + None, + description='Indicates if the node is experimental, subject to changes or removal.', + ) + output_is_list: Optional[List[bool]] = Field( + None, description='Boolean values indicating if each output is a list.' + ) + return_names: Optional[str] = Field( + None, description='Names of the outputs for clarity in workflows.' + ) + return_types: Optional[str] = Field( + None, description='Specifies the types of outputs produced by the node.' + ) + function: Optional[str] = Field( + None, description='Name of the entry-point function to execute the node.' + ) + + +class ComfyNodeCloudBuildInfo(BaseModel): + project_id: Optional[str] = None + project_number: Optional[str] = None + location: Optional[str] = None + build_id: Optional[str] = None + + +class Error(BaseModel): + message: Optional[str] = Field( + None, description='A clear and concise description of the error.' + ) + details: Optional[List[str]] = Field( + None, + description='Optional detailed information about the error or hints for resolving it.', + ) + + +class NodeVersionUpdateRequest(BaseModel): + changelog: Optional[str] = Field( + None, description='The changelog describing the version changes.' + ) + deprecated: Optional[bool] = Field( + None, description='Whether the version is deprecated.' + ) + + +class NodeStatus(str, Enum): + NodeStatusActive = 'NodeStatusActive' + NodeStatusDeleted = 'NodeStatusDeleted' + NodeStatusBanned = 'NodeStatusBanned' + + +class NodeVersionStatus(str, Enum): + NodeVersionStatusActive = 'NodeVersionStatusActive' + NodeVersionStatusDeleted = 'NodeVersionStatusDeleted' + NodeVersionStatusBanned = 'NodeVersionStatusBanned' + NodeVersionStatusPending = 'NodeVersionStatusPending' + NodeVersionStatusFlagged = 'NodeVersionStatusFlagged' + + +class PublisherStatus(str, Enum): + PublisherStatusActive = 'PublisherStatusActive' + PublisherStatusBanned = 'PublisherStatusBanned' + + +class WorkflowRunStatus(str, Enum): + WorkflowRunStatusStarted = 'WorkflowRunStatusStarted' + WorkflowRunStatusFailed = 'WorkflowRunStatusFailed' + WorkflowRunStatusCompleted = 'WorkflowRunStatusCompleted' + + +class MachineStats(BaseModel): + machine_name: Optional[str] = Field(None, description='Name of the machine.') + os_version: Optional[str] = Field( + None, description='The operating system version. eg. Ubuntu Linux 20.04' + ) + gpu_type: Optional[str] = Field( + None, description='The GPU type. eg. NVIDIA Tesla K80' + ) + cpu_capacity: Optional[str] = Field(None, description='Total CPU on the machine.') + initial_cpu: Optional[str] = Field( + None, description='Initial CPU available before the job starts.' + ) + memory_capacity: Optional[str] = Field( + None, description='Total memory on the machine.' + ) + initial_ram: Optional[str] = Field( + None, description='Initial RAM available before the job starts.' + ) + vram_time_series: Optional[Dict[str, Any]] = Field( + None, description='Time series of VRAM usage.' + ) + disk_capacity: Optional[str] = Field( + None, description='Total disk capacity on the machine.' + ) + initial_disk: Optional[str] = Field( + None, description='Initial disk available before the job starts.' + ) + pip_freeze: Optional[str] = Field(None, description='The pip freeze output') + + +class Customer(BaseModel): + id: str = Field(..., description='The firebase UID of the user') + email: Optional[str] = Field(None, description='The email address for this user') + name: Optional[str] = Field(None, description='The name for this user') + createdAt: Optional[datetime] = Field( + None, description='The date and time the user was created' + ) + updatedAt: Optional[datetime] = Field( + None, description='The date and time the user was last updated' + ) + + +class MagicPrompt(str, Enum): + ON = 'ON' + OFF = 'OFF' + + +class ColorPalette(BaseModel): + name: str = Field(..., description='Name of the color palette', examples=['PASTEL']) + + +class StyleCode(RootModel[str]): + root: str = Field(..., pattern='^[0-9A-Fa-f]{8}$') + + +class StyleType(str, Enum): + GENERAL = 'GENERAL' + + class IdeogramColorPalette1(BaseModel): - name: str = Field(..., description="Name of the preset color palette") + name: str = Field(..., description='Name of the preset color palette') class Member(BaseModel): color: Optional[str] = Field( - None, description="Hexadecimal color code", pattern="^#[0-9A-Fa-f]{6}$" + None, description='Hexadecimal color code', pattern='^#[0-9A-Fa-f]{6}$' ) weight: Optional[float] = Field( - None, description="Optional weight for the color (0-1)", ge=0.0, le=1.0 + None, description='Optional weight for the color (0-1)', ge=0.0, le=1.0 ) class IdeogramColorPalette2(BaseModel): members: List[Member] = Field( - ..., description="Array of color definitions with optional weights" + ..., description='Array of color definitions with optional weights' ) @@ -71,42 +257,25 @@ class IdeogramColorPalette( ): root: Union[IdeogramColorPalette1, IdeogramColorPalette2] = Field( ..., - description="A color palette specification that can either use a preset name or explicit color definitions with weights", + description='A color palette specification that can either use a preset name or explicit color definitions with weights', ) class ImageRequest(BaseModel): + prompt: str = Field( + ..., description='Required. The prompt to use to generate the image.' + ) aspect_ratio: Optional[str] = Field( None, description="Optional. The aspect ratio (e.g., 'ASPECT_16_9', 'ASPECT_1_1'). Cannot be used with resolution. Defaults to 'ASPECT_1_1' if unspecified.", ) - color_palette: Optional[Dict[str, Any]] = Field( - None, description="Optional. Color palette object. Only for V_2, V_2_TURBO." - ) + model: str = Field(..., description="The model used (e.g., 'V_2', 'V_2A_TURBO')") magic_prompt_option: Optional[str] = Field( None, description="Optional. MagicPrompt usage ('AUTO', 'ON', 'OFF')." ) - model: str = Field(..., description="The model used (e.g., 'V_2', 'V_2A_TURBO')") - negative_prompt: Optional[str] = Field( - None, - description="Optional. Description of what to exclude. Only for V_1, V_1_TURBO, V_2, V_2_TURBO.", - ) - num_images: Optional[int] = Field( - 1, - description="Optional. Number of images to generate (1-8). Defaults to 1.", - ge=1, - le=8, - ) - prompt: str = Field( - ..., description="Required. The prompt to use to generate the image." - ) - resolution: Optional[str] = Field( - None, - description="Optional. Resolution (e.g., 'RESOLUTION_1024_1024'). Only for model V_2. Cannot be used with aspect_ratio.", - ) seed: Optional[int] = Field( None, - description="Optional. A number between 0 and 2147483647.", + description='Optional. A number between 0 and 2147483647.', ge=0, le=2147483647, ) @@ -114,105 +283,208 @@ class ImageRequest(BaseModel): None, description="Optional. Style type ('AUTO', 'GENERAL', 'REALISTIC', 'DESIGN', 'RENDER_3D', 'ANIME'). Only for models V_2 and above.", ) + negative_prompt: Optional[str] = Field( + None, + description='Optional. Description of what to exclude. Only for V_1, V_1_TURBO, V_2, V_2_TURBO.', + ) + num_images: Optional[int] = Field( + 1, + description='Optional. Number of images to generate (1-8). Defaults to 1.', + ge=1, + le=8, + ) + resolution: Optional[str] = Field( + None, + description="Optional. Resolution (e.g., 'RESOLUTION_1024_1024'). Only for model V_2. Cannot be used with aspect_ratio.", + ) + color_palette: Optional[Dict[str, Any]] = Field( + None, description='Optional. Color palette object. Only for V_2, V_2_TURBO.' + ) class IdeogramGenerateRequest(BaseModel): image_request: ImageRequest = Field( - ..., description="The image generation request parameters." + ..., description='The image generation request parameters.' ) class Datum(BaseModel): - is_image_safe: Optional[bool] = Field( - None, description="Indicates whether the image is considered safe." - ) prompt: Optional[str] = Field( - None, description="The prompt used to generate this image." + None, description='The prompt used to generate this image.' ) resolution: Optional[str] = Field( None, description="The resolution of the generated image (e.g., '1024x1024')." ) - seed: Optional[int] = Field( - None, description="The seed value used for this generation." + is_image_safe: Optional[bool] = Field( + None, description='Indicates whether the image is considered safe.' ) + seed: Optional[int] = Field( + None, description='The seed value used for this generation.' + ) + url: Optional[str] = Field(None, description='URL to the generated image.') style_type: Optional[str] = Field( None, description="The style type used for generation (e.g., 'REALISTIC', 'ANIME').", ) - url: Optional[str] = Field(None, description="URL to the generated image.") class IdeogramGenerateResponse(BaseModel): created: Optional[datetime] = Field( - None, description="Timestamp when the generation was created." + None, description='Timestamp when the generation was created.' ) data: Optional[List[Datum]] = Field( - None, description="Array of generated image information." + None, description='Array of generated image information.' ) -class StyleCode(RootModel[str]): - root: str = Field(..., pattern="^[0-9A-Fa-f]{8}$") +class RenderingSpeed1(str, Enum): + TURBO = 'TURBO' + DEFAULT = 'DEFAULT' + QUALITY = 'QUALITY' -class ColorPalette(BaseModel): - name: str = Field(..., description="Name of the color palette", examples=["PASTEL"]) +class MagicPrompt1(str, Enum): + AUTO = 'AUTO' + ON = 'ON' + OFF = 'OFF' -class MagicPrompt(str, Enum): - ON = "ON" - OFF = "OFF" +class StyleType1(str, Enum): + AUTO = 'AUTO' + GENERAL = 'GENERAL' + REALISTIC = 'REALISTIC' + DESIGN = 'DESIGN' -class StyleType(str, Enum): - GENERAL = "GENERAL" +class IdeogramV3RemixRequest(BaseModel): + image: Optional[StrictBytes] = None + prompt: str + image_weight: Optional[int] = Field(50, ge=1, le=100) + seed: Optional[int] = Field(None, ge=0, le=2147483647) + resolution: Optional[str] = None + aspect_ratio: Optional[str] = None + rendering_speed: Optional[RenderingSpeed1] = None + magic_prompt: Optional[MagicPrompt1] = None + negative_prompt: Optional[str] = None + num_images: Optional[int] = Field(None, ge=1, le=8) + color_palette: Optional[Dict[str, Any]] = None + style_codes: Optional[List[str]] = None + style_type: Optional[StyleType1] = None + style_reference_images: Optional[List[StrictBytes]] = None -class KlingErrorResponse(BaseModel): - code: int = Field( +class Datum1(BaseModel): + prompt: Optional[str] = None + resolution: Optional[str] = None + is_image_safe: Optional[bool] = None + seed: Optional[int] = None + url: Optional[str] = None + style_type: Optional[str] = None + + +class IdeogramV3IdeogramResponse(BaseModel): + created: Optional[datetime] = None + data: Optional[List[Datum1]] = None + + +class IdeogramV3ReframeRequest(BaseModel): + image: Optional[StrictBytes] = None + resolution: str + num_images: Optional[int] = Field(None, ge=1, le=8) + seed: Optional[int] = Field(None, ge=0, le=2147483647) + rendering_speed: Optional[RenderingSpeed1] = None + color_palette: Optional[Dict[str, Any]] = None + style_codes: Optional[List[str]] = None + style_reference_images: Optional[List[StrictBytes]] = None + + +class IdeogramV3ReplaceBackgroundRequest(BaseModel): + image: Optional[StrictBytes] = None + prompt: str + magic_prompt: Optional[MagicPrompt1] = None + num_images: Optional[int] = Field(None, ge=1, le=8) + seed: Optional[int] = Field(None, ge=0, le=2147483647) + rendering_speed: Optional[RenderingSpeed1] = None + color_palette: Optional[Dict[str, Any]] = None + style_codes: Optional[List[str]] = None + style_reference_images: Optional[List[StrictBytes]] = None + + +class KlingTaskStatus(str, Enum): + submitted = 'submitted' + processing = 'processing' + succeed = 'succeed' + failed = 'failed' + + +class KlingVideoGenModelName(str, Enum): + kling_v1 = 'kling-v1' + kling_v1_5 = 'kling-v1-5' + kling_v1_6 = 'kling-v1-6' + kling_v2_master = 'kling-v2-master' + + +class KlingVideoGenMode(str, Enum): + std = 'std' + pro = 'pro' + + +class KlingVideoGenAspectRatio(str, Enum): + field_16_9 = '16:9' + field_9_16 = '9:16' + field_1_1 = '1:1' + + +class KlingVideoGenDuration(str, Enum): + field_5 = '5' + field_10 = '10' + + +class KlingVideoGenCfgScale(RootModel[float]): + root: float = Field( ..., - description="- 1000: Authentication failed\n- 1001: Authorization is empty\n- 1002: Authorization is invalid\n- 1003: Authorization is not yet valid\n- 1004: Authorization has expired\n- 1100: Account exception\n- 1101: Account in arrears (postpaid scenario)\n- 1102: Resource pack depleted or expired (prepaid scenario)\n- 1103: Unauthorized access to requested resource\n- 1200: Invalid request parameters\n- 1201: Invalid parameters\n- 1202: Invalid request method\n- 1203: Requested resource does not exist\n- 1300: Trigger platform strategy\n- 1301: Trigger content security policy\n- 1302: API request too frequent\n- 1303: Concurrency/QPS exceeds limit\n- 1304: Trigger IP whitelist policy\n- 5000: Internal server error\n- 5001: Service temporarily unavailable\n- 5002: Server internal timeout\n", - ) - message: str = Field(..., description="Human-readable error message") - request_id: str = Field( - ..., description="Request ID for tracking and troubleshooting" + description="Flexibility in video generation. The higher the value, the lower the model's degree of flexibility, and the stronger the relevance to the user's prompt.", + ge=0.0, + le=1.0, ) -class AspectRatio(str, Enum): - field_16_9 = "16:9" - field_9_16 = "9:16" - field_1_1 = "1:1" +class KlingCameraControlType(str, Enum): + simple = 'simple' + down_back = 'down_back' + forward_up = 'forward_up' + right_turn_forward = 'right_turn_forward' + left_turn_forward = 'left_turn_forward' -class Config(BaseModel): +class KlingCameraConfig(BaseModel): horizontal: Optional[float] = Field( None, description="Controls camera's movement along horizontal axis (x-axis). Negative indicates left, positive indicates right.", ge=-10.0, le=10.0, ) + vertical: Optional[float] = Field( + None, + description="Controls camera's movement along vertical axis (y-axis). Negative indicates downward, positive indicates upward.", + ge=-10.0, + le=10.0, + ) pan: Optional[float] = Field( None, description="Controls camera's rotation in vertical plane (x-axis). Negative indicates downward rotation, positive indicates upward rotation.", ge=-10.0, le=10.0, ) - roll: Optional[float] = Field( - None, - description="Controls camera's rolling amount (z-axis). Negative indicates counterclockwise, positive indicates clockwise.", - ge=-10.0, - le=10.0, - ) tilt: Optional[float] = Field( None, description="Controls camera's rotation in horizontal plane (y-axis). Negative indicates left rotation, positive indicates right rotation.", ge=-10.0, le=10.0, ) - vertical: Optional[float] = Field( + roll: Optional[float] = Field( None, - description="Controls camera's movement along vertical axis (y-axis). Negative indicates downward, positive indicates upward.", + description="Controls camera's rolling amount (z-axis). Negative indicates counterclockwise, positive indicates clockwise.", ge=-10.0, le=10.0, ) @@ -224,342 +496,1038 @@ class Config(BaseModel): ) -class Type(str, Enum): - simple = "simple" - down_back = "down_back" - forward_up = "forward_up" - right_turn_forward = "right_turn_forward" - left_turn_forward = "left_turn_forward" +class KlingVideoResult(BaseModel): + id: Optional[str] = Field(None, description='Generated video ID') + url: Optional[AnyUrl] = Field(None, description='URL for generated video') + duration: Optional[str] = Field(None, description='Total video duration') -class CameraControl(BaseModel): - config: Optional[Config] = None - type: Optional[Type] = Field( - None, - description="Predefined camera movements type. simple: Customizable camera movement. down_back: Camera descends and moves backward. forward_up: Camera moves forward and tilts up. right_turn_forward: Rotate right and move forward. left_turn_forward: Rotate left and move forward.", - ) +class KlingAudioUploadType(str, Enum): + file = 'file' + url = 'url' -class Duration(str, Enum): - field_5 = "5" - field_10 = "10" +class KlingLipSyncMode(str, Enum): + text2video = 'text2video' + audio2video = 'audio2video' -class Trajectory(BaseModel): - x: Optional[int] = Field( - None, - description="The horizontal coordinate of trajectory point. Based on bottom-left corner of image as origin (0,0).", - ) - y: Optional[int] = Field( - None, - description="The vertical coordinate of trajectory point. Based on bottom-left corner of image as origin (0,0).", - ) +class KlingLipSyncVoiceLanguage(str, Enum): + zh = 'zh' + en = 'en' -class DynamicMask(BaseModel): - mask: Optional[AnyUrl] = Field( - None, - description="Dynamic Brush Application Area (Mask image created by users using the motion brush). The aspect ratio must match the input image.", - ) - trajectories: Optional[List[Trajectory]] = None +class KlingDualCharacterEffectsScene(str, Enum): + hug = 'hug' + kiss = 'kiss' + heart_gesture = 'heart_gesture' -class Mode(str, Enum): - std = "std" - pro = "pro" +class KlingSingleImageEffectsScene(str, Enum): + bloombloom = 'bloombloom' + dizzydizzy = 'dizzydizzy' + fuzzyfuzzy = 'fuzzyfuzzy' + squish = 'squish' + expansion = 'expansion' -class ModelName(str, Enum): - kling_v1 = "kling-v1" - kling_v1_5 = "kling-v1-5" - kling_v1_6 = "kling-v1-6" - kling_v2_master = "kling-v2-master" +class KlingCharacterEffectModelName(str, Enum): + kling_v1 = 'kling-v1' + kling_v1_5 = 'kling-v1-5' + kling_v1_6 = 'kling-v1-6' -class KlingImage2VideoRequest(BaseModel): - aspect_ratio: Optional[AspectRatio] = "16:9" - callback_url: Optional[AnyUrl] = Field( - None, - description="The callback notification address. Server will notify when the task status changes.", - ) - camera_control: Optional[CameraControl] = None - cfg_scale: Optional[float] = Field( - 0.5, - description="Flexibility in video generation. The higher the value, the lower the model's degree of flexibility, and the stronger the relevance to the user's prompt.", - ge=0.0, - le=1.0, - ) - duration: Optional[Duration] = Field("5", description="Video length in seconds") - dynamic_masks: Optional[List[DynamicMask]] = Field( - None, - description="Dynamic Brush Configuration List (up to 6 groups). For 5-second videos, trajectory length must not exceed 77 coordinates.", - ) - external_task_id: Optional[str] = Field( - None, - description="Customized Task ID. Must be unique within a single user account.", - ) - image: Optional[str] = Field( - None, - description="Reference Image - URL or Base64 encoded string, cannot exceed 10MB, resolution not less than 300*300px, aspect ratio between 1:2.5 ~ 2.5:1. Base64 should not include data:image prefix.", - ) - image_tail: Optional[str] = Field( - None, - description="Reference Image - End frame control. URL or Base64 encoded string, cannot exceed 10MB, resolution not less than 300*300px. Base64 should not include data:image prefix.", - ) - mode: Optional[Mode] = Field( - "std", - description="Video generation mode. std: Standard Mode, which is cost-effective. pro: Professional Mode, generates videos with longer duration but higher quality output.", - ) - model_name: Optional[ModelName] = Field("kling-v1", description="Model Name") - negative_prompt: Optional[str] = Field( - None, description="Negative text prompt", max_length=2500 - ) - prompt: Optional[str] = Field( - None, description="Positive text prompt", max_length=2500 - ) - static_mask: Optional[AnyUrl] = Field( - None, - description="Static Brush Application Area (Mask image created by users using the motion brush). The aspect ratio must match the input image.", - ) +class KlingSingleImageEffectModelName(str, Enum): + kling_v1_6 = 'kling-v1-6' + + +class KlingSingleImageEffectDuration(str, Enum): + field_5 = '5' + + +class KlingDualCharacterImages(RootModel[List[str]]): + root: List[str] = Field(..., max_length=2, min_length=2) + + +class KlingImageGenAspectRatio(str, Enum): + field_16_9 = '16:9' + field_9_16 = '9:16' + field_1_1 = '1:1' + field_4_3 = '4:3' + field_3_4 = '3:4' + field_3_2 = '3:2' + field_2_3 = '2:3' + field_21_9 = '21:9' + + +class KlingImageGenImageReferenceType(str, Enum): + subject = 'subject' + face = 'face' + + +class KlingImageGenModelName(str, Enum): + kling_v1 = 'kling-v1' + kling_v1_5 = 'kling-v1-5' + + +class KlingImageResult(BaseModel): + index: Optional[int] = Field(None, description='Image Number (0-9)') + url: Optional[AnyUrl] = Field(None, description='URL for generated image') + + +class KlingVirtualTryOnModelName(str, Enum): + kolors_virtual_try_on_v1 = 'kolors-virtual-try-on-v1' + kolors_virtual_try_on_v1_5 = 'kolors-virtual-try-on-v1-5' class TaskInfo(BaseModel): external_task_id: Optional[str] = None -class Video(BaseModel): - duration: Optional[str] = Field(None, description="Total video duration") - id: Optional[str] = Field(None, description="Generated video ID") - url: Optional[AnyUrl] = Field(None, description="URL for generated video") - - class TaskResult(BaseModel): - videos: Optional[List[Video]] = None - - -class TaskStatus(str, Enum): - submitted = "submitted" - processing = "processing" - succeed = "succeed" - failed = "failed" + videos: Optional[List[KlingVideoResult]] = None class Data(BaseModel): - created_at: Optional[int] = Field(None, description="Task creation time") - task_id: Optional[str] = Field(None, description="Task ID") + task_id: Optional[str] = Field(None, description='Task ID') + task_status: Optional[KlingTaskStatus] = None task_info: Optional[TaskInfo] = None + created_at: Optional[int] = Field(None, description='Task creation time') + updated_at: Optional[int] = Field(None, description='Task update time') task_result: Optional[TaskResult] = None - task_status: Optional[TaskStatus] = None - updated_at: Optional[int] = Field(None, description="Task update time") - - -class KlingImage2VideoResponse(BaseModel): - code: Optional[int] = Field(None, description="Error code") - data: Optional[Data] = None - message: Optional[str] = Field(None, description="Error message") - request_id: Optional[str] = Field(None, description="Request ID") - - -class Config1(BaseModel): - horizontal: Optional[float] = Field(None, ge=-10.0, le=10.0) - pan: Optional[float] = Field(None, ge=-10.0, le=10.0) - roll: Optional[float] = Field(None, ge=-10.0, le=10.0) - tilt: Optional[float] = Field(None, ge=-10.0, le=10.0) - vertical: Optional[float] = Field(None, ge=-10.0, le=10.0) - zoom: Optional[float] = Field(None, ge=-10.0, le=10.0) - - -class CameraControl1(BaseModel): - config: Optional[Config1] = None - type: Optional[Type] = Field(None, description="Predefined camera movements type") - - -class ModelName1(str, Enum): - kling_v1 = "kling-v1" - kling_v1_6 = "kling-v1-6" - kling_v2_master = "kling-v2-master" - - -class KlingText2VideoRequest(BaseModel): - aspect_ratio: Optional[AspectRatio] = "16:9" - callback_url: Optional[AnyUrl] = Field( - None, description="The callback notification address" - ) - camera_control: Optional[CameraControl1] = None - cfg_scale: Optional[float] = Field( - 0.5, description="Flexibility in video generation", ge=0.0, le=1.0 - ) - duration: Optional[Duration] = "5" - external_task_id: Optional[str] = Field(None, description="Customized Task ID") - mode: Optional[Mode] = Field("std", description="Video generation mode") - model_name: Optional[ModelName1] = Field("kling-v1", description="Model Name") - negative_prompt: Optional[str] = Field( - None, description="Negative text prompt", max_length=2500 - ) - prompt: Optional[str] = Field( - None, description="Positive text prompt", max_length=2500 - ) - - -class TaskResult1(BaseModel): - videos: Optional[List[Video]] = None - - -class Data1(BaseModel): - created_at: Optional[int] = Field(None, description="Task creation time") - task_id: Optional[str] = Field(None, description="Task ID") - task_info: Optional[TaskInfo] = None - task_result: Optional[TaskResult1] = None - task_status: Optional[TaskStatus] = None - updated_at: Optional[int] = Field(None, description="Task update time") class KlingText2VideoResponse(BaseModel): - code: Optional[int] = Field(None, description="Error code") + code: Optional[int] = Field(None, description='Error code') + message: Optional[str] = Field(None, description='Error message') + request_id: Optional[str] = Field(None, description='Request ID') + data: Optional[Data] = None + + +class Trajectory(BaseModel): + x: Optional[int] = Field( + None, + description='The horizontal coordinate of trajectory point. Based on bottom-left corner of image as origin (0,0).', + ) + y: Optional[int] = Field( + None, + description='The vertical coordinate of trajectory point. Based on bottom-left corner of image as origin (0,0).', + ) + + +class DynamicMask(BaseModel): + mask: Optional[AnyUrl] = Field( + None, + description='Dynamic Brush Application Area (Mask image created by users using the motion brush). The aspect ratio must match the input image.', + ) + trajectories: Optional[List[Trajectory]] = None + + +class Data1(BaseModel): + task_id: Optional[str] = Field(None, description='Task ID') + task_status: Optional[KlingTaskStatus] = None + task_info: Optional[TaskInfo] = None + created_at: Optional[int] = Field(None, description='Task creation time') + updated_at: Optional[int] = Field(None, description='Task update time') + task_result: Optional[TaskResult] = None + + +class KlingImage2VideoResponse(BaseModel): + code: Optional[int] = Field(None, description='Error code') + message: Optional[str] = Field(None, description='Error message') + request_id: Optional[str] = Field(None, description='Request ID') data: Optional[Data1] = None - message: Optional[str] = Field(None, description="Error message") - request_id: Optional[str] = Field(None, description="Request ID") + + +class KlingVideoExtendRequest(BaseModel): + video_id: Optional[str] = Field( + None, + description='The ID of the video to be extended. Supports videos generated by text-to-video, image-to-video, and previous video extension operations. Cannot exceed 3 minutes total duration after extension.', + ) + prompt: Optional[str] = Field( + None, + description='Positive text prompt for guiding the video extension', + max_length=2500, + ) + negative_prompt: Optional[str] = Field( + None, + description='Negative text prompt for elements to avoid in the extended video', + max_length=2500, + ) + cfg_scale: Optional[KlingVideoGenCfgScale] = Field( + default_factory=lambda: KlingVideoGenCfgScale.model_validate(0.5) + ) + callback_url: Optional[AnyUrl] = Field( + None, + description='The callback notification address. Server will notify when the task status changes.', + ) + + +class Data2(BaseModel): + task_id: Optional[str] = Field(None, description='Task ID') + task_status: Optional[KlingTaskStatus] = None + task_info: Optional[TaskInfo] = None + created_at: Optional[int] = Field(None, description='Task creation time') + updated_at: Optional[int] = Field(None, description='Task update time') + task_result: Optional[TaskResult] = None + + +class KlingVideoExtendResponse(BaseModel): + code: Optional[int] = Field(None, description='Error code') + message: Optional[str] = Field(None, description='Error message') + request_id: Optional[str] = Field(None, description='Request ID') + data: Optional[Data2] = None + + +class KlingLipSyncInputObject(BaseModel): + video_id: Optional[str] = Field( + None, + description='The ID of the video generated by Kling AI. Only supports 5-second and 10-second videos generated within the last 30 days.', + ) + video_url: Optional[AnyUrl] = Field( + None, + description='Get link for uploaded video. Video files support .mp4/.mov, file size does not exceed 100MB, video length between 2-10s.', + ) + mode: KlingLipSyncMode + text: Optional[str] = Field( + None, + description='Text Content for Lip-Sync Video Generation. Required when mode is text2video. Maximum length is 120 characters.', + ) + voice_id: Optional[str] = Field( + None, + description='Voice ID. Required when mode is text2video. The system offers a variety of voice options to choose from.', + ) + voice_language: Optional[KlingLipSyncVoiceLanguage] = 'en' + voice_speed: Optional[float] = Field( + 1, + description='Speech Rate. Valid range: 0.8~2.0, accurate to one decimal place.', + ge=0.8, + le=2.0, + ) + audio_type: Optional[KlingAudioUploadType] = None + audio_file: Optional[str] = Field( + None, + description='Local Path of Audio File. Supported formats: .mp3/.wav/.m4a/.aac, maximum file size of 5MB. Base64 code.', + ) + audio_url: Optional[AnyUrl] = Field( + None, + description='Audio File Download URL. Supported formats: .mp3/.wav/.m4a/.aac, maximum file size of 5MB.', + ) + + +class KlingLipSyncRequest(BaseModel): + input: KlingLipSyncInputObject + callback_url: Optional[AnyUrl] = Field( + None, + description='The callback notification address. Server will notify when the task status changes.', + ) + + +class Data3(BaseModel): + task_id: Optional[str] = Field(None, description='Task ID') + task_status: Optional[KlingTaskStatus] = None + task_info: Optional[TaskInfo] = None + created_at: Optional[int] = Field(None, description='Task creation time') + updated_at: Optional[int] = Field(None, description='Task update time') + task_result: Optional[TaskResult] = None + + +class KlingLipSyncResponse(BaseModel): + code: Optional[int] = Field(None, description='Error code') + message: Optional[str] = Field(None, description='Error message') + request_id: Optional[str] = Field(None, description='Request ID') + data: Optional[Data3] = None + + +class KlingSingleImageEffectInput(BaseModel): + model_name: KlingSingleImageEffectModelName + image: str = Field( + ..., + description='Reference Image. URL or Base64 encoded string (without data:image prefix). File size cannot exceed 10MB, resolution not less than 300*300px, aspect ratio between 1:2.5 ~ 2.5:1.', + ) + duration: KlingSingleImageEffectDuration + + +class KlingDualCharacterEffectInput(BaseModel): + model_name: Optional[KlingCharacterEffectModelName] = 'kling-v1' + mode: Optional[KlingVideoGenMode] = 'std' + images: KlingDualCharacterImages + duration: KlingVideoGenDuration + + +class Data4(BaseModel): + task_id: Optional[str] = Field(None, description='Task ID') + task_status: Optional[KlingTaskStatus] = None + task_info: Optional[TaskInfo] = None + created_at: Optional[int] = Field(None, description='Task creation time') + updated_at: Optional[int] = Field(None, description='Task update time') + task_result: Optional[TaskResult] = None + + +class KlingVideoEffectsResponse(BaseModel): + code: Optional[int] = Field(None, description='Error code') + message: Optional[str] = Field(None, description='Error message') + request_id: Optional[str] = Field(None, description='Request ID') + data: Optional[Data4] = None + + +class KlingImageGenerationsRequest(BaseModel): + model_name: Optional[KlingImageGenModelName] = 'kling-v1' + prompt: str = Field(..., description='Positive text prompt', max_length=500) + negative_prompt: Optional[str] = Field( + None, description='Negative text prompt', max_length=200 + ) + image: Optional[str] = Field( + None, description='Reference Image - Base64 encoded string or image URL' + ) + image_reference: Optional[KlingImageGenImageReferenceType] = None + image_fidelity: Optional[float] = Field( + 0.5, description='Reference intensity for user-uploaded images', ge=0.0, le=1.0 + ) + human_fidelity: Optional[float] = Field( + 0.45, description='Subject reference similarity', ge=0.0, le=1.0 + ) + n: Optional[int] = Field(1, description='Number of generated images', ge=1, le=9) + aspect_ratio: Optional[KlingImageGenAspectRatio] = '16:9' + callback_url: Optional[AnyUrl] = Field( + None, description='The callback notification address' + ) + + +class TaskResult5(BaseModel): + images: Optional[List[KlingImageResult]] = None + + +class Data5(BaseModel): + task_id: Optional[str] = Field(None, description='Task ID') + task_status: Optional[KlingTaskStatus] = None + task_status_msg: Optional[str] = Field(None, description='Task status information') + created_at: Optional[int] = Field(None, description='Task creation time') + updated_at: Optional[int] = Field(None, description='Task update time') + task_result: Optional[TaskResult5] = None + + +class KlingImageGenerationsResponse(BaseModel): + code: Optional[int] = Field(None, description='Error code') + message: Optional[str] = Field(None, description='Error message') + request_id: Optional[str] = Field(None, description='Request ID') + data: Optional[Data5] = None + + +class KlingVirtualTryOnRequest(BaseModel): + model_name: Optional[KlingVirtualTryOnModelName] = 'kolors-virtual-try-on-v1' + human_image: str = Field( + ..., description='Reference human image - Base64 encoded string or image URL' + ) + cloth_image: Optional[str] = Field( + None, + description='Reference clothing image - Base64 encoded string or image URL', + ) + callback_url: Optional[AnyUrl] = Field( + None, description='The callback notification address' + ) + + +class Data6(BaseModel): + task_id: Optional[str] = Field(None, description='Task ID') + task_status: Optional[KlingTaskStatus] = None + task_status_msg: Optional[str] = Field(None, description='Task status information') + created_at: Optional[int] = Field(None, description='Task creation time') + updated_at: Optional[int] = Field(None, description='Task update time') + task_result: Optional[TaskResult5] = None + + +class KlingVirtualTryOnResponse(BaseModel): + code: Optional[int] = Field(None, description='Error code') + message: Optional[str] = Field(None, description='Error message') + request_id: Optional[str] = Field(None, description='Request ID') + data: Optional[Data6] = None + + +class ResourcePackType(str, Enum): + decreasing_total = 'decreasing_total' + constant_period = 'constant_period' + + +class Status(str, Enum): + toBeOnline = 'toBeOnline' + online = 'online' + expired = 'expired' + runOut = 'runOut' + + +class ResourcePackSubscribeInfo(BaseModel): + resource_pack_name: Optional[str] = Field(None, description='Resource package name') + resource_pack_id: Optional[str] = Field(None, description='Resource package ID') + resource_pack_type: Optional[ResourcePackType] = Field( + None, + description='Resource package type (decreasing_total=decreasing total, constant_period=constant periodicity)', + ) + total_quantity: Optional[float] = Field(None, description='Total quantity') + remaining_quantity: Optional[float] = Field( + None, description='Remaining quantity (updated with a 12-hour delay)' + ) + purchase_time: Optional[int] = Field( + None, description='Purchase time, Unix timestamp in ms' + ) + effective_time: Optional[int] = Field( + None, description='Effective time, Unix timestamp in ms' + ) + invalid_time: Optional[int] = Field( + None, description='Expiration time, Unix timestamp in ms' + ) + status: Optional[Status] = Field(None, description='Resource Package Status') + + +class Data7(BaseModel): + code: Optional[int] = Field(None, description='Error code; 0 indicates success') + msg: Optional[str] = Field(None, description='Error information') + resource_pack_subscribe_infos: Optional[List[ResourcePackSubscribeInfo]] = Field( + None, description='Resource package list' + ) + + +class KlingResourcePackageResponse(BaseModel): + code: Optional[int] = Field(None, description='Error code; 0 indicates success') + message: Optional[str] = Field(None, description='Error information') + request_id: Optional[str] = Field( + None, + description='Request ID, generated by the system, used to track requests and troubleshoot problems', + ) + data: Optional[Data7] = None + + +class Object(str, Enum): + event = 'event' + + +class Type(str, Enum): + payment_intent_succeeded = 'payment_intent.succeeded' + + +class StripeRequestInfo(BaseModel): + id: Optional[str] = None + idempotency_key: Optional[str] = None + + +class Object1(str, Enum): + payment_intent = 'payment_intent' + + +class StripeAmountDetails(BaseModel): + tip: Optional[Dict[str, Any]] = None + + +class Object2(str, Enum): + charge = 'charge' + + +class StripeAddress(BaseModel): + city: Optional[str] = None + country: Optional[str] = None + line1: Optional[str] = None + line2: Optional[str] = None + postal_code: Optional[str] = None + state: Optional[str] = None + + +class StripeOutcome(BaseModel): + advice_code: Optional[Any] = None + network_advice_code: Optional[Any] = None + network_decline_code: Optional[Any] = None + network_status: Optional[str] = None + reason: Optional[Any] = None + risk_level: Optional[str] = None + risk_score: Optional[int] = None + seller_message: Optional[str] = None + type: Optional[str] = None + + +class Checks(BaseModel): + address_line1_check: Optional[Any] = None + address_postal_code_check: Optional[Any] = None + cvc_check: Optional[str] = None + + +class ExtendedAuthorization(BaseModel): + status: Optional[str] = None + + +class IncrementalAuthorization(BaseModel): + status: Optional[str] = None + + +class Multicapture(BaseModel): + status: Optional[str] = None + + +class NetworkToken(BaseModel): + used: Optional[bool] = None + + +class Overcapture(BaseModel): + maximum_amount_capturable: Optional[int] = None + status: Optional[str] = None + + +class StripeCardDetails(BaseModel): + amount_authorized: Optional[int] = None + authorization_code: Optional[Any] = None + brand: Optional[str] = None + checks: Optional[Checks] = None + country: Optional[str] = None + exp_month: Optional[int] = None + exp_year: Optional[int] = None + extended_authorization: Optional[ExtendedAuthorization] = None + fingerprint: Optional[str] = None + funding: Optional[str] = None + incremental_authorization: Optional[IncrementalAuthorization] = None + installments: Optional[Any] = None + last4: Optional[str] = None + mandate: Optional[Any] = None + multicapture: Optional[Multicapture] = None + network: Optional[str] = None + network_token: Optional[NetworkToken] = None + network_transaction_id: Optional[str] = None + overcapture: Optional[Overcapture] = None + regulated_status: Optional[str] = None + three_d_secure: Optional[Any] = None + wallet: Optional[Any] = None + + +class StripeRefundList(BaseModel): + object: Optional[str] = None + data: Optional[List[Dict[str, Any]]] = None + has_more: Optional[bool] = None + total_count: Optional[int] = None + url: Optional[str] = None + + +class Card(BaseModel): + installments: Optional[Any] = None + mandate_options: Optional[Any] = None + network: Optional[Any] = None + request_three_d_secure: Optional[str] = None + + +class StripePaymentMethodOptions(BaseModel): + card: Optional[Card] = None + + +class StripeShipping(BaseModel): + address: Optional[StripeAddress] = None + carrier: Optional[str] = None + name: Optional[str] = None + phone: Optional[str] = None + tracking_number: Optional[str] = None + + +class Model(str, Enum): + T2V_01_Director = 'T2V-01-Director' + I2V_01_Director = 'I2V-01-Director' + S2V_01 = 'S2V-01' + I2V_01 = 'I2V-01' + I2V_01_live = 'I2V-01-live' + T2V_01 = 'T2V-01' + + +class SubjectReferenceItem(BaseModel): + image: Optional[str] = Field( + None, description='URL or base64 encoding of the subject reference image.' + ) + mask: Optional[str] = Field( + None, + description='URL or base64 encoding of the mask for the subject reference image.', + ) + + +class MinimaxVideoGenerationRequest(BaseModel): + model: Model = Field( + ..., + description='Required. ID of model. Options: T2V-01-Director, I2V-01-Director, S2V-01, I2V-01, I2V-01-live, T2V-01', + ) + prompt: Optional[str] = Field( + None, + description='Description of the video. Should be less than 2000 characters. Supports camera movement instructions in [brackets].', + max_length=2000, + ) + prompt_optimizer: Optional[bool] = Field( + True, + description='If true (default), the model will automatically optimize the prompt. Set to false for more precise control.', + ) + first_frame_image: Optional[str] = Field( + None, + description='URL or base64 encoding of the first frame image. Required when model is I2V-01, I2V-01-Director, or I2V-01-live.', + ) + subject_reference: Optional[List[SubjectReferenceItem]] = Field( + None, + description='Only available when model is S2V-01. The model will generate a video based on the subject uploaded through this parameter.', + ) + callback_url: Optional[str] = Field( + None, + description='Optional. URL to receive real-time status updates about the video generation task.', + ) + + +class MinimaxBaseResponse(BaseModel): + status_code: int = Field( + ..., + description='Status code. 0 indicates success, other values indicate errors.', + ) + status_msg: str = Field( + ..., description='Specific error details or success message.' + ) + + +class MinimaxVideoGenerationResponse(BaseModel): + task_id: str = Field( + ..., description='The task ID for the asynchronous video generation task.' + ) + base_resp: MinimaxBaseResponse + + +class File(BaseModel): + file_id: Optional[int] = Field(None, description='Unique identifier for the file') + bytes: Optional[int] = Field(None, description='File size in bytes') + created_at: Optional[int] = Field( + None, description='Unix timestamp when the file was created, in seconds' + ) + filename: Optional[str] = Field(None, description='The name of the file') + purpose: Optional[str] = Field(None, description='The purpose of using the file') + download_url: Optional[str] = Field( + None, description='The URL to download the video' + ) + + +class MinimaxFileRetrieveResponse(BaseModel): + file: File + base_resp: MinimaxBaseResponse + + +class Status1(str, Enum): + Queueing = 'Queueing' + Preparing = 'Preparing' + Processing = 'Processing' + Success = 'Success' + Fail = 'Fail' + + +class MinimaxTaskResultResponse(BaseModel): + task_id: str = Field(..., description='The task ID being queried.') + status: Status1 = Field( + ..., + description="Task status: 'Queueing' (in queue), 'Preparing' (task is preparing), 'Processing' (generating), 'Success' (task completed successfully), or 'Fail' (task failed).", + ) + file_id: Optional[str] = Field( + None, + description='After the task status changes to Success, this field returns the file ID corresponding to the generated video.', + ) + base_resp: MinimaxBaseResponse + + +class OutputFormat(str, Enum): + jpeg = 'jpeg' + png = 'png' + + +class BFLFluxPro11GenerateRequest(BaseModel): + prompt: str = Field(..., description='The main text prompt for image generation') + image_prompt: Optional[str] = Field(None, description='Optional image prompt') + width: int = Field(..., description='Width of the generated image') + height: int = Field(..., description='Height of the generated image') + prompt_upsampling: Optional[bool] = Field( + None, description='Whether to use prompt upsampling' + ) + seed: Optional[int] = Field(None, description='Random seed for reproducibility') + safety_tolerance: Optional[int] = Field(None, description='Safety tolerance level') + output_format: Optional[OutputFormat] = Field( + None, description='Output image format' + ) + webhook_url: Optional[str] = Field( + None, description='Optional webhook URL for async processing' + ) + webhook_secret: Optional[str] = Field( + None, description='Optional webhook secret for async processing' + ) + + +class BFLFluxPro11GenerateResponse(BaseModel): + id: str = Field(..., description='Job ID for tracking') + polling_url: str = Field(..., description='URL to poll for results') + + +class BFLFluxProGenerateRequest(BaseModel): + prompt: str = Field(..., description='The text prompt for image generation.') + negative_prompt: Optional[str] = Field( + None, description='The negative prompt for image generation.' + ) + width: int = Field( + ..., description='The width of the image to generate.', ge=64, le=2048 + ) + height: int = Field( + ..., description='The height of the image to generate.', ge=64, le=2048 + ) + num_inference_steps: Optional[int] = Field( + None, description='The number of inference steps.', ge=1, le=100 + ) + guidance_scale: Optional[float] = Field( + None, description='The guidance scale for generation.', ge=1.0, le=20.0 + ) + seed: Optional[int] = Field(None, description='The seed value for reproducibility.') + num_images: Optional[int] = Field( + None, description='The number of images to generate.', ge=1, le=4 + ) + + +class BFLFluxProGenerateResponse(BaseModel): + id: str = Field(..., description='The unique identifier for the generation task.') + polling_url: str = Field(..., description='URL to poll for the generation result.') + + +class Steps(RootModel[int]): + root: int = Field( + ..., + description='Number of steps for the image generation process', + examples=[50], + ge=15, + le=50, + title='Steps', + ) + + +class Guidance(RootModel[float]): + root: float = Field( + ..., + description='Guidance strength for the image generation process', + ge=1.5, + le=100.0, + title='Guidance', + ) + + +class WebhookUrl(RootModel[AnyUrl]): + root: AnyUrl = Field( + ..., description='URL to receive webhook notifications', title='Webhook Url' + ) + + +class BFLAsyncResponse(BaseModel): + id: str = Field(..., title='Id') + polling_url: str = Field(..., title='Polling Url') + + +class BFLAsyncWebhookResponse(BaseModel): + id: str = Field(..., title='Id') + status: str = Field(..., title='Status') + webhook_url: str = Field(..., title='Webhook Url') + + +class Top(RootModel[int]): + root: int = Field( + ..., + description='Number of pixels to expand at the top of the image', + ge=0, + le=2048, + title='Top', + ) + + +class Bottom(RootModel[int]): + root: int = Field( + ..., + description='Number of pixels to expand at the bottom of the image', + ge=0, + le=2048, + title='Bottom', + ) + + +class Left(RootModel[int]): + root: int = Field( + ..., + description='Number of pixels to expand on the left side of the image', + ge=0, + le=2048, + title='Left', + ) + + +class Right(RootModel[int]): + root: int = Field( + ..., + description='Number of pixels to expand on the right side of the image', + ge=0, + le=2048, + title='Right', + ) + + +class CannyLowThreshold(RootModel[int]): + root: int = Field( + ..., + description='Low threshold for Canny edge detection', + ge=0, + le=500, + title='Canny Low Threshold', + ) + + +class CannyHighThreshold(RootModel[int]): + root: int = Field( + ..., + description='High threshold for Canny edge detection', + ge=0, + le=500, + title='Canny High Threshold', + ) + + +class Steps2(RootModel[int]): + root: int = Field( + ..., + description='Number of steps for the image generation process', + ge=15, + le=50, + title='Steps', + ) + + +class Guidance2(RootModel[float]): + root: float = Field( + ..., + description='Guidance strength for the image generation process', + ge=1.0, + le=100.0, + title='Guidance', + ) + + +class BFLOutputFormat(str, Enum): + jpeg = 'jpeg' + png = 'png' + + +class BFLValidationError(BaseModel): + loc: List[Union[str, int]] = Field(..., title='Location') + msg: str = Field(..., title='Message') + type: str = Field(..., title='Error Type') + + +class Datum2(BaseModel): + image_id: Optional[str] = Field( + None, description='Unique identifier for the generated image' + ) + url: Optional[str] = Field(None, description='URL to access the generated image') + + +class RecraftImageGenerationResponse(BaseModel): + created: int = Field( + ..., description='Unix timestamp when the generation was created' + ) + credits: int = Field(..., description='Number of credits used for the generation') + data: List[Datum2] = Field(..., description='Array of generated image information') + + +class RecraftImageFeatures(BaseModel): + nsfw_score: Optional[float] = None + + +class RecraftTextLayoutItem(BaseModel): + bbox: List[List[float]] + text: str + + +class RecraftImageColor(BaseModel): + rgb: Optional[List[int]] = None + std: Optional[List[float]] = None + weight: Optional[float] = None + + +class RecraftImageStyle(str, Enum): + digital_illustration = 'digital_illustration' + icon = 'icon' + realistic_image = 'realistic_image' + vector_illustration = 'vector_illustration' + + +class RecraftImageSubStyle(str, Enum): + field_2d_art_poster = '2d_art_poster' + field_3d = '3d' + field_80s = '80s' + glow = 'glow' + grain = 'grain' + hand_drawn = 'hand_drawn' + infantile_sketch = 'infantile_sketch' + kawaii = 'kawaii' + pixel_art = 'pixel_art' + psychedelic = 'psychedelic' + seamless = 'seamless' + voxel = 'voxel' + watercolor = 'watercolor' + broken_line = 'broken_line' + colored_outline = 'colored_outline' + colored_shapes = 'colored_shapes' + colored_shapes_gradient = 'colored_shapes_gradient' + doodle_fill = 'doodle_fill' + doodle_offset_fill = 'doodle_offset_fill' + offset_fill = 'offset_fill' + outline = 'outline' + outline_gradient = 'outline_gradient' + uneven_fill = 'uneven_fill' + field_70s = '70s' + cartoon = 'cartoon' + doodle_line_art = 'doodle_line_art' + engraving = 'engraving' + flat_2 = 'flat_2' + kawaii_1 = 'kawaii' + line_art = 'line_art' + linocut = 'linocut' + seamless_1 = 'seamless' + b_and_w = 'b_and_w' + enterprise = 'enterprise' + hard_flash = 'hard_flash' + hdr = 'hdr' + motion_blur = 'motion_blur' + natural_light = 'natural_light' + studio_portrait = 'studio_portrait' + line_circuit = 'line_circuit' + field_2d_art_poster_2 = '2d_art_poster_2' + engraving_color = 'engraving_color' + flat_air_art = 'flat_air_art' + hand_drawn_outline = 'hand_drawn_outline' + handmade_3d = 'handmade_3d' + stickers_drawings = 'stickers_drawings' + plastic = 'plastic' + pictogram = 'pictogram' + + +class RecraftTransformModel(str, Enum): + refm1 = 'refm1' + recraft20b = 'recraft20b' + recraftv2 = 'recraftv2' + recraftv3 = 'recraftv3' + flux1_1pro = 'flux1_1pro' + flux1dev = 'flux1dev' + imagen3 = 'imagen3' + hidream_i1_dev = 'hidream_i1_dev' + + +class RecraftImageFormat(str, Enum): + webp = 'webp' + png = 'png' + + +class RecraftResponseFormat(str, Enum): + url = 'url' + b64_json = 'b64_json' + + +class RecraftImage(BaseModel): + b64_json: Optional[str] = None + features: Optional[RecraftImageFeatures] = None + image_id: UUID + revised_prompt: Optional[str] = None + url: Optional[str] = None + + +class RecraftUserControls(BaseModel): + artistic_level: Optional[int] = None + background_color: Optional[RecraftImageColor] = None + colors: Optional[List[RecraftImageColor]] = None + no_text: Optional[bool] = None + + +class RecraftTextLayout(RootModel[List[RecraftTextLayoutItem]]): + root: List[RecraftTextLayoutItem] + + +class RecraftProcessImageRequest(BaseModel): + image: StrictBytes + image_format: Optional[RecraftImageFormat] = None + response_format: Optional[RecraftResponseFormat] = None + + +class RecraftProcessImageResponse(BaseModel): + created: int + credits: int + image: RecraftImage + + +class RecraftImageToImageRequest(BaseModel): + block_nsfw: Optional[bool] = None + calculate_features: Optional[bool] = None + controls: Optional[RecraftUserControls] = None + image: StrictBytes + image_format: Optional[RecraftImageFormat] = None + model: Optional[RecraftTransformModel] = None + n: Optional[int] = None + negative_prompt: Optional[str] = None + prompt: str + random_seed: Optional[int] = None + response_format: Optional[RecraftResponseFormat] = None + strength: float + style: Optional[RecraftImageStyle] = None + style_id: Optional[UUID] = None + substyle: Optional[RecraftImageSubStyle] = None + text_layout: Optional[RecraftTextLayout] = None + + +class RecraftGenerateImageResponse(BaseModel): + created: int + credits: int + data: List[RecraftImage] + + +class RecraftTransformImageWithMaskRequest(BaseModel): + block_nsfw: Optional[bool] = None + calculate_features: Optional[bool] = None + image: StrictBytes + image_format: Optional[RecraftImageFormat] = None + mask: StrictBytes + model: Optional[RecraftTransformModel] = None + n: Optional[int] = None + negative_prompt: Optional[str] = None + prompt: str + random_seed: Optional[int] = None + response_format: Optional[RecraftResponseFormat] = None + style: Optional[RecraftImageStyle] = None + style_id: Optional[UUID] = None + substyle: Optional[RecraftImageSubStyle] = None + text_layout: Optional[RecraftTextLayout] = None + + +class KlingErrorResponse(BaseModel): + code: int = Field( + ..., + description='- 1000: Authentication failed\n- 1001: Authorization is empty\n- 1002: Authorization is invalid\n- 1003: Authorization is not yet valid\n- 1004: Authorization has expired\n- 1100: Account exception\n- 1101: Account in arrears (postpaid scenario)\n- 1102: Resource pack depleted or expired (prepaid scenario)\n- 1103: Unauthorized access to requested resource\n- 1200: Invalid request parameters\n- 1201: Invalid parameters\n- 1202: Invalid request method\n- 1203: Requested resource does not exist\n- 1300: Trigger platform strategy\n- 1301: Trigger content security policy\n- 1302: API request too frequent\n- 1303: Concurrency/QPS exceeds limit\n- 1304: Trigger IP whitelist policy\n- 5000: Internal server error\n- 5001: Service temporarily unavailable\n- 5002: Server internal timeout\n', + ) + message: str = Field(..., description='Human-readable error message') + request_id: str = Field( + ..., description='Request ID for tracking and troubleshooting' + ) class LumaAspectRatio(str, Enum): - field_1_1 = "1:1" - field_16_9 = "16:9" - field_9_16 = "9:16" - field_4_3 = "4:3" - field_3_4 = "3:4" - field_21_9 = "21:9" - field_9_21 = "9:21" - - -class LumaAssets(BaseModel): - image: Optional[AnyUrl] = Field(None, description="The URL of the image") - progress_video: Optional[AnyUrl] = Field( - None, description="The URL of the progress video" - ) - video: Optional[AnyUrl] = Field(None, description="The URL of the video") - - -class GenerationType(str, Enum): - add_audio = "add_audio" - - -class LumaAudioGenerationRequest(BaseModel): - callback_url: Optional[AnyUrl] = Field( - None, description="The callback URL for the audio" - ) - generation_type: Optional[GenerationType] = "add_audio" - negative_prompt: Optional[str] = Field( - None, description="The negative prompt of the audio" - ) - prompt: Optional[str] = Field(None, description="The prompt of the audio") - - -class LumaError(BaseModel): - detail: Optional[str] = Field(None, description="The error message") - - -class Type2(str, Enum): - generation = "generation" - - -class LumaGenerationReference(BaseModel): - id: UUID = Field(..., description="The ID of the generation") - type: Literal["generation"] - - -class GenerationType1(str, Enum): - video = "video" - - -class LumaGenerationType(str, Enum): - video = "video" - image = "image" - - -class GenerationType2(str, Enum): - image = "image" - - -class LumaImageIdentity(BaseModel): - images: Optional[List[AnyUrl]] = Field( - None, description="The URLs of the image identity" - ) - - -class LumaImageModel(str, Enum): - photon_1 = "photon-1" - photon_flash_1 = "photon-flash-1" - - -class LumaImageRef(BaseModel): - url: Optional[AnyUrl] = Field(None, description="The URL of the image reference") - weight: Optional[float] = Field( - None, description="The weight of the image reference" - ) - - -class Type3(str, Enum): - image = "image" - - -class LumaImageReference(BaseModel): - type: Literal["image"] - url: AnyUrl = Field(..., description="The URL of the image") - - -class LumaKeyframe(RootModel[Union[LumaGenerationReference, LumaImageReference]]): - root: Union[LumaGenerationReference, LumaImageReference] = Field( - ..., - description="A keyframe can be either a Generation reference, an Image, or a Video", - discriminator="type", - ) - - -class LumaKeyframes(BaseModel): - frame0: Optional[LumaKeyframe] = None - frame1: Optional[LumaKeyframe] = None - - -class LumaModifyImageRef(BaseModel): - url: Optional[AnyUrl] = Field(None, description="The URL of the image reference") - weight: Optional[float] = Field( - None, description="The weight of the modify image reference" - ) - - -class LumaState(str, Enum): - queued = "queued" - dreaming = "dreaming" - completed = "completed" - failed = "failed" - - -class GenerationType3(str, Enum): - upscale_video = "upscale_video" + field_1_1 = '1:1' + field_16_9 = '16:9' + field_9_16 = '9:16' + field_4_3 = '4:3' + field_3_4 = '3:4' + field_21_9 = '21:9' + field_9_21 = '9:21' class LumaVideoModel(str, Enum): - ray_2 = "ray-2" - ray_flash_2 = "ray-flash-2" - ray_1_6 = "ray-1-6" - - -class LumaVideoModelOutputDuration1(str, Enum): - field_5s = "5s" - field_9s = "9s" - - -class LumaVideoModelOutputDuration( - RootModel[Union[LumaVideoModelOutputDuration1, str]] -): - root: Union[LumaVideoModelOutputDuration1, str] + ray_2 = 'ray-2' + ray_flash_2 = 'ray-flash-2' + ray_1_6 = 'ray-1-6' class LumaVideoModelOutputResolution1(str, Enum): - field_540p = "540p" - field_720p = "720p" - field_1080p = "1080p" - field_4k = "4k" + field_540p = '540p' + field_720p = '720p' + field_1080p = '1080p' + field_4k = '4k' class LumaVideoModelOutputResolution( @@ -568,397 +1536,244 @@ class LumaVideoModelOutputResolution( root: Union[LumaVideoModelOutputResolution1, str] -class MinimaxBaseResponse(BaseModel): - status_code: int = Field( +class LumaVideoModelOutputDuration1(str, Enum): + field_5s = '5s' + field_9s = '9s' + + +class LumaVideoModelOutputDuration( + RootModel[Union[LumaVideoModelOutputDuration1, str]] +): + root: Union[LumaVideoModelOutputDuration1, str] + + +class LumaImageModel(str, Enum): + photon_1 = 'photon-1' + photon_flash_1 = 'photon-flash-1' + + +class LumaImageRef(BaseModel): + url: Optional[AnyUrl] = Field(None, description='The URL of the image reference') + weight: Optional[float] = Field( + None, description='The weight of the image reference' + ) + + +class LumaImageIdentity(BaseModel): + images: Optional[List[AnyUrl]] = Field( + None, description='The URLs of the image identity' + ) + + +class LumaModifyImageRef(BaseModel): + url: Optional[AnyUrl] = Field(None, description='The URL of the image reference') + weight: Optional[float] = Field( + None, description='The weight of the modify image reference' + ) + + +class Type1(str, Enum): + generation = 'generation' + + +class LumaGenerationReference(BaseModel): + type: Literal['generation'] + id: UUID = Field(..., description='The ID of the generation') + + +class Type2(str, Enum): + image = 'image' + + +class LumaImageReference(BaseModel): + type: Literal['image'] + url: AnyUrl = Field(..., description='The URL of the image') + + +class LumaKeyframe(RootModel[Union[LumaGenerationReference, LumaImageReference]]): + root: Union[LumaGenerationReference, LumaImageReference] = Field( ..., - description="Status code. 0 indicates success, other values indicate errors.", - ) - status_msg: str = Field( - ..., description="Specific error details or success message." + description='A keyframe can be either a Generation reference, an Image, or a Video', + discriminator='type', ) -class File(BaseModel): - bytes: Optional[int] = Field(None, description="File size in bytes") - created_at: Optional[int] = Field( - None, description="Unix timestamp when the file was created, in seconds" - ) - download_url: Optional[str] = Field( - None, description="The URL to download the video" - ) - file_id: Optional[int] = Field(None, description="Unique identifier for the file") - filename: Optional[str] = Field(None, description="The name of the file") - purpose: Optional[str] = Field(None, description="The purpose of using the file") +class LumaGenerationType(str, Enum): + video = 'video' + image = 'image' -class MinimaxFileRetrieveResponse(BaseModel): - base_resp: MinimaxBaseResponse - file: File +class LumaState(str, Enum): + queued = 'queued' + dreaming = 'dreaming' + completed = 'completed' + failed = 'failed' -class Status(str, Enum): - Queueing = "Queueing" - Preparing = "Preparing" - Processing = "Processing" - Success = "Success" - Fail = "Fail" - - -class MinimaxTaskResultResponse(BaseModel): - base_resp: MinimaxBaseResponse - file_id: Optional[str] = Field( - None, - description="After the task status changes to Success, this field returns the file ID corresponding to the generated video.", - ) - status: Status = Field( - ..., - description="Task status: 'Queueing' (in queue), 'Preparing' (task is preparing), 'Processing' (generating), 'Success' (task completed successfully), or 'Fail' (task failed).", - ) - task_id: str = Field(..., description="The task ID being queried.") - - -class Model(str, Enum): - T2V_01_Director = "T2V-01-Director" - I2V_01_Director = "I2V-01-Director" - S2V_01 = "S2V-01" - I2V_01 = "I2V-01" - I2V_01_live = "I2V-01-live" - T2V_01 = "T2V-01" - - -class SubjectReferenceItem(BaseModel): - image: Optional[str] = Field( - None, description="URL or base64 encoding of the subject reference image." - ) - mask: Optional[str] = Field( - None, - description="URL or base64 encoding of the mask for the subject reference image.", +class LumaAssets(BaseModel): + video: Optional[AnyUrl] = Field(None, description='The URL of the video') + image: Optional[AnyUrl] = Field(None, description='The URL of the image') + progress_video: Optional[AnyUrl] = Field( + None, description='The URL of the progress video' ) -class MinimaxVideoGenerationRequest(BaseModel): - callback_url: Optional[str] = Field( - None, - description="Optional. URL to receive real-time status updates about the video generation task.", +class GenerationType(str, Enum): + video = 'video' + + +class GenerationType1(str, Enum): + image = 'image' + + +class CharacterRef(BaseModel): + identity0: Optional[LumaImageIdentity] = None + + +class LumaImageGenerationRequest(BaseModel): + generation_type: Optional[GenerationType1] = 'image' + model: Optional[LumaImageModel] = 'photon-1' + prompt: Optional[str] = Field(None, description='The prompt of the generation') + aspect_ratio: Optional[LumaAspectRatio] = '16:9' + callback_url: Optional[AnyUrl] = Field( + None, description='The callback URL for the generation' ) - first_frame_image: Optional[str] = Field( - None, - description="URL or base64 encoding of the first frame image. Required when model is I2V-01, I2V-01-Director, or I2V-01-live.", - ) - model: Model = Field( - ..., - description="Required. ID of model. Options: T2V-01-Director, I2V-01-Director, S2V-01, I2V-01, I2V-01-live, T2V-01", - ) - prompt: Optional[str] = Field( - None, - description="Description of the video. Should be less than 2000 characters. Supports camera movement instructions in [brackets].", - max_length=2000, - ) - prompt_optimizer: Optional[bool] = Field( - True, - description="If true (default), the model will automatically optimize the prompt. Set to false for more precise control.", - ) - subject_reference: Optional[List[SubjectReferenceItem]] = Field( - None, - description="Only available when model is S2V-01. The model will generate a video based on the subject uploaded through this parameter.", + image_ref: Optional[List[LumaImageRef]] = None + style_ref: Optional[List[LumaImageRef]] = None + character_ref: Optional[CharacterRef] = None + modify_image_ref: Optional[LumaModifyImageRef] = None + + +class GenerationType2(str, Enum): + upscale_video = 'upscale_video' + + +class LumaUpscaleVideoGenerationRequest(BaseModel): + generation_type: Optional[GenerationType2] = 'upscale_video' + resolution: Optional[LumaVideoModelOutputResolution] = None + callback_url: Optional[AnyUrl] = Field( + None, description='The callback URL for the upscale' ) -class MinimaxVideoGenerationResponse(BaseModel): - base_resp: MinimaxBaseResponse - task_id: str = Field( - ..., description="The task ID for the asynchronous video generation task." +class GenerationType3(str, Enum): + add_audio = 'add_audio' + + +class LumaAudioGenerationRequest(BaseModel): + generation_type: Optional[GenerationType3] = 'add_audio' + prompt: Optional[str] = Field(None, description='The prompt of the audio') + negative_prompt: Optional[str] = Field( + None, description='The negative prompt of the audio' + ) + callback_url: Optional[AnyUrl] = Field( + None, description='The callback URL for the audio' ) -class Moderation(str, Enum): - low = "low" - auto = "auto" +class LumaError(BaseModel): + detail: Optional[str] = Field(None, description='The error message') -class OutputFormat(str, Enum): - png = "png" - webp = "webp" - jpeg = "jpeg" +class AspectRatio(str, Enum): + field_16_9 = '16:9' + field_4_3 = '4:3' + field_1_1 = '1:1' + field_3_4 = '3:4' + field_9_16 = '9:16' -class OpenAIImageEditRequest(BaseModel): - background: Optional[str] = Field( - None, description="Background transparency", examples=["opaque"] - ) - model: str = Field( - ..., description="The model to use for image editing", examples=["gpt-image-1"] - ) - moderation: Optional[Moderation] = Field( - None, description="Content moderation setting", examples=["auto"] - ) - n: Optional[int] = Field( - None, description="The number of images to generate", examples=[1] - ) - output_compression: Optional[int] = Field( - None, description="Compression level for JPEG or WebP (0-100)", examples=[100] - ) - output_format: Optional[OutputFormat] = Field( - None, description="Format of the output image", examples=["png"] - ) - prompt: str = Field( - ..., - description="A text description of the desired edit", - examples=["Give the rocketship rainbow coloring"], - ) - quality: Optional[str] = Field( - None, description="The quality of the edited image", examples=["low"] - ) - size: Optional[str] = Field( - None, description="Size of the output image", examples=["1024x1024"] - ) - user: Optional[str] = Field( - None, - description="A unique identifier for end-user monitoring", - examples=["user-1234"], - ) - - -class Background(str, Enum): - transparent = "transparent" - opaque = "opaque" - - -class Quality(str, Enum): - low = "low" - medium = "medium" - high = "high" - standard = "standard" - hd = "hd" - - -class ResponseFormat(str, Enum): - url = "url" - b64_json = "b64_json" - - -class Style(str, Enum): - vivid = "vivid" - natural = "natural" - - -class OpenAIImageGenerationRequest(BaseModel): - background: Optional[Background] = Field( - None, description="Background transparency", examples=["opaque"] - ) - model: Optional[str] = Field( - None, description="The model to use for image generation", examples=["dall-e-3"] - ) - moderation: Optional[Moderation] = Field( - None, description="Content moderation setting", examples=["auto"] - ) - n: Optional[int] = Field( - None, - description="The number of images to generate (1-10). Only 1 supported for dall-e-3.", - examples=[1], - ) - output_compression: Optional[int] = Field( - None, description="Compression level for JPEG or WebP (0-100)", examples=[100] - ) - output_format: Optional[OutputFormat] = Field( - None, description="Format of the output image", examples=["png"] - ) - prompt: str = Field( - ..., - description="A text description of the desired image", - examples=["Draw a rocket in front of a blackhole in deep space"], - ) - quality: Optional[Quality] = Field( - None, description="The quality of the generated image", examples=["high"] - ) - response_format: Optional[ResponseFormat] = Field( - None, description="Response format of image data", examples=["b64_json"] - ) - size: Optional[str] = Field( - None, - description="Size of the image (e.g., 1024x1024, 1536x1024, auto)", - examples=["1024x1536"], - ) - style: Optional[Style] = Field( - None, description="Style of the image (only for dall-e-3)", examples=["vivid"] - ) - user: Optional[str] = Field( - None, - description="A unique identifier for end-user monitoring", - examples=["user-1234"], - ) - - -class Datum1(BaseModel): - b64_json: Optional[str] = Field(None, description="Base64 encoded image data") - revised_prompt: Optional[str] = Field(None, description="Revised prompt") - url: Optional[str] = Field(None, description="URL of the image") - - -class InputTokensDetails(BaseModel): - image_tokens: Optional[int] = None - text_tokens: Optional[int] = None - - -class Usage(BaseModel): - input_tokens: Optional[int] = None - input_tokens_details: Optional[InputTokensDetails] = None - output_tokens: Optional[int] = None - total_tokens: Optional[int] = None - - -class OpenAIImageGenerationResponse(BaseModel): - data: Optional[List[Datum1]] = None - usage: Optional[Usage] = None - - -class AspectRatio2(RootModel[float]): - root: float = Field( - ..., - description="Aspect ratio (width / height)", - ge=0.4, - le=2.5, - title="Aspectratio", - ) - - -class IngredientsMode(str, Enum): - creative = "creative" - precise = "precise" - - -class PikaDurationEnum(int, Enum): - integer_5 = 5 - integer_10 = 10 - - -class PikaGenerateResponse(BaseModel): - video_id: str = Field(..., title="Video Id") - - -class PikaResolutionEnum(str, Enum): - field_1080p = "1080p" - field_720p = "720p" - - -class PikaStatusEnum(str, Enum): - queued = "queued" - started = "started" - finished = "finished" - - -class PikaValidationError(BaseModel): - loc: List[Union[str, int]] = Field(..., title="Location") - msg: str = Field(..., title="Message") - type: str = Field(..., title="Error Type") - - -class PikaVideoResponse(BaseModel): - id: str = Field(..., title="Id") - progress: Optional[int] = Field(None, title="Progress") - status: PikaStatusEnum - url: Optional[str] = Field(None, title="Url") - - -class Resp(BaseModel): - img_id: Optional[int] = None - - -class PixverseImageUploadResponse(BaseModel): - ErrCode: Optional[int] = None - ErrMsg: Optional[str] = None - Resp_1: Optional[Resp] = Field(None, alias="Resp") - - -class Duration2(int, Enum): +class Duration(int, Enum): integer_5 = 5 integer_8 = 8 class Model1(str, Enum): - v3_5 = "v3.5" + v3_5 = 'v3.5' class MotionMode(str, Enum): - normal = "normal" - fast = "fast" + normal = 'normal' + fast = 'fast' -class Quality1(str, Enum): - field_360p = "360p" - field_540p = "540p" - field_720p = "720p" - field_1080p = "1080p" +class Quality(str, Enum): + field_360p = '360p' + field_540p = '540p' + field_720p = '720p' + field_1080p = '1080p' -class Style1(str, Enum): - anime = "anime" - field_3d_animation = "3d_animation" - clay = "clay" - comic = "comic" - cyberpunk = "cyberpunk" - - -class PixverseImageVideoRequest(BaseModel): - duration: Duration2 - img_id: int - model: Model1 - motion_mode: Optional[MotionMode] = None - prompt: str - quality: Quality1 - seed: Optional[int] = None - style: Optional[Style1] = None - template_id: Optional[int] = None - water_mark: Optional[bool] = None - - -class AspectRatio3(str, Enum): - field_16_9 = "16:9" - field_4_3 = "4:3" - field_1_1 = "1:1" - field_3_4 = "3:4" - field_9_16 = "9:16" +class Style(str, Enum): + anime = 'anime' + field_3d_animation = '3d_animation' + clay = 'clay' + comic = 'comic' + cyberpunk = 'cyberpunk' class PixverseTextVideoRequest(BaseModel): - aspect_ratio: AspectRatio3 - duration: Duration2 + aspect_ratio: AspectRatio + duration: Duration model: Model1 motion_mode: Optional[MotionMode] = None negative_prompt: Optional[str] = None prompt: str - quality: Quality1 + quality: Quality seed: Optional[int] = None - style: Optional[Style1] = None + style: Optional[Style] = None template_id: Optional[int] = None water_mark: Optional[bool] = None -class PixverseTransitionVideoRequest(BaseModel): - duration: Duration2 - first_frame_img: int - last_frame_img: int - model: Model1 - motion_mode: MotionMode - prompt: str - quality: Quality1 - seed: int - style: Optional[Style1] = None - template_id: Optional[int] = None - water_mark: Optional[bool] = None - - -class Resp1(BaseModel): +class Resp(BaseModel): video_id: Optional[int] = None class PixverseVideoResponse(BaseModel): + ErrCode: Optional[int] = None + ErrMsg: Optional[str] = None + Resp_1: Optional[Resp] = Field(None, alias='Resp') + + +class Resp1(BaseModel): + img_id: Optional[int] = None + + +class PixverseImageUploadResponse(BaseModel): ErrCode: Optional[int] = None ErrMsg: Optional[str] = None Resp: Optional[Resp1] = None -class Status1(int, Enum): +class PixverseImageVideoRequest(BaseModel): + img_id: int + model: Model1 + prompt: str + duration: Duration + quality: Quality + motion_mode: Optional[MotionMode] = None + seed: Optional[int] = None + style: Optional[Style] = None + template_id: Optional[int] = None + water_mark: Optional[bool] = None + + +class PixverseTransitionVideoRequest(BaseModel): + first_frame_img: int + last_frame_img: int + model: Model1 + duration: Duration + quality: Quality + motion_mode: MotionMode + seed: int + prompt: str + style: Optional[Style] = None + template_id: Optional[int] = None + water_mark: Optional[bool] = None + + +class Status2(int, Enum): integer_1 = 1 integer_5 = 5 integer_6 = 6 @@ -977,9 +1792,9 @@ class Resp2(BaseModel): resolution_ratio: Optional[int] = None seed: Optional[int] = None size: Optional[int] = None - status: Optional[Status1] = Field( + status: Optional[Status2] = Field( None, - description="Video generation status codes:\n* 1 - Generation successful\n* 5 - Generating\n* 6 - Deleted\n* 7 - Contents moderation failed\n* 8 - Generation failed\n", + description='Video generation status codes:\n* 1 - Generation successful\n* 5 - Generating\n* 6 - Deleted\n* 7 - Contents moderation failed\n* 8 - Generation failed\n', ) style: Optional[str] = None url: Optional[str] = None @@ -991,124 +1806,6 @@ class PixverseVideoResultResponse(BaseModel): Resp: Optional[Resp2] = None -class RgbItem(RootModel[int]): - root: int = Field(..., ge=0, le=255) - - -class RGBColor(BaseModel): - rgb: List[RgbItem] = Field(..., max_length=3, min_length=3) - - -class Controls(BaseModel): - artistic_level: Optional[int] = Field( - None, - description="Defines artistic tone of your image. At a simple level, the person looks straight at the camera in a static and clean style. Dynamic and eccentric levels introduce movement and creativity.", - ge=0, - le=5, - ) - background_color: Optional[RGBColor] = None - colors: Optional[List[RGBColor]] = Field( - None, description="An array of preferable colors" - ) - no_text: Optional[bool] = Field(None, description="Do not embed text layouts") - - -class RecraftImageGenerationRequest(BaseModel): - controls: Optional[Controls] = Field( - None, description="The controls for the generated image" - ) - model: str = Field( - ..., description='The model to use for generation (e.g., "recraftv3")' - ) - n: int = Field(..., description="The number of images to generate", ge=1, le=4) - prompt: str = Field( - ..., description="The text prompt describing the image to generate" - ) - size: str = Field( - ..., description='The size of the generated image (e.g., "1024x1024")' - ) - style: Optional[str] = Field( - None, - description='The style to apply to the generated image (e.g., "digital_illustration")', - ) - style_id: Optional[str] = Field( - None, - description='The style ID to apply to the generated image (e.g., "123e4567-e89b-12d3-a456-426614174000"). If style_id is provided, style should not be provided.', - ) - - -class Datum2(BaseModel): - image_id: Optional[str] = Field( - None, description="Unique identifier for the generated image" - ) - url: Optional[str] = Field(None, description="URL to access the generated image") - - -class RecraftImageGenerationResponse(BaseModel): - created: int = Field( - ..., description="Unix timestamp when the generation was created" - ) - credits: int = Field(..., description="Number of credits used for the generation") - data: List[Datum2] = Field(..., description="Array of generated image information") - - -class RenderingSpeed(str, Enum): - BALANCED = "BALANCED" - TURBO = "TURBO" - QUALITY = "QUALITY" - - -class Veo2GenVidPollRequest(BaseModel): - operationName: str = Field( - ..., - description="Full operation name (from predict response)", - examples=[ - "projects/PROJECT_ID/locations/us-central1/publishers/google/models/MODEL_ID/operations/OPERATION_ID" - ], - ) - - -class Error(BaseModel): - code: Optional[int] = Field(None, description="Error code") - message: Optional[str] = Field(None, description="Error message") - - -class Video2(BaseModel): - bytesBase64Encoded: Optional[str] = Field( - None, description="Base64-encoded video content" - ) - gcsUri: Optional[str] = Field(None, description="Cloud Storage URI of the video") - mimeType: Optional[str] = Field(None, description="Video MIME type") - - -class Response(BaseModel): - field_type: Optional[str] = Field( - None, - alias="@type", - examples=[ - "type.googleapis.com/cloud.ai.large_models.vision.GenerateVideoResponse" - ], - ) - raiMediaFilteredCount: Optional[int] = Field( - None, description="Count of media filtered by responsible AI policies" - ) - raiMediaFilteredReasons: Optional[List[str]] = Field( - None, description="Reasons why media was filtered by responsible AI policies" - ) - videos: Optional[List[Video2]] = None - - -class Veo2GenVidPollResponse(BaseModel): - done: Optional[bool] = None - error: Optional[Error] = Field( - None, description="Error details if operation failed" - ) - name: Optional[str] = None - response: Optional[Response] = Field( - None, description="The actual prediction response if done is true" - ) - - class Image(BaseModel): bytesBase64Encoded: str gcsUri: Optional[str] = None @@ -1122,28 +1819,28 @@ class Image1(BaseModel): class Instance(BaseModel): + prompt: str = Field(..., description='Text description of the video') image: Optional[Union[Image, Image1]] = Field( - None, description="Optional image to guide video generation" + None, description='Optional image to guide video generation' ) - prompt: str = Field(..., description="Text description of the video") class PersonGeneration(str, Enum): - ALLOW = "ALLOW" - BLOCK = "BLOCK" + ALLOW = 'ALLOW' + BLOCK = 'BLOCK' class Parameters(BaseModel): - aspectRatio: Optional[str] = Field(None, examples=["16:9"]) - durationSeconds: Optional[int] = None - enhancePrompt: Optional[bool] = None + aspectRatio: Optional[str] = Field(None, examples=['16:9']) negativePrompt: Optional[str] = None personGeneration: Optional[PersonGeneration] = None sampleCount: Optional[int] = None seed: Optional[int] = None storageUri: Optional[str] = Field( - None, description="Optional Cloud Storage URI to upload the video" + None, description='Optional Cloud Storage URI to upload the video' ) + durationSeconds: Optional[int] = None + enhancePrompt: Optional[bool] = None class Veo2GenVidRequest(BaseModel): @@ -1154,186 +1851,1720 @@ class Veo2GenVidRequest(BaseModel): class Veo2GenVidResponse(BaseModel): name: str = Field( ..., - description="Operation resource name", + description='Operation resource name', examples=[ - "projects/PROJECT_ID/locations/us-central1/publishers/google/models/MODEL_ID/operations/a1b07c8e-7b5a-4aba-bb34-3e1ccb8afcc8" + 'projects/PROJECT_ID/locations/us-central1/publishers/google/models/MODEL_ID/operations/a1b07c8e-7b5a-4aba-bb34-3e1ccb8afcc8' ], ) -class IdeogramV3EditRequest(BaseModel): - color_palette: Optional[IdeogramColorPalette] = None - image: Optional[bytes_aliased] = Field( - None, - description="The image being edited (max size 10MB); only JPEG, WebP and PNG formats are supported at this time.", +class Veo2GenVidPollRequest(BaseModel): + operationName: str = Field( + ..., + description='Full operation name (from predict response)', + examples=[ + 'projects/PROJECT_ID/locations/us-central1/publishers/google/models/MODEL_ID/operations/OPERATION_ID' + ], ) - magic_prompt: Optional[str] = Field( - None, - description="Determine if MagicPrompt should be used in generating the request or not.", + + +class Video(BaseModel): + gcsUri: Optional[str] = Field(None, description='Cloud Storage URI of the video') + bytesBase64Encoded: Optional[str] = Field( + None, description='Base64-encoded video content' ) - mask: Optional[bytes_aliased] = Field( + mimeType: Optional[str] = Field(None, description='Video MIME type') + + +class Response(BaseModel): + field_type: Optional[str] = Field( None, - description="A black and white image of the same size as the image being edited (max size 10MB). Black regions in the mask should match up with the regions of the image that you would like to edit; only JPEG, WebP and PNG formats are supported at this time.", + alias='@type', + examples=[ + 'type.googleapis.com/cloud.ai.large_models.vision.GenerateVideoResponse' + ], ) - num_images: Optional[int] = Field( - None, description="The number of images to generate." + raiMediaFilteredCount: Optional[int] = Field( + None, description='Count of media filtered by responsible AI policies' + ) + raiMediaFilteredReasons: Optional[List[str]] = Field( + None, description='Reasons why media was filtered by responsible AI policies' + ) + videos: Optional[List[Video]] = None + + +class Error1(BaseModel): + code: Optional[int] = Field(None, description='Error code') + message: Optional[str] = Field(None, description='Error message') + + +class Veo2GenVidPollResponse(BaseModel): + name: Optional[str] = None + done: Optional[bool] = None + response: Optional[Response] = Field( + None, description='The actual prediction response if done is true' + ) + error: Optional[Error1] = Field( + None, description='Error details if operation failed' + ) + + +class RunwayImageToVideoResponse(BaseModel): + id: Optional[str] = Field(None, description='Task ID') + + +class RunwayTaskStatusEnum(str, Enum): + SUCCEEDED = 'SUCCEEDED' + RUNNING = 'RUNNING' + FAILED = 'FAILED' + PENDING = 'PENDING' + CANCELLED = 'CANCELLED' + THROTTLED = 'THROTTLED' + + +class RunwayModelEnum(str, Enum): + gen4_turbo = 'gen4_turbo' + gen3a_turbo = 'gen3a_turbo' + + +class Position(str, Enum): + first = 'first' + last = 'last' + + +class RunwayPromptImageDetailedObject(BaseModel): + uri: AnyUrl = Field( + ..., description='A HTTPS URL or data URI containing an encoded image.' + ) + position: Position = Field( + ..., + description="The position of the image in the output video. 'last' is currently supported for gen3a_turbo only.", + ) + + +class RunwayDurationEnum(int, Enum): + integer_5 = 5 + integer_10 = 10 + + +class RunwayAspectRatioEnum(str, Enum): + field_1280_720 = '1280:720' + field_720_1280 = '720:1280' + field_1104_832 = '1104:832' + field_832_1104 = '832:1104' + field_960_960 = '960:960' + field_1584_672 = '1584:672' + field_1280_768 = '1280:768' + field_768_1280 = '768:1280' + + +class RunwayPromptImageObject( + RootModel[Union[AnyUrl, List[RunwayPromptImageDetailedObject]]] +): + root: Union[AnyUrl, List[RunwayPromptImageDetailedObject]] = Field( + ..., + description='Image(s) to use for the video generation. Can be a single URI or an array of image objects with positions.', + ) + + +class Datum3(BaseModel): + b64_json: Optional[str] = Field(None, description='Base64 encoded image data') + url: Optional[str] = Field(None, description='URL of the image') + revised_prompt: Optional[str] = Field(None, description='Revised prompt') + + +class InputTokensDetails(BaseModel): + text_tokens: Optional[int] = None + image_tokens: Optional[int] = None + + +class Usage(BaseModel): + input_tokens: Optional[int] = None + input_tokens_details: Optional[InputTokensDetails] = None + output_tokens: Optional[int] = None + total_tokens: Optional[int] = None + + +class OpenAIImageGenerationResponse(BaseModel): + data: Optional[List[Datum3]] = None + usage: Optional[Usage] = None + + +class Quality3(str, Enum): + low = 'low' + medium = 'medium' + high = 'high' + standard = 'standard' + hd = 'hd' + + +class OutputFormat1(str, Enum): + png = 'png' + webp = 'webp' + jpeg = 'jpeg' + + +class Moderation(str, Enum): + low = 'low' + auto = 'auto' + + +class Background(str, Enum): + transparent = 'transparent' + opaque = 'opaque' + + +class ResponseFormat(str, Enum): + url = 'url' + b64_json = 'b64_json' + + +class Style3(str, Enum): + vivid = 'vivid' + natural = 'natural' + + +class OpenAIImageGenerationRequest(BaseModel): + model: Optional[str] = Field( + None, description='The model to use for image generation', examples=['dall-e-3'] ) prompt: str = Field( - ..., description="The prompt used to describe the edited result." + ..., + description='A text description of the desired image', + examples=['Draw a rocket in front of a blackhole in deep space'], ) - rendering_speed: RenderingSpeed - seed: Optional[int] = Field( - None, description="Random seed. Set for reproducible generation." - ) - style_codes: Optional[List[StyleCode]] = Field( + n: Optional[int] = Field( None, - description="A list of 8 character hexadecimal codes representing the style of the image. Cannot be used in conjunction with style_reference_images or style_type.", + description='The number of images to generate (1-10). Only 1 supported for dall-e-3.', + examples=[1], ) - style_reference_images: Optional[List[bytes_aliased]] = Field( + quality: Optional[Quality3] = Field( + None, description='The quality of the generated image', examples=['high'] + ) + size: Optional[str] = Field( None, - description="A set of images to use as style references (maximum total size 10MB across all style references). The images should be in JPEG, PNG or WebP format.", + description='Size of the image (e.g., 1024x1024, 1536x1024, auto)', + examples=['1024x1536'], + ) + output_format: Optional[OutputFormat1] = Field( + None, description='Format of the output image', examples=['png'] + ) + output_compression: Optional[int] = Field( + None, description='Compression level for JPEG or WebP (0-100)', examples=[100] + ) + moderation: Optional[Moderation] = Field( + None, description='Content moderation setting', examples=['auto'] + ) + background: Optional[Background] = Field( + None, description='Background transparency', examples=['opaque'] + ) + response_format: Optional[ResponseFormat] = Field( + None, description='Response format of image data', examples=['b64_json'] + ) + style: Optional[Style3] = Field( + None, description='Style of the image (only for dall-e-3)', examples=['vivid'] + ) + user: Optional[str] = Field( + None, + description='A unique identifier for end-user monitoring', + examples=['user-1234'], + ) + + +class OpenAIImageEditRequest(BaseModel): + model: str = Field( + ..., description='The model to use for image editing', examples=['gpt-image-1'] + ) + prompt: str = Field( + ..., + description='A text description of the desired edit', + examples=['Give the rocketship rainbow coloring'], + ) + n: Optional[int] = Field( + None, description='The number of images to generate', examples=[1] + ) + quality: Optional[str] = Field( + None, description='The quality of the edited image', examples=['low'] + ) + size: Optional[str] = Field( + None, description='Size of the output image', examples=['1024x1024'] + ) + output_format: Optional[OutputFormat1] = Field( + None, description='Format of the output image', examples=['png'] + ) + output_compression: Optional[int] = Field( + None, description='Compression level for JPEG or WebP (0-100)', examples=[100] + ) + moderation: Optional[Moderation] = Field( + None, description='Content moderation setting', examples=['auto'] + ) + background: Optional[str] = Field( + None, description='Background transparency', examples=['opaque'] + ) + user: Optional[str] = Field( + None, + description='A unique identifier for end-user monitoring', + examples=['user-1234'], + ) + + +class CustomerStorageResourceResponse(BaseModel): + download_url: Optional[str] = Field( + None, + description='The signed URL to use for downloading the file from the specified path', + ) + upload_url: Optional[str] = Field( + None, + description='The signed URL to use for uploading the file to the specified path', + ) + expires_at: Optional[datetime] = Field( + None, description='When the signed URL will expire' + ) + existing_file: Optional[bool] = Field( + None, description='Whether an existing file with the same hash was found' + ) + + +class PikaBodyGeneratePikaffectsGeneratePikaffectsPost(BaseModel): + image: StrictBytes = Field(..., title='Image') + pikaffect: Optional[str] = Field(None, title='Pikaffect') + promptText: Optional[str] = Field(None, title='Prompttext') + negativePrompt: Optional[str] = Field(None, title='Negativeprompt') + seed: Optional[int] = Field(None, title='Seed') + + +class PikaGenerateResponse(BaseModel): + video_id: str = Field(..., title='Video Id') + + +class PikaBodyGeneratePikadditionsGeneratePikadditionsPost(BaseModel): + video: StrictBytes = Field(..., title='Video') + image: StrictBytes = Field(..., title='Image') + promptText: Optional[str] = Field(None, title='Prompttext') + negativePrompt: Optional[str] = Field(None, title='Negativeprompt') + seed: Optional[int] = Field(None, title='Seed') + + +class PikaBodyGeneratePikaswapsGeneratePikaswapsPost(BaseModel): + video: StrictBytes = Field(..., title='Video') + image: Optional[StrictBytes] = Field(None, title='Image') + promptText: Optional[str] = Field(None, title='Prompttext') + modifyRegionMask: Optional[StrictBytes] = Field( + None, + description='A mask image that specifies the region to modify, where the mask is white and the background is black', + title='Modifyregionmask', + ) + modifyRegionRoi: Optional[str] = Field( + None, + description='Plaintext description of the object / region to modify', + title='Modifyregionroi', + ) + negativePrompt: Optional[str] = Field(None, title='Negativeprompt') + seed: Optional[int] = Field(None, title='Seed') + + +class IngredientsMode(str, Enum): + creative = 'creative' + precise = 'precise' + + +class AspectRatio1(RootModel[float]): + root: float = Field( + ..., + description='Aspect ratio (width / height)', + ge=0.4, + le=2.5, + title='Aspectratio', + ) + + +class PikaBodyGenerate22C2vGenerate22PikascenesPost(BaseModel): + images: List[StrictBytes] = Field(..., title='Images') + ingredientsMode: IngredientsMode = Field(..., title='Ingredientsmode') + promptText: Optional[str] = Field(None, title='Prompttext') + negativePrompt: Optional[str] = Field(None, title='Negativeprompt') + seed: Optional[int] = Field(None, title='Seed') + resolution: Optional[str] = Field('1080p', title='Resolution') + duration: Optional[int] = Field(5, title='Duration') + aspectRatio: Optional[AspectRatio1] = Field( + None, description='Aspect ratio (width / height)', title='Aspectratio' + ) + + +class PikaStatusEnum(str, Enum): + queued = 'queued' + started = 'started' + finished = 'finished' + + +class PikaValidationError(BaseModel): + loc: List[Union[str, int]] = Field(..., title='Location') + msg: str = Field(..., title='Message') + type: str = Field(..., title='Error Type') + + +class PikaResolutionEnum(str, Enum): + field_1080p = '1080p' + field_720p = '720p' + + +class PikaDurationEnum(int, Enum): + integer_5 = 5 + integer_10 = 10 + + +class RgbItem(RootModel[int]): + root: int = Field(..., ge=0, le=255) + + +class RGBColor(BaseModel): + rgb: List[RgbItem] = Field(..., max_length=3, min_length=3) + + +class StabilityStabilityClientID(RootModel[str]): + root: str = Field( + ..., + description='The name of your application, used to help us communicate app-specific debugging or moderation issues to you.', + examples=['my-awesome-app'], + max_length=256, + ) + + +class StabilityStabilityClientUserID(RootModel[str]): + root: str = Field( + ..., + description='A unique identifier for your end user. Used to help us communicate user-specific debugging or moderation issues to you. Feel free to obfuscate this value to protect user privacy.', + examples=['DiscordUser#9999'], + max_length=256, + ) + + +class StabilityStabilityClientVersion(RootModel[str]): + root: str = Field( + ..., + description='The version of your application, used to help us communicate version-specific debugging or moderation issues to you.', + examples=['1.2.1'], + max_length=256, + ) + + +class Name(str, Enum): + content_moderation = 'content_moderation' + + +class StabilityContentModerationResponse(BaseModel): + id: str = Field( + ..., + description='A unique identifier associated with this error. Please include this in any [support tickets](https://kb.stability.ai/knowledge-base/kb-tickets/new) you file, as it will greatly assist us in diagnosing the root cause of the problem.', + examples=['a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4'], + min_length=1, + ) + name: Name = Field( + ..., + description='Our content moderation system has flagged some part of your request and subsequently denied it. You were not charged for this request. While this may at times be frustrating, it is necessary to maintain the integrity of our platform and ensure a safe experience for all users. If you would like to provide feedback, please use the [Support Form](https://kb.stability.ai/knowledge-base/kb-tickets/new).', + ) + errors: List[str] = Field( + ..., + description='One or more error messages indicating what went wrong.', + examples=[['some-field: is required']], + min_length=1, + ) + + +class RenderingSpeed(str, Enum): + BALANCED = 'BALANCED' + TURBO = 'TURBO' + QUALITY = 'QUALITY' + + +class StabilityCreativity(RootModel[float]): + root: float = Field( + ..., + description='Controls the likelihood of creating additional details not heavily conditioned by the init image.', + ge=0.2, + le=0.5, + ) + + +class StabilityGenerationID(RootModel[str]): + root: str = Field( + ..., + description='The `id` of a generation, typically used for async generations, that can be used to check the status of the generation or retrieve the result.', + examples=['a6dc6c6e20acda010fe14d71f180658f2896ed9b4ec25aa99a6ff06c796987c4'], + max_length=64, + min_length=64, + ) + + +class Mode(str, Enum): + text_to_image = 'text-to-image' + image_to_image = 'image-to-image' + + +class AspectRatio2(str, Enum): + field_21_9 = '21:9' + field_16_9 = '16:9' + field_3_2 = '3:2' + field_5_4 = '5:4' + field_1_1 = '1:1' + field_4_5 = '4:5' + field_2_3 = '2:3' + field_9_16 = '9:16' + field_9_21 = '9:21' + + +class Model4(str, Enum): + sd3_5_large = 'sd3.5-large' + sd3_5_large_turbo = 'sd3.5-large-turbo' + sd3_5_medium = 'sd3.5-medium' + + +class OutputFormat3(str, Enum): + png = 'png' + jpeg = 'jpeg' + + +class StylePreset(str, Enum): + enhance = 'enhance' + anime = 'anime' + photographic = 'photographic' + digital_art = 'digital-art' + comic_book = 'comic-book' + fantasy_art = 'fantasy-art' + line_art = 'line-art' + analog_film = 'analog-film' + neon_punk = 'neon-punk' + isometric = 'isometric' + low_poly = 'low-poly' + origami = 'origami' + modeling_compound = 'modeling-compound' + cinematic = 'cinematic' + field_3d_model = '3d-model' + pixel_art = 'pixel-art' + tile_texture = 'tile-texture' + + +class StabilityImageGenrationSD3Request(BaseModel): + prompt: str = Field( + ..., + description='What you wish to see in the output image. A strong, descriptive prompt that clearly defines\nelements, colors, and subjects will lead to better results.', + max_length=10000, + min_length=1, + ) + mode: Optional[Mode] = Field( + 'text-to-image', + description='Controls whether this is a text-to-image or image-to-image generation, which affects which parameters are required:\n- **text-to-image** requires only the `prompt` parameter\n- **image-to-image** requires the `prompt`, `image`, and `strength` parameters', + title='GenerationMode', + ) + image: Optional[StrictBytes] = Field( + None, + description='The image to use as the starting point for the generation.\n\nSupported formats:\n\n\n\n - jpeg\n - png\n - webp\n\nSupported dimensions:\n\n\n\n - Every side must be at least 64 pixels\n\n> **Important:** This parameter is only valid for **image-to-image** requests.', + ) + strength: Optional[float] = Field( + None, + description='Sometimes referred to as _denoising_, this parameter controls how much influence the\n`image` parameter has on the generated image. A value of 0 would yield an image that\nis identical to the input. A value of 1 would be as if you passed in no image at all.\n\n> **Important:** This parameter is only valid for **image-to-image** requests.', + ge=0.0, + le=1.0, + ) + aspect_ratio: Optional[AspectRatio2] = Field( + '1:1', + description='Controls the aspect ratio of the generated image. Defaults to 1:1.\n\n> **Important:** This parameter is only valid for **text-to-image** requests.', + ) + model: Optional[Model4] = Field( + 'sd3.5-large', + description='The model to use for generation.\n\n- `sd3.5-large` requires 6.5 credits per generation\n- `sd3.5-large-turbo` requires 4 credits per generation\n- `sd3.5-medium` requires 3.5 credits per generation\n- As of the April 17, 2025, `sd3-large`, `sd3-large-turbo` and `sd3-medium`\n\n\n\n are re-routed to their `sd3.5-[model version]` equivalent, at the same price.', + ) + seed: Optional[float] = Field( + 0, + description="A specific value that is used to guide the 'randomness' of the generation. (Omit this parameter or pass `0` to use a random seed.)", + ge=0.0, + le=4294967294.0, + ) + output_format: Optional[OutputFormat3] = Field( + 'png', description='Dictates the `content-type` of the generated image.' + ) + style_preset: Optional[StylePreset] = Field( + None, description='Guides the image model towards a particular style.' + ) + negative_prompt: Optional[str] = Field( + None, + description='Keywords of what you **do not** wish to see in the output image.\nThis is an advanced feature.', + max_length=10000, + ) + cfg_scale: Optional[float] = Field( + None, + description='How strictly the diffusion process adheres to the prompt text (higher values keep your image closer to your prompt). The _Large_ and _Medium_ models use a default of `4`. The _Turbo_ model uses a default of `1`.', + ge=1.0, + le=10.0, + ) + + +class FinishReason(str, Enum): + SUCCESS = 'SUCCESS' + CONTENT_FILTERED = 'CONTENT_FILTERED' + + +class StabilityImageGenrationSD3Response200(BaseModel): + image: str = Field( + ..., + description='The generated image, encoded to base64.', + examples=['AAAAIGZ0eXBpc29tAAACAGlzb21pc28yYXZjMW1...'], + ) + seed: Optional[float] = Field( + 0, + description='The seed used as random noise for this generation.', + examples=[343940597], + ge=0.0, + le=4294967294.0, + ) + finish_reason: FinishReason = Field( + ..., + description='The reason the generation finished.\n\n- `SUCCESS` = successful generation.\n- `CONTENT_FILTERED` = successful generation, however the output violated our content moderation\npolicy and has been blurred as a result.', + examples=['SUCCESS'], + ) + + +class StabilityImageGenrationSD3Response400(BaseModel): + id: str = Field( + ..., + description='A unique identifier associated with this error. Please include this in any [support tickets](https://kb.stability.ai/knowledge-base/kb-tickets/new)\nyou file, as it will greatly assist us in diagnosing the root cause of the problem.', + examples=['a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4'], + min_length=1, + ) + name: str = Field( + ..., + description='Short-hand name for an error, useful for discriminating between errors with the same status code.', + examples=['bad_request'], + min_length=1, + ) + errors: List[str] = Field( + ..., + description='One or more error messages indicating what went wrong.', + examples=[['some-field: is required']], + min_length=1, + ) + + +class StabilityImageGenrationSD3Response413(BaseModel): + id: str = Field( + ..., + description='A unique identifier associated with this error. Please include this in any [support tickets](https://kb.stability.ai/knowledge-base/kb-tickets/new)\nyou file, as it will greatly assist us in diagnosing the root cause of the problem.', + examples=['a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4'], + min_length=1, + ) + name: str = Field( + ..., + description='Short-hand name for an error, useful for discriminating between errors with the same status code.', + examples=['bad_request'], + min_length=1, + ) + errors: List[str] = Field( + ..., + description='One or more error messages indicating what went wrong.', + examples=[['some-field: is required']], + min_length=1, + ) + + +class StabilityImageGenrationSD3Response422(BaseModel): + id: str = Field( + ..., + description='A unique identifier associated with this error. Please include this in any [support tickets](https://kb.stability.ai/knowledge-base/kb-tickets/new)\nyou file, as it will greatly assist us in diagnosing the root cause of the problem.', + examples=['a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4'], + min_length=1, + ) + name: str = Field( + ..., + description='Short-hand name for an error, useful for discriminating between errors with the same status code.', + examples=['bad_request'], + min_length=1, + ) + errors: List[str] = Field( + ..., + description='One or more error messages indicating what went wrong.', + examples=[['some-field: is required']], + min_length=1, + ) + + +class StabilityImageGenrationSD3Response429(BaseModel): + id: str = Field( + ..., + description='A unique identifier associated with this error. Please include this in any [support tickets](https://kb.stability.ai/knowledge-base/kb-tickets/new)\nyou file, as it will greatly assist us in diagnosing the root cause of the problem.', + examples=['a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4'], + min_length=1, + ) + name: str = Field( + ..., + description='Short-hand name for an error, useful for discriminating between errors with the same status code.', + examples=['bad_request'], + min_length=1, + ) + errors: List[str] = Field( + ..., + description='One or more error messages indicating what went wrong.', + examples=[['some-field: is required']], + min_length=1, + ) + + +class StabilityImageGenrationSD3Response500(BaseModel): + id: str = Field( + ..., + description='A unique identifier associated with this error. Please include this in any [support tickets](https://kb.stability.ai/knowledge-base/kb-tickets/new)\nyou file, as it will greatly assist us in diagnosing the root cause of the problem.', + examples=['a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4'], + min_length=1, + ) + name: str = Field( + ..., + description='Short-hand name for an error, useful for discriminating between errors with the same status code.', + examples=['bad_request'], + min_length=1, + ) + errors: List[str] = Field( + ..., + description='One or more error messages indicating what went wrong.', + examples=[['some-field: is required']], + min_length=1, + ) + + +class OutputFormat4(str, Enum): + jpeg = 'jpeg' + png = 'png' + webp = 'webp' + + +class StabilityImageGenrationUpscaleConservativeRequest(BaseModel): + image: StrictBytes = Field( + ..., + description='The image you wish to upscale.\n\nSupported Formats:\n- jpeg\n- png\n- webp\n\nValidation Rules:\n- Every side must be at least 64 pixels\n- Total pixel count must be between 4,096 and 9,437,184 pixels\n- The aspect ratio must be between 1:2.5 and 2.5:1', + examples=['./some/image.png'], + ) + prompt: str = Field( + ..., + description="What you wish to see in the output image. A strong, descriptive prompt that clearly defines\nelements, colors, and subjects will lead to better results.\n\nTo control the weight of a given word use the format `(word:weight)`,\nwhere `word` is the word you'd like to control the weight of and `weight`\nis a value between 0 and 1. For example: `The sky was a crisp (blue:0.3) and (green:0.8)`\nwould convey a sky that was blue and green, but more green than blue.", + max_length=10000, + min_length=1, + ) + negative_prompt: Optional[str] = Field( + None, + description='A blurb of text describing what you **do not** wish to see in the output image.\nThis is an advanced feature.', + max_length=10000, + ) + seed: Optional[float] = Field( + 0, + description="A specific value that is used to guide the 'randomness' of the generation. (Omit this parameter or pass `0` to use a random seed.)", + ge=0.0, + le=4294967294.0, + ) + output_format: Optional[OutputFormat4] = Field( + 'png', description='Dictates the `content-type` of the generated image.' + ) + creativity: Optional[StabilityCreativity] = Field( + default_factory=lambda: StabilityCreativity.model_validate(0.35) + ) + + +class StabilityImageGenrationUpscaleConservativeResponse200(BaseModel): + image: str = Field( + ..., + description='The generated image, encoded to base64.', + examples=['AAAAIGZ0eXBpc29tAAACAGlzb21pc28yYXZjMW1...'], + ) + seed: Optional[float] = Field( + 0, + description='The seed used as random noise for this generation.', + examples=[343940597], + ge=0.0, + le=4294967294.0, + ) + finish_reason: FinishReason = Field( + ..., + description='The reason the generation finished.\n\n- `SUCCESS` = successful generation.\n- `CONTENT_FILTERED` = successful generation, however the output violated our content moderation\npolicy and has been blurred as a result.', + examples=['SUCCESS'], + ) + + +class StabilityImageGenrationUpscaleConservativeResponse400(BaseModel): + id: str = Field( + ..., + description='A unique identifier associated with this error. Please include this in any [support tickets](https://kb.stability.ai/knowledge-base/kb-tickets/new)\nyou file, as it will greatly assist us in diagnosing the root cause of the problem.', + examples=['a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4'], + min_length=1, + ) + name: str = Field( + ..., + description='Short-hand name for an error, useful for discriminating between errors with the same status code.', + examples=['bad_request'], + min_length=1, + ) + errors: List[str] = Field( + ..., + description='One or more error messages indicating what went wrong.', + examples=[['some-field: is required']], + min_length=1, + ) + + +class StabilityImageGenrationUpscaleConservativeResponse413(BaseModel): + id: str = Field( + ..., + description='A unique identifier associated with this error. Please include this in any [support tickets](https://kb.stability.ai/knowledge-base/kb-tickets/new)\nyou file, as it will greatly assist us in diagnosing the root cause of the problem.', + examples=['a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4'], + min_length=1, + ) + name: str = Field( + ..., + description='Short-hand name for an error, useful for discriminating between errors with the same status code.', + examples=['bad_request'], + min_length=1, + ) + errors: List[str] = Field( + ..., + description='One or more error messages indicating what went wrong.', + examples=[['some-field: is required']], + min_length=1, + ) + + +class StabilityImageGenrationUpscaleConservativeResponse422(BaseModel): + id: str = Field( + ..., + description='A unique identifier associated with this error. Please include this in any [support tickets](https://kb.stability.ai/knowledge-base/kb-tickets/new)\nyou file, as it will greatly assist us in diagnosing the root cause of the problem.', + examples=['a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4'], + min_length=1, + ) + name: str = Field( + ..., + description='Short-hand name for an error, useful for discriminating between errors with the same status code.', + examples=['bad_request'], + min_length=1, + ) + errors: List[str] = Field( + ..., + description='One or more error messages indicating what went wrong.', + examples=[['some-field: is required']], + min_length=1, + ) + + +class StabilityImageGenrationUpscaleConservativeResponse429(BaseModel): + id: str = Field( + ..., + description='A unique identifier associated with this error. Please include this in any [support tickets](https://kb.stability.ai/knowledge-base/kb-tickets/new)\nyou file, as it will greatly assist us in diagnosing the root cause of the problem.', + examples=['a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4'], + min_length=1, + ) + name: str = Field( + ..., + description='Short-hand name for an error, useful for discriminating between errors with the same status code.', + examples=['bad_request'], + min_length=1, + ) + errors: List[str] = Field( + ..., + description='One or more error messages indicating what went wrong.', + examples=[['some-field: is required']], + min_length=1, + ) + + +class StabilityImageGenrationUpscaleConservativeResponse500(BaseModel): + id: str = Field( + ..., + description='A unique identifier associated with this error. Please include this in any [support tickets](https://kb.stability.ai/knowledge-base/kb-tickets/new)\nyou file, as it will greatly assist us in diagnosing the root cause of the problem.', + examples=['a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4'], + min_length=1, + ) + name: str = Field( + ..., + description='Short-hand name for an error, useful for discriminating between errors with the same status code.', + examples=['bad_request'], + min_length=1, + ) + errors: List[str] = Field( + ..., + description='One or more error messages indicating what went wrong.', + examples=[['some-field: is required']], + min_length=1, + ) + + +class StabilityImageGenrationUpscaleCreativeRequest(BaseModel): + image: StrictBytes = Field( + ..., + description='The image you wish to upscale.\n\nSupported Formats:\n- jpeg\n- png\n- webp\n\nValidation Rules:\n- Every side must be at least 64 pixels\n- Total pixel count must be between 4,096 and 1,048,576 pixels', + examples=['./some/image.png'], + ) + prompt: str = Field( + ..., + description="What you wish to see in the output image. A strong, descriptive prompt that clearly defines\nelements, colors, and subjects will lead to better results.\n\nTo control the weight of a given word use the format `(word:weight)`,\nwhere `word` is the word you'd like to control the weight of and `weight`\nis a value between 0 and 1. For example: `The sky was a crisp (blue:0.3) and (green:0.8)`\nwould convey a sky that was blue and green, but more green than blue.", + max_length=10000, + min_length=1, + ) + negative_prompt: Optional[str] = Field( + None, + description='A blurb of text describing what you **do not** wish to see in the output image.\nThis is an advanced feature.', + max_length=10000, + ) + output_format: Optional[OutputFormat4] = Field( + 'png', description='Dictates the `content-type` of the generated image.' + ) + seed: Optional[float] = Field( + 0, + description="A specific value that is used to guide the 'randomness' of the generation. (Omit this parameter or pass `0` to use a random seed.)", + ge=0.0, + le=4294967294.0, + ) + creativity: Optional[float] = Field( + 0.3, + description='Indicates how creative the model should be when upscaling an image.\nHigher values will result in more details being added to the image during upscaling.', + ge=0.1, + le=0.5, + ) + style_preset: Optional[StylePreset] = Field( + None, description='Guides the image model towards a particular style.' + ) + + +class StabilityImageGenrationUpscaleCreativeResponse200(BaseModel): + id: StabilityGenerationID + + +class StabilityImageGenrationUpscaleCreativeResponse400(BaseModel): + id: str = Field( + ..., + description='A unique identifier associated with this error. Please include this in any [support tickets](https://kb.stability.ai/knowledge-base/kb-tickets/new)\nyou file, as it will greatly assist us in diagnosing the root cause of the problem.', + examples=['a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4'], + min_length=1, + ) + name: str = Field( + ..., + description='Short-hand name for an error, useful for discriminating between errors with the same status code.', + examples=['bad_request'], + min_length=1, + ) + errors: List[str] = Field( + ..., + description='One or more error messages indicating what went wrong.', + examples=[['some-field: is required']], + min_length=1, + ) + + +class StabilityImageGenrationUpscaleCreativeResponse413(BaseModel): + id: str = Field( + ..., + description='A unique identifier associated with this error. Please include this in any [support tickets](https://kb.stability.ai/knowledge-base/kb-tickets/new)\nyou file, as it will greatly assist us in diagnosing the root cause of the problem.', + examples=['a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4'], + min_length=1, + ) + name: str = Field( + ..., + description='Short-hand name for an error, useful for discriminating between errors with the same status code.', + examples=['bad_request'], + min_length=1, + ) + errors: List[str] = Field( + ..., + description='One or more error messages indicating what went wrong.', + examples=[['some-field: is required']], + min_length=1, + ) + + +class StabilityImageGenrationUpscaleCreativeResponse422(BaseModel): + id: str = Field( + ..., + description='A unique identifier associated with this error. Please include this in any [support tickets](https://kb.stability.ai/knowledge-base/kb-tickets/new)\nyou file, as it will greatly assist us in diagnosing the root cause of the problem.', + examples=['a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4'], + min_length=1, + ) + name: str = Field( + ..., + description='Short-hand name for an error, useful for discriminating between errors with the same status code.', + examples=['bad_request'], + min_length=1, + ) + errors: List[str] = Field( + ..., + description='One or more error messages indicating what went wrong.', + examples=[['some-field: is required']], + min_length=1, + ) + + +class StabilityImageGenrationUpscaleCreativeResponse429(BaseModel): + id: str = Field( + ..., + description='A unique identifier associated with this error. Please include this in any [support tickets](https://kb.stability.ai/knowledge-base/kb-tickets/new)\nyou file, as it will greatly assist us in diagnosing the root cause of the problem.', + examples=['a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4'], + min_length=1, + ) + name: str = Field( + ..., + description='Short-hand name for an error, useful for discriminating between errors with the same status code.', + examples=['bad_request'], + min_length=1, + ) + errors: List[str] = Field( + ..., + description='One or more error messages indicating what went wrong.', + examples=[['some-field: is required']], + min_length=1, + ) + + +class StabilityImageGenrationUpscaleCreativeResponse500(BaseModel): + id: str = Field( + ..., + description='A unique identifier associated with this error. Please include this in any [support tickets](https://kb.stability.ai/knowledge-base/kb-tickets/new)\nyou file, as it will greatly assist us in diagnosing the root cause of the problem.', + examples=['a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4'], + min_length=1, + ) + name: str = Field( + ..., + description='Short-hand name for an error, useful for discriminating between errors with the same status code.', + examples=['bad_request'], + min_length=1, + ) + errors: List[str] = Field( + ..., + description='One or more error messages indicating what went wrong.', + examples=[['some-field: is required']], + min_length=1, + ) + + +class StabilityImageGenrationUpscaleFastRequest(BaseModel): + image: StrictBytes = Field( + ..., + description='The image you wish to upscale.\n\nSupported Formats:\n- jpeg\n- png\n- webp\n\nValidation Rules:\n- Width must be between 32 and 1,536 pixels\n- Height must be between 32 and 1,536 pixels\n- Total pixel count must be between 1,024 and 1,048,576 pixels', + examples=['./some/image.png'], + ) + output_format: Optional[OutputFormat4] = Field( + 'png', description='Dictates the `content-type` of the generated image.' + ) + + +class StabilityImageGenrationUpscaleFastResponse200(BaseModel): + image: str = Field( + ..., + description='The generated image, encoded to base64.', + examples=['AAAAIGZ0eXBpc29tAAACAGlzb21pc28yYXZjMW1...'], + ) + seed: Optional[float] = Field( + 0, + description='The seed used as random noise for this generation.', + examples=[343940597], + ge=0.0, + le=4294967294.0, + ) + finish_reason: FinishReason = Field( + ..., + description='The reason the generation finished.\n\n- `SUCCESS` = successful generation.\n- `CONTENT_FILTERED` = successful generation, however the output violated our content moderation\npolicy and has been blurred as a result.', + examples=['SUCCESS'], + ) + + +class StabilityImageGenrationUpscaleFastResponse400(BaseModel): + id: str = Field( + ..., + description='A unique identifier associated with this error. Please include this in any [support tickets](https://kb.stability.ai/knowledge-base/kb-tickets/new)\nyou file, as it will greatly assist us in diagnosing the root cause of the problem.', + examples=['a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4'], + min_length=1, + ) + name: str = Field( + ..., + description='Short-hand name for an error, useful for discriminating between errors with the same status code.', + examples=['bad_request'], + min_length=1, + ) + errors: List[str] = Field( + ..., + description='One or more error messages indicating what went wrong.', + examples=[['some-field: is required']], + min_length=1, + ) + + +class StabilityImageGenrationUpscaleFastResponse413(BaseModel): + id: str = Field( + ..., + description='A unique identifier associated with this error. Please include this in any [support tickets](https://kb.stability.ai/knowledge-base/kb-tickets/new)\nyou file, as it will greatly assist us in diagnosing the root cause of the problem.', + examples=['a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4'], + min_length=1, + ) + name: str = Field( + ..., + description='Short-hand name for an error, useful for discriminating between errors with the same status code.', + examples=['bad_request'], + min_length=1, + ) + errors: List[str] = Field( + ..., + description='One or more error messages indicating what went wrong.', + examples=[['some-field: is required']], + min_length=1, + ) + + +class StabilityImageGenrationUpscaleFastResponse422(BaseModel): + id: str = Field( + ..., + description='A unique identifier associated with this error. Please include this in any [support tickets](https://kb.stability.ai/knowledge-base/kb-tickets/new)\nyou file, as it will greatly assist us in diagnosing the root cause of the problem.', + examples=['a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4'], + min_length=1, + ) + name: str = Field( + ..., + description='Short-hand name for an error, useful for discriminating between errors with the same status code.', + examples=['bad_request'], + min_length=1, + ) + errors: List[str] = Field( + ..., + description='One or more error messages indicating what went wrong.', + examples=[['some-field: is required']], + min_length=1, + ) + + +class StabilityImageGenrationUpscaleFastResponse429(BaseModel): + id: str = Field( + ..., + description='A unique identifier associated with this error. Please include this in any [support tickets](https://kb.stability.ai/knowledge-base/kb-tickets/new)\nyou file, as it will greatly assist us in diagnosing the root cause of the problem.', + examples=['a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4'], + min_length=1, + ) + name: str = Field( + ..., + description='Short-hand name for an error, useful for discriminating between errors with the same status code.', + examples=['bad_request'], + min_length=1, + ) + errors: List[str] = Field( + ..., + description='One or more error messages indicating what went wrong.', + examples=[['some-field: is required']], + min_length=1, + ) + + +class StabilityImageGenrationUpscaleFastResponse500(BaseModel): + id: str = Field( + ..., + description='A unique identifier associated with this error. Please include this in any [support tickets](https://kb.stability.ai/knowledge-base/kb-tickets/new)\nyou file, as it will greatly assist us in diagnosing the root cause of the problem.', + examples=['a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4'], + min_length=1, + ) + name: str = Field( + ..., + description='Short-hand name for an error, useful for discriminating between errors with the same status code.', + examples=['bad_request'], + min_length=1, + ) + errors: List[str] = Field( + ..., + description='One or more error messages indicating what went wrong.', + examples=[['some-field: is required']], + min_length=1, + ) + + +class ActionJobResult(BaseModel): + id: Optional[UUID] = Field(None, description='Unique identifier for the job result') + workflow_name: Optional[str] = Field(None, description='Name of the workflow') + operating_system: Optional[str] = Field(None, description='Operating system used') + python_version: Optional[str] = Field(None, description='PyTorch version used') + pytorch_version: Optional[str] = Field(None, description='PyTorch version used') + action_run_id: Optional[str] = Field( + None, description='Identifier of the run this result belongs to' + ) + action_job_id: Optional[str] = Field( + None, description='Identifier of the job this result belongs to' + ) + cuda_version: Optional[str] = Field(None, description='CUDA version used') + branch_name: Optional[str] = Field( + None, description='Name of the relevant git branch' + ) + commit_hash: Optional[str] = Field(None, description='The hash of the commit') + commit_id: Optional[str] = Field(None, description='The ID of the commit') + commit_time: Optional[int] = Field( + None, description='The Unix timestamp when the commit was made' + ) + commit_message: Optional[str] = Field(None, description='The message of the commit') + comfy_run_flags: Optional[str] = Field( + None, description='The comfy run flags. E.g. `--low-vram`' + ) + git_repo: Optional[str] = Field(None, description='The repository name') + pr_number: Optional[str] = Field(None, description='The pull request number') + start_time: Optional[int] = Field( + None, description='The start time of the job as a Unix timestamp.' + ) + end_time: Optional[int] = Field( + None, description='The end time of the job as a Unix timestamp.' + ) + avg_vram: Optional[int] = Field( + None, description='The average VRAM used by the job' + ) + peak_vram: Optional[int] = Field(None, description='The peak VRAM used by the job') + job_trigger_user: Optional[str] = Field( + None, description='The user who triggered the job.' + ) + author: Optional[str] = Field(None, description='The author of the commit') + machine_stats: Optional[MachineStats] = None + status: Optional[WorkflowRunStatus] = None + storage_file: Optional[StorageFile] = None + + +class Publisher(BaseModel): + name: Optional[str] = None + id: Optional[str] = Field( + None, + description="The unique identifier for the publisher. It's akin to a username. Should be lowercase.", + ) + description: Optional[str] = None + website: Optional[str] = None + support: Optional[str] = None + source_code_repo: Optional[str] = None + logo: Optional[str] = Field(None, description="URL to the publisher's logo.") + createdAt: Optional[datetime] = Field( + None, description='The date and time the publisher was created.' + ) + members: Optional[List[PublisherMember]] = Field( + None, description='A list of members in the publisher.' + ) + status: Optional[PublisherStatus] = Field( + None, description='The status of the publisher.' + ) + + +class NodeVersion(BaseModel): + id: Optional[str] = None + version: Optional[str] = Field( + None, + description='The version identifier, following semantic versioning. Must be unique for the node.', + ) + createdAt: Optional[datetime] = Field( + None, description='The date and time the version was created.' + ) + changelog: Optional[str] = Field( + None, description='Summary of changes made in this version' + ) + dependencies: Optional[List[str]] = Field( + None, description='A list of pip dependencies required by the node.' + ) + downloadUrl: Optional[str] = Field( + None, description='[Output Only] URL to download this version of the node' + ) + deprecated: Optional[bool] = Field( + None, description='Indicates if this version is deprecated.' + ) + status: Optional[NodeVersionStatus] = Field( + None, description='The status of the node version.' + ) + status_reason: Optional[str] = Field( + None, description='The reason for the status change.' + ) + node_id: Optional[str] = Field( + None, description='The unique identifier of the node.' + ) + comfy_node_extract_status: Optional[str] = Field( + None, description='The status of comfy node extraction process.' ) class IdeogramV3Request(BaseModel): - aspect_ratio: Optional[str] = Field( - None, description="Aspect ratio in format WxH", examples=["1x3"] + prompt: str = Field(..., description='The text prompt for image generation') + seed: Optional[int] = Field( + None, description='Seed value for reproducible generation' ) - color_palette: Optional[ColorPalette] = None + resolution: Optional[str] = Field( + None, description='Image resolution in format WxH', examples=['1280x800'] + ) + aspect_ratio: Optional[str] = Field( + None, description='Aspect ratio in format WxH', examples=['1x3'] + ) + rendering_speed: RenderingSpeed magic_prompt: Optional[MagicPrompt] = Field( - None, description="Whether to enable magic prompt enhancement" + None, description='Whether to enable magic prompt enhancement' ) negative_prompt: Optional[str] = Field( - None, description="Text prompt specifying what to avoid in the generation" + None, description='Text prompt specifying what to avoid in the generation' ) num_images: Optional[int] = Field( - None, description="Number of images to generate", ge=1 - ) - prompt: str = Field(..., description="The text prompt for image generation") - rendering_speed: RenderingSpeed - resolution: Optional[str] = Field( - None, description="Image resolution in format WxH", examples=["1280x800"] - ) - seed: Optional[int] = Field( - None, description="Seed value for reproducible generation" + None, description='Number of images to generate', ge=1 ) + color_palette: Optional[ColorPalette] = None style_codes: Optional[List[StyleCode]] = Field( - None, description="Array of style codes in hexadecimal format" - ) - style_reference_images: Optional[List[str]] = Field( - None, description="Array of reference image URLs or identifiers" + None, description='Array of style codes in hexadecimal format' ) style_type: Optional[StyleType] = Field( - None, description="The type of style to apply" + None, description='The type of style to apply' ) + style_reference_images: Optional[List[str]] = Field( + None, description='Array of reference image URLs or identifiers' + ) + + +class IdeogramV3EditRequest(BaseModel): + image: Optional[StrictBytes] = Field( + None, + description='The image being edited (max size 10MB); only JPEG, WebP and PNG formats are supported at this time.', + ) + mask: Optional[StrictBytes] = Field( + None, + description='A black and white image of the same size as the image being edited (max size 10MB). Black regions in the mask should match up with the regions of the image that you would like to edit; only JPEG, WebP and PNG formats are supported at this time.', + ) + prompt: str = Field( + ..., description='The prompt used to describe the edited result.' + ) + magic_prompt: Optional[str] = Field( + None, + description='Determine if MagicPrompt should be used in generating the request or not.', + ) + num_images: Optional[int] = Field( + None, description='The number of images to generate.' + ) + seed: Optional[int] = Field( + None, description='Random seed. Set for reproducible generation.' + ) + rendering_speed: RenderingSpeed + color_palette: Optional[IdeogramColorPalette] = Field( + None, + description='A color palette for generation, must EITHER be specified via one of the presets (name) or explicitly via hexadecimal representations of the color with optional weights (members). Not supported by V_1, V_1_TURBO, V_2A and V_2A_TURBO models.', + ) + style_codes: Optional[List[StyleCode]] = Field( + None, + description='A list of 8 character hexadecimal codes representing the style of the image. Cannot be used in conjunction with style_reference_images or style_type.', + ) + style_reference_images: Optional[List[StrictBytes]] = Field( + None, + description='A set of images to use as style references (maximum total size 10MB across all style references). The images should be in JPEG, PNG or WebP format.', + ) + + +class KlingCameraControl(BaseModel): + type: Optional[KlingCameraControlType] = None + config: Optional[KlingCameraConfig] = None + + +class KlingText2VideoRequest(BaseModel): + model_name: Optional[KlingVideoGenModelName] = 'kling-v2-master' + prompt: Optional[str] = Field( + None, description='Positive text prompt', max_length=2500 + ) + negative_prompt: Optional[str] = Field( + None, description='Negative text prompt', max_length=2500 + ) + cfg_scale: Optional[KlingVideoGenCfgScale] = Field( + default_factory=lambda: KlingVideoGenCfgScale.model_validate(0.5) + ) + mode: Optional[KlingVideoGenMode] = 'std' + camera_control: Optional[KlingCameraControl] = None + aspect_ratio: Optional[KlingVideoGenAspectRatio] = '16:9' + duration: Optional[KlingVideoGenDuration] = '5' + callback_url: Optional[AnyUrl] = Field( + None, description='The callback notification address' + ) + external_task_id: Optional[str] = Field(None, description='Customized Task ID') + + +class KlingImage2VideoRequest(BaseModel): + model_name: Optional[KlingVideoGenModelName] = 'kling-v2-master' + image: Optional[str] = Field( + None, + description='Reference Image - URL or Base64 encoded string, cannot exceed 10MB, resolution not less than 300*300px, aspect ratio between 1:2.5 ~ 2.5:1. Base64 should not include data:image prefix.', + ) + image_tail: Optional[str] = Field( + None, + description='Reference Image - End frame control. URL or Base64 encoded string, cannot exceed 10MB, resolution not less than 300*300px. Base64 should not include data:image prefix.', + ) + prompt: Optional[str] = Field( + None, description='Positive text prompt', max_length=2500 + ) + negative_prompt: Optional[str] = Field( + None, description='Negative text prompt', max_length=2500 + ) + cfg_scale: Optional[KlingVideoGenCfgScale] = Field( + default_factory=lambda: KlingVideoGenCfgScale.model_validate(0.5) + ) + mode: Optional[KlingVideoGenMode] = 'std' + static_mask: Optional[AnyUrl] = Field( + None, + description='Static Brush Application Area (Mask image created by users using the motion brush). The aspect ratio must match the input image.', + ) + dynamic_masks: Optional[List[DynamicMask]] = Field( + None, + description='Dynamic Brush Configuration List (up to 6 groups). For 5-second videos, trajectory length must not exceed 77 coordinates.', + ) + camera_control: Optional[KlingCameraControl] = None + aspect_ratio: Optional[KlingVideoGenAspectRatio] = '16:9' + duration: Optional[KlingVideoGenDuration] = '5' + callback_url: Optional[AnyUrl] = Field( + None, + description='The callback notification address. Server will notify when the task status changes.', + ) + external_task_id: Optional[str] = Field( + None, + description='Customized Task ID. Must be unique within a single user account.', + ) + + +class KlingVideoEffectsInput( + RootModel[Union[KlingSingleImageEffectInput, KlingDualCharacterEffectInput]] +): + root: Union[KlingSingleImageEffectInput, KlingDualCharacterEffectInput] + + +class StripeBillingDetails(BaseModel): + address: Optional[StripeAddress] = None + email: Optional[str] = None + name: Optional[str] = None + phone: Optional[str] = None + tax_id: Optional[Any] = None + + +class StripePaymentMethodDetails(BaseModel): + card: Optional[StripeCardDetails] = None + type: Optional[str] = None + + +class BFLFluxProFillInputs(BaseModel): + image: str = Field( + ..., + description='A Base64-encoded string representing the image you wish to modify. Can contain alpha mask if desired.', + title='Image', + ) + mask: Optional[str] = Field( + None, + description='A Base64-encoded string representing a mask for the areas you want to modify in the image. The mask should be the same dimensions as the image and in black and white. Black areas (0%) indicate no modification, while white areas (100%) specify areas for inpainting. Optional if you provide an alpha mask in the original image. Validation: The endpoint verifies that the dimensions of the mask match the original image.', + title='Mask', + ) + prompt: Optional[str] = Field( + '', + description='The description of the changes you want to make. This text guides the inpainting process, allowing you to specify features, styles, or modifications for the masked area.', + examples=['ein fantastisches bild'], + title='Prompt', + ) + steps: Optional[Steps] = Field( + default_factory=lambda: Steps.model_validate(50), + description='Number of steps for the image generation process', + examples=[50], + title='Steps', + ) + prompt_upsampling: Optional[bool] = Field( + False, + description='Whether to perform upsampling on the prompt. If active, automatically modifies the prompt for more creative generation', + title='Prompt Upsampling', + ) + seed: Optional[int] = Field( + None, description='Optional seed for reproducibility', title='Seed' + ) + guidance: Optional[Guidance] = Field( + default_factory=lambda: Guidance.model_validate(60), + description='Guidance strength for the image generation process', + title='Guidance', + ) + output_format: Optional[BFLOutputFormat] = Field( + 'jpeg', + description="Output format for the generated image. Can be 'jpeg' or 'png'.", + ) + safety_tolerance: Optional[int] = Field( + 2, + description='Tolerance level for input and output moderation. Between 0 and 6, 0 being most strict, 6 being least strict.', + examples=[2], + ge=0, + le=6, + title='Safety Tolerance', + ) + webhook_url: Optional[WebhookUrl] = Field( + None, description='URL to receive webhook notifications', title='Webhook Url' + ) + webhook_secret: Optional[str] = Field( + None, + description='Optional secret for webhook signature verification', + title='Webhook Secret', + ) + + +class BFLHTTPValidationError(BaseModel): + detail: Optional[List[BFLValidationError]] = Field(None, title='Detail') + + +class BFLFluxProExpandInputs(BaseModel): + image: str = Field( + ..., + description='A Base64-encoded string representing the image you wish to expand.', + title='Image', + ) + top: Optional[Top] = Field( + 0, description='Number of pixels to expand at the top of the image', title='Top' + ) + bottom: Optional[Bottom] = Field( + 0, + description='Number of pixels to expand at the bottom of the image', + title='Bottom', + ) + left: Optional[Left] = Field( + 0, + description='Number of pixels to expand on the left side of the image', + title='Left', + ) + right: Optional[Right] = Field( + 0, + description='Number of pixels to expand on the right side of the image', + title='Right', + ) + prompt: Optional[str] = Field( + '', + description='The description of the changes you want to make. This text guides the expansion process, allowing you to specify features, styles, or modifications for the expanded areas.', + examples=['ein fantastisches bild'], + title='Prompt', + ) + steps: Optional[Steps] = Field( + default_factory=lambda: Steps.model_validate(50), + description='Number of steps for the image generation process', + examples=[50], + title='Steps', + ) + prompt_upsampling: Optional[bool] = Field( + False, + description='Whether to perform upsampling on the prompt. If active, automatically modifies the prompt for more creative generation', + title='Prompt Upsampling', + ) + seed: Optional[int] = Field( + None, description='Optional seed for reproducibility', title='Seed' + ) + guidance: Optional[Guidance] = Field( + default_factory=lambda: Guidance.model_validate(60), + description='Guidance strength for the image generation process', + title='Guidance', + ) + output_format: Optional[BFLOutputFormat] = Field( + 'jpeg', + description="Output format for the generated image. Can be 'jpeg' or 'png'.", + ) + safety_tolerance: Optional[int] = Field( + 2, + description='Tolerance level for input and output moderation. Between 0 and 6, 0 being most strict, 6 being least strict.', + examples=[2], + ge=0, + le=6, + title='Safety Tolerance', + ) + webhook_url: Optional[WebhookUrl] = Field( + None, description='URL to receive webhook notifications', title='Webhook Url' + ) + webhook_secret: Optional[str] = Field( + None, + description='Optional secret for webhook signature verification', + title='Webhook Secret', + ) + + +class BFLCannyInputs(BaseModel): + prompt: str = Field( + ..., + description='Text prompt for image generation', + examples=['ein fantastisches bild'], + title='Prompt', + ) + control_image: Optional[str] = Field( + None, + description='Base64 encoded image to use as control input if no preprocessed image is provided', + title='Control Image', + ) + preprocessed_image: Optional[str] = Field( + None, + description='Optional pre-processed image that will bypass the control preprocessing step', + title='Preprocessed Image', + ) + canny_low_threshold: Optional[CannyLowThreshold] = Field( + default_factory=lambda: CannyLowThreshold.model_validate(50), + description='Low threshold for Canny edge detection', + title='Canny Low Threshold', + ) + canny_high_threshold: Optional[CannyHighThreshold] = Field( + default_factory=lambda: CannyHighThreshold.model_validate(200), + description='High threshold for Canny edge detection', + title='Canny High Threshold', + ) + prompt_upsampling: Optional[bool] = Field( + False, + description='Whether to perform upsampling on the prompt', + title='Prompt Upsampling', + ) + seed: Optional[int] = Field( + None, + description='Optional seed for reproducibility', + examples=[42], + title='Seed', + ) + steps: Optional[Steps2] = Field( + default_factory=lambda: Steps2.model_validate(50), + description='Number of steps for the image generation process', + title='Steps', + ) + output_format: Optional[BFLOutputFormat] = Field( + 'jpeg', + description="Output format for the generated image. Can be 'jpeg' or 'png'.", + ) + guidance: Optional[Guidance2] = Field( + default_factory=lambda: Guidance2.model_validate(30), + description='Guidance strength for the image generation process', + title='Guidance', + ) + safety_tolerance: Optional[int] = Field( + 2, + description='Tolerance level for input and output moderation. Between 0 and 6, 0 being most strict, 6 being least strict.', + ge=0, + le=6, + title='Safety Tolerance', + ) + webhook_url: Optional[WebhookUrl] = Field( + None, description='URL to receive webhook notifications', title='Webhook Url' + ) + webhook_secret: Optional[str] = Field( + None, + description='Optional secret for webhook signature verification', + title='Webhook Secret', + ) + + +class BFLDepthInputs(BaseModel): + prompt: str = Field( + ..., + description='Text prompt for image generation', + examples=['ein fantastisches bild'], + title='Prompt', + ) + control_image: Optional[str] = Field( + None, + description='Base64 encoded image to use as control input', + title='Control Image', + ) + preprocessed_image: Optional[str] = Field( + None, + description='Optional pre-processed image that will bypass the control preprocessing step', + title='Preprocessed Image', + ) + prompt_upsampling: Optional[bool] = Field( + False, + description='Whether to perform upsampling on the prompt', + title='Prompt Upsampling', + ) + seed: Optional[int] = Field( + None, + description='Optional seed for reproducibility', + examples=[42], + title='Seed', + ) + steps: Optional[Steps2] = Field( + default_factory=lambda: Steps2.model_validate(50), + description='Number of steps for the image generation process', + title='Steps', + ) + output_format: Optional[BFLOutputFormat] = Field( + 'jpeg', + description="Output format for the generated image. Can be 'jpeg' or 'png'.", + ) + guidance: Optional[Guidance2] = Field( + default_factory=lambda: Guidance2.model_validate(15), + description='Guidance strength for the image generation process', + title='Guidance', + ) + safety_tolerance: Optional[int] = Field( + 2, + description='Tolerance level for input and output moderation. Between 0 and 6, 0 being most strict, 6 being least strict.', + ge=0, + le=6, + title='Safety Tolerance', + ) + webhook_url: Optional[WebhookUrl] = Field( + None, description='URL to receive webhook notifications', title='Webhook Url' + ) + webhook_secret: Optional[str] = Field( + None, + description='Optional secret for webhook signature verification', + title='Webhook Secret', + ) + + +class Controls(BaseModel): + artistic_level: Optional[int] = Field( + None, + description='Defines artistic tone of your image. At a simple level, the person looks straight at the camera in a static and clean style. Dynamic and eccentric levels introduce movement and creativity.', + ge=0, + le=5, + ) + colors: Optional[List[RGBColor]] = Field( + None, description='An array of preferable colors' + ) + background_color: Optional[RGBColor] = Field( + None, description='Use given color as a desired background color' + ) + no_text: Optional[bool] = Field(None, description='Do not embed text layouts') + + +class RecraftImageGenerationRequest(BaseModel): + prompt: str = Field( + ..., description='The text prompt describing the image to generate' + ) + model: str = Field( + ..., description='The model to use for generation (e.g., "recraftv3")' + ) + style: Optional[str] = Field( + None, + description='The style to apply to the generated image (e.g., "digital_illustration")', + ) + style_id: Optional[str] = Field( + None, + description='The style ID to apply to the generated image (e.g., "123e4567-e89b-12d3-a456-426614174000"). If style_id is provided, style should not be provided.', + ) + size: str = Field( + ..., description='The size of the generated image (e.g., "1024x1024")' + ) + controls: Optional[Controls] = Field( + None, description='The controls for the generated image' + ) + n: int = Field(..., description='The number of images to generate', ge=1, le=4) + + +class LumaKeyframes(BaseModel): + frame0: Optional[LumaKeyframe] = None + frame1: Optional[LumaKeyframe] = None class LumaGenerationRequest(BaseModel): + generation_type: Optional[GenerationType] = 'video' + prompt: str = Field(..., description='The prompt of the generation') aspect_ratio: LumaAspectRatio + loop: Optional[bool] = Field(None, description='Whether to loop the video') + keyframes: Optional[LumaKeyframes] = None callback_url: Optional[AnyUrl] = Field( None, - description="The callback URL of the generation, a POST request with Generation object will be sent to the callback URL when the generation is dreaming, completed, or failed", + description='The callback URL of the generation, a POST request with Generation object will be sent to the callback URL when the generation is dreaming, completed, or failed', ) - duration: LumaVideoModelOutputDuration - generation_type: Optional[GenerationType1] = "video" - keyframes: Optional[LumaKeyframes] = None - loop: Optional[bool] = Field(None, description="Whether to loop the video") model: LumaVideoModel - prompt: str = Field(..., description="The prompt of the generation") resolution: LumaVideoModelOutputResolution - - -class CharacterRef(BaseModel): - identity0: Optional[LumaImageIdentity] = None - - -class LumaImageGenerationRequest(BaseModel): - aspect_ratio: Optional[LumaAspectRatio] = "16:9" - callback_url: Optional[AnyUrl] = Field( - None, description="The callback URL for the generation" - ) - character_ref: Optional[CharacterRef] = None - generation_type: Optional[GenerationType2] = "image" - image_ref: Optional[List[LumaImageRef]] = None - model: Optional[LumaImageModel] = "photon-1" - modify_image_ref: Optional[LumaModifyImageRef] = None - prompt: Optional[str] = Field(None, description="The prompt of the generation") - style_ref: Optional[List[LumaImageRef]] = None - - -class LumaUpscaleVideoGenerationRequest(BaseModel): - callback_url: Optional[AnyUrl] = Field( - None, description="The callback URL for the upscale" - ) - generation_type: Optional[GenerationType3] = "upscale_video" - resolution: Optional[LumaVideoModelOutputResolution] = None - - -class PikaBodyGenerate22C2vGenerate22PikascenesPost(BaseModel): - aspectRatio: Optional[AspectRatio2] = Field( - None, description="Aspect ratio (width / height)", title="Aspectratio" - ) - duration: Optional[PikaDurationEnum] = 5 - images: Optional[List[bytes_aliased]] = Field( - None, description="Array of images to process", title="Images" - ) - ingredientsMode: IngredientsMode = Field(..., title="Ingredientsmode") - negativePrompt: Optional[str] = Field(None, title="Negativeprompt") - promptText: Optional[str] = Field(None, title="Prompttext") - resolution: Optional[PikaResolutionEnum] = "1080p" - seed: Optional[int] = Field(None, title="Seed") - - -class PikaBodyGenerate22I2vGenerate22I2vPost(BaseModel): - duration: Optional[PikaDurationEnum] = 5 - image: Optional[str] = Field(None, title="Image") - negativePrompt: Optional[str] = Field(None, title="Negativeprompt") - promptText: Optional[str] = Field(None, title="Prompttext") - resolution: Optional[PikaResolutionEnum] = "1080p" - seed: Optional[int] = Field(None, title="Seed") - - -class PikaBodyGenerate22KeyframeGenerate22PikaframesPost(BaseModel): - duration: Optional[int] = Field(None, ge=5, le=10, title="Duration") - keyFrames: List[bytes_aliased] = Field( - ..., description="Array of keyframe images", title="Keyframes" - ) - negativePrompt: Optional[str] = Field(None, title="Negativeprompt") - promptText: str = Field(..., title="Prompttext") - resolution: Optional[PikaResolutionEnum] = "1080p" - seed: Optional[int] = Field(None, title="Seed") - - -class PikaBodyGenerate22T2vGenerate22T2vPost(BaseModel): - aspectRatio: Optional[float] = Field( - 1.7777777777777777, - description="Aspect ratio (width / height)", - ge=0.4, - le=2.5, - title="Aspectratio", - ) - duration: Optional[PikaDurationEnum] = 5 - negativePrompt: Optional[str] = Field(None, title="Negativeprompt") - promptText: str = Field(..., title="Prompttext") - resolution: Optional[PikaResolutionEnum] = "1080p" - seed: Optional[int] = Field(None, title="Seed") - - -class PikaHTTPValidationError(BaseModel): - detail: Optional[List[PikaValidationError]] = Field(None, title="Detail") + duration: LumaVideoModelOutputDuration class LumaGeneration(BaseModel): - assets: Optional[LumaAssets] = None - created_at: Optional[datetime] = Field( - None, description="The date and time when the generation was created" - ) - failure_reason: Optional[str] = Field( - None, description="The reason for the state of the generation" - ) + id: Optional[UUID] = Field(None, description='The ID of the generation') generation_type: Optional[LumaGenerationType] = None - id: Optional[UUID] = Field(None, description="The ID of the generation") - model: Optional[str] = Field(None, description="The model used for the generation") + state: Optional[LumaState] = None + failure_reason: Optional[str] = Field( + None, description='The reason for the state of the generation' + ) + created_at: Optional[datetime] = Field( + None, description='The date and time when the generation was created' + ) + assets: Optional[LumaAssets] = None + model: Optional[str] = Field(None, description='The model used for the generation') request: Optional[ Union[ LumaGenerationRequest, @@ -1341,5 +3572,238 @@ class LumaGeneration(BaseModel): LumaUpscaleVideoGenerationRequest, LumaAudioGenerationRequest, ] - ] = Field(None, description="The request of the generation") - state: Optional[LumaState] = None + ] = Field(None, description='The request of the generation') + + +class RunwayImageToVideoRequest(BaseModel): + promptImage: RunwayPromptImageObject + seed: int = Field( + ..., description='Random seed for generation', ge=0, le=4294967295 + ) + model: RunwayModelEnum = Field(..., description='Model to use for generation') + promptText: Optional[str] = Field( + None, description='Text prompt for the generation', max_length=1000 + ) + duration: RunwayDurationEnum = Field( + ..., description='The number of seconds of duration for the output video.' + ) + ratio: RunwayAspectRatioEnum = Field( + ..., + description='The resolution (aspect ratio) of the output video. Allowable values depend on the selected model. 1280:768 and 768:1280 are only supported for gen3a_turbo.', + ) + + +class RunwayTaskStatusResponse(BaseModel): + id: Optional[str] = Field(None, description='Task ID') + status: Optional[RunwayTaskStatusEnum] = Field(None, description='Task status') + createdAt: Optional[datetime] = Field(None, description='Task creation timestamp') + output: Optional[List[str]] = Field(None, description='Array of output video URLs') + + +class PikaHTTPValidationError(BaseModel): + detail: Optional[List[PikaValidationError]] = Field(None, title='Detail') + + +class PikaBodyGenerate22T2vGenerate22T2vPost(BaseModel): + promptText: str = Field(..., title='Prompttext') + negativePrompt: Optional[str] = Field(None, title='Negativeprompt') + seed: Optional[int] = Field(None, title='Seed') + resolution: Optional[PikaResolutionEnum] = Field('1080p', title='Resolution') + duration: Optional[PikaDurationEnum] = Field(5, title='Duration') + aspectRatio: Optional[float] = Field( + 1.7777777777777777, + description='Aspect ratio (width / height)', + ge=0.4, + le=2.5, + title='Aspectratio', + ) + + +class PikaBodyGenerate22I2vGenerate22I2vPost(BaseModel): + image: Optional[StrictBytes] = Field(None, title='Image') + promptText: Optional[str] = Field(None, title='Prompttext') + negativePrompt: Optional[str] = Field(None, title='Negativeprompt') + seed: Optional[int] = Field(None, title='Seed') + resolution: Optional[PikaResolutionEnum] = Field('1080p', title='Resolution') + duration: Optional[PikaDurationEnum] = Field(5, title='Duration') + + +class PikaBodyGenerate22KeyframeGenerate22PikaframesPost(BaseModel): + keyFrames: List[StrictBytes] = Field( + ..., description='Array of keyframe images', title='Keyframes' + ) + promptText: str = Field(..., title='Prompttext') + negativePrompt: Optional[str] = Field(None, title='Negativeprompt') + seed: Optional[int] = Field(None, title='Seed') + resolution: Optional[PikaResolutionEnum] = Field('1080p', title='Resolution') + duration: Optional[int] = Field(None, ge=5, le=10, title='Duration') + + +class PikaVideoResponse(BaseModel): + id: str = Field(..., title='Id') + status: PikaStatusEnum = Field( + ..., description='The status of the video', title='Status' + ) + url: Optional[str] = Field(None, title='Url') + progress: Optional[int] = Field(None, title='Progress') + + +class Node(BaseModel): + id: Optional[str] = Field(None, description='The unique identifier of the node.') + name: Optional[str] = Field(None, description='The display name of the node.') + category: Optional[str] = Field(None, description='The category of the node.') + description: Optional[str] = None + author: Optional[str] = None + license: Optional[str] = Field( + None, description="The path to the LICENSE file in the node's repository." + ) + icon: Optional[str] = Field(None, description="URL to the node's icon.") + repository: Optional[str] = Field(None, description="URL to the node's repository.") + tags: Optional[List[str]] = None + latest_version: Optional[NodeVersion] = Field( + None, description='The latest version of the node.' + ) + rating: Optional[float] = Field(None, description='The average rating of the node.') + downloads: Optional[int] = Field( + None, description='The number of downloads of the node.' + ) + publisher: Optional[Publisher] = Field( + None, description='The publisher of the node.' + ) + status: Optional[NodeStatus] = Field(None, description='The status of the node.') + status_detail: Optional[str] = Field( + None, description='The status detail of the node.' + ) + translations: Optional[Dict[str, Dict[str, Any]]] = None + + +class KlingVideoEffectsRequest(BaseModel): + effect_scene: Union[KlingDualCharacterEffectsScene, KlingSingleImageEffectsScene] + input: KlingVideoEffectsInput + callback_url: Optional[AnyUrl] = Field( + None, + description='The callback notification address for the result of this task.', + ) + external_task_id: Optional[str] = Field( + None, + description='Customized Task ID. Must be unique within a single user account.', + ) + + +class StripeCharge(BaseModel): + id: Optional[str] = None + object: Optional[Object2] = None + amount: Optional[int] = None + amount_captured: Optional[int] = None + amount_refunded: Optional[int] = None + application: Optional[str] = None + application_fee: Optional[str] = None + application_fee_amount: Optional[int] = None + balance_transaction: Optional[str] = None + billing_details: Optional[StripeBillingDetails] = None + calculated_statement_descriptor: Optional[str] = None + captured: Optional[bool] = None + created: Optional[int] = None + currency: Optional[str] = None + customer: Optional[str] = None + description: Optional[str] = None + destination: Optional[Any] = None + dispute: Optional[Any] = None + disputed: Optional[bool] = None + failure_balance_transaction: Optional[Any] = None + failure_code: Optional[Any] = None + failure_message: Optional[Any] = None + fraud_details: Optional[Dict[str, Any]] = None + invoice: Optional[Any] = None + livemode: Optional[bool] = None + metadata: Optional[Dict[str, Any]] = None + on_behalf_of: Optional[Any] = None + order: Optional[Any] = None + outcome: Optional[StripeOutcome] = None + paid: Optional[bool] = None + payment_intent: Optional[str] = None + payment_method: Optional[str] = None + payment_method_details: Optional[StripePaymentMethodDetails] = None + radar_options: Optional[Dict[str, Any]] = None + receipt_email: Optional[str] = None + receipt_number: Optional[str] = None + receipt_url: Optional[str] = None + refunded: Optional[bool] = None + refunds: Optional[StripeRefundList] = None + review: Optional[Any] = None + shipping: Optional[StripeShipping] = None + source: Optional[Any] = None + source_transfer: Optional[Any] = None + statement_descriptor: Optional[Any] = None + statement_descriptor_suffix: Optional[Any] = None + status: Optional[str] = None + transfer_data: Optional[Any] = None + transfer_group: Optional[Any] = None + + +class StripeChargeList(BaseModel): + object: Optional[str] = None + data: Optional[List[StripeCharge]] = None + has_more: Optional[bool] = None + total_count: Optional[int] = None + url: Optional[str] = None + + +class StripePaymentIntent(BaseModel): + id: Optional[str] = None + object: Optional[Object1] = None + amount: Optional[int] = None + amount_capturable: Optional[int] = None + amount_details: Optional[StripeAmountDetails] = None + amount_received: Optional[int] = None + application: Optional[str] = None + application_fee_amount: Optional[int] = None + automatic_payment_methods: Optional[Any] = None + canceled_at: Optional[int] = None + cancellation_reason: Optional[str] = None + capture_method: Optional[str] = None + charges: Optional[StripeChargeList] = None + client_secret: Optional[str] = None + confirmation_method: Optional[str] = None + created: Optional[int] = None + currency: Optional[str] = None + customer: Optional[str] = None + description: Optional[str] = None + invoice: Optional[str] = None + last_payment_error: Optional[Any] = None + latest_charge: Optional[str] = None + livemode: Optional[bool] = None + metadata: Optional[Dict[str, Any]] = None + next_action: Optional[Any] = None + on_behalf_of: Optional[Any] = None + payment_method: Optional[str] = None + payment_method_configuration_details: Optional[Any] = None + payment_method_options: Optional[StripePaymentMethodOptions] = None + payment_method_types: Optional[List[str]] = None + processing: Optional[Any] = None + receipt_email: Optional[str] = None + review: Optional[Any] = None + setup_future_usage: Optional[Any] = None + shipping: Optional[StripeShipping] = None + source: Optional[Any] = None + statement_descriptor: Optional[Any] = None + statement_descriptor_suffix: Optional[Any] = None + status: Optional[str] = None + transfer_data: Optional[Any] = None + transfer_group: Optional[Any] = None + + +class Data8(BaseModel): + object: Optional[StripePaymentIntent] = None + + +class StripeEvent(BaseModel): + id: str + object: Object + api_version: Optional[str] = None + created: Optional[int] = None + data: Data8 + livemode: Optional[bool] = None + pending_webhooks: Optional[int] = None + request: Optional[StripeRequestInfo] = None + type: Type diff --git a/comfy_api_nodes/nodes_kling.py b/comfy_api_nodes/nodes_kling.py index 4b2492908..208a3730a 100644 --- a/comfy_api_nodes/nodes_kling.py +++ b/comfy_api_nodes/nodes_kling.py @@ -1,21 +1,74 @@ -from typing import Optional +""" +Kling API Nodes + +Compatibility Table +| Mode | Duration | Model Name | Camera Control | Image Tail | +|------|----------|------------------|----------------|------------| +| std | 5 | kling-v1 | No | Yes | +| std | 5 | kling-v1-5 | No | Yes | +| std | 5 | kling-v1-6 | No | No | +| std | 5 | kling-v2-master | No | No | +| std | 10 | kling-v1 | No | No | +| std | 10 | kling-v1-5 | No | No | +| std | 10 | kling-v1-6 | No | No | +| std | 10 | kling-v2-master | No | No | +| pro | 5 | kling-v1 | No | Yes | +| pro | 5 | kling-v1-5 | Yes | Yes | +| pro | 5 | kling-v1-6 | No | Yes | +| pro | 5 | kling-v2-master | No | No | +| pro | 10 | kling-v1 | No | No | +| pro | 10 | kling-v1-5 | No | Yes | +| pro | 10 | kling-v1-6 | No | Yes | +| pro | 10 | kling-v2-master | No | No | + +**Note**: Although the combo of pro mode, kling-v1-5 model, and 5s duration +supports both camera_control and image_tail, you can only use one feature +at a time. +""" + +from typing import Optional, TypeVar, Any import math import logging + import torch from comfy_api_nodes.apis import ( + KlingTaskStatus, + KlingCameraControl, + KlingCameraConfig, + KlingCameraControlType, + KlingVideoGenDuration, + KlingVideoGenMode, + KlingVideoGenAspectRatio, + KlingVideoGenModelName, KlingText2VideoRequest, KlingText2VideoResponse, - TaskStatus, - CameraControl, - Config as CameraConfig, - Type as CameraType, - Duration, - Mode, - AspectRatio, - ModelName, KlingImage2VideoRequest, KlingImage2VideoResponse, + KlingVideoExtendRequest, + KlingVideoExtendResponse, + KlingLipSyncVoiceLanguage, + KlingLipSyncInputObject, + KlingLipSyncRequest, + KlingLipSyncResponse, + KlingVirtualTryOnModelName, + KlingVirtualTryOnRequest, + KlingVirtualTryOnResponse, + KlingVideoResult, + KlingImageResult, + KlingImageGenerationsRequest, + KlingImageGenerationsResponse, + KlingImageGenImageReferenceType, + KlingImageGenModelName, + KlingImageGenAspectRatio, + KlingVideoEffectsRequest, + KlingVideoEffectsResponse, + KlingDualCharacterEffectsScene, + KlingSingleImageEffectsScene, + KlingDualCharacterEffectInput, + KlingSingleImageEffectInput, + KlingCharacterEffectModelName, + KlingSingleImageEffectModelName, ) from comfy_api_nodes.apis.client import ( ApiEndpoint, @@ -27,10 +80,15 @@ from comfy_api_nodes.apis.client import ( from comfy_api_nodes.apinode_utils import ( tensor_to_base64_string, download_url_to_video_output, + upload_video_to_comfyapi, + upload_audio_to_comfyapi, + download_url_to_image_tensor, ) from comfy_api_nodes.mapper_utils import model_field_to_node_input -from comfy.comfy_types.node_typing import IO, InputTypeOptions, ComfyNodeABC +from comfy_api.input.basic_types import AudioInput +from comfy_api.input.video_types import VideoInput from comfy_api.input_impl import VideoFromFile +from comfy.comfy_types.node_typing import IO, InputTypeOptions, ComfyNodeABC KLING_API_VERSION = "v1" PATH_TEXT_TO_VIDEO = f"/proxy/kling/{KLING_API_VERSION}/videos/text2video" @@ -40,9 +98,16 @@ PATH_LIP_SYNC = f"/proxy/kling/{KLING_API_VERSION}/videos/lip-sync" PATH_VIDEO_EFFECTS = f"/proxy/kling/{KLING_API_VERSION}/videos/effects" PATH_CHARACTER_IMAGE = f"/proxy/kling/{KLING_API_VERSION}/images/generations" PATH_VIRTUAL_TRY_ON = f"/proxy/kling/{KLING_API_VERSION}/images/kolors-virtual-try-on" +PATH_IMAGE_GENERATIONS = f"/proxy/kling/{KLING_API_VERSION}/images/generations" + MAX_PROMPT_LENGTH_T2V = 2500 MAX_PROMPT_LENGTH_I2V = 500 +MAX_PROMPT_LENGTH_IMAGE_GEN = 500 +MAX_NEGATIVE_PROMPT_LENGTH_IMAGE_GEN = 200 +MAX_PROMPT_LENGTH_LIP_SYNC = 120 + +R = TypeVar("R") class KlingApiError(Exception): @@ -51,6 +116,23 @@ class KlingApiError(Exception): pass +def poll_until_finished(auth_token: str, api_endpoint: ApiEndpoint[Any, R]) -> R: + """Polls the Kling API endpoint until the task reaches a terminal state, then returns the response.""" + return PollingOperation( + poll_endpoint=api_endpoint, + completed_statuses=[ + KlingTaskStatus.succeed.value, + ], + failed_statuses=[KlingTaskStatus.failed.value], + status_extractor=lambda response: ( + response.data.task_status.value + if response.data and response.data.task_status + else None + ), + auth_token=auth_token, + ).execute() + + def is_valid_camera_control_configs(configs: list[float]) -> bool: """Verifies that at least one camera control configuration is non-zero.""" return any(not math.isclose(value, 0.0) for value in configs) @@ -61,7 +143,7 @@ def is_valid_prompt(prompt: str) -> bool: return bool(prompt) -def is_valid_initial_response(response: KlingText2VideoResponse) -> bool: +def is_valid_task_creation_response(response: KlingText2VideoResponse) -> bool: """Verifies that the initial response contains a task ID.""" return bool(response.data.task_id) @@ -69,12 +151,21 @@ def is_valid_initial_response(response: KlingText2VideoResponse) -> bool: def is_valid_video_response(response: KlingText2VideoResponse) -> bool: """Verifies that the response contains a task result with at least one video.""" return ( - response.data.task_result - and response.data.task_result.videos + "task_result" in response.data + and "videos" in response.data.task_result and len(response.data.task_result.videos) > 0 ) +def is_valid_image_response(response: KlingVirtualTryOnResponse) -> bool: + """Verifies that the response contains a task result with at least one image.""" + return ( + "task_result" in response.data + and "images" in response.data.task_result + and len(response.data.task_result.images) > 0 + ) + + def validate_prompts(prompt: str, negative_prompt: str, max_length: int) -> bool: """Verifies that the positive prompt is not empty and that neither promt is too long.""" if not prompt: @@ -88,6 +179,30 @@ def validate_prompts(prompt: str, negative_prompt: str, max_length: int) -> bool return True +def validate_task_creation_response(response): + """Validates that the Kling task creation request was successful.""" + if not is_valid_task_creation_response(response): + error_msg = f"Kling initial request failed. Code: {response.code}, Message: {response.message}, Data: {response.data}" + logging.error(error_msg) + raise KlingApiError(error_msg) + + +def validate_video_result_response(response): + """Validates that the Kling task result contains a video.""" + if not is_valid_video_response(response): + error_msg = f"Kling task {response.data.task_id} succeeded but no video data found in response." + logging.error(error_msg) + raise KlingApiError(error_msg) + + +def validate_image_result_response(response): + """Validates that the Kling task result contains an image.""" + if not is_valid_image_response(response): + error_msg = f"Kling task {response.data.task_id} succeeded but no image data found in response." + logging.error(error_msg) + raise KlingApiError(error_msg) + + def get_camera_control_input_config( tooltip: str, default: float = 0.0 ) -> tuple[IO, InputTypeOptions]: @@ -103,34 +218,46 @@ def get_camera_control_input_config( return IO.FLOAT, input_config +def get_video_from_response(response) -> KlingVideoResult: + """Returns the first video object from the Kling video generation task result.""" + video = response.data.task_result.videos[0] + logging.debug( + "Kling task %s succeeded. Video URL: %s", response.data.task_id, video.url + ) + return video + + +def get_images_from_response(response) -> list[KlingImageResult]: + images = response.data.task_result.images + logging.debug("Kling task %s succeeded. Images: %s", response.data.task_id, images) + return images + + +def video_result_to_node_output( + video: KlingVideoResult, +) -> tuple[VideoFromFile, str, str]: + """Converts a KlingVideoResult to a tuple of (VideoFromFile, str, str) to be used as a ComfyUI node output.""" + return ( + download_url_to_video_output(video.url), + str(video.id), + str(video.duration), + ) + + +def image_result_to_node_output( + image: KlingImageResult, +) -> torch.Tensor: + """ + Converts a KlingImageResult to a tuple containing a [B, H, W, C] tensor. + If multiple images are returned, they will be stacked along the batch dimension. + """ + return (download_url_to_image_tensor(image.url),) + + class KlingNodeBase(ComfyNodeABC): """ Base class for Kling nodes. - Compatibility Table - =================== - | Mode | Duration | Model Name | Camera Control | Image Tail | - |------|----------|------------------|----------------|------------| - | std | 5 | kling-v1 | No | Yes | - | std | 5 | kling-v1-5 | No | Yes | - | std | 5 | kling-v1-6 | No | No | - | std | 5 | kling-v2-master | No | No | - | std | 10 | kling-v1 | No | No | - | std | 10 | kling-v1-5 | No | No | - | std | 10 | kling-v1-6 | No | No | - | std | 10 | kling-v2-master | No | No | - | pro | 5 | kling-v1 | No | Yes | - | pro | 5 | kling-v1-5 | Yes | Yes | - | pro | 5 | kling-v1-6 | No | Yes | - | pro | 5 | kling-v2-master | No | No | - | pro | 10 | kling-v1 | No | No | - | pro | 10 | kling-v1-5 | No | Yes | - | pro | 10 | kling-v1-6 | No | Yes | - | pro | 10 | kling-v2-master | No | No | - - **Note**: Although the combo of pro mode, kling-v1-5 model, and 5s duration - supports both camera_control and image_tail, you can only use one feature - at a time. """ FUNCTION = "api_call" @@ -145,16 +272,11 @@ class KlingCameraControls(KlingNodeBase): def INPUT_TYPES(cls): return { "required": { - "camera_control_type": ( + "camera_control_type": model_field_to_node_input( IO.COMBO, - { - "options": [ - camera_control_type.value - for camera_control_type in CameraType - ], - "default": "simple", - "tooltip": "Predefined camera movements type. simple: Customizable camera movement. down_back: Camera descends and moves backward. forward_up: Camera moves forward and tilts up. right_turn_forward: Rotate right and move forward. left_turn_forward: Rotate left and move forward.", - }, + KlingCameraControl, + "type", + enum_type=KlingCameraControlType, ), "horizontal_movement": get_camera_control_input_config( "Controls camera's movement along horizontal axis (x-axis). Negative indicates left, positive indicates right" @@ -178,7 +300,7 @@ class KlingCameraControls(KlingNodeBase): } } - DESCRIPTION = "Kling Camera Controls Node. Not all model and mode combinations support camera control. Please refer to the Kling API documentation for more information." + DESCRIPTION = "Allows specifying configuration options for Kling Camera Controls and motion control effects." RETURN_TYPES = ("CAMERA_CONTROL",) RETURN_NAMES = ("camera_control",) FUNCTION = "main" @@ -215,11 +337,11 @@ class KlingCameraControls(KlingNodeBase): tilt: float, roll: float, zoom: float, - ) -> tuple[CameraControl]: + ) -> tuple[KlingCameraControl]: return ( - CameraControl( - type=CameraType(camera_control_type), - config=CameraConfig( + KlingCameraControl( + type=KlingCameraControlType(camera_control_type), + config=KlingCameraConfig( horizontal=horizontal_movement, vertical=vertical_movement, pan=pan, @@ -234,29 +356,6 @@ class KlingCameraControls(KlingNodeBase): class KlingTextToVideoNode(KlingNodeBase): """Kling Text to Video Node""" - @staticmethod - def poll_for_task_status(task_id: str, auth_token: str) -> KlingText2VideoResponse: - """Polls the Kling API endpoint until the task reaches a terminal state.""" - polling_operation = PollingOperation( - poll_endpoint=ApiEndpoint( - path=f"{PATH_TEXT_TO_VIDEO}/{task_id}", - method=HttpMethod.GET, - request_model=EmptyRequest, - response_model=KlingText2VideoResponse, - ), - completed_statuses=[ - TaskStatus.succeed.value, - ], - failed_statuses=[TaskStatus.failed.value], - status_extractor=lambda response: ( - response.data.task_status.value - if response.data and response.data.task_status - else None - ), - auth_token=auth_token, - ) - return polling_operation.execute() - @classmethod def INPUT_TYPES(s): return { @@ -271,32 +370,48 @@ class KlingTextToVideoNode(KlingNodeBase): IO.COMBO, KlingText2VideoRequest, "model_name", - enum_type=ModelName, - default="kling-v2-master", + enum_type=KlingVideoGenModelName, ), "cfg_scale": model_field_to_node_input( IO.FLOAT, KlingText2VideoRequest, "cfg_scale" ), "mode": model_field_to_node_input( - IO.COMBO, KlingText2VideoRequest, "mode", enum_type=Mode + IO.COMBO, + KlingText2VideoRequest, + "mode", + enum_type=KlingVideoGenMode, ), "duration": model_field_to_node_input( - IO.COMBO, KlingText2VideoRequest, "duration", enum_type=Duration + IO.COMBO, + KlingText2VideoRequest, + "duration", + enum_type=KlingVideoGenDuration, ), "aspect_ratio": model_field_to_node_input( IO.COMBO, KlingText2VideoRequest, "aspect_ratio", - enum_type=AspectRatio, + enum_type=KlingVideoGenAspectRatio, ), }, "hidden": {"auth_token": "AUTH_TOKEN_COMFY_ORG"}, } RETURN_TYPES = ("VIDEO", "STRING", "STRING") - RETURN_NAMES = ("VIDEO", "Kling ID", "Duration (sec)") + RETURN_NAMES = ("VIDEO", "video_id", "duration") DESCRIPTION = "Kling Text to Video Node" + def get_response(self, task_id: str, auth_token: str) -> KlingText2VideoResponse: + return poll_until_finished( + auth_token, + ApiEndpoint( + path=f"{PATH_TEXT_TO_VIDEO}/{task_id}", + method=HttpMethod.GET, + request_model=EmptyRequest, + response_model=KlingText2VideoResponse, + ), + ) + def api_call( self, prompt: str, @@ -306,7 +421,7 @@ class KlingTextToVideoNode(KlingNodeBase): mode: str, duration: int, aspect_ratio: str, - camera_control: Optional[CameraControl] = None, + camera_control: Optional[KlingCameraControl] = None, auth_token: Optional[str] = None, ) -> tuple[VideoFromFile, str, str]: validate_prompts(prompt, negative_prompt, MAX_PROMPT_LENGTH_T2V) @@ -320,38 +435,25 @@ class KlingTextToVideoNode(KlingNodeBase): request=KlingText2VideoRequest( prompt=prompt if prompt else None, negative_prompt=negative_prompt if negative_prompt else None, - duration=Duration(duration), - mode=Mode(mode), - model_name=ModelName(model_name), + duration=KlingVideoGenDuration(duration), + mode=KlingVideoGenMode(mode), + model_name=KlingVideoGenModelName(model_name), cfg_scale=cfg_scale, - aspect_ratio=AspectRatio(aspect_ratio), + aspect_ratio=KlingVideoGenAspectRatio(aspect_ratio), camera_control=camera_control, ), auth_token=auth_token, ) - initial_response = initial_operation.execute() - if not is_valid_initial_response(initial_response): - error_msg = f"Kling initial request failed. Code: {initial_response.code}, Message: {initial_response.message}, Data: {initial_response.data}" - logging.error(error_msg) - raise KlingApiError(error_msg) + task_creation_response = initial_operation.execute() + validate_task_creation_response(task_creation_response) - task_id = initial_response.data.task_id - final_response = self.poll_for_task_status(task_id, auth_token) - if not is_valid_video_response(final_response): - error_msg = ( - f"Kling task {task_id} succeeded but no video data found in response." - ) - logging.error(error_msg) - raise KlingApiError(error_msg) + task_id = task_creation_response.data.task_id + final_response = self.get_response(task_id, auth_token) + validate_video_result_response(final_response) - video = final_response.data.task_result.videos[0] - logging.debug("Kling task %s succeeded. Video URL: %s", task_id, video.url) - return ( - download_url_to_video_output(video.url), - str(video.id), - str(video.duration), - ) + video = get_video_from_response(final_response) + return video_result_to_node_output(video) class KlingCameraControlT2VNode(KlingTextToVideoNode): @@ -380,7 +482,7 @@ class KlingCameraControlT2VNode(KlingTextToVideoNode): IO.COMBO, KlingText2VideoRequest, "aspect_ratio", - enum_type=AspectRatio, + enum_type=KlingVideoGenAspectRatio, ), "camera_control": ( "CAMERA_CONTROL", @@ -400,15 +502,15 @@ class KlingCameraControlT2VNode(KlingTextToVideoNode): negative_prompt: str, cfg_scale: float, aspect_ratio: str, - camera_control: Optional[CameraControl] = None, + camera_control: Optional[KlingCameraControl] = None, auth_token: Optional[str] = None, ): return super().api_call( - model_name="kling-v1-5", + model_name=KlingVideoGenModelName.kling_v1_5, cfg_scale=cfg_scale, - mode="pro", - aspect_ratio=aspect_ratio, - duration="5", + mode=KlingVideoGenMode.pro, + aspect_ratio=KlingVideoGenAspectRatio(aspect_ratio), + duration=KlingVideoGenDuration.field_5, prompt=prompt, negative_prompt=negative_prompt, camera_control=camera_control, @@ -419,27 +521,6 @@ class KlingCameraControlT2VNode(KlingTextToVideoNode): class KlingImage2VideoNode(KlingNodeBase): """Kling Image to Video Node""" - @staticmethod - def poll_for_task_status(task_id: str, auth_token: str) -> KlingImage2VideoResponse: - """Polls the Kling API endpoint until the task reaches a terminal state.""" - polling_operation = PollingOperation( - poll_endpoint=ApiEndpoint( - path=f"{PATH_IMAGE_TO_VIDEO}/{task_id}", - method=HttpMethod.GET, - request_model=KlingImage2VideoRequest, - response_model=KlingImage2VideoResponse, - ), - completed_statuses=[TaskStatus.succeed.value], - failed_statuses=[TaskStatus.failed.value], - status_extractor=lambda response: ( - response.data.task_status.value - if response.data and response.data.task_status - else None - ), - auth_token=auth_token, - ) - return polling_operation.execute() - @classmethod def INPUT_TYPES(s): return { @@ -460,32 +541,48 @@ class KlingImage2VideoNode(KlingNodeBase): IO.COMBO, KlingImage2VideoRequest, "model_name", - enum_type=ModelName, - default="kling-v2-master", + enum_type=KlingVideoGenModelName, ), "cfg_scale": model_field_to_node_input( IO.FLOAT, KlingImage2VideoRequest, "cfg_scale" ), "mode": model_field_to_node_input( - IO.COMBO, KlingImage2VideoRequest, "mode", enum_type=Mode + IO.COMBO, + KlingImage2VideoRequest, + "mode", + enum_type=KlingVideoGenMode, ), "aspect_ratio": model_field_to_node_input( IO.COMBO, KlingImage2VideoRequest, "aspect_ratio", - enum_type=AspectRatio, + enum_type=KlingVideoGenAspectRatio, ), "duration": model_field_to_node_input( - IO.COMBO, KlingImage2VideoRequest, "duration", enum_type=Duration + IO.COMBO, + KlingImage2VideoRequest, + "duration", + enum_type=KlingVideoGenDuration, ), }, "hidden": {"auth_token": "AUTH_TOKEN_COMFY_ORG"}, } RETURN_TYPES = ("VIDEO", "STRING", "STRING") - RETURN_NAMES = ("VIDEO", "Kling ID", "Duration (sec)") + RETURN_NAMES = ("VIDEO", "video_id", "duration") DESCRIPTION = "Kling Image to Video Node" + def get_response(self, task_id: str, auth_token: str) -> KlingImage2VideoResponse: + return poll_until_finished( + auth_token, + ApiEndpoint( + path=f"{PATH_IMAGE_TO_VIDEO}/{task_id}", + method=HttpMethod.GET, + request_model=KlingImage2VideoRequest, + response_model=KlingImage2VideoResponse, + ), + ) + def api_call( self, start_frame: torch.Tensor, @@ -496,7 +593,7 @@ class KlingImage2VideoNode(KlingNodeBase): mode: str, aspect_ratio: str, duration: str, - camera_control: Optional[CameraControl] = None, + camera_control: Optional[KlingCameraControl] = None, end_frame: Optional[torch.Tensor] = None, auth_token: Optional[str] = None, ) -> tuple[VideoFromFile]: @@ -509,7 +606,7 @@ class KlingImage2VideoNode(KlingNodeBase): response_model=KlingImage2VideoResponse, ), request=KlingImage2VideoRequest( - model_name=ModelName(model_name), + model_name=KlingVideoGenModelName(model_name), image=tensor_to_base64_string(start_frame), image_tail=( tensor_to_base64_string(end_frame) @@ -519,36 +616,23 @@ class KlingImage2VideoNode(KlingNodeBase): prompt=prompt, negative_prompt=negative_prompt if negative_prompt else None, cfg_scale=cfg_scale, - mode=Mode(mode), - aspect_ratio=AspectRatio(aspect_ratio), - duration=Duration(duration), + mode=KlingVideoGenMode(mode), + aspect_ratio=KlingVideoGenAspectRatio(aspect_ratio), + duration=KlingVideoGenDuration(duration), camera_control=camera_control, ), auth_token=auth_token, ) - initial_response = initial_operation.execute() - if not is_valid_initial_response(initial_response): - error_msg = f"Kling initial request failed. Code: {initial_response.code}, Message: {initial_response.message}, Data: {initial_response.data}" - logging.error(error_msg) - raise KlingApiError(error_msg) - task_id = initial_response.data.task_id - final_response = KlingImage2VideoNode.poll_for_task_status(task_id, auth_token) - if not is_valid_video_response(final_response): - error_msg = ( - f"Kling task {task_id} succeeded but no video data found in response." - ) - logging.error(error_msg) - raise KlingApiError(error_msg) + task_creation_response = initial_operation.execute() + validate_task_creation_response(task_creation_response) + task_id = task_creation_response.data.task_id - video = final_response.data.task_result.videos[0] - logging.info("Kling task %s succeeded. Video URL: %s", task_id, video.url) + final_response = self.get_response(task_id, auth_token) + validate_video_result_response(final_response) - return ( - download_url_to_video_output(video.url), - str(video.id), - str(video.duration), - ) + video = get_video_from_response(final_response) + return video_result_to_node_output(video) class KlingCameraControlI2VNode(KlingImage2VideoNode): @@ -580,7 +664,7 @@ class KlingCameraControlI2VNode(KlingImage2VideoNode): IO.COMBO, KlingImage2VideoRequest, "aspect_ratio", - enum_type=AspectRatio, + enum_type=KlingVideoGenAspectRatio, ), "camera_control": ( "CAMERA_CONTROL", @@ -601,16 +685,16 @@ class KlingCameraControlI2VNode(KlingImage2VideoNode): negative_prompt: str, cfg_scale: float, aspect_ratio: str, - camera_control: CameraControl, + camera_control: KlingCameraControl, auth_token: Optional[str] = None, ): return super().api_call( - model_name="kling-v1-5", + model_name=KlingVideoGenModelName.kling_v1_5, start_frame=start_frame, cfg_scale=cfg_scale, - mode="pro", - aspect_ratio=aspect_ratio, - duration="5", + mode=KlingVideoGenMode.pro, + aspect_ratio=KlingVideoGenAspectRatio(aspect_ratio), + duration=KlingVideoGenDuration.field_5, prompt=prompt, negative_prompt=negative_prompt, camera_control=camera_control, @@ -666,7 +750,7 @@ class KlingStartEndFrameNode(KlingImage2VideoNode): IO.COMBO, KlingImage2VideoRequest, "aspect_ratio", - enum_type=AspectRatio, + enum_type=KlingVideoGenAspectRatio, ), "mode": ( modes, @@ -681,10 +765,6 @@ class KlingStartEndFrameNode(KlingImage2VideoNode): DESCRIPTION = "Generate a video sequence that transitions between your provided start and end images. The node creates all frames in between, producing a smooth transformation from the first frame to the last." - def parse_inputs_from_mode(self, mode: str) -> tuple[str, str, str]: - """Parses the mode input into a tuple of (model_name, duration, mode).""" - return KlingStartEndFrameNode.get_mode_string_mapping()[mode] - def api_call( self, start_frame: torch.Tensor, @@ -696,7 +776,9 @@ class KlingStartEndFrameNode(KlingImage2VideoNode): mode: str, auth_token: Optional[str] = None, ): - mode, duration, model_name = self.parse_inputs_from_mode(mode) + mode, duration, model_name = KlingStartEndFrameNode.get_mode_string_mapping()[ + mode + ] return super().api_call( prompt=prompt, negative_prompt=negative_prompt, @@ -711,6 +793,711 @@ class KlingStartEndFrameNode(KlingImage2VideoNode): ) +class KlingVideoExtendNode(KlingNodeBase): + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "prompt": model_field_to_node_input( + IO.STRING, KlingVideoExtendRequest, "prompt", multiline=True + ), + "negative_prompt": model_field_to_node_input( + IO.STRING, + KlingVideoExtendRequest, + "negative_prompt", + multiline=True, + ), + "cfg_scale": model_field_to_node_input( + IO.FLOAT, KlingVideoExtendRequest, "cfg_scale" + ), + "video_id": model_field_to_node_input( + IO.STRING, KlingVideoExtendRequest, "video_id", forceInput=True + ), + }, + "hidden": {"auth_token": "AUTH_TOKEN_COMFY_ORG"}, + } + + RETURN_TYPES = ("VIDEO", "STRING", "STRING") + RETURN_NAMES = ("VIDEO", "video_id", "duration") + DESCRIPTION = "Kling Video Extend Node. Extend videos made by other Kling nodes. The Kling ID is output by Kling Nodes." + + def get_response(self, task_id: str, auth_token: str) -> KlingVideoExtendResponse: + return poll_until_finished( + auth_token, + ApiEndpoint( + path=f"{PATH_VIDEO_EXTEND}/{task_id}", + method=HttpMethod.GET, + request_model=EmptyRequest, + response_model=KlingVideoExtendResponse, + ), + ) + + def api_call( + self, + prompt: str, + negative_prompt: str, + cfg_scale: float, + video_id: str, + auth_token: Optional[str] = None, + ) -> tuple[VideoFromFile, str, str]: + validate_prompts(prompt, negative_prompt, MAX_PROMPT_LENGTH_T2V) + initial_operation = SynchronousOperation( + endpoint=ApiEndpoint( + path=PATH_VIDEO_EXTEND, + method=HttpMethod.POST, + request_model=KlingVideoExtendRequest, + response_model=KlingVideoExtendResponse, + ), + request=KlingVideoExtendRequest( + prompt=prompt if prompt else None, + negative_prompt=negative_prompt if negative_prompt else None, + cfg_scale=cfg_scale, + video_id=video_id, + ), + auth_token=auth_token, + ) + + task_creation_response = initial_operation.execute() + validate_task_creation_response(task_creation_response) + task_id = task_creation_response.data.task_id + + final_response = self.get_response(task_id, auth_token) + validate_video_result_response(final_response) + + video = get_video_from_response(final_response) + return video_result_to_node_output(video) + + +class KlingVideoEffectsBase(KlingNodeBase): + """Kling Video Effects Base""" + + RETURN_TYPES = ("VIDEO", "STRING", "STRING") + RETURN_NAMES = ("VIDEO", "video_id", "duration") + + def get_response(self, task_id: str, auth_token: str) -> KlingVideoEffectsResponse: + return poll_until_finished( + auth_token, + ApiEndpoint( + path=f"{PATH_VIDEO_EFFECTS}/{task_id}", + method=HttpMethod.GET, + request_model=EmptyRequest, + response_model=KlingVideoEffectsResponse, + ), + ) + + def api_call( + self, + dual_character: bool, + effect_scene: KlingDualCharacterEffectsScene | KlingSingleImageEffectsScene, + model_name: str, + duration: KlingVideoGenDuration, + image_1: torch.Tensor, + image_2: Optional[torch.Tensor] = None, + mode: Optional[KlingVideoGenMode] = None, + auth_token: Optional[str] = None, + ): + if dual_character: + request_input_field = KlingDualCharacterEffectInput( + model_name=model_name, + mode=mode, + images=[ + tensor_to_base64_string(image_1), + tensor_to_base64_string(image_2), + ], + duration=duration, + ) + else: + request_input_field = KlingSingleImageEffectInput( + model_name=model_name, + image=tensor_to_base64_string(image_1), + duration=duration, + ) + + initial_operation = SynchronousOperation( + endpoint=ApiEndpoint( + path=PATH_VIDEO_EFFECTS, + method=HttpMethod.POST, + request_model=KlingVideoEffectsRequest, + response_model=KlingVideoEffectsResponse, + ), + request=KlingVideoEffectsRequest( + effect_scene=effect_scene, + input=request_input_field, + ), + auth_token=auth_token, + ) + + task_creation_response = initial_operation.execute() + validate_task_creation_response(task_creation_response) + task_id = task_creation_response.data.task_id + + final_response = self.get_response(task_id, auth_token) + validate_video_result_response(final_response) + + video = get_video_from_response(final_response) + return video_result_to_node_output(video) + + +class KlingDualCharacterVideoEffectNode(KlingVideoEffectsBase): + """Kling Dual Character Video Effect Node""" + + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "image_left": (IO.IMAGE, {"tooltip": "Left side image"}), + "image_right": (IO.IMAGE, {"tooltip": "Right side image"}), + "effect_scene": model_field_to_node_input( + IO.COMBO, + KlingVideoEffectsRequest, + "effect_scene", + enum_type=KlingDualCharacterEffectsScene, + ), + "model_name": model_field_to_node_input( + IO.COMBO, + KlingDualCharacterEffectInput, + "model_name", + enum_type=KlingCharacterEffectModelName, + ), + "mode": model_field_to_node_input( + IO.COMBO, + KlingDualCharacterEffectInput, + "mode", + enum_type=KlingVideoGenMode, + ), + "duration": model_field_to_node_input( + IO.COMBO, + KlingDualCharacterEffectInput, + "duration", + enum_type=KlingVideoGenDuration, + ), + }, + "hidden": {"auth_token": "AUTH_TOKEN_COMFY_ORG"}, + } + + DESCRIPTION = "Achieve different special effects when generating a video based on the effect_scene. First image will be positioned on left side, second on right side of the composite." + + def api_call( + self, + image_left: torch.Tensor, + image_right: torch.Tensor, + effect_scene: KlingDualCharacterEffectsScene, + model_name: KlingCharacterEffectModelName, + mode: KlingVideoGenMode, + duration: KlingVideoGenDuration, + auth_token: Optional[str] = None, + ): + return super().api_call( + dual_character=True, + effect_scene=effect_scene, + model_name=model_name, + mode=mode, + duration=duration, + image_1=image_left, + image_2=image_right, + auth_token=auth_token, + ) + + +class KlingSingleImageVideoEffectNode(KlingVideoEffectsBase): + """Kling Single Image Video Effect Node""" + + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "image": ( + IO.IMAGE, + { + "tooltip": " Reference Image. URL or Base64 encoded string (without data:image prefix). File size cannot exceed 10MB, resolution not less than 300*300px, aspect ratio between 1:2.5 ~ 2.5:1" + }, + ), + "effect_scene": model_field_to_node_input( + IO.COMBO, + KlingVideoEffectsRequest, + "effect_scene", + enum_type=KlingSingleImageEffectsScene, + ), + "model_name": model_field_to_node_input( + IO.COMBO, + KlingSingleImageEffectInput, + "model_name", + enum_type=KlingSingleImageEffectModelName, + ), + "duration": model_field_to_node_input( + IO.COMBO, + KlingSingleImageEffectInput, + "duration", + enum_type=KlingVideoGenDuration, + ), + }, + "hidden": {"auth_token": "AUTH_TOKEN_COMFY_ORG"}, + } + + DESCRIPTION = "Achieve different special effects when generating a video based on the effect_scene." + + def api_call( + self, + image: torch.Tensor, + effect_scene: KlingSingleImageEffectsScene, + model_name: KlingSingleImageEffectModelName, + duration: KlingVideoGenDuration, + auth_token: Optional[str] = None, + ): + return super().api_call( + dual_character=False, + effect_scene=effect_scene, + model_name=model_name, + duration=duration, + image_1=image, + auth_token=auth_token, + ) + + +class KlingLipSyncBase(KlingNodeBase): + """Kling Lip Sync Base""" + + RETURN_TYPES = ("VIDEO", "STRING", "STRING") + RETURN_NAMES = ("VIDEO", "video_id", "duration") + + def validate_text(self, text: str): + if not text: + raise ValueError("Text is required") + if len(text) > MAX_PROMPT_LENGTH_LIP_SYNC: + raise ValueError( + f"Text is too long. Maximum length is {MAX_PROMPT_LENGTH_LIP_SYNC} characters." + ) + + def get_response(self, task_id: str, auth_token: str) -> KlingLipSyncResponse: + """Polls the Kling API endpoint until the task reaches a terminal state.""" + return poll_until_finished( + auth_token, + ApiEndpoint( + path=f"{PATH_LIP_SYNC}/{task_id}", + method=HttpMethod.GET, + request_model=EmptyRequest, + response_model=KlingLipSyncResponse, + ), + ) + + def api_call( + self, + video: VideoInput, + audio: Optional[AudioInput] = None, + voice_language: Optional[str] = None, + mode: Optional[str] = None, + text: Optional[str] = None, + voice_speed: Optional[float] = None, + voice_id: Optional[str] = None, + auth_token: Optional[str] = None, + ) -> tuple[VideoFromFile, str, str]: + if text: + self.validate_text(text) + + # Upload video to Comfy API and get download URL + video_url = upload_video_to_comfyapi(video, auth_token) + logging.info("Uploaded video to Comfy API. URL: %s", video_url) + + # Upload the audio file to Comfy API and get download URL + if audio: + audio_url = upload_audio_to_comfyapi(audio, auth_token) + logging.info("Uploaded audio to Comfy API. URL: %s", audio_url) + else: + audio_url = None + + initial_operation = SynchronousOperation( + endpoint=ApiEndpoint( + path=PATH_LIP_SYNC, + method=HttpMethod.POST, + request_model=KlingLipSyncRequest, + response_model=KlingLipSyncResponse, + ), + request=KlingLipSyncRequest( + input=KlingLipSyncInputObject( + video_url=video_url, + mode=mode, + text=text, + voice_language=voice_language, + voice_speed=voice_speed, + audio_type="url", + audio_url=audio_url, + voice_id=voice_id, + ), + ), + auth_token=auth_token, + ) + + task_creation_response = initial_operation.execute() + validate_task_creation_response(task_creation_response) + task_id = task_creation_response.data.task_id + + final_response = self.get_response(task_id, auth_token) + validate_video_result_response(final_response) + + video = get_video_from_response(final_response) + return video_result_to_node_output(video) + + +class KlingLipSyncAudioToVideoNode(KlingLipSyncBase): + """Kling Lip Sync Audio to Video Node. Syncs mouth movements in a video file to the audio content of an audio file.""" + + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "video": (IO.VIDEO, {}), + "audio": (IO.AUDIO, {}), + "voice_language": model_field_to_node_input( + IO.COMBO, + KlingLipSyncInputObject, + "voice_language", + enum_type=KlingLipSyncVoiceLanguage, + ), + }, + "hidden": {"auth_token": "AUTH_TOKEN_COMFY_ORG"}, + } + + DESCRIPTION = "Kling Lip Sync Audio to Video Node. Syncs mouth movements in a video file to the audio content of an audio file." + + def api_call( + self, + video: VideoInput, + audio: AudioInput, + voice_language: str, + auth_token: Optional[str] = None, + ): + return super().api_call( + video=video, + audio=audio, + voice_language=voice_language, + mode="audio2video", + auth_token=auth_token, + ) + + +class KlingLipSyncTextToVideoNode(KlingLipSyncBase): + """Kling Lip Sync Text to Video Node. Syncs mouth movements in a video file to a text prompt.""" + + @staticmethod + def get_voice_config() -> dict[str, tuple[str, str]]: + return { + # English voices + "Melody": ("girlfriend_4_speech02", "en"), + "Sunny": ("genshin_vindi2", "en"), + "Sage": ("zhinen_xuesheng", "en"), + "Ace": ("AOT", "en"), + "Blossom": ("ai_shatang", "en"), + "Peppy": ("genshin_klee2", "en"), + "Dove": ("genshin_kirara", "en"), + "Shine": ("ai_kaiya", "en"), + "Anchor": ("oversea_male1", "en"), + "Lyric": ("ai_chenjiahao_712", "en"), + "Tender": ("chat1_female_new-3", "en"), + "Siren": ("chat_0407_5-1", "en"), + "Zippy": ("cartoon-boy-07", "en"), + "Bud": ("uk_boy1", "en"), + "Sprite": ("cartoon-girl-01", "en"), + "Candy": ("PeppaPig_platform", "en"), + "Beacon": ("ai_huangzhong_712", "en"), + "Rock": ("ai_huangyaoshi_712", "en"), + "Titan": ("ai_laoguowang_712", "en"), + "Grace": ("chengshu_jiejie", "en"), + "Helen": ("you_pingjing", "en"), + "Lore": ("calm_story1", "en"), + "Crag": ("uk_man2", "en"), + "Prattle": ("laopopo_speech02", "en"), + "Hearth": ("heainainai_speech02", "en"), + "The Reader": ("reader_en_m-v1", "en"), + "Commercial Lady": ("commercial_lady_en_f-v1", "en"), + # Chinese voices + "阳光少年": ("genshin_vindi2", "zh"), + "懂事小弟": ("zhinen_xuesheng", "zh"), + "运动少年": ("tiyuxi_xuedi", "zh"), + "青春少女": ("ai_shatang", "zh"), + "温柔小妹": ("genshin_klee2", "zh"), + "元气少女": ("genshin_kirara", "zh"), + "阳光男生": ("ai_kaiya", "zh"), + "幽默小哥": ("tiexin_nanyou", "zh"), + "文艺小哥": ("ai_chenjiahao_712", "zh"), + "甜美邻家": ("girlfriend_1_speech02", "zh"), + "温柔姐姐": ("chat1_female_new-3", "zh"), + "职场女青": ("girlfriend_2_speech02", "zh"), + "活泼男童": ("cartoon-boy-07", "zh"), + "俏皮女童": ("cartoon-girl-01", "zh"), + "稳重老爸": ("ai_huangyaoshi_712", "zh"), + "温柔妈妈": ("you_pingjing", "zh"), + "严肃上司": ("ai_laoguowang_712", "zh"), + "优雅贵妇": ("chengshu_jiejie", "zh"), + "慈祥爷爷": ("zhuxi_speech02", "zh"), + "唠叨爷爷": ("uk_oldman3", "zh"), + "唠叨奶奶": ("laopopo_speech02", "zh"), + "和蔼奶奶": ("heainainai_speech02", "zh"), + "东北老铁": ("dongbeilaotie_speech02", "zh"), + "重庆小伙": ("chongqingxiaohuo_speech02", "zh"), + "四川妹子": ("chuanmeizi_speech02", "zh"), + "潮汕大叔": ("chaoshandashu_speech02", "zh"), + "台湾男生": ("ai_taiwan_man2_speech02", "zh"), + "西安掌柜": ("xianzhanggui_speech02", "zh"), + "天津姐姐": ("tianjinjiejie_speech02", "zh"), + "新闻播报男": ("diyinnansang_DB_CN_M_04-v2", "zh"), + "译制片男": ("yizhipiannan-v1", "zh"), + "撒娇女友": ("tianmeixuemei-v1", "zh"), + "刀片烟嗓": ("daopianyansang-v1", "zh"), + "乖巧正太": ("mengwa-v1", "zh"), + } + + @classmethod + def INPUT_TYPES(s): + voice_options = list(s.get_voice_config().keys()) + return { + "required": { + "video": (IO.VIDEO, {}), + "text": model_field_to_node_input( + IO.STRING, KlingLipSyncInputObject, "text", multiline=True + ), + "voice": (voice_options, {"default": voice_options[0]}), + "voice_speed": model_field_to_node_input( + IO.FLOAT, KlingLipSyncInputObject, "voice_speed", slider=True + ), + }, + "hidden": {"auth_token": "AUTH_TOKEN_COMFY_ORG"}, + } + + DESCRIPTION = "Kling Lip Sync Text to Video Node. Syncs mouth movements in a video file to a text prompt." + + def api_call( + self, + video: VideoInput, + text: str, + voice: str, + voice_speed: float, + auth_token: Optional[str] = None, + ): + voice_id, voice_language = KlingLipSyncTextToVideoNode.get_voice_config()[voice] + return super().api_call( + video=video, + text=text, + voice_language=voice_language, + voice_id=voice_id, + voice_speed=voice_speed, + mode="text2video", + auth_token=auth_token, + ) + + +class KlingImageGenerationBase(KlingNodeBase): + """Kling Image Generation Base Node.""" + + RETURN_TYPES = ("IMAGE",) + CATEGORY = "api node/image/Kling" + + def validate_prompt(self, prompt: str, negative_prompt: Optional[str] = None): + if not prompt or len(prompt) > MAX_PROMPT_LENGTH_IMAGE_GEN: + raise ValueError( + f"Prompt must be less than {MAX_PROMPT_LENGTH_IMAGE_GEN} characters" + ) + if negative_prompt and len(negative_prompt) > MAX_PROMPT_LENGTH_IMAGE_GEN: + raise ValueError( + f"Negative prompt must be less than {MAX_PROMPT_LENGTH_IMAGE_GEN} characters" + ) + + +class KlingVirtualTryOnNode(KlingImageGenerationBase): + """Kling Virtual Try On Node.""" + + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "human_image": (IO.IMAGE, {}), + "cloth_image": (IO.IMAGE, {}), + "model_name": model_field_to_node_input( + IO.COMBO, + KlingVirtualTryOnRequest, + "model_name", + enum_type=KlingVirtualTryOnModelName, + ), + }, + "hidden": {"auth_token": "AUTH_TOKEN_COMFY_ORG"}, + } + + DESCRIPTION = "Kling Virtual Try On Node. Input a human image and a cloth image to try on the cloth on the human." + + def get_response( + self, task_id: str, auth_token: Optional[str] = None + ) -> KlingVirtualTryOnResponse: + return poll_until_finished( + auth_token, + ApiEndpoint( + path=f"{PATH_VIRTUAL_TRY_ON}/{task_id}", + method=HttpMethod.GET, + request_model=EmptyRequest, + response_model=KlingVirtualTryOnResponse, + ), + ) + + def api_call( + self, + human_image: torch.Tensor, + cloth_image: torch.Tensor, + model_name: KlingVirtualTryOnModelName, + auth_token: Optional[str] = None, + ): + initial_operation = SynchronousOperation( + endpoint=ApiEndpoint( + path=PATH_VIRTUAL_TRY_ON, + method=HttpMethod.POST, + request_model=KlingVirtualTryOnRequest, + response_model=KlingVirtualTryOnResponse, + ), + request=KlingVirtualTryOnRequest( + human_image=tensor_to_base64_string(human_image), + cloth_image=tensor_to_base64_string(cloth_image), + model_name=model_name, + ), + auth_token=auth_token, + ) + + task_creation_response = initial_operation.execute() + validate_task_creation_response(task_creation_response) + task_id = task_creation_response.data.task_id + + final_response = self.get_response(task_id, auth_token) + validate_image_result_response(final_response) + + image = get_images_from_response(final_response) + return image_result_to_node_output(image) + + +class KlingImageGenerationNode(KlingImageGenerationBase): + """Kling Image Generation Node. Generate an image from a text prompt with an optional reference image.""" + + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "prompt": model_field_to_node_input( + IO.STRING, + KlingImageGenerationsRequest, + "prompt", + multiline=True, + max_length=MAX_PROMPT_LENGTH_IMAGE_GEN, + ), + "negative_prompt": model_field_to_node_input( + IO.STRING, + KlingImageGenerationsRequest, + "negative_prompt", + multiline=True, + ), + "image_type": model_field_to_node_input( + IO.COMBO, + KlingImageGenerationsRequest, + "image_reference", + enum_type=KlingImageGenImageReferenceType, + ), + "image_fidelity": model_field_to_node_input( + IO.FLOAT, + KlingImageGenerationsRequest, + "image_fidelity", + slider=True, + step=0.01, + ), + "human_fidelity": model_field_to_node_input( + IO.FLOAT, + KlingImageGenerationsRequest, + "human_fidelity", + slider=True, + step=0.01, + ), + "model_name": model_field_to_node_input( + IO.COMBO, + KlingImageGenerationsRequest, + "model_name", + enum_type=KlingImageGenModelName, + ), + "aspect_ratio": model_field_to_node_input( + IO.COMBO, + KlingImageGenerationsRequest, + "aspect_ratio", + enum_type=KlingImageGenAspectRatio, + ), + "n": model_field_to_node_input( + IO.INT, + KlingImageGenerationsRequest, + "n", + ), + }, + "optional": { + "image": (IO.IMAGE, {}), + }, + "hidden": {"auth_token": "AUTH_TOKEN_COMFY_ORG"}, + } + + DESCRIPTION = "Kling Image Generation Node. Generate an image from a text prompt with an optional reference image." + + def get_response( + self, task_id: str, auth_token: Optional[str] = None + ) -> KlingImageGenerationsResponse: + return poll_until_finished( + auth_token, + ApiEndpoint( + path=f"{PATH_IMAGE_GENERATIONS}/{task_id}", + method=HttpMethod.GET, + request_model=EmptyRequest, + response_model=KlingImageGenerationsResponse, + ), + ) + + def api_call( + self, + model_name: KlingImageGenModelName, + prompt: str, + negative_prompt: str, + image_type: KlingImageGenImageReferenceType, + image_fidelity: float, + human_fidelity: float, + n: int, + aspect_ratio: KlingImageGenAspectRatio, + image: Optional[torch.Tensor] = None, + auth_token: Optional[str] = None, + ): + self.validate_prompt(prompt, negative_prompt) + + if image is not None: + image = tensor_to_base64_string(image) + + initial_operation = SynchronousOperation( + endpoint=ApiEndpoint( + path=PATH_IMAGE_GENERATIONS, + method=HttpMethod.POST, + request_model=KlingImageGenerationsRequest, + response_model=KlingImageGenerationsResponse, + ), + request=KlingImageGenerationsRequest( + model_name=model_name, + prompt=prompt, + negative_prompt=negative_prompt, + image=image, + image_reference=image_type, + image_fidelity=image_fidelity, + human_fidelity=human_fidelity, + n=n, + aspect_ratio=aspect_ratio, + ), + auth_token=auth_token, + ) + + task_creation_response = initial_operation.execute() + validate_task_creation_response(task_creation_response) + task_id = task_creation_response.data.task_id + + final_response = self.get_response(task_id, auth_token) + validate_image_result_response(final_response) + + image = get_images_from_response(final_response) + return image_result_to_node_output(image) + + NODE_CLASS_MAPPINGS = { "KlingCameraControls": KlingCameraControls, "KlingTextToVideoNode": KlingTextToVideoNode, @@ -718,6 +1505,13 @@ NODE_CLASS_MAPPINGS = { "KlingCameraControlI2VNode": KlingCameraControlI2VNode, "KlingCameraControlT2VNode": KlingCameraControlT2VNode, "KlingStartEndFrameNode": KlingStartEndFrameNode, + "KlingVideoExtendNode": KlingVideoExtendNode, + "KlingLipSyncAudioToVideoNode": KlingLipSyncAudioToVideoNode, + "KlingLipSyncTextToVideoNode": KlingLipSyncTextToVideoNode, + "KlingVirtualTryOnNode": KlingVirtualTryOnNode, + "KlingImageGenerationNode": KlingImageGenerationNode, + "KlingSingleImageVideoEffectNode": KlingSingleImageVideoEffectNode, + "KlingDualCharacterVideoEffectNode": KlingDualCharacterVideoEffectNode, } NODE_DISPLAY_NAME_MAPPINGS = { @@ -727,4 +1521,11 @@ NODE_DISPLAY_NAME_MAPPINGS = { "KlingCameraControlI2VNode": "Kling Image to Video (Camera Control)", "KlingCameraControlT2VNode": "Kling Text to Video (Camera Control)", "KlingStartEndFrameNode": "Kling Start-End Frame to Video", + "KlingVideoExtendNode": "Kling Video Extend", + "KlingLipSyncAudioToVideoNode": "Kling Lip Sync Video with Audio", + "KlingLipSyncTextToVideoNode": "Kling Lip Sync Video with Text", + "KlingVirtualTryOnNode": "Kling Virtual Try On", + "KlingImageGenerationNode": "Kling Image Generation", + "KlingSingleImageVideoEffectNode": "Kling Video Effects", + "KlingDualCharacterVideoEffectNode": "Kling Dual Character Video Effects", } From 00f9679bd1e2cefc2114e77208141d93babac487 Mon Sep 17 00:00:00 2001 From: Jedrzej Kosinski Date: Sat, 3 May 2025 18:23:27 -0500 Subject: [PATCH 088/121] Add 8 nodes - 4 BFL, 4 Stability (#117) --- comfy_api_nodes/apinode_utils.py | 18 ++ comfy_api_nodes/apis/stability_api.py | 37 ++- comfy_api_nodes/nodes_bfl.py | 52 +++-- comfy_api_nodes/nodes_recraft.py | 10 +- comfy_api_nodes/nodes_stability.py | 319 +++++++++++++++++++++++++- 5 files changed, 399 insertions(+), 37 deletions(-) diff --git a/comfy_api_nodes/apinode_utils.py b/comfy_api_nodes/apinode_utils.py index baeaa0ceb..13f900a61 100644 --- a/comfy_api_nodes/apinode_utils.py +++ b/comfy_api_nodes/apinode_utils.py @@ -524,3 +524,21 @@ def upload_images_to_comfyapi( if idx_image >= batch_length: break return download_urls + + +def resize_mask_to_image(mask: torch.Tensor, image: torch.Tensor, + upscale_method="nearest-exact", crop="disabled", + allow_gradient=True, add_channel_dim=False): + """ + Resize mask to be the same dimensions as an image, while maintaining proper format for API calls. + """ + _, H, W, _ = image.shape + mask = mask.unsqueeze(-1) + mask = mask.movedim(-1,1) + mask = common_upscale(mask, width=W, height=H, upscale_method=upscale_method, crop=crop) + mask = mask.movedim(1,-1) + if not add_channel_dim: + mask = mask.squeeze(-1) + if not allow_gradient: + mask = (mask > 0.5).float() + return mask diff --git a/comfy_api_nodes/apis/stability_api.py b/comfy_api_nodes/apis/stability_api.py index d8d4f4e6a..47c87daec 100644 --- a/comfy_api_nodes/apis/stability_api.py +++ b/comfy_api_nodes/apis/stability_api.py @@ -53,8 +53,8 @@ class StabilityStylePreset(str, Enum): class Stability_SD3_5_Model(str, Enum): sd3_5_large = "sd3.5-large" - sd3_5_large_turbo = "sd3.5-large-turbo" - #sd3_5_medium = "sd3.5-medium" + # sd3_5_large_turbo = "sd3.5-large-turbo" + sd3_5_medium = "sd3.5-medium" class Stability_SD3_5_GenerationMode(str, Enum): @@ -76,6 +76,25 @@ class StabilityStable3_5Request(BaseModel): strength: Optional[confloat(ge=0.0, le=1.0)] = Field(None) +class StabilityUpscaleConservativeRequest(BaseModel): + prompt: str = Field(...) + negative_prompt: Optional[str] = Field(None) + seed: Optional[int] = Field(None) + output_format: Optional[str] = Field(StabilityFormat.png.value) + image: Optional[str] = Field(None) + creativity: Optional[confloat(ge=0.2, le=0.5)] = Field(None) + + +class StabilityUpscaleCreativeRequest(BaseModel): + prompt: str = Field(...) + negative_prompt: Optional[str] = Field(None) + seed: Optional[int] = Field(None) + output_format: Optional[str] = Field(StabilityFormat.png.value) + image: Optional[str] = Field(None) + creativity: Optional[confloat(ge=0.1, le=0.5)] = Field(None) + style_preset: Optional[str] = Field(None) + + class StabilityStableUltraRequest(BaseModel): prompt: str = Field(...) negative_prompt: Optional[str] = Field(None) @@ -92,3 +111,17 @@ class StabilityStableUltraResponse(BaseModel): finish_reason: Optional[str] = Field(None) seed: Optional[int] = Field(None) + +class StabilityResultsGetResponse(BaseModel): + image: Optional[str] = Field(None) + finish_reason: Optional[str] = Field(None) + seed: Optional[int] = Field(None) + id: Optional[str] = Field(None) + name: Optional[str] = Field(None) + errors: Optional[list[str]] = Field(None) + status: Optional[str] = Field(None) + result: Optional[str] = Field(None) + + +class StabilityAsyncResponse(BaseModel): + id: Optional[str] = Field(None) diff --git a/comfy_api_nodes/nodes_bfl.py b/comfy_api_nodes/nodes_bfl.py index fd58249c8..f1014459a 100644 --- a/comfy_api_nodes/nodes_bfl.py +++ b/comfy_api_nodes/nodes_bfl.py @@ -20,6 +20,7 @@ from comfy_api_nodes.apinode_utils import ( downscale_image_tensor, validate_aspect_ratio, process_image_response, + resize_mask_to_image, ) import numpy as np @@ -589,9 +590,11 @@ class FluxProFillNode(ComfyNodeABC): auth_token=None, **kwargs, ): + # prepare mask + mask = resize_mask_to_image(mask, image) + mask = convert_image_to_base64(convert_mask_to_image(mask)) # make sure image will have alpha channel removed image = convert_image_to_base64(image[:,:,:,:3]) - mask = convert_image_to_base64(convert_mask_to_image(mask)) operation = SynchronousOperation( endpoint=ApiEndpoint( @@ -641,20 +644,22 @@ class FluxProCannyNode(ComfyNodeABC): }, ), "canny_low_threshold": ( - IO.INT, + IO.FLOAT, { - "default": 0, - "min": 0, - "max": 500, + "default": 0.1, + "min": 0.01, + "max": 0.99, + "step": 0.01, "tooltip": "Low threshold for Canny edge detection; ignored if skip_processing is True" }, ), "canny_high_threshold": ( - IO.INT, + IO.FLOAT, { - "default": 0, - "min": 0, - "max": 500, + "default": 0.4, + "min": 0.01, + "max": 0.99, + "step": 0.01, "tooltip": "High threshold for Canny edge detection; ignored if skip_processing is True" }, ), @@ -712,8 +717,8 @@ class FluxProCannyNode(ComfyNodeABC): control_image: torch.Tensor, prompt: str, prompt_upsampling: bool, - canny_low_threshold: int, - canny_high_threshold: int, + canny_low_threshold: float, + canny_high_threshold: float, skip_preprocessing: bool, steps: int, guidance: float, @@ -724,6 +729,13 @@ class FluxProCannyNode(ComfyNodeABC): control_image = convert_image_to_base64(control_image[:,:,:,:3]) preprocessed_image = None + # scale canny threshold between 0-500, to match BFL's API + def scale_value(value: float, min_val=0, max_val=500): + return min_val + value * (max_val - min_val) + canny_low_threshold = int(round(scale_value(canny_low_threshold))) + canny_high_threshold = int(round(scale_value(canny_high_threshold))) + + if skip_preprocessing: preprocessed_image = control_image control_image = None @@ -849,7 +861,7 @@ class FluxProDepthNode(ComfyNodeABC): operation = SynchronousOperation( endpoint=ApiEndpoint( - path="/proxy/bfl/flux-pro-1.0-canny/generate", + path="/proxy/bfl/flux-pro-1.0-depth/generate", method=HttpMethod.POST, request_model=BFLFluxDepthImageRequest, response_model=BFLFluxProGenerateResponse, @@ -874,18 +886,18 @@ class FluxProDepthNode(ComfyNodeABC): NODE_CLASS_MAPPINGS = { "FluxProUltraImageNode": FluxProUltraImageNode, # "FluxProImageNode": FluxProImageNode, - # "FluxProExpandNode": FluxProExpandNode, - # "FluxProFillNode": FluxProFillNode, - # "FluxProCannyNode": FluxProCannyNode, - # "FluxProDepthNode": FluxProDepthNode, + "FluxProExpandNode": FluxProExpandNode, + "FluxProFillNode": FluxProFillNode, + "FluxProCannyNode": FluxProCannyNode, + "FluxProDepthNode": FluxProDepthNode, } # A dictionary that contains the friendly/humanly readable titles for the nodes NODE_DISPLAY_NAME_MAPPINGS = { "FluxProUltraImageNode": "Flux 1.1 [pro] Ultra Image", # "FluxProImageNode": "Flux 1.1 [pro] Image", - # "FluxProExpandNode": "Flux.1 Expand Image", - # "FluxProFillNode": "Flux.1 Fill Image", - # "FluxProCannyNode": "Flux.1 Canny Control Image", - # "FluxProDepthNode": "Flux.1 Depth Control Image", + "FluxProExpandNode": "Flux.1 Expand Image", + "FluxProFillNode": "Flux.1 Fill Image", + "FluxProCannyNode": "Flux.1 Canny Control Image", + "FluxProDepthNode": "Flux.1 Depth Control Image", } diff --git a/comfy_api_nodes/nodes_recraft.py b/comfy_api_nodes/nodes_recraft.py index 62cc503e2..eaa15f92b 100644 --- a/comfy_api_nodes/nodes_recraft.py +++ b/comfy_api_nodes/nodes_recraft.py @@ -1,6 +1,6 @@ from __future__ import annotations from inspect import cleandoc -from comfy.utils import ProgressBar, common_upscale +from comfy.utils import ProgressBar from comfy.comfy_types.node_typing import IO from comfy_api_nodes.apis.recraft_api import ( RecraftImageGenerationRequest, @@ -25,6 +25,7 @@ from comfy_api_nodes.apinode_utils import ( bytesio_to_image_tensor, download_url_to_bytesio, tensor_to_bytesio, + resize_mask_to_image, ) import folder_paths import json @@ -654,12 +655,7 @@ class RecraftImageInpaintingNode: ) # prepare mask tensor - _, H, W, _ = image.shape - mask = mask.unsqueeze(-1) - mask = mask.movedim(-1,1) - mask = common_upscale(mask, width=W, height=H, upscale_method="nearest-exact", crop="disabled") - mask = mask.movedim(1,-1) - mask = (mask > 0.5).float() + mask = resize_mask_to_image(mask, image, allow_gradient=False, add_channel_dim=True) images = [] total = image.shape[0] diff --git a/comfy_api_nodes/nodes_stability.py b/comfy_api_nodes/nodes_stability.py index 2ac6b30a2..50d264b5d 100644 --- a/comfy_api_nodes/nodes_stability.py +++ b/comfy_api_nodes/nodes_stability.py @@ -1,6 +1,10 @@ from inspect import cleandoc from comfy.comfy_types.node_typing import IO from comfy_api_nodes.apis.stability_api import ( + StabilityUpscaleConservativeRequest, + StabilityUpscaleCreativeRequest, + StabilityAsyncResponse, + StabilityResultsGetResponse, StabilityStable3_5Request, StabilityStableUltraRequest, StabilityStableUltraResponse, @@ -13,6 +17,8 @@ from comfy_api_nodes.apis.client import ( ApiEndpoint, HttpMethod, SynchronousOperation, + PollingOperation, + EmptyRequest, ) from comfy_api_nodes.apinode_utils import ( bytesio_to_image_tensor, @@ -22,8 +28,22 @@ from comfy_api_nodes.apinode_utils import ( import torch import base64 from io import BytesIO +from enum import Enum +class StabilityPollStatus(str, Enum): + finished = "finished" + in_progress = "in_progress" + failed = "failed" + + +def get_async_dummy_status(x: StabilityResultsGetResponse): + if x.name is not None or x.errors is not None: + return StabilityPollStatus.failed + elif x.finish_reason is not None: + return StabilityPollStatus.finished + return StabilityPollStatus.in_progress + class StabilityStableImageUltraNode: """ @@ -108,7 +128,7 @@ class StabilityStableImageUltraNode: # prepare image binary if image present image_binary = None if image is not None: - image_binary = tensor_to_bytesio(image, 1504 * 1504).read() + image_binary = tensor_to_bytesio(image, total_pixels=1504*1504).read() else: image_denoise = None @@ -174,6 +194,7 @@ class StabilityStableImageSD_3_5Node: "tooltip": "What you wish to see in the output image. A strong, descriptive prompt that clearly defines elements, colors, and subjects will lead to better results." }, ), + "model": ([x.value for x in Stability_SD3_5_Model],), "aspect_ratio": ([x.value for x in StabilityAspectRatio], { "default": StabilityAspectRatio.ratio_1_1, @@ -232,16 +253,16 @@ class StabilityStableImageSD_3_5Node: }, } - def api_call(self, prompt: str, aspect_ratio: str, style_preset: str, seed: int, cfg_scale: float, + def api_call(self, model: str, prompt: str, aspect_ratio: str, style_preset: str, seed: int, cfg_scale: float, negative_prompt: str=None, image: torch.Tensor = None, image_denoise: float=None, auth_token=None): - model = Stability_SD3_5_Model.sd3_5_large.value # prepare image binary if image present image_binary = None - mode = Stability_SD3_5_GenerationMode.text_to_image.value + mode = Stability_SD3_5_GenerationMode.text_to_image if image is not None: - image_binary = tensor_to_bytesio(image, 1504 * 1504).read() - mode = Stability_SD3_5_GenerationMode.image_to_image.value + image_binary = tensor_to_bytesio(image, total_pixels=1504*1504).read() + mode = Stability_SD3_5_GenerationMode.image_to_image + aspect_ratio = None else: image_denoise = None @@ -287,15 +308,297 @@ class StabilityStableImageSD_3_5Node: return (returned_image,) +class StabilityUpscaleConservativeNode: + """ + Upscale image with minimal alterations to 4K resolution. + """ + + RETURN_TYPES = (IO.IMAGE,) + DESCRIPTION = cleandoc(__doc__ or "") # Handle potential None value + FUNCTION = "api_call" + API_NODE = True + CATEGORY = "api node/image/stability" + + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "image": (IO.IMAGE,), + "prompt": ( + IO.STRING, + { + "multiline": True, + "default": "", + "tooltip": "What you wish to see in the output image. A strong, descriptive prompt that clearly defines elements, colors, and subjects will lead to better results." + }, + ), + "creativity": ( + IO.FLOAT, + { + "default": 0.35, + "min": 0.2, + "max": 0.5, + "step": 0.01, + "tooltip": "Controls the likelihood of creating additional details not heavily conditioned by the init image.", + }, + ), + "seed": ( + IO.INT, + { + "default": 0, + "min": 0, + "max": 4294967294, + "control_after_generate": True, + "tooltip": "The random seed used for creating the noise.", + }, + ), + }, + "optional": { + "negative_prompt": ( + IO.STRING, + { + "default": "", + "forceInput": True, + "tooltip": "Keywords of what you do not wish to see in the output image. This is an advanced feature." + }, + ), + }, + "hidden": { + "auth_token": "AUTH_TOKEN_COMFY_ORG", + }, + } + + def api_call(self, image: torch.Tensor, prompt: str, creativity: float, seed: int, negative_prompt: str=None, + auth_token=None): + image_binary = tensor_to_bytesio(image, total_pixels=1024*1024).read() + + if not negative_prompt: + negative_prompt = None + + files = { + "image": image_binary + } + + operation = SynchronousOperation( + endpoint=ApiEndpoint( + path="/proxy/stability/v2beta/stable-image/upscale/conservative", + method=HttpMethod.POST, + request_model=StabilityUpscaleConservativeRequest, + response_model=StabilityStableUltraResponse, + ), + request=StabilityUpscaleConservativeRequest( + prompt=prompt, + negative_prompt=negative_prompt, + creativity=round(creativity,2), + seed=seed, + ), + files=files, + content_type="multipart/form-data", + auth_token=auth_token, + ) + response_api = operation.execute() + + if response_api.finish_reason != "SUCCESS": + raise Exception(f"Stability Upscale Conservative generation failed: {response_api.finish_reason}.") + + image_data = base64.b64decode(response_api.image) + returned_image = bytesio_to_image_tensor(BytesIO(image_data)) + + return (returned_image,) + + +class StabilityUpscaleCreativeNode: + """ + Upscale image with minimal alterations to 4K resolution. + """ + + RETURN_TYPES = (IO.IMAGE,) + DESCRIPTION = cleandoc(__doc__ or "") # Handle potential None value + FUNCTION = "api_call" + API_NODE = True + CATEGORY = "api node/image/stability" + + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "image": (IO.IMAGE,), + "prompt": ( + IO.STRING, + { + "multiline": True, + "default": "", + "tooltip": "What you wish to see in the output image. A strong, descriptive prompt that clearly defines elements, colors, and subjects will lead to better results." + }, + ), + "creativity": ( + IO.FLOAT, + { + "default": 0.3, + "min": 0.1, + "max": 0.5, + "step": 0.01, + "tooltip": "Controls the likelihood of creating additional details not heavily conditioned by the init image.", + }, + ), + "style_preset": (get_stability_style_presets(), + { + "tooltip": "Optional desired style of generated image.", + }, + ), + "seed": ( + IO.INT, + { + "default": 0, + "min": 0, + "max": 4294967294, + "control_after_generate": True, + "tooltip": "The random seed used for creating the noise.", + }, + ), + }, + "optional": { + "negative_prompt": ( + IO.STRING, + { + "default": "", + "forceInput": True, + "tooltip": "Keywords of what you do not wish to see in the output image. This is an advanced feature." + }, + ), + }, + "hidden": { + "auth_token": "AUTH_TOKEN_COMFY_ORG", + }, + } + + def api_call(self, image: torch.Tensor, prompt: str, creativity: float, style_preset: str, seed: int, negative_prompt: str=None, + auth_token=None): + image_binary = tensor_to_bytesio(image, total_pixels=1024*1024).read() + + if not negative_prompt: + negative_prompt = None + if style_preset == "None": + style_preset = None + + files = { + "image": image_binary + } + + operation = SynchronousOperation( + endpoint=ApiEndpoint( + path="/proxy/stability/v2beta/stable-image/upscale/creative", + method=HttpMethod.POST, + request_model=StabilityUpscaleCreativeRequest, + response_model=StabilityAsyncResponse, + ), + request=StabilityUpscaleCreativeRequest( + prompt=prompt, + negative_prompt=negative_prompt, + creativity=round(creativity,2), + style_preset=style_preset, + seed=seed, + ), + files=files, + content_type="multipart/form-data", + auth_token=auth_token, + ) + response_api = operation.execute() + + operation = PollingOperation( + poll_endpoint=ApiEndpoint( + path=f"/proxy/stability/v2beta/results/{response_api.id}", + method=HttpMethod.GET, + request_model=EmptyRequest, + response_model=StabilityResultsGetResponse, + ), + poll_interval=3, + completed_statuses=[StabilityPollStatus.finished], + failed_statuses=[StabilityPollStatus.failed], + status_extractor=lambda x: get_async_dummy_status(x), + auth_token=auth_token, + ) + response_poll: StabilityResultsGetResponse = operation.execute() + + if response_poll.finish_reason != "SUCCESS": + raise Exception(f"Stability Upscale Creative generation failed: {response_poll.finish_reason}.") + + image_data = base64.b64decode(response_poll.result) + returned_image = bytesio_to_image_tensor(BytesIO(image_data)) + + return (returned_image,) + + +class StabilityUpscaleFastNode: + """ + Quickly upscales an image via Stability API call to 4x its original size; intended for upscaling low-quality/compressed images. + """ + + RETURN_TYPES = (IO.IMAGE,) + DESCRIPTION = cleandoc(__doc__ or "") # Handle potential None value + FUNCTION = "api_call" + API_NODE = True + CATEGORY = "api node/image/stability" + + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "image": (IO.IMAGE,), + }, + "optional": { + }, + "hidden": { + "auth_token": "AUTH_TOKEN_COMFY_ORG", + }, + } + + def api_call(self, image: torch.Tensor, + auth_token=None): + image_binary = tensor_to_bytesio(image, total_pixels=4096*4096).read() + + files = { + "image": image_binary + } + + operation = SynchronousOperation( + endpoint=ApiEndpoint( + path="/proxy/stability/v2beta/stable-image/upscale/fast", + method=HttpMethod.POST, + request_model=EmptyRequest, + response_model=StabilityStableUltraResponse, + ), + request=EmptyRequest(), + files=files, + content_type="multipart/form-data", + auth_token=auth_token, + ) + response_api = operation.execute() + + if response_api.finish_reason != "SUCCESS": + raise Exception(f"Stability Upscale Fast failed: {response_api.finish_reason}.") + + image_data = base64.b64decode(response_api.image) + returned_image = bytesio_to_image_tensor(BytesIO(image_data)) + + return (returned_image,) + + # A dictionary that contains all nodes you want to export with their names # NOTE: names should be globally unique NODE_CLASS_MAPPINGS = { "StabilityStableImageUltraNode": StabilityStableImageUltraNode, - # "StabilityStableImageSD_3_5Node": StabilityStableImageSD_3_5Node, + "StabilityStableImageSD_3_5Node": StabilityStableImageSD_3_5Node, + "StabilityUpscaleConservativeNode": StabilityUpscaleConservativeNode, + "StabilityUpscaleCreativeNode": StabilityUpscaleCreativeNode, + "StabilityUpscaleFastNode": StabilityUpscaleFastNode, } # A dictionary that contains the friendly/humanly readable titles for the nodes NODE_DISPLAY_NAME_MAPPINGS = { "StabilityStableImageUltraNode": "Stability Stable Image Ultra", - # "StabilityStableImageSD_3_5Node": "Stability Stable Diffusion 3.5 Image", + "StabilityStableImageSD_3_5Node": "Stability Stable Diffusion 3.5 Image", + "StabilityUpscaleConservativeNode": "Stability Upscale Conservative", + "StabilityUpscaleCreativeNode": "Stability Upscale Creative", + "StabilityUpscaleFastNode": "Stability Upscale Fast", } From ef111e758737e174dd0cc4ac085edfa24942edc2 Mon Sep 17 00:00:00 2001 From: Jedrzej Kosinski Date: Sat, 3 May 2025 19:18:39 -0500 Subject: [PATCH 089/121] Fix error for Recraft ImageToImage error for nonexistent random_seed param (#118) --- comfy_api_nodes/nodes_recraft.py | 1 - 1 file changed, 1 deletion(-) diff --git a/comfy_api_nodes/nodes_recraft.py b/comfy_api_nodes/nodes_recraft.py index eaa15f92b..5a60debc9 100644 --- a/comfy_api_nodes/nodes_recraft.py +++ b/comfy_api_nodes/nodes_recraft.py @@ -543,7 +543,6 @@ class RecraftImageToImageNode: substyle=recraft_style.substyle, style_id=recraft_style.style_id, controls=controls_api, - random_seed=seed, ) images = [] From eca1467fd9e884ec49012ea0d30449aa40e57160 Mon Sep 17 00:00:00 2001 From: Christian Byrne Date: Sat, 3 May 2025 21:53:59 -0700 Subject: [PATCH 090/121] Add remaining Pika nodes (#119) --- comfy_api_nodes/apis/__init__.py | 37 ++- comfy_api_nodes/nodes_pika.py | 381 +++++++++++++++++++++++++++++-- 2 files changed, 390 insertions(+), 28 deletions(-) diff --git a/comfy_api_nodes/apis/__init__.py b/comfy_api_nodes/apis/__init__.py index 9998bb07a..063d176e6 100644 --- a/comfy_api_nodes/apis/__init__.py +++ b/comfy_api_nodes/apis/__init__.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: filtered-openapi.yaml -# timestamp: 2025-05-03T21:40:30+00:00 +# timestamp: 2025-05-04T04:12:39+00:00 from __future__ import annotations @@ -2125,9 +2125,28 @@ class CustomerStorageResourceResponse(BaseModel): ) +class Pikaffect(str, Enum): + Cake_ify = 'Cake-ify' + Crumble = 'Crumble' + Crush = 'Crush' + Decapitate = 'Decapitate' + Deflate = 'Deflate' + Dissolve = 'Dissolve' + Explode = 'Explode' + Eye_pop = 'Eye-pop' + Inflate = 'Inflate' + Levitate = 'Levitate' + Melt = 'Melt' + Peel = 'Peel' + Poke = 'Poke' + Squish = 'Squish' + Ta_da = 'Ta-da' + Tear = 'Tear' + + class PikaBodyGeneratePikaffectsGeneratePikaffectsPost(BaseModel): - image: StrictBytes = Field(..., title='Image') - pikaffect: Optional[str] = Field(None, title='Pikaffect') + image: Optional[StrictBytes] = Field(None, title='Image') + pikaffect: Optional[Pikaffect] = Field(None, title='Pikaffect') promptText: Optional[str] = Field(None, title='Prompttext') negativePrompt: Optional[str] = Field(None, title='Negativeprompt') seed: Optional[int] = Field(None, title='Seed') @@ -2138,15 +2157,15 @@ class PikaGenerateResponse(BaseModel): class PikaBodyGeneratePikadditionsGeneratePikadditionsPost(BaseModel): - video: StrictBytes = Field(..., title='Video') - image: StrictBytes = Field(..., title='Image') + video: Optional[StrictBytes] = Field(None, title='Video') + image: Optional[StrictBytes] = Field(None, title='Image') promptText: Optional[str] = Field(None, title='Prompttext') negativePrompt: Optional[str] = Field(None, title='Negativeprompt') seed: Optional[int] = Field(None, title='Seed') class PikaBodyGeneratePikaswapsGeneratePikaswapsPost(BaseModel): - video: StrictBytes = Field(..., title='Video') + video: Optional[StrictBytes] = Field(None, title='Video') image: Optional[StrictBytes] = Field(None, title='Image') promptText: Optional[str] = Field(None, title='Prompttext') modifyRegionMask: Optional[StrictBytes] = Field( @@ -2179,7 +2198,7 @@ class AspectRatio1(RootModel[float]): class PikaBodyGenerate22C2vGenerate22PikascenesPost(BaseModel): - images: List[StrictBytes] = Field(..., title='Images') + images: Optional[List[StrictBytes]] = Field(None, title='Images') ingredientsMode: IngredientsMode = Field(..., title='Ingredientsmode') promptText: Optional[str] = Field(None, title='Prompttext') negativePrompt: Optional[str] = Field(None, title='Negativeprompt') @@ -3629,8 +3648,8 @@ class PikaBodyGenerate22I2vGenerate22I2vPost(BaseModel): class PikaBodyGenerate22KeyframeGenerate22PikaframesPost(BaseModel): - keyFrames: List[StrictBytes] = Field( - ..., description='Array of keyframe images', title='Keyframes' + keyFrames: Optional[List[StrictBytes]] = Field( + None, description='Array of keyframe images', title='Keyframes' ) promptText: str = Field(..., title='Prompttext') negativePrompt: Optional[str] = Field(None, title='Negativeprompt') diff --git a/comfy_api_nodes/nodes_pika.py b/comfy_api_nodes/nodes_pika.py index 825d570d7..64f9645da 100644 --- a/comfy_api_nodes/nodes_pika.py +++ b/comfy_api_nodes/nodes_pika.py @@ -1,8 +1,14 @@ -"""Pika API docs: https://pika-827374fb.mintlify.app/api-reference""" +""" +Pika x ComfyUI API Nodes +Pika API docs: https://pika-827374fb.mintlify.app/api-reference +""" + +import io from typing import Optional, TypeVar import logging import torch +import numpy as np from comfy_api_nodes.apis import ( PikaBodyGenerate22T2vGenerate22T2vPost, PikaGenerateResponse, @@ -12,6 +18,11 @@ from comfy_api_nodes.apis import ( IngredientsMode, PikaDurationEnum, PikaResolutionEnum, + PikaBodyGeneratePikaffectsGeneratePikaffectsPost, + PikaBodyGeneratePikadditionsGeneratePikadditionsPost, + PikaBodyGeneratePikaswapsGeneratePikaswapsPost, + PikaBodyGenerate22KeyframeGenerate22PikaframesPost, + Pikaffect, ) from comfy_api_nodes.apis.client import ( ApiEndpoint, @@ -25,16 +36,22 @@ from comfy_api_nodes.apinode_utils import ( download_url_to_video_output, ) from comfy_api_nodes.mapper_utils import model_field_to_node_input -from comfy.comfy_types.node_typing import IO, ComfyNodeABC, InputTypeOptions +from comfy_api.input_impl.video_types import VideoInput, VideoContainer, VideoCodec from comfy_api.input_impl import VideoFromFile +from comfy.comfy_types.node_typing import IO, ComfyNodeABC, InputTypeOptions R = TypeVar("R") +PATH_PIKADDITIONS = "/proxy/pika/generate/pikadditions" +PATH_PIKASWAPS = "/proxy/pika/generate/pikaswaps" +PATH_PIKAFFECTS = "/proxy/pika/generate/pikaffects" + PIKA_API_VERSION = "2.2" PATH_TEXT_TO_VIDEO = f"/proxy/pika/generate/{PIKA_API_VERSION}/t2v" PATH_IMAGE_TO_VIDEO = f"/proxy/pika/generate/{PIKA_API_VERSION}/i2v" PATH_PIKAFRAMES = f"/proxy/pika/generate/{PIKA_API_VERSION}/pikaframes" PATH_PIKASCENES = f"/proxy/pika/generate/{PIKA_API_VERSION}/pikascenes" + PATH_VIDEO_GET = "/proxy/pika/videos" @@ -100,11 +117,11 @@ class PikaNodeBase(ComfyNodeABC): CATEGORY = "api node/video/Pika" API_NODE = True FUNCTION = "api_call" + RETURN_TYPES = ("VIDEO",) def poll_for_task_status( self, task_id: str, auth_token: str ) -> PikaGenerateResponse: - """Polls the Pika API endpoint until the task reaches a terminal state.""" polling_operation = PollingOperation( poll_endpoint=ApiEndpoint( path=f"{PATH_VIDEO_GET}/{task_id}", @@ -180,7 +197,6 @@ class PikaImageToVideoV2_2(PikaNodeBase): } DESCRIPTION = "Sends an image and prompt to the Pika API v2.2 to generate a video." - RETURN_TYPES = ("VIDEO",) def api_call( self, @@ -192,15 +208,13 @@ class PikaImageToVideoV2_2(PikaNodeBase): duration: int, auth_token: Optional[str] = None, ) -> tuple[VideoFromFile]: - """API call for Pika 2.2 Image to Video.""" # Convert image to BytesIO image_bytes_io = tensor_to_bytesio(image) - image_bytes_io.seek(0) # Reset stream position + image_bytes_io.seek(0) - # Prepare file data for multipart upload pika_files = {"image": ("image.png", image_bytes_io, "image/png")} - # Prepare non-file data using the Pydantic model + # Prepare non-file data pika_request_data = PikaBodyGenerate22I2vGenerate22I2vPost( promptText=prompt_text, negativePrompt=negative_prompt, @@ -226,7 +240,7 @@ class PikaImageToVideoV2_2(PikaNodeBase): class PikaTextToVideoNodeV2_2(PikaNodeBase): - """Pika 2.2 Text to Video Node.""" + """Pika Text2Video v2.2 Node.""" @classmethod def INPUT_TYPES(cls): @@ -248,7 +262,6 @@ class PikaTextToVideoNodeV2_2(PikaNodeBase): }, } - RETURN_TYPES = ("VIDEO",) DESCRIPTION = "Sends a text prompt to the Pika API v2.2 to generate a video." def api_call( @@ -261,7 +274,6 @@ class PikaTextToVideoNodeV2_2(PikaNodeBase): aspect_ratio: float, auth_token: Optional[str] = None, ) -> tuple[VideoFromFile]: - """API call for Pika 2.2 Text to Video.""" initial_operation = SynchronousOperation( endpoint=ApiEndpoint( path=PATH_TEXT_TO_VIDEO, @@ -285,7 +297,7 @@ class PikaTextToVideoNodeV2_2(PikaNodeBase): class PikaScenesV2_2(PikaNodeBase): - """Pika 2.2 Scenes Node.""" + """PikaScenes v2.2 Node.""" @classmethod def INPUT_TYPES(cls): @@ -328,7 +340,6 @@ class PikaScenesV2_2(PikaNodeBase): } DESCRIPTION = "Combine your images to create a video with the objects in them. Upload multiple images as ingredients and generate a high-quality video that incorporates all of them." - RETURN_TYPES = ("VIDEO",) def api_call( self, @@ -346,7 +357,7 @@ class PikaScenesV2_2(PikaNodeBase): image_ingredient_5: Optional[torch.Tensor] = None, auth_token: Optional[str] = None, ) -> tuple[VideoFromFile]: - """API call for Pika Scenes 2.2.""" + # Convert all passed images to BytesIO all_image_bytes_io = [] for image in [ image_ingredient_1, @@ -360,13 +371,11 @@ class PikaScenesV2_2(PikaNodeBase): image_bytes_io.seek(0) all_image_bytes_io.append(image_bytes_io) - # Prepare files data for multipart upload pika_files = [ ("images", (f"image_{i}.png", image_bytes_io, "image/png")) for i, image_bytes_io in enumerate(all_image_bytes_io) ] - # Prepare non-file data using the Pydantic model pika_request_data = PikaBodyGenerate22C2vGenerate22PikascenesPost( ingredientsMode=ingredients_mode, promptText=prompt_text, @@ -393,14 +402,348 @@ class PikaScenesV2_2(PikaNodeBase): return self.execute_task(initial_operation, auth_token) +class PikAdditionsNode(PikaNodeBase): + """Pika Pikadditions Node. Add an image into a video.""" + + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "video": (IO.VIDEO, {"tooltip": "The video to add an image to."}), + "image": (IO.IMAGE, {"tooltip": "The image to add to the video."}), + "prompt_text": model_field_to_node_input( + IO.STRING, + PikaBodyGeneratePikadditionsGeneratePikadditionsPost, + "promptText", + multiline=True, + ), + "negative_prompt": model_field_to_node_input( + IO.STRING, + PikaBodyGeneratePikadditionsGeneratePikadditionsPost, + "negativePrompt", + multiline=True, + ), + "seed": model_field_to_node_input( + IO.INT, + PikaBodyGeneratePikadditionsGeneratePikadditionsPost, + "seed", + min=0, + max=0xFFFFFFFF, + control_after_generate=True, + ), + }, + "hidden": { + "auth_token": "AUTH_TOKEN_COMFY_ORG", + }, + } + + DESCRIPTION = "Add any object or image into your video. Upload a video and specify what you’d like to add to create a seamlessly integrated result." + + def api_call( + self, + video: VideoInput, + image: torch.Tensor, + prompt_text: str, + negative_prompt: str, + seed: int, + auth_token: Optional[str] = None, + ) -> tuple[VideoFromFile]: + # Convert video to BytesIO + video_bytes_io = io.BytesIO() + video.save_to(video_bytes_io, format=VideoContainer.MP4, codec=VideoCodec.H264) + video_bytes_io.seek(0) + + # Convert image to BytesIO + image_bytes_io = tensor_to_bytesio(image) + image_bytes_io.seek(0) + + pika_files = [ + ("video", ("video.mp4", video_bytes_io, "video/mp4")), + ("image", ("image.png", image_bytes_io, "image/png")), + ] + + # Prepare non-file data + pika_request_data = PikaBodyGeneratePikadditionsGeneratePikadditionsPost( + promptText=prompt_text, + negativePrompt=negative_prompt, + seed=seed, + ) + + initial_operation = SynchronousOperation( + endpoint=ApiEndpoint( + path=PATH_PIKADDITIONS, + method=HttpMethod.POST, + request_model=PikaBodyGeneratePikadditionsGeneratePikadditionsPost, + response_model=PikaGenerateResponse, + ), + request=pika_request_data, + files=pika_files, + content_type="multipart/form-data", + auth_token=auth_token, + ) + + return self.execute_task(initial_operation, auth_token) + + +class PikaSwapsNode(PikaNodeBase): + """Pika Pikaswaps Node.""" + + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "video": (IO.VIDEO, {"tooltip": "The video to swap an object in."}), + "image": ( + IO.IMAGE, + { + "tooltip": "The image used to replace the masked object in the video." + }, + ), + "mask": ( + IO.MASK, + {"tooltip": "Use the mask to define areas in the video to replace"}, + ), + "prompt_text": model_field_to_node_input( + IO.STRING, + PikaBodyGeneratePikaswapsGeneratePikaswapsPost, + "promptText", + multiline=True, + ), + "negative_prompt": model_field_to_node_input( + IO.STRING, + PikaBodyGeneratePikaswapsGeneratePikaswapsPost, + "negativePrompt", + multiline=True, + ), + "seed": model_field_to_node_input( + IO.INT, + PikaBodyGeneratePikaswapsGeneratePikaswapsPost, + "seed", + min=0, + max=0xFFFFFFFF, + control_after_generate=True, + ), + }, + "hidden": { + "auth_token": "AUTH_TOKEN_COMFY_ORG", + }, + } + + DESCRIPTION = "Swap out any object or region of your video with a new image or object. Define areas to replace either with a mask or coordinates." + RETURN_TYPES = ("VIDEO",) + + def api_call( + self, + video: VideoInput, + image: torch.Tensor, + mask: torch.Tensor, + prompt_text: str, + negative_prompt: str, + seed: int, + auth_token: Optional[str] = None, + ) -> tuple[VideoFromFile]: + # Convert video to BytesIO + video_bytes_io = io.BytesIO() + video.save_to(video_bytes_io, format=VideoContainer.MP4, codec=VideoCodec.H264) + video_bytes_io.seek(0) + + # Convert mask to binary mask with three channels + mask = torch.round(mask) + mask = mask.repeat(1, 3, 1, 1) + + # Convert 3-channel binary mask to BytesIO + mask_bytes_io = io.BytesIO() + mask_bytes_io.write(mask.numpy().astype(np.uint8)) + mask_bytes_io.seek(0) + + # Convert image to BytesIO + image_bytes_io = tensor_to_bytesio(image) + image_bytes_io.seek(0) + + pika_files = [ + ("video", ("video.mp4", video_bytes_io, "video/mp4")), + ("image", ("image.png", image_bytes_io, "image/png")), + ("modifyRegionMask", ("mask.png", mask_bytes_io, "image/png")), + ] + + # Prepare non-file data + pika_request_data = PikaBodyGeneratePikaswapsGeneratePikaswapsPost( + promptText=prompt_text, + negativePrompt=negative_prompt, + seed=seed, + ) + + initial_operation = SynchronousOperation( + endpoint=ApiEndpoint( + path=PATH_PIKADDITIONS, + method=HttpMethod.POST, + request_model=PikaBodyGeneratePikadditionsGeneratePikadditionsPost, + response_model=PikaGenerateResponse, + ), + request=pika_request_data, + files=pika_files, + content_type="multipart/form-data", + auth_token=auth_token, + ) + + return self.execute_task(initial_operation, auth_token) + + +class PikaffectsNode(PikaNodeBase): + """Pika Pikaffects Node.""" + + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "image": ( + IO.IMAGE, + {"tooltip": "The reference image to apply the Pikaffect to."}, + ), + "pikaffect": model_field_to_node_input( + IO.COMBO, + PikaBodyGeneratePikaffectsGeneratePikaffectsPost, + "pikaffect", + enum_type=Pikaffect, + default="Cake-ify", + ), + "prompt_text": model_field_to_node_input( + IO.STRING, + PikaBodyGeneratePikaffectsGeneratePikaffectsPost, + "promptText", + multiline=True, + ), + "negative_prompt": model_field_to_node_input( + IO.STRING, + PikaBodyGeneratePikaffectsGeneratePikaffectsPost, + "negativePrompt", + multiline=True, + ), + "seed": model_field_to_node_input( + IO.INT, + PikaBodyGeneratePikaffectsGeneratePikaffectsPost, + "seed", + min=0, + max=0xFFFFFFFF, + control_after_generate=True, + ), + }, + "hidden": { + "auth_token": "AUTH_TOKEN_COMFY_ORG", + }, + } + + DESCRIPTION = "Generate a video with a specific Pikaffect. Supported Pikaffects: Cake-ify, Crumble, Crush, Decapitate, Deflate, Dissolve, Explode, Eye-pop, Inflate, Levitate, Melt, Peel, Poke, Squish, Ta-da, Tear" + + def api_call( + self, + image: torch.Tensor, + pikaffect: str, + prompt_text: str, + negative_prompt: str, + seed: int, + auth_token: Optional[str] = None, + ) -> tuple[VideoFromFile]: + + initial_operation = SynchronousOperation( + endpoint=ApiEndpoint( + path=PATH_PIKAFFECTS, + method=HttpMethod.POST, + request_model=PikaBodyGeneratePikaffectsGeneratePikaffectsPost, + response_model=PikaGenerateResponse, + ), + request=PikaBodyGeneratePikaffectsGeneratePikaffectsPost( + pikaffect=pikaffect, + promptText=prompt_text, + negativePrompt=negative_prompt, + seed=seed, + ), + files={"image": ("image.png", tensor_to_bytesio(image), "image/png")}, + content_type="multipart/form-data", + auth_token=auth_token, + ) + + return self.execute_task(initial_operation, auth_token) + + +class PikaStartEndFrameNode2_2(PikaNodeBase): + """PikaFrames v2.2 Node.""" + + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "image_start": (IO.IMAGE, {"tooltip": "The first image to combine."}), + "image_end": (IO.IMAGE, {"tooltip": "The last image to combine."}), + **cls.get_base_inputs_types( + PikaBodyGenerate22KeyframeGenerate22PikaframesPost + ), + }, + "hidden": { + "auth_token": "AUTH_TOKEN_COMFY_ORG", + }, + } + + DESCRIPTION = "Generate a video by combining your first and last frame. Upload two images to define the start and end points, and let the AI create a smooth transition between them." + + def api_call( + self, + image_start: torch.Tensor, + image_end: torch.Tensor, + prompt_text: str, + negative_prompt: str, + seed: int, + resolution: str, + duration: int, + auth_token: Optional[str] = None, + ) -> tuple[VideoFromFile]: + + pika_files = [ + ( + "keyFrames", + ("image_start.png", tensor_to_bytesio(image_start), "image/png"), + ), + ("keyFrames", ("image_end.png", tensor_to_bytesio(image_end), "image/png")), + ] + + initial_operation = SynchronousOperation( + endpoint=ApiEndpoint( + path=PATH_PIKAFRAMES, + method=HttpMethod.POST, + request_model=PikaBodyGenerate22KeyframeGenerate22PikaframesPost, + response_model=PikaGenerateResponse, + ), + request=PikaBodyGenerate22KeyframeGenerate22PikaframesPost( + promptText=prompt_text, + negativePrompt=negative_prompt, + seed=seed, + resolution=resolution, + duration=duration, + ), + files=pika_files, + content_type="multipart/form-data", + auth_token=auth_token, + ) + + return self.execute_task(initial_operation, auth_token) + + NODE_CLASS_MAPPINGS = { "PikaImageToVideoNode2_2": PikaImageToVideoV2_2, "PikaTextToVideoNode2_2": PikaTextToVideoNodeV2_2, "PikaScenesV2_2": PikaScenesV2_2, + "Pikadditions": PikAdditionsNode, + "Pikaswaps": PikaSwapsNode, + "Pikaffects": PikaffectsNode, + "PikaStartEndFrameNode2_2": PikaStartEndFrameNode2_2, } NODE_DISPLAY_NAME_MAPPINGS = { - "PikaImageToVideoNode2_2": "Pika 2.2 Image to Video", - "PikaTextToVideoNode2_2": "Pika 2.2 Text to Video", - "PikaScenesV2_2": "Pika 2.2 Scenes", + "PikaImageToVideoNode2_2": "Pika Image to Video", + "PikaTextToVideoNode2_2": "Pika Text to Video", + "PikaScenesV2_2": "Pika Scenes (Video Image Composition)", + "Pikadditions": "Pikadditions (Video Object Insertion)", + "Pikaswaps": "Pika Swaps (Video Object Replacement)", + "Pikaffects": "Pikaffects (Video Effects)", + "PikaStartEndFrameNode2_2": "Pika Start and End Frame to Video", } From 96753dc180969a19fd889d1021981d16a2a516f4 Mon Sep 17 00:00:00 2001 From: Jedrzej Kosinski Date: Sun, 4 May 2025 04:13:23 -0500 Subject: [PATCH 091/121] Make controls input work for Recraft Image to Image node (#120) --- comfy_api_nodes/apis/client.py | 10 ++++- comfy_api_nodes/nodes_recraft.py | 67 ++++++++++++++++++++++++++++++++ 2 files changed, 76 insertions(+), 1 deletion(-) diff --git a/comfy_api_nodes/apis/client.py b/comfy_api_nodes/apis/client.py index 1d9f40881..2a7a3adab 100644 --- a/comfy_api_nodes/apis/client.py +++ b/comfy_api_nodes/apis/client.py @@ -180,10 +180,14 @@ class ApiClient: data: Dict[str, Any], files: Dict[str, Any], headers: Optional[Dict[str, str]] = None, + multipart_parser = None, ) -> Dict[str, Any]: if headers and "Content-Type" in headers: del headers["Content-Type"] + if multipart_parser: + data = multipart_parser(data) + return { "data": data, "files": files, @@ -222,6 +226,7 @@ class ApiClient: files: Optional[Dict[str, Any]] = None, headers: Optional[Dict[str, str]] = None, content_type: str = "application/json", + multipart_parser: Callable = None, ) -> Dict[str, Any]: """ Make an HTTP request to the API @@ -261,7 +266,7 @@ class ApiClient: case "application/x-www-form-urlencoded": payload_args = self._create_urlencoded_form_data_args(data, request_headers) case "multipart/form-data": - payload_args = self._create_form_data_args(data, files, request_headers) + payload_args = self._create_form_data_args(data, files, request_headers, multipart_parser) case _: payload_args = self._create_json_payload_args(data, request_headers) @@ -400,6 +405,7 @@ class SynchronousOperation(Generic[T, R]): timeout: float = 604800.0, verify_ssl: bool = True, content_type: str = "application/json", + multipart_parser: Callable = None, ): self.endpoint = endpoint self.request = request @@ -411,6 +417,7 @@ class SynchronousOperation(Generic[T, R]): self.verify_ssl = verify_ssl self.files = files self.content_type = content_type + self.multipart_parser = multipart_parser def execute(self, client: Optional[ApiClient] = None) -> R: """Execute the API operation using the provided client or create one""" try: @@ -454,6 +461,7 @@ class SynchronousOperation(Generic[T, R]): params=self.endpoint.query_params, files=self.files, content_type=self.content_type, + multipart_parser=self.multipart_parser ) # Debug log for response diff --git a/comfy_api_nodes/nodes_recraft.py b/comfy_api_nodes/nodes_recraft.py index 5a60debc9..d8ac68797 100644 --- a/comfy_api_nodes/nodes_recraft.py +++ b/comfy_api_nodes/nodes_recraft.py @@ -66,6 +66,7 @@ def handle_recraft_file_request( files=files, content_type="multipart/form-data", auth_token=auth_token, + multipart_parser=recraft_multipart_parser, ) response: RecraftImageGenerationResponse = operation.execute() all_bytesio = [] @@ -78,6 +79,72 @@ def handle_recraft_file_request( return all_bytesio +def recraft_multipart_parser(data, parent_key=None, formatter: callable=None, converted_to_check: list[list]=None, is_list=False) -> dict: + """ + Formats data such that multipart/form-data will work with requests library + when both files and data are present. + + The OpenAI client that Recraft uses has a bizarre way of serializing lists: + + It does NOT keep track of indeces of each list, so for background_color, that must be serialized as: + 'background_color[rgb][]' = [0, 0, 255] + where the array is assigned to a key that has '[]' at the end, to signal it's an array. + + This has the consequence of nested lists having the exact same key, forcing arrays to merge; all colors inputs fall under the same key: + if 1 color -> 'controls[colors][][rgb][]' = [0, 0, 255] + if 2 colors -> 'controls[colors][][rgb][]' = [0, 0, 255, 255, 0, 0] + if 3 colors -> 'controls[colors][][rgb][]' = [0, 0, 255, 255, 0, 0, 0, 255, 0] + etc. + Whoever made this serialization up at OpenAI added the constraint that lists must be of uniform length on objects of same 'type'. + """ + # Modification of a function that handled a different type of multipart parsing, big ups: + # https://gist.github.com/kazqvaizer/4cebebe5db654a414132809f9f88067b + + def handle_converted_lists(data, parent_key, lists_to_check=tuple[list]): + # if list already exists exists, just extend list with data + for check_list in lists_to_check: + for conv_tuple in check_list: + if conv_tuple[0] == parent_key and type(conv_tuple[1]) is list: + conv_tuple[1].append(formatter(data)) + return True + return False + + if converted_to_check is None: + converted_to_check = [] + + + if formatter is None: + formatter = lambda v: v # Multipart representation of value + + if type(data) is not dict: + # if list already exists exists, just extend list with data + added = handle_converted_lists(data, parent_key, converted_to_check) + if added: + return {} + # otherwise if is_list, create new list with data + if is_list: + return {parent_key: [formatter(data)]} + # return new key with data + return {parent_key: formatter(data)} + + converted = [] + next_check = [converted] + next_check.extend(converted_to_check) + + for key, value in data.items(): + current_key = key if parent_key is None else f"{parent_key}[{key}]" + if type(value) is dict: + converted.extend(recraft_multipart_parser(value, current_key, formatter, next_check).items()) + elif type(value) is list: + for ind, list_value in enumerate(value): + iter_key = f"{current_key}[]" + converted.extend(recraft_multipart_parser(list_value, iter_key, formatter, next_check, is_list=True).items()) + else: + converted.append((current_key, formatter(value))) + + return dict(converted) + + class SVG: """ Stores SVG representations via a list of BytesIO objects. From cb9f13ea08bc004bd600c3cce8b1a394d407a1e8 Mon Sep 17 00:00:00 2001 From: Christian Byrne Date: Sun, 4 May 2025 14:45:41 -0700 Subject: [PATCH 092/121] Use upstream PR: Support saving Comfy VIDEO type to buffer (#123) --- comfy_api/input_impl/video_types.py | 56 +++++++++++++++++++++-------- 1 file changed, 41 insertions(+), 15 deletions(-) diff --git a/comfy_api/input_impl/video_types.py b/comfy_api/input_impl/video_types.py index 146d6daf8..d0b0b36d2 100644 --- a/comfy_api/input_impl/video_types.py +++ b/comfy_api/input_impl/video_types.py @@ -12,6 +12,46 @@ import torch from comfy_api.input import VideoInput from comfy_api.util import VideoContainer, VideoCodec, VideoComponents + +def container_to_output_format(container_format: str | None) -> str | None: + """ + A container's `format` may be a comma-separated list of formats. + E.g., iso container's `format` may be `mov,mp4,m4a,3gp,3g2,mj2`. + However, writing to a file/stream with `av.open` requires a single format, + or `None` to auto-detect. + """ + if not container_format: + return None # Auto-detect + + if "," not in container_format: + return container_format + + formats = container_format.split(",") + return formats[0] + + +def get_open_write_kwargs( + dest: str | io.BytesIO, container_format: str, to_format: str | None +) -> dict: + """Get kwargs for writing a `VideoFromFile` to a file/stream with `av.open`""" + open_kwargs = { + "mode": "w", + # If isobmff, preserve custom metadata tags (workflow, prompt, extra_pnginfo) + "options": {"movflags": "use_metadata_tags"}, + } + + is_write_to_buffer = isinstance(dest, io.BytesIO) + if is_write_to_buffer: + # Set output format explicitly, since it cannot be inferred from file extension + if to_format == VideoContainer.AUTO: + to_format = container_format.lower() + elif isinstance(to_format, str): + to_format = to_format.lower() + open_kwargs["format"] = container_to_output_format(to_format) + + return open_kwargs + + class VideoFromFile(VideoInput): """ Class representing video input from a file. @@ -116,22 +156,8 @@ class VideoFromFile(VideoInput): ) streams = container.streams - open_kwargs = { - "mode": "w", - "options": {"movflags": "use_metadata_tags"} - } - - if not isinstance(path, str): - # Explicit format is needed for non-path destinations (like BytesIO) - output_format_str = ( - format.value.lower() - if format != VideoContainer.AUTO - else container.format.name - ) - if "," in output_format_str: - output_format_str = output_format_str.split(",")[0] - open_kwargs["format"] = output_format_str + open_kwargs = get_open_write_kwargs(path, container_format, format) with av.open(path, **open_kwargs) as output_container: # Copy over the original metadata for key, value in container.metadata.items(): From 3a04204713ad0b7df00e032cf98059adafe4441b Mon Sep 17 00:00:00 2001 From: Christian Byrne Date: Sun, 4 May 2025 20:13:19 -0700 Subject: [PATCH 093/121] Use Upstream PR: "Fix: Error creating video when sliced audio tensor chunks are non-c-contiguous" (#127) --- comfy_api/input_impl/video_types.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/comfy_api/input_impl/video_types.py b/comfy_api/input_impl/video_types.py index d0b0b36d2..ae48dbaa4 100644 --- a/comfy_api/input_impl/video_types.py +++ b/comfy_api/input_impl/video_types.py @@ -253,7 +253,12 @@ class VideoFromComponents(VideoInput): start = i * samples_per_frame end = start + samples_per_frame # TODO(Feature) - Add support for stereo audio - chunk = self.__components.audio['waveform'][0, 0, start:end].unsqueeze(0).numpy() + chunk = ( + self.__components.audio["waveform"][0, 0, start:end] + .unsqueeze(0) + .contiguous() + .numpy() + ) audio_frame = av.AudioFrame.from_ndarray(chunk, format='fltp', layout='mono') audio_frame.sample_rate = audio_sample_rate audio_frame.pts = i * samples_per_frame From 7ead535d463499cff8312b699a1badae8bedf3c1 Mon Sep 17 00:00:00 2001 From: Christian Byrne Date: Sun, 4 May 2025 20:58:43 -0700 Subject: [PATCH 094/121] Improve audio upload utils (#128) --- comfy_api_nodes/apinode_utils.py | 78 ++++++++++++++++++++------------ 1 file changed, 49 insertions(+), 29 deletions(-) diff --git a/comfy_api_nodes/apinode_utils.py b/comfy_api_nodes/apinode_utils.py index 13f900a61..04884d610 100644 --- a/comfy_api_nodes/apinode_utils.py +++ b/comfy_api_nodes/apinode_utils.py @@ -361,7 +361,7 @@ def upload_video_to_comfyapi( Uses the specified container and codec for saving the video before upload. Args: - video: Input VideoInput object. + video: VideoInput object (Comfy VIDEO type). auth_token: Optional authentication token. container: The video container format to use (default: MP4). codec: The video codec to use (default: H264). @@ -394,47 +394,41 @@ def upload_video_to_comfyapi( ) -def upload_audio_to_comfyapi( - audio: AudioInput, - auth_token: Optional[str] = None, -) -> str: +def audio_tensor_to_contiguous_ndarray(waveform: torch.Tensor) -> np.ndarray: """ - Uploads a single audio input to ComfyUI API and returns its download URL. - Encodes the raw waveform into MP4/AAC format before uploading. + Prepares audio waveform for av library by converting to a contiguous numpy array. Args: - audio: Input AudioInput object (containing waveform tensor and sample_rate). - auth_token: Optional authentication token. + waveform: a tensor of shape (1, channels, samples) derived from a Comfy `AUDIO` type. Returns: - The download URL for the uploaded audio file. + Contiguous numpy array of the audio waveform. If the audio was batched, + the first item is taken. """ - waveform: torch.Tensor = audio["waveform"] - sample_rate: int = audio["sample_rate"] + if waveform.ndim != 3 or waveform.shape[0] != 1: + raise ValueError("Expected waveform tensor shape (1, channels, samples)") # If batch is > 1, take first item if waveform.shape[0] > 1: waveform = waveform[0] - # Check waveform tensor shape - if waveform.ndim != 3 or waveform.shape[0] != 1: - raise ValueError("Expected waveform tensor shape (1, channels, samples)") - - # Prepare data for av library - audio_data_np = waveform.squeeze(0).cpu().numpy() + # Prepare for av: remove batch dim, move to CPU, make contiguous, convert to numpy array + audio_data_np = waveform.squeeze(0).cpu().contiguous().numpy() if audio_data_np.dtype != np.float32: audio_data_np = audio_data_np.astype(np.float32) - # Ensure the array is C-contiguous - if not audio_data_np.flags["C_CONTIGUOUS"]: - audio_data_np = np.ascontiguousarray(audio_data_np) + return audio_data_np - # Default to MP4/AAC - container_format = "mp4" - codec_name = "aac" - upload_mime_type = "audio/mp4" - filename = "uploaded_audio.mp4" +def audio_ndarray_to_bytesio( + audio_data_np: np.ndarray, + sample_rate: int, + container_format: str = "mp4", + codec_name: str = "aac", +) -> BytesIO: + """ + Encodes a numpy array of audio data into a BytesIO object. + """ audio_bytes_io = io.BytesIO() with av.open(audio_bytes_io, mode="w", format=container_format) as output_container: audio_stream = output_container.add_stream(codec_name, rate=sample_rate) @@ -453,12 +447,38 @@ def upload_audio_to_comfyapi( for packet in audio_stream.encode(None): output_container.mux(packet) - audio_bytes_io.seek(0) # Reset buffer position for reading + audio_bytes_io.seek(0) + return audio_bytes_io - return upload_file_to_comfyapi( - audio_bytes_io, filename, upload_mime_type, auth_token + +def upload_audio_to_comfyapi( + audio: AudioInput, + auth_token: Optional[str] = None, + container_format: str = "mp4", + codec_name: str = "aac", + mime_type: str = "audio/mp4", + filename: str = "uploaded_audio.mp4", +) -> str: + """ + Uploads a single audio input to ComfyUI API and returns its download URL. + Encodes the raw waveform into the specified format before uploading. + + Args: + audio: a Comfy `AUDIO` type (contains waveform tensor and sample_rate) + auth_token: Optional authentication token. + + Returns: + The download URL for the uploaded audio file. + """ + sample_rate: int = audio["sample_rate"] + waveform: torch.Tensor = audio["waveform"] + audio_data_np = audio_tensor_to_contiguous_ndarray(waveform) + audio_bytes_io = audio_ndarray_to_bytesio( + audio_data_np, sample_rate, container_format, codec_name ) + return upload_file_to_comfyapi(audio_bytes_io, filename, mime_type, auth_token) + def upload_images_to_comfyapi( image: torch.Tensor, max_images=8, auth_token=None, mime_type: Optional[str] = None From 337e70710382bdd615aa5d7fa2d062a7274b5870 Mon Sep 17 00:00:00 2001 From: Christian Byrne Date: Sun, 4 May 2025 21:04:21 -0700 Subject: [PATCH 095/121] Fix: Nested `AnyUrl` in request model cannot be serialized (Kling, Runway) (#129) --- comfy_api_nodes/apis/__init__.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/comfy_api_nodes/apis/__init__.py b/comfy_api_nodes/apis/__init__.py index 063d176e6..cb448f4fc 100644 --- a/comfy_api_nodes/apis/__init__.py +++ b/comfy_api_nodes/apis/__init__.py @@ -684,7 +684,7 @@ class KlingLipSyncInputObject(BaseModel): None, description='The ID of the video generated by Kling AI. Only supports 5-second and 10-second videos generated within the last 30 days.', ) - video_url: Optional[AnyUrl] = Field( + video_url: Optional[str] = Field( None, description='Get link for uploaded video. Video files support .mp4/.mov, file size does not exceed 100MB, video length between 2-10s.', ) @@ -709,7 +709,7 @@ class KlingLipSyncInputObject(BaseModel): None, description='Local Path of Audio File. Supported formats: .mp3/.wav/.m4a/.aac, maximum file size of 5MB. Base64 code.', ) - audio_url: Optional[AnyUrl] = Field( + audio_url: Optional[str] = Field( None, description='Audio File Download URL. Supported formats: .mp3/.wav/.m4a/.aac, maximum file size of 5MB.', ) @@ -1933,7 +1933,7 @@ class Position(str, Enum): class RunwayPromptImageDetailedObject(BaseModel): - uri: AnyUrl = Field( + uri: str = Field( ..., description='A HTTPS URL or data URI containing an encoded image.' ) position: Position = Field( @@ -1959,9 +1959,9 @@ class RunwayAspectRatioEnum(str, Enum): class RunwayPromptImageObject( - RootModel[Union[AnyUrl, List[RunwayPromptImageDetailedObject]]] + RootModel[Union[str, List[RunwayPromptImageDetailedObject]]] ): - root: Union[AnyUrl, List[RunwayPromptImageDetailedObject]] = Field( + root: Union[str, List[RunwayPromptImageDetailedObject]] = Field( ..., description='Image(s) to use for the video generation. Can be a single URI or an array of image objects with positions.', ) @@ -3211,7 +3211,7 @@ class KlingImage2VideoRequest(BaseModel): default_factory=lambda: KlingVideoGenCfgScale.model_validate(0.5) ) mode: Optional[KlingVideoGenMode] = 'std' - static_mask: Optional[AnyUrl] = Field( + static_mask: Optional[str] = Field( None, description='Static Brush Application Area (Mask image created by users using the motion brush). The aspect ratio must match the input image.', ) From d010dd9d8042481b4f11fee131d29e5b4bc15474 Mon Sep 17 00:00:00 2001 From: Christian Byrne Date: Mon, 5 May 2025 01:22:49 -0700 Subject: [PATCH 096/121] Show errors and API output URLs to the user (change log levels) (#131) --- comfy_api_nodes/apis/client.py | 36 ++++++++++++---------------------- comfy_api_nodes/nodes_kling.py | 15 ++++++-------- comfy_api_nodes/nodes_pika.py | 2 +- 3 files changed, 20 insertions(+), 33 deletions(-) diff --git a/comfy_api_nodes/apis/client.py b/comfy_api_nodes/apis/client.py index 2a7a3adab..496831ac4 100644 --- a/comfy_api_nodes/apis/client.py +++ b/comfy_api_nodes/apis/client.py @@ -1,11 +1,3 @@ -import logging -import time -from typing import Callable -import io - -from comfy.cli_args import args -from comfy import utils - """ API Client Framework for api.comfy.org. @@ -97,21 +89,18 @@ operation = PollingOperation( result = operation.execute(client=api_client) # Returns the final ImageGenerationResult when done """ -from typing import ( - Dict, - Type, - Optional, - Any, - TypeVar, - Generic, -) -from pydantic import BaseModel, Field +import logging +import time +import io +from typing import Dict, Type, Optional, Any, TypeVar, Generic, Callable from enum import Enum import json import requests from urllib.parse import urljoin +from pydantic import BaseModel, Field -# Import models from your generated stubs +from comfy.cli_args import args +from comfy import utils T = TypeVar("T", bound=BaseModel) R = TypeVar("R", bound=BaseModel) @@ -475,7 +464,7 @@ class SynchronousOperation(Generic[T, R]): return self._parse_response(resp) except Exception as e: - logging.debug(f"[DEBUG] API Exception: {str(e)}") + logging.error(f"[DEBUG] API Exception: {str(e)}") raise Exception(str(e)) def _parse_response(self, resp): @@ -554,7 +543,7 @@ class PollingOperation(Generic[T, R]): return TaskStatus.FAILED return TaskStatus.PENDING except Exception as e: - logging.debug(f"Error extracting status: {e}") + logging.error(f"Error extracting status: {e}") return TaskStatus.PENDING def _poll_until_complete(self, client: ApiClient) -> R: @@ -609,8 +598,9 @@ class PollingOperation(Generic[T, R]): progress.update(100) return self.final_response elif status == TaskStatus.FAILED: - logging.debug(f"[DEBUG] Task failed: {json.dumps(resp)}") - raise Exception(f"Task failed: {json.dumps(resp)}") + message = f"Task failed: {json.dumps(resp)}" + logging.error(f"[DEBUG] {message}") + raise Exception(message) else: logging.debug("[DEBUG] Task still pending, continuing to poll...") @@ -621,5 +611,5 @@ class PollingOperation(Generic[T, R]): time.sleep(self.poll_interval) except Exception as e: - logging.debug(f"[DEBUG] Polling error: {str(e)}") + logging.error(f"[DEBUG] Polling error: {str(e)}") raise Exception(f"Error while polling: {str(e)}") diff --git a/comfy_api_nodes/nodes_kling.py b/comfy_api_nodes/nodes_kling.py index 208a3730a..b36c1c544 100644 --- a/comfy_api_nodes/nodes_kling.py +++ b/comfy_api_nodes/nodes_kling.py @@ -179,7 +179,7 @@ def validate_prompts(prompt: str, negative_prompt: str, max_length: int) -> bool return True -def validate_task_creation_response(response): +def validate_task_creation_response(response) -> None: """Validates that the Kling task creation request was successful.""" if not is_valid_task_creation_response(response): error_msg = f"Kling initial request failed. Code: {response.code}, Message: {response.message}, Data: {response.data}" @@ -187,7 +187,7 @@ def validate_task_creation_response(response): raise KlingApiError(error_msg) -def validate_video_result_response(response): +def validate_video_result_response(response) -> None: """Validates that the Kling task result contains a video.""" if not is_valid_video_response(response): error_msg = f"Kling task {response.data.task_id} succeeded but no video data found in response." @@ -195,7 +195,7 @@ def validate_video_result_response(response): raise KlingApiError(error_msg) -def validate_image_result_response(response): +def validate_image_result_response(response) -> None: """Validates that the Kling task result contains an image.""" if not is_valid_image_response(response): error_msg = f"Kling task {response.data.task_id} succeeded but no image data found in response." @@ -221,7 +221,7 @@ def get_camera_control_input_config( def get_video_from_response(response) -> KlingVideoResult: """Returns the first video object from the Kling video generation task result.""" video = response.data.task_result.videos[0] - logging.debug( + logging.info( "Kling task %s succeeded. Video URL: %s", response.data.task_id, video.url ) return video @@ -229,7 +229,7 @@ def get_video_from_response(response) -> KlingVideoResult: def get_images_from_response(response) -> list[KlingImageResult]: images = response.data.task_result.images - logging.debug("Kling task %s succeeded. Images: %s", response.data.task_id, images) + logging.info("Kling task %s succeeded. Images: %s", response.data.task_id, images) return images @@ -255,10 +255,7 @@ def image_result_to_node_output( class KlingNodeBase(ComfyNodeABC): - """ - Base class for Kling nodes. - - """ + """Base class for Kling nodes.""" FUNCTION = "api_call" CATEGORY = "api node/video/Kling" diff --git a/comfy_api_nodes/nodes_pika.py b/comfy_api_nodes/nodes_pika.py index 64f9645da..ba4e8457d 100644 --- a/comfy_api_nodes/nodes_pika.py +++ b/comfy_api_nodes/nodes_pika.py @@ -173,7 +173,7 @@ class PikaNodeBase(ComfyNodeABC): raise PikaApiError(error_msg) video_url = str(final_response.url) - logging.debug("Pika task %s succeeded. Video URL: %s", task_id, video_url) + logging.info("Pika task %s succeeded. Video URL: %s", task_id, video_url) return (download_url_to_video_output(video_url),) From 8825835ae94f451cc8782a546c4314a21792b6c9 Mon Sep 17 00:00:00 2001 From: Christian Byrne Date: Mon, 5 May 2025 01:29:47 -0700 Subject: [PATCH 097/121] Fix: Luma I2I fails when weight is <=0.01 (#132) --- comfy_api_nodes/nodes_luma.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/comfy_api_nodes/nodes_luma.py b/comfy_api_nodes/nodes_luma.py index 7b95a1f41..32c054c61 100644 --- a/comfy_api_nodes/nodes_luma.py +++ b/comfy_api_nodes/nodes_luma.py @@ -327,7 +327,7 @@ class LumaImageModifyNode(ComfyNodeABC): IO.FLOAT, { "default": 1.0, - "min": 0.0, + "min": 0.2, "max": 1.0, "step": 0.01, "tooltip": "Weight of the image; the closer to 0.0, the less the image will be modified.", From c0a348d778357cd21c2e4fbfad7537cc4a984c1d Mon Sep 17 00:00:00 2001 From: Christian Byrne Date: Mon, 5 May 2025 01:48:03 -0700 Subject: [PATCH 098/121] Change category of `LumaConcepts` node from image to video (#133) --- comfy_api_nodes/nodes_luma.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/comfy_api_nodes/nodes_luma.py b/comfy_api_nodes/nodes_luma.py index 32c054c61..874a414ad 100644 --- a/comfy_api_nodes/nodes_luma.py +++ b/comfy_api_nodes/nodes_luma.py @@ -94,7 +94,7 @@ class LumaConceptsNode(ComfyNodeABC): RETURN_NAMES = ("luma_concepts",) DESCRIPTION = cleandoc(__doc__ or "") # Handle potential None value FUNCTION = "create_concepts" - CATEGORY = "api node/image/Luma" + CATEGORY = "api node/video/Luma" @classmethod def INPUT_TYPES(s): From 0b0eead4ae9735a9255d5f0fc14cbecfd87b27a2 Mon Sep 17 00:00:00 2001 From: Christian Byrne Date: Mon, 5 May 2025 02:18:24 -0700 Subject: [PATCH 099/121] Fix: `image.shape` accessed before `image` is null-checked (#134) --- comfy_api_nodes/nodes_openai.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/comfy_api_nodes/nodes_openai.py b/comfy_api_nodes/nodes_openai.py index e3fbbb868..de912af11 100644 --- a/comfy_api_nodes/nodes_openai.py +++ b/comfy_api_nodes/nodes_openai.py @@ -427,10 +427,10 @@ class OpenAIGPTImage1(ComfyNodeABC): files.append(("image[]", img_binary)) if mask is not None: - if image.shape[0] != 1: - raise Exception("Cannot use a mask with multiple image") if image is None: raise Exception("Cannot use a mask without an input image") + if image.shape[0] != 1: + raise Exception("Cannot use a mask with multiple image") if mask.shape[1:] != image.shape[1:-1]: raise Exception("Mask and Image must be the same size") batch, height, width = mask.shape From 0e34f00fee44c6ae0d1d64c030cdfc5321319ce5 Mon Sep 17 00:00:00 2001 From: Jedrzej Kosinski Date: Mon, 5 May 2025 06:34:32 -0500 Subject: [PATCH 100/121] Apply small fixes and most prompt validation (if needed to avoid API error) (#135) --- comfy_api_nodes/apinode_utils.py | 11 +++++++++++ comfy_api_nodes/nodes_bfl.py | 3 +++ comfy_api_nodes/nodes_luma.py | 5 ++++- comfy_api_nodes/nodes_minimax.py | 3 +++ comfy_api_nodes/nodes_openai.py | 6 +++++- comfy_api_nodes/nodes_pixverse.py | 4 ++++ comfy_api_nodes/nodes_recraft.py | 6 +++++- comfy_api_nodes/nodes_stability.py | 5 +++++ 8 files changed, 40 insertions(+), 3 deletions(-) diff --git a/comfy_api_nodes/apinode_utils.py b/comfy_api_nodes/apinode_utils.py index 04884d610..bd3b8908b 100644 --- a/comfy_api_nodes/apinode_utils.py +++ b/comfy_api_nodes/apinode_utils.py @@ -562,3 +562,14 @@ def resize_mask_to_image(mask: torch.Tensor, image: torch.Tensor, if not allow_gradient: mask = (mask > 0.5).float() return mask + + +def validate_string(string: str, strip_whitespace=True, field_name="prompt", min_length=None, max_length=None): + if strip_whitespace: + string = string.strip() + if min_length and len(string) < min_length: + raise Exception(f"Field '{field_name}' cannot be shorter than {min_length} characters; was {len(string)} characters long.") + if max_length and len(string) > max_length: + raise Exception(f" Field '{field_name} cannot be longer than {max_length} characters; was {len(string)} characters long.") + if not string: + raise Exception(f"Field '{field_name}' cannot be empty.") diff --git a/comfy_api_nodes/nodes_bfl.py b/comfy_api_nodes/nodes_bfl.py index f1014459a..6b7866741 100644 --- a/comfy_api_nodes/nodes_bfl.py +++ b/comfy_api_nodes/nodes_bfl.py @@ -21,6 +21,7 @@ from comfy_api_nodes.apinode_utils import ( validate_aspect_ratio, process_image_response, resize_mask_to_image, + validate_string, ) import numpy as np @@ -213,6 +214,8 @@ class FluxProUltraImageNode(ComfyNodeABC): auth_token=None, **kwargs, ): + if image_prompt is None: + validate_string(prompt, strip_whitespace=False) operation = SynchronousOperation( endpoint=ApiEndpoint( path="/proxy/bfl/flux-pro-1.1-ultra/generate", diff --git a/comfy_api_nodes/nodes_luma.py b/comfy_api_nodes/nodes_luma.py index 874a414ad..6a9d46fdc 100644 --- a/comfy_api_nodes/nodes_luma.py +++ b/comfy_api_nodes/nodes_luma.py @@ -32,6 +32,7 @@ from comfy_api_nodes.apis.client import ( from comfy_api_nodes.apinode_utils import ( upload_images_to_comfyapi, process_image_response, + validate_string, ) import requests @@ -216,6 +217,7 @@ class LumaImageGenerationNode(ComfyNodeABC): auth_token=None, **kwargs, ): + validate_string(prompt, strip_whitespace=True, min_length=3) # handle image_luma_ref api_image_ref = None if image_luma_ref is not None: @@ -327,7 +329,7 @@ class LumaImageModifyNode(ComfyNodeABC): IO.FLOAT, { "default": 1.0, - "min": 0.2, + "min": 0.02, "max": 1.0, "step": 0.01, "tooltip": "Weight of the image; the closer to 0.0, the less the image will be modified.", @@ -484,6 +486,7 @@ class LumaTextToVideoGenerationNode(ComfyNodeABC): auth_token=None, **kwargs, ): + validate_string(prompt, strip_whitespace=False, min_length=3) duration = duration if model != LumaVideoModel.ray_1_6 else None resolution = resolution if model != LumaVideoModel.ray_1_6 else None diff --git a/comfy_api_nodes/nodes_minimax.py b/comfy_api_nodes/nodes_minimax.py index 449ae1473..37bfce72d 100644 --- a/comfy_api_nodes/nodes_minimax.py +++ b/comfy_api_nodes/nodes_minimax.py @@ -18,6 +18,7 @@ from comfy_api_nodes.apis.client import ( from comfy_api_nodes.apinode_utils import ( download_url_to_bytesio, upload_images_to_comfyapi, + validate_string, ) import torch @@ -88,6 +89,8 @@ class MinimaxTextToVideoNode: ''' Function used between Minimax nodes - supports T2V, I2V, and S2V, based on provided arguments. ''' + if image is None: + validate_string(prompt_text, field_name="prompt_text") # upload image, if passed in image_url = None if image is not None: diff --git a/comfy_api_nodes/nodes_openai.py b/comfy_api_nodes/nodes_openai.py index de912af11..ccdc2bdbf 100644 --- a/comfy_api_nodes/nodes_openai.py +++ b/comfy_api_nodes/nodes_openai.py @@ -21,7 +21,8 @@ from comfy_api_nodes.apis.client import ( from comfy_api_nodes.apinode_utils import ( downscale_image_tensor, - validate_and_cast_response + validate_and_cast_response, + validate_string, ) class OpenAIDalle2(ComfyNodeABC): @@ -114,6 +115,7 @@ class OpenAIDalle2(ComfyNodeABC): size="1024x1024", auth_token=None, ): + validate_string(prompt, strip_whitespace=False) model = "dall-e-2" path = "/proxy/openai/images/generations" content_type = "application/json" @@ -258,6 +260,7 @@ class OpenAIDalle3(ComfyNodeABC): size="1024x1024", auth_token=None, ): + validate_string(prompt, strip_whitespace=False) model = "dall-e-3" # build the operation @@ -393,6 +396,7 @@ class OpenAIGPTImage1(ComfyNodeABC): size="1024x1024", auth_token=None, ): + validate_string(prompt, strip_whitespace=False) model = "gpt-image-1" path = "/proxy/openai/images/generations" content_type="application/json" diff --git a/comfy_api_nodes/nodes_pixverse.py b/comfy_api_nodes/nodes_pixverse.py index 8064b8f9f..b2db3f321 100644 --- a/comfy_api_nodes/nodes_pixverse.py +++ b/comfy_api_nodes/nodes_pixverse.py @@ -24,6 +24,7 @@ from comfy_api_nodes.apis.client import ( ) from comfy_api_nodes.apinode_utils import ( tensor_to_bytesio, + validate_string, ) from comfy.comfy_types.node_typing import IO, ComfyNodeABC from comfy_api.input_impl import VideoFromFile @@ -163,6 +164,7 @@ class PixverseTextToVideoNode(ComfyNodeABC): auth_token=None, **kwargs, ): + validate_string(prompt, strip_whitespace=False) # 1080p is limited to 5 seconds duration # only normal motion_mode supported for 1080p or for non-5 second duration if quality == PixverseQuality.res_1080p: @@ -292,6 +294,7 @@ class PixverseImageToVideoNode(ComfyNodeABC): auth_token=None, **kwargs, ): + validate_string(prompt, strip_whitespace=False) img_id = upload_image_to_pixverse(image, auth_token=auth_token) # 1080p is limited to 5 seconds duration @@ -427,6 +430,7 @@ class PixverseTransitionVideoNode(ComfyNodeABC): auth_token=None, **kwargs, ): + validate_string(prompt, strip_whitespace=False) first_frame_id = upload_image_to_pixverse(first_frame, auth_token=auth_token) last_frame_id = upload_image_to_pixverse(last_frame, auth_token=auth_token) diff --git a/comfy_api_nodes/nodes_recraft.py b/comfy_api_nodes/nodes_recraft.py index d8ac68797..a5e513e05 100644 --- a/comfy_api_nodes/nodes_recraft.py +++ b/comfy_api_nodes/nodes_recraft.py @@ -26,6 +26,7 @@ from comfy_api_nodes.apinode_utils import ( download_url_to_bytesio, tensor_to_bytesio, resize_mask_to_image, + validate_string, ) import folder_paths import json @@ -455,6 +456,7 @@ class RecraftTextToImageNode: auth_token=None, **kwargs, ): + validate_string(prompt, strip_whitespace=False, max_length=1000) default_style = RecraftStyle(RecraftStyleV3.realistic_image) if recraft_style is None: recraft_style = default_style @@ -589,6 +591,7 @@ class RecraftImageToImageNode: recraft_controls: RecraftControls = None, **kwargs, ): + validate_string(prompt, strip_whitespace=False, max_length=1000) default_style = RecraftStyle(RecraftStyleV3.realistic_image) if recraft_style is None: recraft_style = default_style @@ -702,6 +705,7 @@ class RecraftImageInpaintingNode: negative_prompt: str = None, **kwargs, ): + validate_string(prompt, strip_whitespace=False, max_length=1000) default_style = RecraftStyle(RecraftStyleV3.realistic_image) if recraft_style is None: recraft_style = default_style @@ -717,7 +721,6 @@ class RecraftImageInpaintingNode: style=recraft_style.style, substyle=recraft_style.substyle, style_id=recraft_style.style_id, - random_seed=seed, ) # prepare mask tensor @@ -825,6 +828,7 @@ class RecraftTextToVectorNode: auth_token=None, **kwargs, ): + validate_string(prompt, strip_whitespace=False, max_length=1000) # create RecraftStyle so strings will be formatted properly (i.e. "None" will become None) recraft_style = RecraftStyle(RecraftStyleV3.vector_illustration, substyle=substyle) diff --git a/comfy_api_nodes/nodes_stability.py b/comfy_api_nodes/nodes_stability.py index 50d264b5d..4f1676a6a 100644 --- a/comfy_api_nodes/nodes_stability.py +++ b/comfy_api_nodes/nodes_stability.py @@ -23,6 +23,7 @@ from comfy_api_nodes.apis.client import ( from comfy_api_nodes.apinode_utils import ( bytesio_to_image_tensor, tensor_to_bytesio, + validate_string, ) import torch @@ -125,6 +126,7 @@ class StabilityStableImageUltraNode: def api_call(self, prompt: str, aspect_ratio: str, style_preset: str, seed: int, negative_prompt: str=None, image: torch.Tensor = None, image_denoise: float=None, auth_token=None): + validate_string(prompt, strip_whitespace=False) # prepare image binary if image present image_binary = None if image is not None: @@ -256,6 +258,7 @@ class StabilityStableImageSD_3_5Node: def api_call(self, model: str, prompt: str, aspect_ratio: str, style_preset: str, seed: int, cfg_scale: float, negative_prompt: str=None, image: torch.Tensor = None, image_denoise: float=None, auth_token=None): + validate_string(prompt, strip_whitespace=False) # prepare image binary if image present image_binary = None mode = Stability_SD3_5_GenerationMode.text_to_image @@ -370,6 +373,7 @@ class StabilityUpscaleConservativeNode: def api_call(self, image: torch.Tensor, prompt: str, creativity: float, seed: int, negative_prompt: str=None, auth_token=None): + validate_string(prompt, strip_whitespace=False) image_binary = tensor_to_bytesio(image, total_pixels=1024*1024).read() if not negative_prompt: @@ -474,6 +478,7 @@ class StabilityUpscaleCreativeNode: def api_call(self, image: torch.Tensor, prompt: str, creativity: float, style_preset: str, seed: int, negative_prompt: str=None, auth_token=None): + validate_string(prompt, strip_whitespace=False) image_binary = tensor_to_bytesio(image, total_pixels=1024*1024).read() if not negative_prompt: From 961651f373d0489c02e00766ac3d43f9d4c04e55 Mon Sep 17 00:00:00 2001 From: Jedrzej Kosinski Date: Mon, 5 May 2025 11:17:44 -0500 Subject: [PATCH 101/121] Node name/category modifications (#140) --- comfy_api_nodes/nodes_bfl.py | 12 +++++------ comfy_api_nodes/nodes_ideogram.py | 6 +++--- comfy_api_nodes/nodes_minimax.py | 28 +++++++++++++------------- comfy_api_nodes/nodes_openai.py | 6 +++--- comfy_api_nodes/nodes_pixverse.py | 32 +++++++++++++++--------------- comfy_api_nodes/nodes_stability.py | 20 +++++++++---------- 6 files changed, 52 insertions(+), 52 deletions(-) diff --git a/comfy_api_nodes/nodes_bfl.py b/comfy_api_nodes/nodes_bfl.py index 6b7866741..7f02df88e 100644 --- a/comfy_api_nodes/nodes_bfl.py +++ b/comfy_api_nodes/nodes_bfl.py @@ -200,7 +200,7 @@ class FluxProUltraImageNode(ComfyNodeABC): DESCRIPTION = cleandoc(__doc__ or "") # Handle potential None value FUNCTION = "api_call" API_NODE = True - CATEGORY = "api node/image/bfl" + CATEGORY = "api node/image/BFL" def api_call( self, @@ -326,7 +326,7 @@ class FluxProImageNode(ComfyNodeABC): DESCRIPTION = cleandoc(__doc__ or "") # Handle potential None value FUNCTION = "api_call" API_NODE = True - CATEGORY = "api node/image/bfl" + CATEGORY = "api node/image/BFL" def api_call( self, @@ -468,7 +468,7 @@ class FluxProExpandNode(ComfyNodeABC): DESCRIPTION = cleandoc(__doc__ or "") # Handle potential None value FUNCTION = "api_call" API_NODE = True - CATEGORY = "api node/image/bfl" + CATEGORY = "api node/image/BFL" def api_call( self, @@ -579,7 +579,7 @@ class FluxProFillNode(ComfyNodeABC): DESCRIPTION = cleandoc(__doc__ or "") # Handle potential None value FUNCTION = "api_call" API_NODE = True - CATEGORY = "api node/image/bfl" + CATEGORY = "api node/image/BFL" def api_call( self, @@ -713,7 +713,7 @@ class FluxProCannyNode(ComfyNodeABC): DESCRIPTION = cleandoc(__doc__ or "") # Handle potential None value FUNCTION = "api_call" API_NODE = True - CATEGORY = "api node/image/bfl" + CATEGORY = "api node/image/BFL" def api_call( self, @@ -841,7 +841,7 @@ class FluxProDepthNode(ComfyNodeABC): DESCRIPTION = cleandoc(__doc__ or "") # Handle potential None value FUNCTION = "api_call" API_NODE = True - CATEGORY = "api node/image/bfl" + CATEGORY = "api node/image/BFL" def api_call( self, diff --git a/comfy_api_nodes/nodes_ideogram.py b/comfy_api_nodes/nodes_ideogram.py index 1142bd5e5..3f80a62d7 100644 --- a/comfy_api_nodes/nodes_ideogram.py +++ b/comfy_api_nodes/nodes_ideogram.py @@ -307,7 +307,7 @@ class IdeogramV1(ComfyNodeABC): RETURN_TYPES = (IO.IMAGE,) FUNCTION = "api_call" - CATEGORY = "api node/image/ideogram/v1" + CATEGORY = "api node/image/Ideogram/v1" DESCRIPTION = cleandoc(__doc__ or "") API_NODE = True @@ -462,7 +462,7 @@ class IdeogramV2(ComfyNodeABC): RETURN_TYPES = (IO.IMAGE,) FUNCTION = "api_call" - CATEGORY = "api node/image/ideogram/v2" + CATEGORY = "api node/image/Ideogram/v2" DESCRIPTION = cleandoc(__doc__ or "") API_NODE = True @@ -625,7 +625,7 @@ class IdeogramV3(ComfyNodeABC): RETURN_TYPES = (IO.IMAGE,) FUNCTION = "api_call" - CATEGORY = "api node/image/ideogram/v3" + CATEGORY = "api node/image/Ideogram/v3" DESCRIPTION = cleandoc(__doc__ or "") API_NODE = True diff --git a/comfy_api_nodes/nodes_minimax.py b/comfy_api_nodes/nodes_minimax.py index 37bfce72d..cacda22c6 100644 --- a/comfy_api_nodes/nodes_minimax.py +++ b/comfy_api_nodes/nodes_minimax.py @@ -27,7 +27,7 @@ import logging class MinimaxTextToVideoNode: """ - Generates videos synchronously based on a prompt, and optional parameters using Minimax's API. + Generates videos synchronously based on a prompt, and optional parameters using MiniMax's API. """ @classmethod @@ -71,9 +71,9 @@ class MinimaxTextToVideoNode: } RETURN_TYPES = ("VIDEO",) - DESCRIPTION = "Generates videos from prompts using Minimax's API" + DESCRIPTION = "Generates videos from prompts using MiniMax's API" FUNCTION = "generate_video" - CATEGORY = "api node/video/Minimax" + CATEGORY = "api node/video/MiniMax" API_NODE = True OUTPUT_NODE = True @@ -87,7 +87,7 @@ class MinimaxTextToVideoNode: auth_token=None, ): ''' - Function used between Minimax nodes - supports T2V, I2V, and S2V, based on provided arguments. + Function used between MiniMax nodes - supports T2V, I2V, and S2V, based on provided arguments. ''' if image is None: validate_string(prompt_text, field_name="prompt_text") @@ -124,7 +124,7 @@ class MinimaxTextToVideoNode: task_id = response.task_id if not task_id: - raise Exception(f"Minimax generation failed: {response.base_resp}") + raise Exception(f"MiniMax generation failed: {response.base_resp}") video_generate_operation = PollingOperation( poll_endpoint=ApiEndpoint( @@ -174,7 +174,7 @@ class MinimaxTextToVideoNode: class MinimaxImageToVideoNode(MinimaxTextToVideoNode): """ - Generates videos synchronously based on an image and prompt, and optional parameters using Minimax's API. + Generates videos synchronously based on an image and prompt, and optional parameters using MiniMax's API. """ @classmethod @@ -225,16 +225,16 @@ class MinimaxImageToVideoNode(MinimaxTextToVideoNode): } RETURN_TYPES = ("VIDEO",) - DESCRIPTION = "Generates videos from an image and prompts using Minimax's API" + DESCRIPTION = "Generates videos from an image and prompts using MiniMax's API" FUNCTION = "generate_video" - CATEGORY = "api node/video/Minimax" + CATEGORY = "api node/video/MiniMax" API_NODE = True OUTPUT_NODE = True class MinimaxSubjectToVideoNode(MinimaxTextToVideoNode): """ - Generates videos synchronously based on an image and prompt, and optional parameters using Minimax's API. + Generates videos synchronously based on an image and prompt, and optional parameters using MiniMax's API. """ @classmethod @@ -283,9 +283,9 @@ class MinimaxSubjectToVideoNode(MinimaxTextToVideoNode): } RETURN_TYPES = ("VIDEO",) - DESCRIPTION = "Generates videos from an image and prompts using Minimax's API" + DESCRIPTION = "Generates videos from an image and prompts using MiniMax's API" FUNCTION = "generate_video" - CATEGORY = "api node/video/Minimax" + CATEGORY = "api node/video/MiniMax" API_NODE = True OUTPUT_NODE = True @@ -300,7 +300,7 @@ NODE_CLASS_MAPPINGS = { # A dictionary that contains the friendly/humanly readable titles for the nodes NODE_DISPLAY_NAME_MAPPINGS = { - "MinimaxTextToVideoNode": "Minimax Text to Video", - "MinimaxImageToVideoNode": "Minimax Image to Video", - "MinimaxSubjectToVideoNode": "Minimax Subject to Video", + "MinimaxTextToVideoNode": "MiniMax Text to Video", + "MinimaxImageToVideoNode": "MiniMax Image to Video", + "MinimaxSubjectToVideoNode": "MiniMax Subject to Video", } diff --git a/comfy_api_nodes/nodes_openai.py b/comfy_api_nodes/nodes_openai.py index ccdc2bdbf..88db82f09 100644 --- a/comfy_api_nodes/nodes_openai.py +++ b/comfy_api_nodes/nodes_openai.py @@ -101,7 +101,7 @@ class OpenAIDalle2(ComfyNodeABC): RETURN_TYPES = (IO.IMAGE,) FUNCTION = "api_call" - CATEGORY = "api node/image/openai" + CATEGORY = "api node/image/OpenAI" DESCRIPTION = cleandoc(__doc__ or "") API_NODE = True @@ -247,7 +247,7 @@ class OpenAIDalle3(ComfyNodeABC): RETURN_TYPES = (IO.IMAGE,) FUNCTION = "api_call" - CATEGORY = "api node/image/openai" + CATEGORY = "api node/image/OpenAI" DESCRIPTION = cleandoc(__doc__ or "") API_NODE = True @@ -380,7 +380,7 @@ class OpenAIGPTImage1(ComfyNodeABC): RETURN_TYPES = (IO.IMAGE,) FUNCTION = "api_call" - CATEGORY = "api node/image/openai" + CATEGORY = "api node/image/OpenAI" DESCRIPTION = cleandoc(__doc__ or "") API_NODE = True diff --git a/comfy_api_nodes/nodes_pixverse.py b/comfy_api_nodes/nodes_pixverse.py index b2db3f321..864bf40e0 100644 --- a/comfy_api_nodes/nodes_pixverse.py +++ b/comfy_api_nodes/nodes_pixverse.py @@ -54,20 +54,20 @@ def upload_image_to_pixverse(image: torch.Tensor, auth_token=None): response_upload: PixverseImageUploadResponse = operation.execute() if response_upload.Resp is None: - raise Exception(f"Pixverse image upload request failed: '{response_upload.ErrMsg}'") + raise Exception(f"PixVerse image upload request failed: '{response_upload.ErrMsg}'") return response_upload.Resp.img_id class PixverseTemplateNode: """ - Select template for Pixverse Video generation. + Select template for PixVerse Video generation. """ RETURN_TYPES = (PixverseIO.TEMPLATE,) RETURN_NAMES = ("pixverse_template",) FUNCTION = "create_template" - CATEGORY = "api node/video/Pixverse" + CATEGORY = "api node/video/PixVerse" @classmethod def INPUT_TYPES(s): @@ -94,7 +94,7 @@ class PixverseTextToVideoNode(ComfyNodeABC): DESCRIPTION = cleandoc(__doc__ or "") # Handle potential None value FUNCTION = "api_call" API_NODE = True - CATEGORY = "api node/video/Pixverse" + CATEGORY = "api node/video/PixVerse" @classmethod def INPUT_TYPES(s): @@ -142,7 +142,7 @@ class PixverseTextToVideoNode(ComfyNodeABC): "pixverse_template": ( PixverseIO.TEMPLATE, { - "tooltip": "An optional template to influence style of generation, created by the Pixverse Template node." + "tooltip": "An optional template to influence style of generation, created by the PixVerse Template node." } ) }, @@ -195,7 +195,7 @@ class PixverseTextToVideoNode(ComfyNodeABC): response_api = operation.execute() if response_api.Resp is None: - raise Exception(f"Pixverse request failed: '{response_api.ErrMsg}'") + raise Exception(f"PixVerse request failed: '{response_api.ErrMsg}'") operation = PollingOperation( poll_endpoint=ApiEndpoint( @@ -224,7 +224,7 @@ class PixverseImageToVideoNode(ComfyNodeABC): DESCRIPTION = cleandoc(__doc__ or "") # Handle potential None value FUNCTION = "api_call" API_NODE = True - CATEGORY = "api node/video/Pixverse" + CATEGORY = "api node/video/PixVerse" @classmethod def INPUT_TYPES(s): @@ -272,7 +272,7 @@ class PixverseImageToVideoNode(ComfyNodeABC): "pixverse_template": ( PixverseIO.TEMPLATE, { - "tooltip": "An optional template to influence style of generation, created by the Pixverse Template node." + "tooltip": "An optional template to influence style of generation, created by the PixVerse Template node." } ) }, @@ -327,7 +327,7 @@ class PixverseImageToVideoNode(ComfyNodeABC): response_api = operation.execute() if response_api.Resp is None: - raise Exception(f"Pixverse request failed: '{response_api.ErrMsg}'") + raise Exception(f"PixVerse request failed: '{response_api.ErrMsg}'") operation = PollingOperation( poll_endpoint=ApiEndpoint( @@ -356,7 +356,7 @@ class PixverseTransitionVideoNode(ComfyNodeABC): DESCRIPTION = cleandoc(__doc__ or "") # Handle potential None value FUNCTION = "api_call" API_NODE = True - CATEGORY = "api node/video/Pixverse" + CATEGORY = "api node/video/PixVerse" @classmethod def INPUT_TYPES(s): @@ -407,7 +407,7 @@ class PixverseTransitionVideoNode(ComfyNodeABC): "pixverse_template": ( PixverseIO.TEMPLATE, { - "tooltip": "An optional template to influence style of generation, created by the Pixverse Template node." + "tooltip": "An optional template to influence style of generation, created by the PixVerse Template node." } ) }, @@ -465,7 +465,7 @@ class PixverseTransitionVideoNode(ComfyNodeABC): response_api = operation.execute() if response_api.Resp is None: - raise Exception(f"Pixverse request failed: '{response_api.ErrMsg}'") + raise Exception(f"PixVerse request failed: '{response_api.ErrMsg}'") operation = PollingOperation( poll_endpoint=ApiEndpoint( @@ -493,8 +493,8 @@ NODE_CLASS_MAPPINGS = { } NODE_DISPLAY_NAME_MAPPINGS = { - "PixverseTextToVideoNode": "Pixverse Text to Video", - "PixverseImageToVideoNode": "Pixverse Image to Video", - "PixverseTransitionVideoNode": "Pixverse Transition Video", - "PixverseTemplateNode": "Pixverse Template", + "PixverseTextToVideoNode": "PixVerse Text to Video", + "PixverseImageToVideoNode": "PixVerse Image to Video", + "PixverseTransitionVideoNode": "PixVerse Transition Video", + "PixverseTemplateNode": "PixVerse Template", } diff --git a/comfy_api_nodes/nodes_stability.py b/comfy_api_nodes/nodes_stability.py index 4f1676a6a..52fe2417c 100644 --- a/comfy_api_nodes/nodes_stability.py +++ b/comfy_api_nodes/nodes_stability.py @@ -55,7 +55,7 @@ class StabilityStableImageUltraNode: DESCRIPTION = cleandoc(__doc__ or "") # Handle potential None value FUNCTION = "api_call" API_NODE = True - CATEGORY = "api node/image/stability" + CATEGORY = "api node/image/Stability AI" @classmethod def INPUT_TYPES(s): @@ -182,7 +182,7 @@ class StabilityStableImageSD_3_5Node: DESCRIPTION = cleandoc(__doc__ or "") # Handle potential None value FUNCTION = "api_call" API_NODE = True - CATEGORY = "api node/image/stability" + CATEGORY = "api node/image/Stability AI" @classmethod def INPUT_TYPES(s): @@ -320,7 +320,7 @@ class StabilityUpscaleConservativeNode: DESCRIPTION = cleandoc(__doc__ or "") # Handle potential None value FUNCTION = "api_call" API_NODE = True - CATEGORY = "api node/image/stability" + CATEGORY = "api node/image/Stability AI" @classmethod def INPUT_TYPES(s): @@ -420,7 +420,7 @@ class StabilityUpscaleCreativeNode: DESCRIPTION = cleandoc(__doc__ or "") # Handle potential None value FUNCTION = "api_call" API_NODE = True - CATEGORY = "api node/image/stability" + CATEGORY = "api node/image/Stability AI" @classmethod def INPUT_TYPES(s): @@ -543,7 +543,7 @@ class StabilityUpscaleFastNode: DESCRIPTION = cleandoc(__doc__ or "") # Handle potential None value FUNCTION = "api_call" API_NODE = True - CATEGORY = "api node/image/stability" + CATEGORY = "api node/image/Stability AI" @classmethod def INPUT_TYPES(s): @@ -601,9 +601,9 @@ NODE_CLASS_MAPPINGS = { # A dictionary that contains the friendly/humanly readable titles for the nodes NODE_DISPLAY_NAME_MAPPINGS = { - "StabilityStableImageUltraNode": "Stability Stable Image Ultra", - "StabilityStableImageSD_3_5Node": "Stability Stable Diffusion 3.5 Image", - "StabilityUpscaleConservativeNode": "Stability Upscale Conservative", - "StabilityUpscaleCreativeNode": "Stability Upscale Creative", - "StabilityUpscaleFastNode": "Stability Upscale Fast", + "StabilityStableImageUltraNode": "Stability AI Stable Image Ultra", + "StabilityStableImageSD_3_5Node": "Stability AI Stable Diffusion 3.5 Image", + "StabilityUpscaleConservativeNode": "Stability AI Upscale Conservative", + "StabilityUpscaleCreativeNode": "Stability AI Upscale Creative", + "StabilityUpscaleFastNode": "Stability AI Upscale Fast", } From 87047e996f6af19f9823e323c4a9aace07318cd3 Mon Sep 17 00:00:00 2001 From: Jedrzej Kosinski Date: Mon, 5 May 2025 12:10:06 -0500 Subject: [PATCH 102/121] Add back Recraft Style - Infinite Style Library node (#141) --- comfy_api_nodes/nodes_recraft.py | 59 +++++++++++++++++++++++++++++--- 1 file changed, 54 insertions(+), 5 deletions(-) diff --git a/comfy_api_nodes/nodes_recraft.py b/comfy_api_nodes/nodes_recraft.py index a5e513e05..994f377d1 100644 --- a/comfy_api_nodes/nodes_recraft.py +++ b/comfy_api_nodes/nodes_recraft.py @@ -33,6 +33,7 @@ import json import os import torch from io import BytesIO +from PIL import UnidentifiedImageError def handle_recraft_file_request( @@ -146,6 +147,21 @@ def recraft_multipart_parser(data, parent_key=None, formatter: callable=None, co return dict(converted) +class handle_recraft_image_output: + """ + Catch an exception related to receiving SVG data instead of image, when Infinite Style Library style_id is in use. + """ + def __init__(self): + pass + + def __enter__(self): + pass + + def __exit__(self, exc_type, exc_val, exc_tb): + if exc_type is not None and exc_type is UnidentifiedImageError: + raise Exception("Received output data was not an image; likely an SVG. If you used style_id, make sure it is not a Vector art style.") + + class SVG: """ Stores SVG representations via a list of BytesIO objects. @@ -372,6 +388,34 @@ class RecraftStyleV3LogoRasterNode(RecraftStyleV3RealisticImageNode): RECRAFT_STYLE = RecraftStyleV3.logo_raster +class RecraftStyleInfiniteStyleLibrary: + """ + Select style based on preexisting UUID from Recraft's Infinite Style Library. + """ + + RETURN_TYPES = (RecraftIO.STYLEV3,) + RETURN_NAMES = ("recraft_style",) + DESCRIPTION = cleandoc(__doc__ or "") # Handle potential None value + FUNCTION = "create_style" + CATEGORY = "api node/image/Recraft" + + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "style_id": (IO.STRING, { + "default": "", + "tooltip": "UUID of style from Infinite Style Library.", + }) + } + } + + def create_style(self, style_id: str): + if not style_id: + raise Exception("The style_id input cannot be empty.") + return (RecraftStyle(style_id=style_id),) + + class RecraftTextToImageNode: """ Generates images synchronously based on prompt and resolution. @@ -491,9 +535,10 @@ class RecraftTextToImageNode: response: RecraftImageGenerationResponse = operation.execute() images = [] for data in response.data: - image = bytesio_to_image_tensor( - download_url_to_bytesio(data.url, timeout=1024) - ) + with handle_recraft_image_output(): + image = bytesio_to_image_tensor( + download_url_to_bytesio(data.url, timeout=1024) + ) if len(image.shape) < 4: image = image.unsqueeze(0) images.append(image) @@ -625,7 +670,8 @@ class RecraftImageToImageNode: request=request, auth_token=auth_token, ) - images.append(torch.cat([bytesio_to_image_tensor(x) for x in sub_bytes], dim=0)) + with handle_recraft_image_output(): + images.append(torch.cat([bytesio_to_image_tensor(x) for x in sub_bytes], dim=0)) pbar.update(1) images_tensor = torch.cat(images, dim=0) @@ -737,7 +783,8 @@ class RecraftImageInpaintingNode: request=request, auth_token=auth_token, ) - images.append(torch.cat([bytesio_to_image_tensor(x) for x in sub_bytes], dim=0)) + with handle_recraft_image_output(): + images.append(torch.cat([bytesio_to_image_tensor(x) for x in sub_bytes], dim=0)) pbar.update(1) images_tensor = torch.cat(images, dim=0) @@ -1143,6 +1190,7 @@ NODE_CLASS_MAPPINGS = { "RecraftStyleV3RealisticImage": RecraftStyleV3RealisticImageNode, "RecraftStyleV3DigitalIllustration": RecraftStyleV3DigitalIllustrationNode, "RecraftStyleV3LogoRaster": RecraftStyleV3LogoRasterNode, + "RecraftStyleV3InfiniteStyleLibrary": RecraftStyleInfiniteStyleLibrary, "RecraftColorRGB": RecraftColorRGBNode, "RecraftControls": RecraftControlsNode, "SaveSVG": SaveSVGNode, @@ -1162,6 +1210,7 @@ NODE_DISPLAY_NAME_MAPPINGS = { "RecraftStyleV3RealisticImage": "Recraft Style - Realistic Image", "RecraftStyleV3DigitalIllustration": "Recraft Style - Digital Illustration", "RecraftStyleV3LogoRaster": "Recraft Style - Logo Raster", + "RecraftStyleV3InfiniteStyleLibrary": "Recraft Style - Infinite Style Library", "RecraftColorRGB": "Recraft Color RGB", "RecraftControls": "Recraft Controls", "SaveSVG": "Save SVG", From d78764dda967d03af841e19e50861d59b370a938 Mon Sep 17 00:00:00 2001 From: Robin Huang Date: Mon, 5 May 2025 12:36:24 -0700 Subject: [PATCH 103/121] Fixed Kling: Check attributes of pydantic types. (#144) --- comfy_api_nodes/nodes_kling.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/comfy_api_nodes/nodes_kling.py b/comfy_api_nodes/nodes_kling.py index b36c1c544..1bd834fd3 100644 --- a/comfy_api_nodes/nodes_kling.py +++ b/comfy_api_nodes/nodes_kling.py @@ -151,8 +151,9 @@ def is_valid_task_creation_response(response: KlingText2VideoResponse) -> bool: def is_valid_video_response(response: KlingText2VideoResponse) -> bool: """Verifies that the response contains a task result with at least one video.""" return ( - "task_result" in response.data - and "videos" in response.data.task_result + response.data is not None + and response.data.task_result is not None + and response.data.task_result.videos is not None and len(response.data.task_result.videos) > 0 ) From 7361f8789d1c053e584d627261bdf52bc2b0c804 Mon Sep 17 00:00:00 2001 From: Christian Byrne Date: Mon, 5 May 2025 13:17:08 -0700 Subject: [PATCH 104/121] Bump `comfyui-workflow-templates` version (#142) --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index bc79168c1..a0c8fefac 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ comfyui-frontend-package==1.17.11 -comfyui-workflow-templates==0.1.8 +comfyui-workflow-templates==0.1.9 torch torchsde torchvision From 7399187340667f0a64d9436e3fda946522a30d85 Mon Sep 17 00:00:00 2001 From: Christian Byrne Date: Mon, 5 May 2025 13:17:49 -0700 Subject: [PATCH 105/121] [Kling] Print response data when error validating response (#146) --- comfy_api_nodes/nodes_kling.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/comfy_api_nodes/nodes_kling.py b/comfy_api_nodes/nodes_kling.py index 1bd834fd3..2ad1f8949 100644 --- a/comfy_api_nodes/nodes_kling.py +++ b/comfy_api_nodes/nodes_kling.py @@ -192,7 +192,7 @@ def validate_video_result_response(response) -> None: """Validates that the Kling task result contains a video.""" if not is_valid_video_response(response): error_msg = f"Kling task {response.data.task_id} succeeded but no video data found in response." - logging.error(error_msg) + logging.error(f"Error: {error_msg}.\nResponse: {response}") raise KlingApiError(error_msg) @@ -200,7 +200,7 @@ def validate_image_result_response(response) -> None: """Validates that the Kling task result contains an image.""" if not is_valid_image_response(response): error_msg = f"Kling task {response.data.task_id} succeeded but no image data found in response." - logging.error(error_msg) + logging.error(f"Error: {error_msg}.\nResponse: {response}") raise KlingApiError(error_msg) From 1ec10422d75d8fb1f72458c52cfd80563612ee23 Mon Sep 17 00:00:00 2001 From: Christian Byrne Date: Mon, 5 May 2025 13:20:28 -0700 Subject: [PATCH 106/121] Fix: error validating Kling image response, trying to use `"key" in` on Pydantic class instance (#147) --- comfy_api_nodes/nodes_kling.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/comfy_api_nodes/nodes_kling.py b/comfy_api_nodes/nodes_kling.py index 2ad1f8949..a357e4bc5 100644 --- a/comfy_api_nodes/nodes_kling.py +++ b/comfy_api_nodes/nodes_kling.py @@ -161,8 +161,9 @@ def is_valid_video_response(response: KlingText2VideoResponse) -> bool: def is_valid_image_response(response: KlingVirtualTryOnResponse) -> bool: """Verifies that the response contains a task result with at least one image.""" return ( - "task_result" in response.data - and "images" in response.data.task_result + response.data is not None + and response.data.task_result is not None + and response.data.task_result.images is not None and len(response.data.task_result.images) > 0 ) From e493f8a4cd6ddbef78122c6b9640503bd0870fcc Mon Sep 17 00:00:00 2001 From: Christian Byrne Date: Mon, 5 May 2025 15:47:52 -0700 Subject: [PATCH 107/121] [Kling] Fix: Correct/verify supported subset of input combos in Kling nodes (#149) --- comfy_api_nodes/nodes_kling.py | 86 +++++++++++++++------------------- 1 file changed, 38 insertions(+), 48 deletions(-) diff --git a/comfy_api_nodes/nodes_kling.py b/comfy_api_nodes/nodes_kling.py index a357e4bc5..6116cfee3 100644 --- a/comfy_api_nodes/nodes_kling.py +++ b/comfy_api_nodes/nodes_kling.py @@ -1,29 +1,7 @@ -""" -Kling API Nodes +"""Kling API Nodes -Compatibility Table -| Mode | Duration | Model Name | Camera Control | Image Tail | -|------|----------|------------------|----------------|------------| -| std | 5 | kling-v1 | No | Yes | -| std | 5 | kling-v1-5 | No | Yes | -| std | 5 | kling-v1-6 | No | No | -| std | 5 | kling-v2-master | No | No | -| std | 10 | kling-v1 | No | No | -| std | 10 | kling-v1-5 | No | No | -| std | 10 | kling-v1-6 | No | No | -| std | 10 | kling-v2-master | No | No | -| pro | 5 | kling-v1 | No | Yes | -| pro | 5 | kling-v1-5 | Yes | Yes | -| pro | 5 | kling-v1-6 | No | Yes | -| pro | 5 | kling-v2-master | No | No | -| pro | 10 | kling-v1 | No | No | -| pro | 10 | kling-v1-5 | No | Yes | -| pro | 10 | kling-v1-6 | No | Yes | -| pro | 10 | kling-v2-master | No | No | - -**Note**: Although the combo of pro mode, kling-v1-5 model, and 5s duration -supports both camera_control and image_tail, you can only use one feature -at a time. +For source of truth on the allowed permutations of request fields, please reference: +- [Compatibility Table](https://app.klingai.com/global/dev/document-api/apiReference/model/skillsMap) """ from typing import Optional, TypeVar, Any @@ -355,8 +333,26 @@ class KlingCameraControls(KlingNodeBase): class KlingTextToVideoNode(KlingNodeBase): """Kling Text to Video Node""" + @staticmethod + def get_mode_string_mapping() -> dict[str, tuple[str, str, str]]: + """ + Returns a mapping of mode strings to their corresponding (mode, duration, model_name) tuples. + Only includes config combos that support the `image_tail` request field. + + See: [Kling API Docs Capability Map](https://app.klingai.com/global/dev/document-api/apiReference/model/skillsMap) + """ + return { + "standard mode / 5s duration / kling-v1": ("std", "5", "kling-v1"), + "standard mode / 10s duration / kling-v1": ("std", "10", "kling-v1"), + "pro mode / 5s duration / kling-v1": ("pro", "5", "kling-v1"), + "pro mode / 10s duration / kling-v1": ("pro", "10", "kling-v1"), + "standard mode / 5s duration / kling-v1-6": ("std", "5", "kling-v1-6"), + "standard mode / 10s duration / kling-v1-6": ("std", "10", "kling-v1-6"), + } + @classmethod def INPUT_TYPES(s): + modes = list(KlingTextToVideoNode.get_mode_string_mapping().keys()) return { "required": { "prompt": model_field_to_node_input( @@ -365,33 +361,22 @@ class KlingTextToVideoNode(KlingNodeBase): "negative_prompt": model_field_to_node_input( IO.STRING, KlingText2VideoRequest, "negative_prompt", multiline=True ), - "model_name": model_field_to_node_input( - IO.COMBO, - KlingText2VideoRequest, - "model_name", - enum_type=KlingVideoGenModelName, - ), "cfg_scale": model_field_to_node_input( IO.FLOAT, KlingText2VideoRequest, "cfg_scale" ), - "mode": model_field_to_node_input( - IO.COMBO, - KlingText2VideoRequest, - "mode", - enum_type=KlingVideoGenMode, - ), - "duration": model_field_to_node_input( - IO.COMBO, - KlingText2VideoRequest, - "duration", - enum_type=KlingVideoGenDuration, - ), "aspect_ratio": model_field_to_node_input( IO.COMBO, KlingText2VideoRequest, "aspect_ratio", enum_type=KlingVideoGenAspectRatio, ), + "mode": ( + modes, + { + "default": modes[4], + "tooltip": "The configuration to use for the video generation following the format: mode / duration / model_name.", + }, + ), }, "hidden": {"auth_token": "AUTH_TOKEN_COMFY_ORG"}, } @@ -415,15 +400,14 @@ class KlingTextToVideoNode(KlingNodeBase): self, prompt: str, negative_prompt: str, - model_name: str, cfg_scale: float, mode: str, - duration: int, aspect_ratio: str, camera_control: Optional[KlingCameraControl] = None, auth_token: Optional[str] = None, ) -> tuple[VideoFromFile, str, str]: validate_prompts(prompt, negative_prompt, MAX_PROMPT_LENGTH_T2V) + mode, duration, model_name = self.get_mode_string_mapping()[mode] initial_operation = SynchronousOperation( endpoint=ApiEndpoint( path=PATH_TEXT_TO_VIDEO, @@ -543,7 +527,7 @@ class KlingImage2VideoNode(KlingNodeBase): enum_type=KlingVideoGenModelName, ), "cfg_scale": model_field_to_node_input( - IO.FLOAT, KlingImage2VideoRequest, "cfg_scale" + IO.FLOAT, KlingImage2VideoRequest, "cfg_scale", default=0.8 ), "mode": model_field_to_node_input( IO.COMBO, @@ -597,6 +581,11 @@ class KlingImage2VideoNode(KlingNodeBase): auth_token: Optional[str] = None, ) -> tuple[VideoFromFile]: validate_prompts(prompt, negative_prompt, MAX_PROMPT_LENGTH_I2V) + + if camera_control is not None: + # Camera control type for image 2 video is always simple + camera_control.type = KlingCameraControlType.simple + initial_operation = SynchronousOperation( endpoint=ApiEndpoint( path=PATH_IMAGE_TO_VIDEO, @@ -711,14 +700,15 @@ class KlingStartEndFrameNode(KlingImage2VideoNode): """ Returns a mapping of mode strings to their corresponding (mode, duration, model_name) tuples. Only includes config combos that support the `image_tail` request field. + + See: [Kling API Docs Capability Map](https://app.klingai.com/global/dev/document-api/apiReference/model/skillsMap) """ return { "standard mode / 5s duration / kling-v1": ("std", "5", "kling-v1"), - "standard mode / 5s duration / kling-v1-5": ("std", "5", "kling-v1-5"), "pro mode / 5s duration / kling-v1": ("pro", "5", "kling-v1"), "pro mode / 5s duration / kling-v1-5": ("pro", "5", "kling-v1-5"), - "pro mode / 5s duration / kling-v1-6": ("pro", "5", "kling-v1-6"), "pro mode / 10s duration / kling-v1-5": ("pro", "10", "kling-v1-5"), + "pro mode / 5s duration / kling-v1-6": ("pro", "5", "kling-v1-6"), "pro mode / 10s duration / kling-v1-6": ("pro", "10", "kling-v1-6"), } From 7e1ce6a0495c24a2736d675769effb1c3985bbb9 Mon Sep 17 00:00:00 2001 From: Christian Byrne Date: Mon, 5 May 2025 20:00:34 -0700 Subject: [PATCH 108/121] [Kling] Fix typo in node description (#150) --- comfy_api_nodes/nodes_kling.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/comfy_api_nodes/nodes_kling.py b/comfy_api_nodes/nodes_kling.py index 6116cfee3..ffa781249 100644 --- a/comfy_api_nodes/nodes_kling.py +++ b/comfy_api_nodes/nodes_kling.py @@ -808,7 +808,7 @@ class KlingVideoExtendNode(KlingNodeBase): RETURN_TYPES = ("VIDEO", "STRING", "STRING") RETURN_NAMES = ("VIDEO", "video_id", "duration") - DESCRIPTION = "Kling Video Extend Node. Extend videos made by other Kling nodes. The Kling ID is output by Kling Nodes." + DESCRIPTION = "Kling Video Extend Node. Extend videos made by other Kling nodes. The video_id is created by using other Kling Nodes." def get_response(self, task_id: str, auth_token: str) -> KlingVideoExtendResponse: return poll_until_finished( From 80e49682d873c063cccff8295821ee1294abe33f Mon Sep 17 00:00:00 2001 From: Christian Byrne Date: Mon, 5 May 2025 20:00:42 -0700 Subject: [PATCH 109/121] [Kling] Fix: CFG min/max not being enforced (#151) --- comfy_api_nodes/nodes_kling.py | 42 +++++++++++++++++++++++++++++----- 1 file changed, 36 insertions(+), 6 deletions(-) diff --git a/comfy_api_nodes/nodes_kling.py b/comfy_api_nodes/nodes_kling.py index ffa781249..5228e77f1 100644 --- a/comfy_api_nodes/nodes_kling.py +++ b/comfy_api_nodes/nodes_kling.py @@ -362,7 +362,12 @@ class KlingTextToVideoNode(KlingNodeBase): IO.STRING, KlingText2VideoRequest, "negative_prompt", multiline=True ), "cfg_scale": model_field_to_node_input( - IO.FLOAT, KlingText2VideoRequest, "cfg_scale" + IO.FLOAT, + KlingText2VideoRequest, + "cfg_scale", + default=1.0, + min=0.0, + max=1.0, ), "aspect_ratio": model_field_to_node_input( IO.COMBO, @@ -459,7 +464,12 @@ class KlingCameraControlT2VNode(KlingTextToVideoNode): multiline=True, ), "cfg_scale": model_field_to_node_input( - IO.FLOAT, KlingText2VideoRequest, "cfg_scale" + IO.FLOAT, + KlingText2VideoRequest, + "cfg_scale", + default=0.75, + min=0.0, + max=1.0, ), "aspect_ratio": model_field_to_node_input( IO.COMBO, @@ -527,7 +537,12 @@ class KlingImage2VideoNode(KlingNodeBase): enum_type=KlingVideoGenModelName, ), "cfg_scale": model_field_to_node_input( - IO.FLOAT, KlingImage2VideoRequest, "cfg_scale", default=0.8 + IO.FLOAT, + KlingImage2VideoRequest, + "cfg_scale", + default=0.8, + min=0.0, + max=1.0, ), "mode": model_field_to_node_input( IO.COMBO, @@ -646,7 +661,12 @@ class KlingCameraControlI2VNode(KlingImage2VideoNode): multiline=True, ), "cfg_scale": model_field_to_node_input( - IO.FLOAT, KlingImage2VideoRequest, "cfg_scale" + IO.FLOAT, + KlingImage2VideoRequest, + "cfg_scale", + default=0.75, + min=0.0, + max=1.0, ), "aspect_ratio": model_field_to_node_input( IO.COMBO, @@ -733,7 +753,12 @@ class KlingStartEndFrameNode(KlingImage2VideoNode): multiline=True, ), "cfg_scale": model_field_to_node_input( - IO.FLOAT, KlingImage2VideoRequest, "cfg_scale" + IO.FLOAT, + KlingImage2VideoRequest, + "cfg_scale", + default=0.5, + min=0.0, + max=1.0, ), "aspect_ratio": model_field_to_node_input( IO.COMBO, @@ -797,7 +822,12 @@ class KlingVideoExtendNode(KlingNodeBase): multiline=True, ), "cfg_scale": model_field_to_node_input( - IO.FLOAT, KlingVideoExtendRequest, "cfg_scale" + IO.FLOAT, + KlingVideoExtendRequest, + "cfg_scale", + default=0.5, + min=0.0, + max=1.0, ), "video_id": model_field_to_node_input( IO.STRING, KlingVideoExtendRequest, "video_id", forceInput=True From 11a35d6dfb84bd689f786194cf7d60e9b48fc429 Mon Sep 17 00:00:00 2001 From: Jedrzej Kosinski Date: Mon, 5 May 2025 22:20:26 -0500 Subject: [PATCH 110/121] Rebase launch-rebase (private) on prep-branch (public copy of master) (#153) Co-authored-by: comfyanonymous Co-authored-by: AustinMroz Co-authored-by: comfyanonymous <121283862+comfyanonymous@users.noreply.github.com> Co-authored-by: Benjamin Lu Co-authored-by: Andrew Kvochko Co-authored-by: Pam <42671363+pamparamm@users.noreply.github.com> Co-authored-by: chaObserv <154517000+chaObserv@users.noreply.github.com> Co-authored-by: Yoland Yan <4950057+yoland68@users.noreply.github.com> Co-authored-by: guill Co-authored-by: Chenlei Hu Co-authored-by: Terry Jia Co-authored-by: Silver <65376327+silveroxides@users.noreply.github.com> Co-authored-by: Christian Byrne Co-authored-by: catboxanon <122327233+catboxanon@users.noreply.github.com> Co-authored-by: liesen Co-authored-by: Kohaku-Blueleaf <59680068+KohakuBlueleaf@users.noreply.github.com> Co-authored-by: Robin Huang Co-authored-by: thot-experiment Co-authored-by: thot experiment <94414189+thot-experiment@users.noreply.github.com> --- .ci/update_windows/update.py | 7 +- .github/workflows/stable-release.yml | 6 +- .../windows_release_dependencies.yml | 4 +- .github/workflows/windows_release_package.yml | 6 +- README.md | 20 +- app/custom_node_manager.py | 43 +-- app/user_manager.py | 106 +++++++ comfy/cli_args.py | 1 + comfy/comfy_types/node_typing.py | 4 + comfy/k_diffusion/sampling.py | 34 ++- comfy/ldm/chroma/layers.py | 183 ++++++++++++ comfy/ldm/chroma/model.py | 271 ++++++++++++++++++ comfy/ldm/cosmos/blocks.py | 11 +- comfy/ldm/cosmos/model.py | 4 +- .../genmo/joint_model/asymm_models_joint.py | 9 +- comfy/ldm/genmo/joint_model/layers.py | 11 - comfy/ldm/hidream/model.py | 3 + comfy/ldm/hydit/models.py | 4 +- comfy/ldm/lightricks/model.py | 5 +- comfy/ldm/lumina/model.py | 18 +- comfy/ldm/wan/model.py | 8 +- comfy/lora.py | 7 + comfy/model_base.py | 20 +- comfy/model_detection.py | 15 +- comfy/model_management.py | 52 +++- comfy/model_sampling.py | 3 +- comfy/ops.py | 24 +- comfy/samplers.py | 2 +- comfy/sd.py | 13 +- comfy/sd1_clip.py | 17 +- comfy/sdxl_clip.py | 4 +- comfy/supported_models.py | 33 ++- comfy/text_encoders/flux.py | 4 +- comfy/text_encoders/hidream.py | 8 +- comfy/text_encoders/hunyuan_video.py | 4 +- comfy/text_encoders/hydit.py | 4 +- comfy/text_encoders/sd3_clip.py | 6 +- comfy/weight_adapter/boft.py | 34 +-- comfy/weight_adapter/oft.py | 20 +- comfy_extras/nodes_cond.py | 25 +- comfy_extras/nodes_custom_sampler.py | 51 ++++ comfy_extras/nodes_lt.py | 14 +- comfy_extras/nodes_model_merging.py | 5 +- comfy_extras/nodes_optimalsteps.py | 3 +- comfy_extras/nodes_post_processing.py | 1 + comfy_extras/nodes_preview_any.py | 43 +++ comfy_extras/nodes_webcam.py | 2 +- comfyui_version.py | 2 +- hook_breaker_ac10a0.py | 17 ++ main.py | 7 +- nodes.py | 3 +- pyproject.toml | 3 +- requirements.txt | 2 +- tests-unit/comfy_api_test/input_impl_test.py | 91 ++++++ .../prompt_server_test/user_manager_test.py | 58 ++++ 55 files changed, 1205 insertions(+), 150 deletions(-) create mode 100644 comfy/ldm/chroma/layers.py create mode 100644 comfy/ldm/chroma/model.py create mode 100644 comfy_extras/nodes_preview_any.py create mode 100644 hook_breaker_ac10a0.py create mode 100644 tests-unit/comfy_api_test/input_impl_test.py diff --git a/.ci/update_windows/update.py b/.ci/update_windows/update.py index 731b6bc53..51a263203 100755 --- a/.ci/update_windows/update.py +++ b/.ci/update_windows/update.py @@ -63,7 +63,12 @@ except: print("checking out master branch") # noqa: T201 branch = repo.lookup_branch('master') if branch is None: - ref = repo.lookup_reference('refs/remotes/origin/master') + try: + ref = repo.lookup_reference('refs/remotes/origin/master') + except: + print("pulling.") # noqa: T201 + pull(repo) + ref = repo.lookup_reference('refs/remotes/origin/master') repo.checkout(ref) branch = repo.lookup_branch('master') if branch is None: diff --git a/.github/workflows/stable-release.yml b/.github/workflows/stable-release.yml index 40df7ab88..a046ff9ea 100644 --- a/.github/workflows/stable-release.yml +++ b/.github/workflows/stable-release.yml @@ -12,7 +12,7 @@ on: description: 'CUDA version' required: true type: string - default: "126" + default: "128" python_minor: description: 'Python minor version' required: true @@ -22,7 +22,7 @@ on: description: 'Python patch version' required: true type: string - default: "9" + default: "10" jobs: @@ -91,6 +91,8 @@ jobs: cd ComfyUI_windows_portable python_embeded/python.exe -s ComfyUI/main.py --quick-test-for-ci --cpu + python_embeded/python.exe -s ./update/update.py ComfyUI/ + ls - name: Upload binaries to release diff --git a/.github/workflows/windows_release_dependencies.yml b/.github/workflows/windows_release_dependencies.yml index 7a8ec5782..dfdb96d50 100644 --- a/.github/workflows/windows_release_dependencies.yml +++ b/.github/workflows/windows_release_dependencies.yml @@ -17,7 +17,7 @@ on: description: 'cuda version' required: true type: string - default: "126" + default: "128" python_minor: description: 'python minor version' @@ -29,7 +29,7 @@ on: description: 'python patch version' required: true type: string - default: "9" + default: "10" # push: # branches: # - master diff --git a/.github/workflows/windows_release_package.yml b/.github/workflows/windows_release_package.yml index dc79b1f4a..3926a65f3 100644 --- a/.github/workflows/windows_release_package.yml +++ b/.github/workflows/windows_release_package.yml @@ -7,7 +7,7 @@ on: description: 'cuda version' required: true type: string - default: "126" + default: "128" python_minor: description: 'python minor version' @@ -19,7 +19,7 @@ on: description: 'python patch version' required: true type: string - default: "9" + default: "10" # push: # branches: # - master @@ -88,6 +88,8 @@ jobs: cd ComfyUI_windows_portable python_embeded/python.exe -s ComfyUI/main.py --quick-test-for-ci --cpu + python_embeded/python.exe -s ./update/update.py ComfyUI/ + ls - name: Upload binaries to release diff --git a/README.md b/README.md index 62800bb4f..0f39cfce2 100644 --- a/README.md +++ b/README.md @@ -49,7 +49,6 @@ Supports all operating systems and GPU types (NVIDIA, AMD, Intel, Apple Silicon, ## [Examples](https://comfyanonymous.github.io/ComfyUI_examples/) See what ComfyUI can do with the [example workflows](https://comfyanonymous.github.io/ComfyUI_examples/). - ## Features - Nodes/graph/flowchart interface to experiment and create complex Stable Diffusion workflows without needing to code anything. - Image Models @@ -99,6 +98,23 @@ See what ComfyUI can do with the [example workflows](https://comfyanonymous.gith Workflow examples can be found on the [Examples page](https://comfyanonymous.github.io/ComfyUI_examples/) +## Release Process + +ComfyUI follows a weekly release cycle every Friday, with three interconnected repositories: + +1. **[ComfyUI Core](https://github.com/comfyanonymous/ComfyUI)** + - Releases a new stable version (e.g., v0.7.0) + - Serves as the foundation for the desktop release + +2. **[ComfyUI Desktop](https://github.com/Comfy-Org/desktop)** + - Builds a new release using the latest stable core version + - Version numbers match the core release (e.g., Desktop v1.7.0 uses Core v1.7.0) + +3. **[ComfyUI Frontend](https://github.com/Comfy-Org/ComfyUI_frontend)** + - Weekly frontend updates are merged into the core repository + - Features are frozen for the upcoming core release + - Development continues for the next release cycle + ## Shortcuts | Keybind | Explanation | @@ -149,8 +165,6 @@ Simply download, extract with [7-Zip](https://7-zip.org) and run. Make sure you If you have trouble extracting it, right click the file -> properties -> unblock -If you have a 50 series Blackwell card like a 5090 or 5080 see [this discussion thread](https://github.com/comfyanonymous/ComfyUI/discussions/6643) - #### How do I share models between another UI and ComfyUI? See the [Config file](extra_model_paths.yaml.example) to set the search paths for models. In the standalone windows build you can find this file in the ComfyUI directory. Rename this file to extra_model_paths.yaml and edit it with your favorite text editor. diff --git a/app/custom_node_manager.py b/app/custom_node_manager.py index 42b0d75ba..281febca9 100644 --- a/app/custom_node_manager.py +++ b/app/custom_node_manager.py @@ -93,16 +93,20 @@ class CustomNodeManager: def add_routes(self, routes, webapp, loadedModules): + example_workflow_folder_names = ["example_workflows", "example", "examples", "workflow", "workflows"] + @routes.get("/workflow_templates") async def get_workflow_templates(request): """Returns a web response that contains the map of custom_nodes names and their associated workflow templates. The ones without templates are omitted.""" - files = [ - file - for folder in folder_paths.get_folder_paths("custom_nodes") - for file in glob.glob( - os.path.join(folder, "*/example_workflows/*.json") - ) - ] + + files = [] + + for folder in folder_paths.get_folder_paths("custom_nodes"): + for folder_name in example_workflow_folder_names: + pattern = os.path.join(folder, f"*/{folder_name}/*.json") + matched_files = glob.glob(pattern) + files.extend(matched_files) + workflow_templates_dict = ( {} ) # custom_nodes folder name -> example workflow names @@ -118,15 +122,22 @@ class CustomNodeManager: # Serve workflow templates from custom nodes. for module_name, module_dir in loadedModules: - workflows_dir = os.path.join(module_dir, "example_workflows") - if os.path.exists(workflows_dir): - webapp.add_routes( - [ - web.static( - "/api/workflow_templates/" + module_name, workflows_dir - ) - ] - ) + for folder_name in example_workflow_folder_names: + workflows_dir = os.path.join(module_dir, folder_name) + + if os.path.exists(workflows_dir): + if folder_name != "example_workflows": + logging.debug( + "Found example workflow folder '%s' for custom node '%s', consider renaming it to 'example_workflows'", + folder_name, module_name) + + webapp.add_routes( + [ + web.static( + "/api/workflow_templates/" + module_name, workflows_dir + ) + ] + ) @routes.get("/i18n") async def get_i18n(request): diff --git a/app/user_manager.py b/app/user_manager.py index e7381e621..d31da5b9b 100644 --- a/app/user_manager.py +++ b/app/user_manager.py @@ -197,6 +197,112 @@ class UserManager(): return web.json_response(results) + @routes.get("/v2/userdata") + async def list_userdata_v2(request): + """ + List files and directories in a user's data directory. + + This endpoint provides a structured listing of contents within a specified + subdirectory of the user's data storage. + + Query Parameters: + - path (optional): The relative path within the user's data directory + to list. Defaults to the root (''). + + Returns: + - 400: If the requested path is invalid, outside the user's data directory, or is not a directory. + - 404: If the requested path does not exist. + - 403: If the user is invalid. + - 500: If there is an error reading the directory contents. + - 200: JSON response containing a list of file and directory objects. + Each object includes: + - name: The name of the file or directory. + - type: 'file' or 'directory'. + - path: The relative path from the user's data root. + - size (for files): The size in bytes. + - modified (for files): The last modified timestamp (Unix epoch). + """ + requested_rel_path = request.rel_url.query.get('path', '') + + # URL-decode the path parameter + try: + requested_rel_path = parse.unquote(requested_rel_path) + except Exception as e: + logging.warning(f"Failed to decode path parameter: {requested_rel_path}, Error: {e}") + return web.Response(status=400, text="Invalid characters in path parameter") + + + # Check user validity and get the absolute path for the requested directory + try: + base_user_path = self.get_request_user_filepath(request, None, create_dir=False) + + if requested_rel_path: + target_abs_path = self.get_request_user_filepath(request, requested_rel_path, create_dir=False) + else: + target_abs_path = base_user_path + + except KeyError as e: + # Invalid user detected by get_request_user_id inside get_request_user_filepath + logging.warning(f"Access denied for user: {e}") + return web.Response(status=403, text="Invalid user specified in request") + + + if not target_abs_path: + # Path traversal or other issue detected by get_request_user_filepath + return web.Response(status=400, text="Invalid path requested") + + # Handle cases where the user directory or target path doesn't exist + if not os.path.exists(target_abs_path): + # Check if it's the base user directory that's missing (new user case) + if target_abs_path == base_user_path: + # It's okay if the base user directory doesn't exist yet, return empty list + return web.json_response([]) + else: + # A specific subdirectory was requested but doesn't exist + return web.Response(status=404, text="Requested path not found") + + if not os.path.isdir(target_abs_path): + return web.Response(status=400, text="Requested path is not a directory") + + results = [] + try: + for root, dirs, files in os.walk(target_abs_path, topdown=True): + # Process directories + for dir_name in dirs: + dir_path = os.path.join(root, dir_name) + rel_path = os.path.relpath(dir_path, base_user_path).replace(os.sep, '/') + results.append({ + "name": dir_name, + "path": rel_path, + "type": "directory" + }) + + # Process files + for file_name in files: + file_path = os.path.join(root, file_name) + rel_path = os.path.relpath(file_path, base_user_path).replace(os.sep, '/') + entry_info = { + "name": file_name, + "path": rel_path, + "type": "file" + } + try: + stats = os.stat(file_path) # Use os.stat for potentially better performance with os.walk + entry_info["size"] = stats.st_size + entry_info["modified"] = stats.st_mtime + except OSError as stat_error: + logging.warning(f"Could not stat file {file_path}: {stat_error}") + pass # Include file with available info + results.append(entry_info) + except OSError as e: + logging.error(f"Error listing directory {target_abs_path}: {e}") + return web.Response(status=500, text="Error reading directory contents") + + # Sort results alphabetically, directories first then files + results.sort(key=lambda x: (x['type'] != 'directory', x['name'].lower())) + + return web.json_response(results) + def get_user_data_path(request, check_exists = False, param = "file"): file = request.match_info.get(param, None) if not file: diff --git a/comfy/cli_args.py b/comfy/cli_args.py index 155ec53d2..ef5ab6277 100644 --- a/comfy/cli_args.py +++ b/comfy/cli_args.py @@ -128,6 +128,7 @@ vram_group.add_argument("--cpu", action="store_true", help="To use the CPU for e parser.add_argument("--reserve-vram", type=float, default=None, help="Set the amount of vram in GB you want to reserve for use by your OS/other software. By default some amount is reserved depending on your OS.") +parser.add_argument("--async-offload", action="store_true", help="Use async weight offloading.") parser.add_argument("--default-hashing-function", type=str, choices=['md5', 'sha1', 'sha256', 'sha512'], default='sha256', help="Allows you to choose the hash function to use for duplicate filename / contents comparison. Default is sha256.") diff --git a/comfy/comfy_types/node_typing.py b/comfy/comfy_types/node_typing.py index 9a345586e..2ffc9c021 100644 --- a/comfy/comfy_types/node_typing.py +++ b/comfy/comfy_types/node_typing.py @@ -121,6 +121,10 @@ class InputTypeOptions(TypedDict): Available from frontend v1.17.5 Ref: https://github.com/Comfy-Org/ComfyUI_frontend/pull/3548 """ + widgetType: NotRequired[str] + """Specifies a type to be used for widget initialization if different from the input type. + Available from frontend v1.18.0 + https://github.com/Comfy-Org/ComfyUI_frontend/pull/3550""" # class InputTypeNumber(InputTypeOptions): # default: float | int min: NotRequired[float] diff --git a/comfy/k_diffusion/sampling.py b/comfy/k_diffusion/sampling.py index 6388d3faf..77ef748e8 100644 --- a/comfy/k_diffusion/sampling.py +++ b/comfy/k_diffusion/sampling.py @@ -1345,28 +1345,52 @@ def sample_res_multistep_ancestral_cfg_pp(model, x, sigmas, extra_args=None, cal return res_multistep(model, x, sigmas, extra_args=extra_args, callback=callback, disable=disable, s_noise=s_noise, noise_sampler=noise_sampler, eta=eta, cfg_pp=True) @torch.no_grad() -def sample_gradient_estimation(model, x, sigmas, extra_args=None, callback=None, disable=None, ge_gamma=2.): +def sample_gradient_estimation(model, x, sigmas, extra_args=None, callback=None, disable=None, ge_gamma=2., cfg_pp=False): """Gradient-estimation sampler. Paper: https://openreview.net/pdf?id=o2ND9v0CeK""" extra_args = {} if extra_args is None else extra_args s_in = x.new_ones([x.shape[0]]) old_d = None + uncond_denoised = None + def post_cfg_function(args): + nonlocal uncond_denoised + uncond_denoised = args["uncond_denoised"] + return args["denoised"] + + if cfg_pp: + model_options = extra_args.get("model_options", {}).copy() + extra_args["model_options"] = comfy.model_patcher.set_model_options_post_cfg_function(model_options, post_cfg_function, disable_cfg1_optimization=True) + for i in trange(len(sigmas) - 1, disable=disable): denoised = model(x, sigmas[i] * s_in, **extra_args) - d = to_d(x, sigmas[i], denoised) + if cfg_pp: + d = to_d(x, sigmas[i], uncond_denoised) + else: + d = to_d(x, sigmas[i], denoised) if callback is not None: callback({'x': x, 'i': i, 'sigma': sigmas[i], 'sigma_hat': sigmas[i], 'denoised': denoised}) dt = sigmas[i + 1] - sigmas[i] if i == 0: # Euler method - x = x + d * dt + if cfg_pp: + x = denoised + d * sigmas[i + 1] + else: + x = x + d * dt else: # Gradient estimation - d_bar = ge_gamma * d + (1 - ge_gamma) * old_d - x = x + d_bar * dt + if cfg_pp: + d_bar = (ge_gamma - 1) * (d - old_d) + x = denoised + d * sigmas[i + 1] + d_bar * dt + else: + d_bar = ge_gamma * d + (1 - ge_gamma) * old_d + x = x + d_bar * dt old_d = d return x +@torch.no_grad() +def sample_gradient_estimation_cfg_pp(model, x, sigmas, extra_args=None, callback=None, disable=None, ge_gamma=2.): + return sample_gradient_estimation(model, x, sigmas, extra_args=extra_args, callback=callback, disable=disable, ge_gamma=ge_gamma, cfg_pp=True) + @torch.no_grad() def sample_er_sde(model, x, sigmas, extra_args=None, callback=None, disable=None, s_noise=1., noise_sampler=None, noise_scaler=None, max_stage=3): """ diff --git a/comfy/ldm/chroma/layers.py b/comfy/ldm/chroma/layers.py new file mode 100644 index 000000000..35da91ee2 --- /dev/null +++ b/comfy/ldm/chroma/layers.py @@ -0,0 +1,183 @@ +import torch +from torch import Tensor, nn + +from comfy.ldm.flux.math import attention +from comfy.ldm.flux.layers import ( + MLPEmbedder, + RMSNorm, + QKNorm, + SelfAttention, + ModulationOut, +) + + + +class ChromaModulationOut(ModulationOut): + @classmethod + def from_offset(cls, tensor: torch.Tensor, offset: int = 0) -> ModulationOut: + return cls( + shift=tensor[:, offset : offset + 1, :], + scale=tensor[:, offset + 1 : offset + 2, :], + gate=tensor[:, offset + 2 : offset + 3, :], + ) + + + + +class Approximator(nn.Module): + def __init__(self, in_dim: int, out_dim: int, hidden_dim: int, n_layers = 5, dtype=None, device=None, operations=None): + super().__init__() + self.in_proj = operations.Linear(in_dim, hidden_dim, bias=True, dtype=dtype, device=device) + self.layers = nn.ModuleList([MLPEmbedder(hidden_dim, hidden_dim, dtype=dtype, device=device, operations=operations) for x in range( n_layers)]) + self.norms = nn.ModuleList([RMSNorm(hidden_dim, dtype=dtype, device=device, operations=operations) for x in range( n_layers)]) + self.out_proj = operations.Linear(hidden_dim, out_dim, dtype=dtype, device=device) + + @property + def device(self): + # Get the device of the module (assumes all parameters are on the same device) + return next(self.parameters()).device + + def forward(self, x: Tensor) -> Tensor: + x = self.in_proj(x) + + for layer, norms in zip(self.layers, self.norms): + x = x + layer(norms(x)) + + x = self.out_proj(x) + + return x + + +class DoubleStreamBlock(nn.Module): + def __init__(self, hidden_size: int, num_heads: int, mlp_ratio: float, qkv_bias: bool = False, flipped_img_txt=False, dtype=None, device=None, operations=None): + super().__init__() + + mlp_hidden_dim = int(hidden_size * mlp_ratio) + self.num_heads = num_heads + self.hidden_size = hidden_size + self.img_norm1 = operations.LayerNorm(hidden_size, elementwise_affine=False, eps=1e-6, dtype=dtype, device=device) + self.img_attn = SelfAttention(dim=hidden_size, num_heads=num_heads, qkv_bias=qkv_bias, dtype=dtype, device=device, operations=operations) + + self.img_norm2 = operations.LayerNorm(hidden_size, elementwise_affine=False, eps=1e-6, dtype=dtype, device=device) + self.img_mlp = nn.Sequential( + operations.Linear(hidden_size, mlp_hidden_dim, bias=True, dtype=dtype, device=device), + nn.GELU(approximate="tanh"), + operations.Linear(mlp_hidden_dim, hidden_size, bias=True, dtype=dtype, device=device), + ) + + self.txt_norm1 = operations.LayerNorm(hidden_size, elementwise_affine=False, eps=1e-6, dtype=dtype, device=device) + self.txt_attn = SelfAttention(dim=hidden_size, num_heads=num_heads, qkv_bias=qkv_bias, dtype=dtype, device=device, operations=operations) + + self.txt_norm2 = operations.LayerNorm(hidden_size, elementwise_affine=False, eps=1e-6, dtype=dtype, device=device) + self.txt_mlp = nn.Sequential( + operations.Linear(hidden_size, mlp_hidden_dim, bias=True, dtype=dtype, device=device), + nn.GELU(approximate="tanh"), + operations.Linear(mlp_hidden_dim, hidden_size, bias=True, dtype=dtype, device=device), + ) + self.flipped_img_txt = flipped_img_txt + + def forward(self, img: Tensor, txt: Tensor, pe: Tensor, vec: Tensor, attn_mask=None): + (img_mod1, img_mod2), (txt_mod1, txt_mod2) = vec + + # prepare image for attention + img_modulated = self.img_norm1(img) + img_modulated = (1 + img_mod1.scale) * img_modulated + img_mod1.shift + img_qkv = self.img_attn.qkv(img_modulated) + img_q, img_k, img_v = img_qkv.view(img_qkv.shape[0], img_qkv.shape[1], 3, self.num_heads, -1).permute(2, 0, 3, 1, 4) + img_q, img_k = self.img_attn.norm(img_q, img_k, img_v) + + # prepare txt for attention + txt_modulated = self.txt_norm1(txt) + txt_modulated = (1 + txt_mod1.scale) * txt_modulated + txt_mod1.shift + txt_qkv = self.txt_attn.qkv(txt_modulated) + txt_q, txt_k, txt_v = txt_qkv.view(txt_qkv.shape[0], txt_qkv.shape[1], 3, self.num_heads, -1).permute(2, 0, 3, 1, 4) + txt_q, txt_k = self.txt_attn.norm(txt_q, txt_k, txt_v) + + # run actual attention + attn = attention(torch.cat((txt_q, img_q), dim=2), + torch.cat((txt_k, img_k), dim=2), + torch.cat((txt_v, img_v), dim=2), + pe=pe, mask=attn_mask) + + txt_attn, img_attn = attn[:, : txt.shape[1]], attn[:, txt.shape[1] :] + + # calculate the img bloks + img = img + img_mod1.gate * self.img_attn.proj(img_attn) + img = img + img_mod2.gate * self.img_mlp((1 + img_mod2.scale) * self.img_norm2(img) + img_mod2.shift) + + # calculate the txt bloks + txt += txt_mod1.gate * self.txt_attn.proj(txt_attn) + txt += txt_mod2.gate * self.txt_mlp((1 + txt_mod2.scale) * self.txt_norm2(txt) + txt_mod2.shift) + + if txt.dtype == torch.float16: + txt = torch.nan_to_num(txt, nan=0.0, posinf=65504, neginf=-65504) + + return img, txt + + +class SingleStreamBlock(nn.Module): + """ + A DiT block with parallel linear layers as described in + https://arxiv.org/abs/2302.05442 and adapted modulation interface. + """ + + def __init__( + self, + hidden_size: int, + num_heads: int, + mlp_ratio: float = 4.0, + qk_scale: float = None, + dtype=None, + device=None, + operations=None + ): + super().__init__() + self.hidden_dim = hidden_size + self.num_heads = num_heads + head_dim = hidden_size // num_heads + self.scale = qk_scale or head_dim**-0.5 + + self.mlp_hidden_dim = int(hidden_size * mlp_ratio) + # qkv and mlp_in + self.linear1 = operations.Linear(hidden_size, hidden_size * 3 + self.mlp_hidden_dim, dtype=dtype, device=device) + # proj and mlp_out + self.linear2 = operations.Linear(hidden_size + self.mlp_hidden_dim, hidden_size, dtype=dtype, device=device) + + self.norm = QKNorm(head_dim, dtype=dtype, device=device, operations=operations) + + self.hidden_size = hidden_size + self.pre_norm = operations.LayerNorm(hidden_size, elementwise_affine=False, eps=1e-6, dtype=dtype, device=device) + + self.mlp_act = nn.GELU(approximate="tanh") + + def forward(self, x: Tensor, pe: Tensor, vec: Tensor, attn_mask=None) -> Tensor: + mod = vec + x_mod = (1 + mod.scale) * self.pre_norm(x) + mod.shift + qkv, mlp = torch.split(self.linear1(x_mod), [3 * self.hidden_size, self.mlp_hidden_dim], dim=-1) + + q, k, v = qkv.view(qkv.shape[0], qkv.shape[1], 3, self.num_heads, -1).permute(2, 0, 3, 1, 4) + q, k = self.norm(q, k, v) + + # compute attention + attn = attention(q, k, v, pe=pe, mask=attn_mask) + # compute activation in mlp stream, cat again and run second linear layer + output = self.linear2(torch.cat((attn, self.mlp_act(mlp)), 2)) + x += mod.gate * output + if x.dtype == torch.float16: + x = torch.nan_to_num(x, nan=0.0, posinf=65504, neginf=-65504) + return x + + +class LastLayer(nn.Module): + def __init__(self, hidden_size: int, patch_size: int, out_channels: int, dtype=None, device=None, operations=None): + super().__init__() + self.norm_final = operations.LayerNorm(hidden_size, elementwise_affine=False, eps=1e-6, dtype=dtype, device=device) + self.linear = operations.Linear(hidden_size, out_channels, bias=True, dtype=dtype, device=device) + + def forward(self, x: Tensor, vec: Tensor) -> Tensor: + shift, scale = vec + shift = shift.squeeze(1) + scale = scale.squeeze(1) + x = (1 + scale[:, None, :]) * self.norm_final(x) + shift[:, None, :] + x = self.linear(x) + return x diff --git a/comfy/ldm/chroma/model.py b/comfy/ldm/chroma/model.py new file mode 100644 index 000000000..636748fc5 --- /dev/null +++ b/comfy/ldm/chroma/model.py @@ -0,0 +1,271 @@ +#Original code can be found on: https://github.com/black-forest-labs/flux + +from dataclasses import dataclass + +import torch +from torch import Tensor, nn +from einops import rearrange, repeat +import comfy.ldm.common_dit + +from comfy.ldm.flux.layers import ( + EmbedND, + timestep_embedding, +) + +from .layers import ( + DoubleStreamBlock, + LastLayer, + SingleStreamBlock, + Approximator, + ChromaModulationOut, +) + + +@dataclass +class ChromaParams: + in_channels: int + out_channels: int + context_in_dim: int + hidden_size: int + mlp_ratio: float + num_heads: int + depth: int + depth_single_blocks: int + axes_dim: list + theta: int + patch_size: int + qkv_bias: bool + in_dim: int + out_dim: int + hidden_dim: int + n_layers: int + + + + +class Chroma(nn.Module): + """ + Transformer model for flow matching on sequences. + """ + + def __init__(self, image_model=None, final_layer=True, dtype=None, device=None, operations=None, **kwargs): + super().__init__() + self.dtype = dtype + params = ChromaParams(**kwargs) + self.params = params + self.patch_size = params.patch_size + self.in_channels = params.in_channels + self.out_channels = params.out_channels + if params.hidden_size % params.num_heads != 0: + raise ValueError( + f"Hidden size {params.hidden_size} must be divisible by num_heads {params.num_heads}" + ) + pe_dim = params.hidden_size // params.num_heads + if sum(params.axes_dim) != pe_dim: + raise ValueError(f"Got {params.axes_dim} but expected positional dim {pe_dim}") + self.hidden_size = params.hidden_size + self.num_heads = params.num_heads + self.in_dim = params.in_dim + self.out_dim = params.out_dim + self.hidden_dim = params.hidden_dim + self.n_layers = params.n_layers + self.pe_embedder = EmbedND(dim=pe_dim, theta=params.theta, axes_dim=params.axes_dim) + self.img_in = operations.Linear(self.in_channels, self.hidden_size, bias=True, dtype=dtype, device=device) + self.txt_in = operations.Linear(params.context_in_dim, self.hidden_size, dtype=dtype, device=device) + # set as nn identity for now, will overwrite it later. + self.distilled_guidance_layer = Approximator( + in_dim=self.in_dim, + hidden_dim=self.hidden_dim, + out_dim=self.out_dim, + n_layers=self.n_layers, + dtype=dtype, device=device, operations=operations + ) + + + self.double_blocks = nn.ModuleList( + [ + DoubleStreamBlock( + self.hidden_size, + self.num_heads, + mlp_ratio=params.mlp_ratio, + qkv_bias=params.qkv_bias, + dtype=dtype, device=device, operations=operations + ) + for _ in range(params.depth) + ] + ) + + self.single_blocks = nn.ModuleList( + [ + SingleStreamBlock(self.hidden_size, self.num_heads, mlp_ratio=params.mlp_ratio, dtype=dtype, device=device, operations=operations) + for _ in range(params.depth_single_blocks) + ] + ) + + if final_layer: + self.final_layer = LastLayer(self.hidden_size, 1, self.out_channels, dtype=dtype, device=device, operations=operations) + + self.skip_mmdit = [] + self.skip_dit = [] + self.lite = False + + def get_modulations(self, tensor: torch.Tensor, block_type: str, *, idx: int = 0): + # This function slices up the modulations tensor which has the following layout: + # single : num_single_blocks * 3 elements + # double_img : num_double_blocks * 6 elements + # double_txt : num_double_blocks * 6 elements + # final : 2 elements + if block_type == "final": + return (tensor[:, -2:-1, :], tensor[:, -1:, :]) + single_block_count = self.params.depth_single_blocks + double_block_count = self.params.depth + offset = 3 * idx + if block_type == "single": + return ChromaModulationOut.from_offset(tensor, offset) + # Double block modulations are 6 elements so we double 3 * idx. + offset *= 2 + if block_type in {"double_img", "double_txt"}: + # Advance past the single block modulations. + offset += 3 * single_block_count + if block_type == "double_txt": + # Advance past the double block img modulations. + offset += 6 * double_block_count + return ( + ChromaModulationOut.from_offset(tensor, offset), + ChromaModulationOut.from_offset(tensor, offset + 3), + ) + raise ValueError("Bad block_type") + + + def forward_orig( + self, + img: Tensor, + img_ids: Tensor, + txt: Tensor, + txt_ids: Tensor, + timesteps: Tensor, + guidance: Tensor = None, + control = None, + transformer_options={}, + attn_mask: Tensor = None, + ) -> Tensor: + patches_replace = transformer_options.get("patches_replace", {}) + if img.ndim != 3 or txt.ndim != 3: + raise ValueError("Input img and txt tensors must have 3 dimensions.") + + # running on sequences img + img = self.img_in(img) + + # distilled vector guidance + mod_index_length = 344 + distill_timestep = timestep_embedding(timesteps.detach().clone(), 16).to(img.device, img.dtype) + # guidance = guidance * + distil_guidance = timestep_embedding(guidance.detach().clone(), 16).to(img.device, img.dtype) + + # get all modulation index + modulation_index = timestep_embedding(torch.arange(mod_index_length), 32).to(img.device, img.dtype) + # we need to broadcast the modulation index here so each batch has all of the index + modulation_index = modulation_index.unsqueeze(0).repeat(img.shape[0], 1, 1).to(img.device, img.dtype) + # and we need to broadcast timestep and guidance along too + timestep_guidance = torch.cat([distill_timestep, distil_guidance], dim=1).unsqueeze(1).repeat(1, mod_index_length, 1).to(img.dtype).to(img.device, img.dtype) + # then and only then we could concatenate it together + input_vec = torch.cat([timestep_guidance, modulation_index], dim=-1).to(img.device, img.dtype) + + mod_vectors = self.distilled_guidance_layer(input_vec) + + txt = self.txt_in(txt) + + ids = torch.cat((txt_ids, img_ids), dim=1) + pe = self.pe_embedder(ids) + + blocks_replace = patches_replace.get("dit", {}) + for i, block in enumerate(self.double_blocks): + if i not in self.skip_mmdit: + double_mod = ( + self.get_modulations(mod_vectors, "double_img", idx=i), + self.get_modulations(mod_vectors, "double_txt", idx=i), + ) + if ("double_block", i) in blocks_replace: + def block_wrap(args): + out = {} + out["img"], out["txt"] = block(img=args["img"], + txt=args["txt"], + vec=args["vec"], + pe=args["pe"], + attn_mask=args.get("attn_mask")) + return out + + out = blocks_replace[("double_block", i)]({"img": img, + "txt": txt, + "vec": double_mod, + "pe": pe, + "attn_mask": attn_mask}, + {"original_block": block_wrap}) + txt = out["txt"] + img = out["img"] + else: + img, txt = block(img=img, + txt=txt, + vec=double_mod, + pe=pe, + attn_mask=attn_mask) + + if control is not None: # Controlnet + control_i = control.get("input") + if i < len(control_i): + add = control_i[i] + if add is not None: + img += add + + img = torch.cat((txt, img), 1) + + for i, block in enumerate(self.single_blocks): + if i not in self.skip_dit: + single_mod = self.get_modulations(mod_vectors, "single", idx=i) + if ("single_block", i) in blocks_replace: + def block_wrap(args): + out = {} + out["img"] = block(args["img"], + vec=args["vec"], + pe=args["pe"], + attn_mask=args.get("attn_mask")) + return out + + out = blocks_replace[("single_block", i)]({"img": img, + "vec": single_mod, + "pe": pe, + "attn_mask": attn_mask}, + {"original_block": block_wrap}) + img = out["img"] + else: + img = block(img, vec=single_mod, pe=pe, attn_mask=attn_mask) + + if control is not None: # Controlnet + control_o = control.get("output") + if i < len(control_o): + add = control_o[i] + if add is not None: + img[:, txt.shape[1] :, ...] += add + + img = img[:, txt.shape[1] :, ...] + final_mod = self.get_modulations(mod_vectors, "final") + img = self.final_layer(img, vec=final_mod) # (N, T, patch_size ** 2 * out_channels) + return img + + def forward(self, x, timestep, context, guidance, control=None, transformer_options={}, **kwargs): + bs, c, h, w = x.shape + patch_size = 2 + x = comfy.ldm.common_dit.pad_to_patch_size(x, (patch_size, patch_size)) + + img = rearrange(x, "b c (h ph) (w pw) -> b (h w) (c ph pw)", ph=patch_size, pw=patch_size) + + h_len = ((h + (patch_size // 2)) // patch_size) + w_len = ((w + (patch_size // 2)) // patch_size) + img_ids = torch.zeros((h_len, w_len, 3), device=x.device, dtype=x.dtype) + img_ids[:, :, 1] = img_ids[:, :, 1] + torch.linspace(0, h_len - 1, steps=h_len, device=x.device, dtype=x.dtype).unsqueeze(1) + img_ids[:, :, 2] = img_ids[:, :, 2] + torch.linspace(0, w_len - 1, steps=w_len, device=x.device, dtype=x.dtype).unsqueeze(0) + img_ids = repeat(img_ids, "h w c -> b (h w) c", b=bs) + + txt_ids = torch.zeros((bs, context.shape[1], 3), device=x.device, dtype=x.dtype) + out = self.forward_orig(img, img_ids, context, txt_ids, timestep, guidance, control, transformer_options, attn_mask=kwargs.get("attention_mask", None)) + return rearrange(out, "b (h w) (c ph pw) -> b c (h ph) (w pw)", h=h_len, w=w_len, ph=2, pw=2)[:,:,:h,:w] diff --git a/comfy/ldm/cosmos/blocks.py b/comfy/ldm/cosmos/blocks.py index 84fd6d839..a12f892d2 100644 --- a/comfy/ldm/cosmos/blocks.py +++ b/comfy/ldm/cosmos/blocks.py @@ -23,7 +23,6 @@ from einops import rearrange, repeat from einops.layers.torch import Rearrange from torch import nn -from comfy.ldm.modules.diffusionmodules.mmdit import RMSNorm from comfy.ldm.modules.attention import optimized_attention @@ -37,11 +36,11 @@ def apply_rotary_pos_emb( return t_out -def get_normalization(name: str, channels: int, weight_args={}): +def get_normalization(name: str, channels: int, weight_args={}, operations=None): if name == "I": return nn.Identity() elif name == "R": - return RMSNorm(channels, elementwise_affine=True, eps=1e-6, **weight_args) + return operations.RMSNorm(channels, elementwise_affine=True, eps=1e-6, **weight_args) else: raise ValueError(f"Normalization {name} not found") @@ -120,15 +119,15 @@ class Attention(nn.Module): self.to_q = nn.Sequential( operations.Linear(query_dim, inner_dim, bias=qkv_bias, **weight_args), - get_normalization(qkv_norm[0], norm_dim), + get_normalization(qkv_norm[0], norm_dim, weight_args=weight_args, operations=operations), ) self.to_k = nn.Sequential( operations.Linear(context_dim, inner_dim, bias=qkv_bias, **weight_args), - get_normalization(qkv_norm[1], norm_dim), + get_normalization(qkv_norm[1], norm_dim, weight_args=weight_args, operations=operations), ) self.to_v = nn.Sequential( operations.Linear(context_dim, inner_dim, bias=qkv_bias, **weight_args), - get_normalization(qkv_norm[2], norm_dim), + get_normalization(qkv_norm[2], norm_dim, weight_args=weight_args, operations=operations), ) self.to_out = nn.Sequential( diff --git a/comfy/ldm/cosmos/model.py b/comfy/ldm/cosmos/model.py index 06d0baef3..4836e0b69 100644 --- a/comfy/ldm/cosmos/model.py +++ b/comfy/ldm/cosmos/model.py @@ -27,8 +27,6 @@ from torchvision import transforms from enum import Enum import logging -from comfy.ldm.modules.diffusionmodules.mmdit import RMSNorm - from .blocks import ( FinalLayer, GeneralDITTransformerBlock, @@ -195,7 +193,7 @@ class GeneralDIT(nn.Module): if self.affline_emb_norm: logging.debug("Building affine embedding normalization layer") - self.affline_norm = RMSNorm(model_channels, elementwise_affine=True, eps=1e-6) + self.affline_norm = operations.RMSNorm(model_channels, elementwise_affine=True, eps=1e-6, device=device, dtype=dtype) else: self.affline_norm = nn.Identity() diff --git a/comfy/ldm/genmo/joint_model/asymm_models_joint.py b/comfy/ldm/genmo/joint_model/asymm_models_joint.py index 2c46c24bf..366a8b713 100644 --- a/comfy/ldm/genmo/joint_model/asymm_models_joint.py +++ b/comfy/ldm/genmo/joint_model/asymm_models_joint.py @@ -13,7 +13,6 @@ from comfy.ldm.modules.attention import optimized_attention from .layers import ( FeedForward, PatchEmbed, - RMSNorm, TimestepEmbedder, ) @@ -90,10 +89,10 @@ class AsymmetricAttention(nn.Module): # Query and key normalization for stability. assert qk_norm - self.q_norm_x = RMSNorm(self.head_dim, device=device, dtype=dtype) - self.k_norm_x = RMSNorm(self.head_dim, device=device, dtype=dtype) - self.q_norm_y = RMSNorm(self.head_dim, device=device, dtype=dtype) - self.k_norm_y = RMSNorm(self.head_dim, device=device, dtype=dtype) + self.q_norm_x = operations.RMSNorm(self.head_dim, eps=1e-5, device=device, dtype=dtype) + self.k_norm_x = operations.RMSNorm(self.head_dim, eps=1e-5, device=device, dtype=dtype) + self.q_norm_y = operations.RMSNorm(self.head_dim, eps=1e-5, device=device, dtype=dtype) + self.k_norm_y = operations.RMSNorm(self.head_dim, eps=1e-5, device=device, dtype=dtype) # Output layers. y features go back down from dim_x -> dim_y. self.proj_x = operations.Linear(dim_x, dim_x, bias=out_bias, device=device, dtype=dtype) diff --git a/comfy/ldm/genmo/joint_model/layers.py b/comfy/ldm/genmo/joint_model/layers.py index 51d979559..e310bd717 100644 --- a/comfy/ldm/genmo/joint_model/layers.py +++ b/comfy/ldm/genmo/joint_model/layers.py @@ -151,14 +151,3 @@ class PatchEmbed(nn.Module): x = self.norm(x) return x - - -class RMSNorm(torch.nn.Module): - def __init__(self, hidden_size, eps=1e-5, device=None, dtype=None): - super().__init__() - self.eps = eps - self.weight = torch.nn.Parameter(torch.empty(hidden_size, device=device, dtype=dtype)) - self.register_parameter("bias", None) - - def forward(self, x): - return comfy.ldm.common_dit.rms_norm(x, self.weight, self.eps) diff --git a/comfy/ldm/hidream/model.py b/comfy/ldm/hidream/model.py index fcb5a9c51..0305747bf 100644 --- a/comfy/ldm/hidream/model.py +++ b/comfy/ldm/hidream/model.py @@ -699,10 +699,13 @@ class HiDreamImageTransformer2DModel(nn.Module): y: Optional[torch.Tensor] = None, context: Optional[torch.Tensor] = None, encoder_hidden_states_llama3=None, + image_cond=None, control = None, transformer_options = {}, ) -> torch.Tensor: bs, c, h, w = x.shape + if image_cond is not None: + x = torch.cat([x, image_cond], dim=-1) hidden_states = comfy.ldm.common_dit.pad_to_patch_size(x, (self.patch_size, self.patch_size)) timesteps = t pooled_embeds = y diff --git a/comfy/ldm/hydit/models.py b/comfy/ldm/hydit/models.py index 359f6a965..5ba2b76e0 100644 --- a/comfy/ldm/hydit/models.py +++ b/comfy/ldm/hydit/models.py @@ -3,7 +3,7 @@ import torch import torch.nn as nn import comfy.ops -from comfy.ldm.modules.diffusionmodules.mmdit import Mlp, TimestepEmbedder, PatchEmbed, RMSNorm +from comfy.ldm.modules.diffusionmodules.mmdit import Mlp, TimestepEmbedder, PatchEmbed from comfy.ldm.modules.diffusionmodules.util import timestep_embedding from torch.utils import checkpoint @@ -51,7 +51,7 @@ class HunYuanDiTBlock(nn.Module): if norm_type == "layer": norm_layer = operations.LayerNorm elif norm_type == "rms": - norm_layer = RMSNorm + norm_layer = operations.RMSNorm else: raise ValueError(f"Unknown norm_type: {norm_type}") diff --git a/comfy/ldm/lightricks/model.py b/comfy/ldm/lightricks/model.py index 6e8e06181..056e101a4 100644 --- a/comfy/ldm/lightricks/model.py +++ b/comfy/ldm/lightricks/model.py @@ -1,7 +1,6 @@ import torch from torch import nn import comfy.ldm.modules.attention -from comfy.ldm.genmo.joint_model.layers import RMSNorm import comfy.ldm.common_dit from einops import rearrange import math @@ -262,8 +261,8 @@ class CrossAttention(nn.Module): self.heads = heads self.dim_head = dim_head - self.q_norm = RMSNorm(inner_dim, dtype=dtype, device=device) - self.k_norm = RMSNorm(inner_dim, dtype=dtype, device=device) + self.q_norm = operations.RMSNorm(inner_dim, dtype=dtype, device=device) + self.k_norm = operations.RMSNorm(inner_dim, dtype=dtype, device=device) self.to_q = operations.Linear(query_dim, inner_dim, bias=True, dtype=dtype, device=device) self.to_k = operations.Linear(context_dim, inner_dim, bias=True, dtype=dtype, device=device) diff --git a/comfy/ldm/lumina/model.py b/comfy/ldm/lumina/model.py index ccd5d2c0e..f8dc4d7db 100644 --- a/comfy/ldm/lumina/model.py +++ b/comfy/ldm/lumina/model.py @@ -8,7 +8,7 @@ import torch.nn as nn import torch.nn.functional as F import comfy.ldm.common_dit -from comfy.ldm.modules.diffusionmodules.mmdit import TimestepEmbedder, RMSNorm +from comfy.ldm.modules.diffusionmodules.mmdit import TimestepEmbedder from comfy.ldm.modules.attention import optimized_attention_masked from comfy.ldm.flux.layers import EmbedND @@ -64,8 +64,8 @@ class JointAttention(nn.Module): ) if qk_norm: - self.q_norm = RMSNorm(self.head_dim, elementwise_affine=True, **operation_settings) - self.k_norm = RMSNorm(self.head_dim, elementwise_affine=True, **operation_settings) + self.q_norm = operation_settings.get("operations").RMSNorm(self.head_dim, elementwise_affine=True, device=operation_settings.get("device"), dtype=operation_settings.get("dtype")) + self.k_norm = operation_settings.get("operations").RMSNorm(self.head_dim, elementwise_affine=True, device=operation_settings.get("device"), dtype=operation_settings.get("dtype")) else: self.q_norm = self.k_norm = nn.Identity() @@ -242,11 +242,11 @@ class JointTransformerBlock(nn.Module): operation_settings=operation_settings, ) self.layer_id = layer_id - self.attention_norm1 = RMSNorm(dim, eps=norm_eps, elementwise_affine=True, **operation_settings) - self.ffn_norm1 = RMSNorm(dim, eps=norm_eps, elementwise_affine=True, **operation_settings) + self.attention_norm1 = operation_settings.get("operations").RMSNorm(dim, eps=norm_eps, elementwise_affine=True, device=operation_settings.get("device"), dtype=operation_settings.get("dtype")) + self.ffn_norm1 = operation_settings.get("operations").RMSNorm(dim, eps=norm_eps, elementwise_affine=True, device=operation_settings.get("device"), dtype=operation_settings.get("dtype")) - self.attention_norm2 = RMSNorm(dim, eps=norm_eps, elementwise_affine=True, **operation_settings) - self.ffn_norm2 = RMSNorm(dim, eps=norm_eps, elementwise_affine=True, **operation_settings) + self.attention_norm2 = operation_settings.get("operations").RMSNorm(dim, eps=norm_eps, elementwise_affine=True, device=operation_settings.get("device"), dtype=operation_settings.get("dtype")) + self.ffn_norm2 = operation_settings.get("operations").RMSNorm(dim, eps=norm_eps, elementwise_affine=True, device=operation_settings.get("device"), dtype=operation_settings.get("dtype")) self.modulation = modulation if modulation: @@ -431,7 +431,7 @@ class NextDiT(nn.Module): self.t_embedder = TimestepEmbedder(min(dim, 1024), **operation_settings) self.cap_embedder = nn.Sequential( - RMSNorm(cap_feat_dim, eps=norm_eps, elementwise_affine=True, **operation_settings), + operation_settings.get("operations").RMSNorm(cap_feat_dim, eps=norm_eps, elementwise_affine=True, device=operation_settings.get("device"), dtype=operation_settings.get("dtype")), operation_settings.get("operations").Linear( cap_feat_dim, dim, @@ -457,7 +457,7 @@ class NextDiT(nn.Module): for layer_id in range(n_layers) ] ) - self.norm_final = RMSNorm(dim, eps=norm_eps, elementwise_affine=True, **operation_settings) + self.norm_final = operation_settings.get("operations").RMSNorm(dim, eps=norm_eps, elementwise_affine=True, device=operation_settings.get("device"), dtype=operation_settings.get("dtype")) self.final_layer = FinalLayer(dim, patch_size, self.out_channels, operation_settings=operation_settings) assert (dim // n_heads) == sum(axes_dims) diff --git a/comfy/ldm/wan/model.py b/comfy/ldm/wan/model.py index b8eec3afb..fc5ff40c5 100644 --- a/comfy/ldm/wan/model.py +++ b/comfy/ldm/wan/model.py @@ -9,7 +9,6 @@ from einops import repeat from comfy.ldm.modules.attention import optimized_attention from comfy.ldm.flux.layers import EmbedND from comfy.ldm.flux.math import apply_rope -from comfy.ldm.modules.diffusionmodules.mmdit import RMSNorm import comfy.ldm.common_dit import comfy.model_management @@ -49,8 +48,8 @@ class WanSelfAttention(nn.Module): self.k = operation_settings.get("operations").Linear(dim, dim, device=operation_settings.get("device"), dtype=operation_settings.get("dtype")) self.v = operation_settings.get("operations").Linear(dim, dim, device=operation_settings.get("device"), dtype=operation_settings.get("dtype")) self.o = operation_settings.get("operations").Linear(dim, dim, device=operation_settings.get("device"), dtype=operation_settings.get("dtype")) - self.norm_q = RMSNorm(dim, eps=eps, elementwise_affine=True, device=operation_settings.get("device"), dtype=operation_settings.get("dtype")) if qk_norm else nn.Identity() - self.norm_k = RMSNorm(dim, eps=eps, elementwise_affine=True, device=operation_settings.get("device"), dtype=operation_settings.get("dtype")) if qk_norm else nn.Identity() + self.norm_q = operation_settings.get("operations").RMSNorm(dim, eps=eps, elementwise_affine=True, device=operation_settings.get("device"), dtype=operation_settings.get("dtype")) if qk_norm else nn.Identity() + self.norm_k = operation_settings.get("operations").RMSNorm(dim, eps=eps, elementwise_affine=True, device=operation_settings.get("device"), dtype=operation_settings.get("dtype")) if qk_norm else nn.Identity() def forward(self, x, freqs): r""" @@ -114,7 +113,7 @@ class WanI2VCrossAttention(WanSelfAttention): self.k_img = operation_settings.get("operations").Linear(dim, dim, device=operation_settings.get("device"), dtype=operation_settings.get("dtype")) self.v_img = operation_settings.get("operations").Linear(dim, dim, device=operation_settings.get("device"), dtype=operation_settings.get("dtype")) # self.alpha = nn.Parameter(torch.zeros((1, ))) - self.norm_k_img = RMSNorm(dim, eps=eps, elementwise_affine=True, device=operation_settings.get("device"), dtype=operation_settings.get("dtype")) if qk_norm else nn.Identity() + self.norm_k_img = operation_settings.get("operations").RMSNorm(dim, eps=eps, elementwise_affine=True, device=operation_settings.get("device"), dtype=operation_settings.get("dtype")) if qk_norm else nn.Identity() def forward(self, x, context, context_img_len): r""" @@ -631,6 +630,7 @@ class VaceWanModel(WanModel): if ii is not None: c_skip, c = self.vace_blocks[ii](c, x=x_orig, e=e0, freqs=freqs, context=context, context_img_len=context_img_len) x += c_skip * vace_strength + del c_skip # head x = self.head(x, e) diff --git a/comfy/lora.py b/comfy/lora.py index 8760a21fb..fff524be2 100644 --- a/comfy/lora.py +++ b/comfy/lora.py @@ -279,6 +279,13 @@ def model_lora_keys_unet(model, key_map={}): key_map["transformer.{}".format(key_lora)] = k key_map["diffusion_model.{}".format(key_lora)] = k # Old loras + if isinstance(model, comfy.model_base.HiDream): + for k in sdk: + if k.startswith("diffusion_model."): + if k.endswith(".weight"): + key_lora = k[len("diffusion_model."):-len(".weight")].replace(".", "_") + key_map["lycoris_{}".format(key_lora)] = k #SimpleTuner lycoris format + return key_map diff --git a/comfy/model_base.py b/comfy/model_base.py index b0c6a465b..3d33086d8 100644 --- a/comfy/model_base.py +++ b/comfy/model_base.py @@ -38,6 +38,7 @@ import comfy.ldm.lumina.model import comfy.ldm.wan.model import comfy.ldm.hunyuan3d.model import comfy.ldm.hidream.model +import comfy.ldm.chroma.model import comfy.model_management import comfy.patcher_extension @@ -786,8 +787,8 @@ class PixArt(BaseModel): return out class Flux(BaseModel): - def __init__(self, model_config, model_type=ModelType.FLUX, device=None): - super().__init__(model_config, model_type, device=device, unet_model=comfy.ldm.flux.model.Flux) + def __init__(self, model_config, model_type=ModelType.FLUX, device=None, unet_model=comfy.ldm.flux.model.Flux): + super().__init__(model_config, model_type, device=device, unet_model=unet_model) def concat_cond(self, **kwargs): try: @@ -1104,4 +1105,19 @@ class HiDream(BaseModel): conditioning_llama3 = kwargs.get("conditioning_llama3", None) if conditioning_llama3 is not None: out['encoder_hidden_states_llama3'] = comfy.conds.CONDRegular(conditioning_llama3) + image_cond = kwargs.get("concat_latent_image", None) + if image_cond is not None: + out['image_cond'] = comfy.conds.CONDNoiseShape(self.process_latent_in(image_cond)) + return out + +class Chroma(Flux): + def __init__(self, model_config, model_type=ModelType.FLOW, device=None): + super().__init__(model_config, model_type, device=device, unet_model=comfy.ldm.chroma.model.Chroma) + + def extra_conds(self, **kwargs): + out = super().extra_conds(**kwargs) + + guidance = kwargs.get("guidance", 0) + if guidance is not None: + out['guidance'] = comfy.conds.CONDRegular(torch.FloatTensor([guidance])) return out diff --git a/comfy/model_detection.py b/comfy/model_detection.py index 76de78a8a..9254843ea 100644 --- a/comfy/model_detection.py +++ b/comfy/model_detection.py @@ -164,7 +164,9 @@ def detect_unet_config(state_dict, key_prefix, metadata=None): if in_key in state_dict_keys: dit_config["in_channels"] = state_dict[in_key].shape[1] // (patch_size * patch_size) dit_config["out_channels"] = 16 - dit_config["vec_in_dim"] = 768 + vec_in_key = '{}vector_in.in_layer.weight'.format(key_prefix) + if vec_in_key in state_dict_keys: + dit_config["vec_in_dim"] = state_dict[vec_in_key].shape[1] dit_config["context_in_dim"] = 4096 dit_config["hidden_size"] = 3072 dit_config["mlp_ratio"] = 4.0 @@ -174,7 +176,16 @@ def detect_unet_config(state_dict, key_prefix, metadata=None): dit_config["axes_dim"] = [16, 56, 56] dit_config["theta"] = 10000 dit_config["qkv_bias"] = True - dit_config["guidance_embed"] = "{}guidance_in.in_layer.weight".format(key_prefix) in state_dict_keys + if '{}distilled_guidance_layer.0.norms.0.scale'.format(key_prefix) in state_dict_keys or '{}distilled_guidance_layer.norms.0.scale'.format(key_prefix) in state_dict_keys: #Chroma + dit_config["image_model"] = "chroma" + dit_config["in_channels"] = 64 + dit_config["out_channels"] = 64 + dit_config["in_dim"] = 64 + dit_config["out_dim"] = 3072 + dit_config["hidden_dim"] = 5120 + dit_config["n_layers"] = 5 + else: + dit_config["guidance_embed"] = "{}guidance_in.in_layer.weight".format(key_prefix) in state_dict_keys return dit_config if '{}t5_yproj.weight'.format(key_prefix) in state_dict_keys: #Genmo mochi preview diff --git a/comfy/model_management.py b/comfy/model_management.py index 43e402243..44aff3762 100644 --- a/comfy/model_management.py +++ b/comfy/model_management.py @@ -939,15 +939,61 @@ def force_channels_last(): #TODO return False -def cast_to(weight, dtype=None, device=None, non_blocking=False, copy=False): + +STREAMS = {} +NUM_STREAMS = 1 +if args.async_offload: + NUM_STREAMS = 2 + logging.info("Using async weight offloading with {} streams".format(NUM_STREAMS)) + +stream_counters = {} +def get_offload_stream(device): + stream_counter = stream_counters.get(device, 0) + if NUM_STREAMS <= 1: + return None + + if device in STREAMS: + ss = STREAMS[device] + s = ss[stream_counter] + stream_counter = (stream_counter + 1) % len(ss) + if is_device_cuda(device): + ss[stream_counter].wait_stream(torch.cuda.current_stream()) + stream_counters[device] = stream_counter + return s + elif is_device_cuda(device): + ss = [] + for k in range(NUM_STREAMS): + ss.append(torch.cuda.Stream(device=device, priority=0)) + STREAMS[device] = ss + s = ss[stream_counter] + stream_counter = (stream_counter + 1) % len(ss) + stream_counters[device] = stream_counter + return s + return None + +def sync_stream(device, stream): + if stream is None: + return + if is_device_cuda(device): + torch.cuda.current_stream().wait_stream(stream) + +def cast_to(weight, dtype=None, device=None, non_blocking=False, copy=False, stream=None): if device is None or weight.device == device: if not copy: if dtype is None or weight.dtype == dtype: return weight + if stream is not None: + with stream: + return weight.to(dtype=dtype, copy=copy) return weight.to(dtype=dtype, copy=copy) - r = torch.empty_like(weight, dtype=dtype, device=device) - r.copy_(weight, non_blocking=non_blocking) + if stream is not None: + with stream: + r = torch.empty_like(weight, dtype=dtype, device=device) + r.copy_(weight, non_blocking=non_blocking) + else: + r = torch.empty_like(weight, dtype=dtype, device=device) + r.copy_(weight, non_blocking=non_blocking) return r def cast_to_device(tensor, device, dtype, copy=False): diff --git a/comfy/model_sampling.py b/comfy/model_sampling.py index b79af1e92..7e7291476 100644 --- a/comfy/model_sampling.py +++ b/comfy/model_sampling.py @@ -111,13 +111,14 @@ class ModelSamplingDiscrete(torch.nn.Module): self.num_timesteps = int(timesteps) self.linear_start = linear_start self.linear_end = linear_end + self.zsnr = zsnr # self.register_buffer('betas', torch.tensor(betas, dtype=torch.float32)) # self.register_buffer('alphas_cumprod', torch.tensor(alphas_cumprod, dtype=torch.float32)) # self.register_buffer('alphas_cumprod_prev', torch.tensor(alphas_cumprod_prev, dtype=torch.float32)) sigmas = ((1 - alphas_cumprod) / alphas_cumprod) ** 0.5 - if zsnr: + if self.zsnr: sigmas = rescale_zero_terminal_snr_sigmas(sigmas) self.set_sigmas(sigmas) diff --git a/comfy/ops.py b/comfy/ops.py index aae6cafac..032787915 100644 --- a/comfy/ops.py +++ b/comfy/ops.py @@ -22,6 +22,7 @@ import comfy.model_management from comfy.cli_args import args, PerformanceFeature import comfy.float import comfy.rmsnorm +import contextlib cast_to = comfy.model_management.cast_to #TODO: remove once no more references @@ -37,20 +38,31 @@ def cast_bias_weight(s, input=None, dtype=None, device=None, bias_dtype=None): if device is None: device = input.device + offload_stream = comfy.model_management.get_offload_stream(device) + if offload_stream is not None: + wf_context = offload_stream + else: + wf_context = contextlib.nullcontext() + bias = None non_blocking = comfy.model_management.device_supports_non_blocking(device) if s.bias is not None: has_function = len(s.bias_function) > 0 - bias = comfy.model_management.cast_to(s.bias, bias_dtype, device, non_blocking=non_blocking, copy=has_function) + bias = comfy.model_management.cast_to(s.bias, bias_dtype, device, non_blocking=non_blocking, copy=has_function, stream=offload_stream) + if has_function: - for f in s.bias_function: - bias = f(bias) + with wf_context: + for f in s.bias_function: + bias = f(bias) has_function = len(s.weight_function) > 0 - weight = comfy.model_management.cast_to(s.weight, dtype, device, non_blocking=non_blocking, copy=has_function) + weight = comfy.model_management.cast_to(s.weight, dtype, device, non_blocking=non_blocking, copy=has_function, stream=offload_stream) if has_function: - for f in s.weight_function: - weight = f(weight) + with wf_context: + for f in s.weight_function: + weight = f(weight) + + comfy.model_management.sync_stream(device, offload_stream) return weight, bias class CastWeightBiasOp: diff --git a/comfy/samplers.py b/comfy/samplers.py index 27dfce45a..67ae09a25 100644 --- a/comfy/samplers.py +++ b/comfy/samplers.py @@ -710,7 +710,7 @@ KSAMPLER_NAMES = ["euler", "euler_cfg_pp", "euler_ancestral", "euler_ancestral_c "lms", "dpm_fast", "dpm_adaptive", "dpmpp_2s_ancestral", "dpmpp_2s_ancestral_cfg_pp", "dpmpp_sde", "dpmpp_sde_gpu", "dpmpp_2m", "dpmpp_2m_cfg_pp", "dpmpp_2m_sde", "dpmpp_2m_sde_gpu", "dpmpp_3m_sde", "dpmpp_3m_sde_gpu", "ddpm", "lcm", "ipndm", "ipndm_v", "deis", "res_multistep", "res_multistep_cfg_pp", "res_multistep_ancestral", "res_multistep_ancestral_cfg_pp", - "gradient_estimation", "er_sde", "seeds_2", "seeds_3"] + "gradient_estimation", "gradient_estimation_cfg_pp", "er_sde", "seeds_2", "seeds_3"] class KSAMPLER(Sampler): def __init__(self, sampler_function, extra_options={}, inpaint_options={}): diff --git a/comfy/sd.py b/comfy/sd.py index 8aba5d655..da9b36d0e 100644 --- a/comfy/sd.py +++ b/comfy/sd.py @@ -120,6 +120,7 @@ class CLIP: self.layer_idx = None self.use_clip_schedule = False logging.info("CLIP/text encoder model load device: {}, offload device: {}, current: {}, dtype: {}".format(load_device, offload_device, params['device'], dtype)) + self.tokenizer_options = {} def clone(self): n = CLIP(no_init=True) @@ -127,6 +128,7 @@ class CLIP: n.cond_stage_model = self.cond_stage_model n.tokenizer = self.tokenizer n.layer_idx = self.layer_idx + n.tokenizer_options = self.tokenizer_options.copy() n.use_clip_schedule = self.use_clip_schedule n.apply_hooks_to_conds = self.apply_hooks_to_conds return n @@ -134,10 +136,18 @@ class CLIP: def add_patches(self, patches, strength_patch=1.0, strength_model=1.0): return self.patcher.add_patches(patches, strength_patch, strength_model) + def set_tokenizer_option(self, option_name, value): + self.tokenizer_options[option_name] = value + def clip_layer(self, layer_idx): self.layer_idx = layer_idx def tokenize(self, text, return_word_ids=False, **kwargs): + tokenizer_options = kwargs.get("tokenizer_options", {}) + if len(self.tokenizer_options) > 0: + tokenizer_options = {**self.tokenizer_options, **tokenizer_options} + if len(tokenizer_options) > 0: + kwargs["tokenizer_options"] = tokenizer_options return self.tokenizer.tokenize_with_weights(text, return_word_ids, **kwargs) def add_hooks_to_dict(self, pooled_dict: dict[str]): @@ -704,6 +714,7 @@ class CLIPType(Enum): LUMINA2 = 12 WAN = 13 HIDREAM = 14 + CHROMA = 15 def load_clip(ckpt_paths, embedding_directory=None, clip_type=CLIPType.STABLE_DIFFUSION, model_options={}): @@ -808,7 +819,7 @@ def load_text_encoder_state_dicts(state_dicts=[], embedding_directory=None, clip elif clip_type == CLIPType.LTXV: clip_target.clip = comfy.text_encoders.lt.ltxv_te(**t5xxl_detect(clip_data)) clip_target.tokenizer = comfy.text_encoders.lt.LTXVT5Tokenizer - elif clip_type == CLIPType.PIXART: + elif clip_type == CLIPType.PIXART or clip_type == CLIPType.CHROMA: clip_target.clip = comfy.text_encoders.pixart_t5.pixart_te(**t5xxl_detect(clip_data)) clip_target.tokenizer = comfy.text_encoders.pixart_t5.PixArtTokenizer elif clip_type == CLIPType.WAN: diff --git a/comfy/sd1_clip.py b/comfy/sd1_clip.py index 2ca5ed9ba..ac61babe9 100644 --- a/comfy/sd1_clip.py +++ b/comfy/sd1_clip.py @@ -457,13 +457,14 @@ def load_embed(embedding_name, embedding_directory, embedding_size, embed_key=No return embed_out class SDTokenizer: - def __init__(self, tokenizer_path=None, max_length=77, pad_with_end=True, embedding_directory=None, embedding_size=768, embedding_key='clip_l', tokenizer_class=CLIPTokenizer, has_start_token=True, has_end_token=True, pad_to_max_length=True, min_length=None, pad_token=None, end_token=None, tokenizer_data={}, tokenizer_args={}): + def __init__(self, tokenizer_path=None, max_length=77, pad_with_end=True, embedding_directory=None, embedding_size=768, embedding_key='clip_l', tokenizer_class=CLIPTokenizer, has_start_token=True, has_end_token=True, pad_to_max_length=True, min_length=None, pad_token=None, end_token=None, min_padding=None, tokenizer_data={}, tokenizer_args={}): if tokenizer_path is None: tokenizer_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "sd1_tokenizer") self.tokenizer = tokenizer_class.from_pretrained(tokenizer_path, **tokenizer_args) self.max_length = tokenizer_data.get("{}_max_length".format(embedding_key), max_length) self.min_length = min_length self.end_token = None + self.min_padding = min_padding empty = self.tokenizer('')["input_ids"] self.tokenizer_adds_end_token = has_end_token @@ -518,13 +519,15 @@ class SDTokenizer: return (embed, leftover) - def tokenize_with_weights(self, text:str, return_word_ids=False, **kwargs): + def tokenize_with_weights(self, text:str, return_word_ids=False, tokenizer_options={}, **kwargs): ''' Takes a prompt and converts it to a list of (token, weight, word id) elements. Tokens can both be integer tokens and pre computed CLIP tensors. Word id values are unique per word and embedding, where the id 0 is reserved for non word tokens. Returned list has the dimensions NxM where M is the input size of CLIP ''' + min_length = tokenizer_options.get("{}_min_length".format(self.embedding_key), self.min_length) + min_padding = tokenizer_options.get("{}_min_padding".format(self.embedding_key), self.min_padding) text = escape_important(text) parsed_weights = token_weights(text, 1.0) @@ -603,10 +606,12 @@ class SDTokenizer: #fill last batch if self.end_token is not None: batch.append((self.end_token, 1.0, 0)) - if self.pad_to_max_length: + if min_padding is not None: + batch.extend([(self.pad_token, 1.0, 0)] * min_padding) + if self.pad_to_max_length and len(batch) < self.max_length: batch.extend([(self.pad_token, 1.0, 0)] * (self.max_length - len(batch))) - if self.min_length is not None and len(batch) < self.min_length: - batch.extend([(self.pad_token, 1.0, 0)] * (self.min_length - len(batch))) + if min_length is not None and len(batch) < min_length: + batch.extend([(self.pad_token, 1.0, 0)] * (min_length - len(batch))) if not return_word_ids: batched_tokens = [[(t, w) for t, w,_ in x] for x in batched_tokens] @@ -634,7 +639,7 @@ class SD1Tokenizer: def tokenize_with_weights(self, text:str, return_word_ids=False, **kwargs): out = {} - out[self.clip_name] = getattr(self, self.clip).tokenize_with_weights(text, return_word_ids) + out[self.clip_name] = getattr(self, self.clip).tokenize_with_weights(text, return_word_ids, **kwargs) return out def untokenize(self, token_weight_pair): diff --git a/comfy/sdxl_clip.py b/comfy/sdxl_clip.py index ea7f5d10f..c8cef14e4 100644 --- a/comfy/sdxl_clip.py +++ b/comfy/sdxl_clip.py @@ -28,8 +28,8 @@ class SDXLTokenizer: def tokenize_with_weights(self, text:str, return_word_ids=False, **kwargs): out = {} - out["g"] = self.clip_g.tokenize_with_weights(text, return_word_ids) - out["l"] = self.clip_l.tokenize_with_weights(text, return_word_ids) + out["g"] = self.clip_g.tokenize_with_weights(text, return_word_ids, **kwargs) + out["l"] = self.clip_l.tokenize_with_weights(text, return_word_ids, **kwargs) return out def untokenize(self, token_weight_pair): diff --git a/comfy/supported_models.py b/comfy/supported_models.py index 5e55035cf..d5210cfac 100644 --- a/comfy/supported_models.py +++ b/comfy/supported_models.py @@ -993,6 +993,10 @@ class WAN21_Vace(WAN21_T2V): "model_type": "vace", } + def __init__(self, unet_config): + super().__init__(unet_config) + self.memory_usage_factor = 1.2 * self.memory_usage_factor + def get_model(self, state_dict, prefix="", device=None): out = model_base.WAN21_Vace(self, image_to_video=False, device=device) return out @@ -1064,7 +1068,34 @@ class HiDream(supported_models_base.BASE): def clip_target(self, state_dict={}): return None # TODO +class Chroma(supported_models_base.BASE): + unet_config = { + "image_model": "chroma", + } -models = [LotusD, Stable_Zero123, SD15_instructpix2pix, SD15, SD20, SD21UnclipL, SD21UnclipH, SDXL_instructpix2pix, SDXLRefiner, SDXL, SSD1B, KOALA_700M, KOALA_1B, Segmind_Vega, SD_X4Upscaler, Stable_Cascade_C, Stable_Cascade_B, SV3D_u, SV3D_p, SD3, StableAudio, AuraFlow, PixArtAlpha, PixArtSigma, HunyuanDiT, HunyuanDiT1, FluxInpaint, Flux, FluxSchnell, GenmoMochi, LTXV, HunyuanVideoSkyreelsI2V, HunyuanVideoI2V, HunyuanVideo, CosmosT2V, CosmosI2V, Lumina2, WAN21_T2V, WAN21_I2V, WAN21_FunControl2V, WAN21_Vace, Hunyuan3Dv2mini, Hunyuan3Dv2, HiDream] + unet_extra_config = { + } + + sampling_settings = { + "multiplier": 1.0, + } + + latent_format = comfy.latent_formats.Flux + + memory_usage_factor = 3.2 + + supported_inference_dtypes = [torch.bfloat16, torch.float16, torch.float32] + + + def get_model(self, state_dict, prefix="", device=None): + out = model_base.Chroma(self, device=device) + return out + + def clip_target(self, state_dict={}): + pref = self.text_encoder_key_prefix[0] + t5_detect = comfy.text_encoders.sd3_clip.t5_xxl_detect(state_dict, "{}t5xxl.transformer.".format(pref)) + return supported_models_base.ClipTarget(comfy.text_encoders.pixart_t5.PixArtTokenizer, comfy.text_encoders.pixart_t5.pixart_te(**t5_detect)) + +models = [LotusD, Stable_Zero123, SD15_instructpix2pix, SD15, SD20, SD21UnclipL, SD21UnclipH, SDXL_instructpix2pix, SDXLRefiner, SDXL, SSD1B, KOALA_700M, KOALA_1B, Segmind_Vega, SD_X4Upscaler, Stable_Cascade_C, Stable_Cascade_B, SV3D_u, SV3D_p, SD3, StableAudio, AuraFlow, PixArtAlpha, PixArtSigma, HunyuanDiT, HunyuanDiT1, FluxInpaint, Flux, FluxSchnell, GenmoMochi, LTXV, HunyuanVideoSkyreelsI2V, HunyuanVideoI2V, HunyuanVideo, CosmosT2V, CosmosI2V, Lumina2, WAN21_T2V, WAN21_I2V, WAN21_FunControl2V, WAN21_Vace, Hunyuan3Dv2mini, Hunyuan3Dv2, HiDream, Chroma] models += [SVD_img2vid] diff --git a/comfy/text_encoders/flux.py b/comfy/text_encoders/flux.py index 0666dde7f..d61ef6668 100644 --- a/comfy/text_encoders/flux.py +++ b/comfy/text_encoders/flux.py @@ -19,8 +19,8 @@ class FluxTokenizer: def tokenize_with_weights(self, text:str, return_word_ids=False, **kwargs): out = {} - out["l"] = self.clip_l.tokenize_with_weights(text, return_word_ids) - out["t5xxl"] = self.t5xxl.tokenize_with_weights(text, return_word_ids) + out["l"] = self.clip_l.tokenize_with_weights(text, return_word_ids, **kwargs) + out["t5xxl"] = self.t5xxl.tokenize_with_weights(text, return_word_ids, **kwargs) return out def untokenize(self, token_weight_pair): diff --git a/comfy/text_encoders/hidream.py b/comfy/text_encoders/hidream.py index 8e1abcfc1..dbcf52784 100644 --- a/comfy/text_encoders/hidream.py +++ b/comfy/text_encoders/hidream.py @@ -16,11 +16,11 @@ class HiDreamTokenizer: def tokenize_with_weights(self, text:str, return_word_ids=False, **kwargs): out = {} - out["g"] = self.clip_g.tokenize_with_weights(text, return_word_ids) - out["l"] = self.clip_l.tokenize_with_weights(text, return_word_ids) - t5xxl = self.t5xxl.tokenize_with_weights(text, return_word_ids) + out["g"] = self.clip_g.tokenize_with_weights(text, return_word_ids, **kwargs) + out["l"] = self.clip_l.tokenize_with_weights(text, return_word_ids, **kwargs) + t5xxl = self.t5xxl.tokenize_with_weights(text, return_word_ids, **kwargs) out["t5xxl"] = [t5xxl[0]] # Use only first 128 tokens - out["llama"] = self.llama.tokenize_with_weights(text, return_word_ids) + out["llama"] = self.llama.tokenize_with_weights(text, return_word_ids, **kwargs) return out def untokenize(self, token_weight_pair): diff --git a/comfy/text_encoders/hunyuan_video.py b/comfy/text_encoders/hunyuan_video.py index 33ac22497..b02148b33 100644 --- a/comfy/text_encoders/hunyuan_video.py +++ b/comfy/text_encoders/hunyuan_video.py @@ -49,13 +49,13 @@ class HunyuanVideoTokenizer: def tokenize_with_weights(self, text, return_word_ids=False, llama_template=None, image_embeds=None, image_interleave=1, **kwargs): out = {} - out["l"] = self.clip_l.tokenize_with_weights(text, return_word_ids) + out["l"] = self.clip_l.tokenize_with_weights(text, return_word_ids, **kwargs) if llama_template is None: llama_text = self.llama_template.format(text) else: llama_text = llama_template.format(text) - llama_text_tokens = self.llama.tokenize_with_weights(llama_text, return_word_ids) + llama_text_tokens = self.llama.tokenize_with_weights(llama_text, return_word_ids, **kwargs) embed_count = 0 for r in llama_text_tokens: for i in range(len(r)): diff --git a/comfy/text_encoders/hydit.py b/comfy/text_encoders/hydit.py index e7273f425..ac6994529 100644 --- a/comfy/text_encoders/hydit.py +++ b/comfy/text_encoders/hydit.py @@ -41,8 +41,8 @@ class HyditTokenizer: def tokenize_with_weights(self, text:str, return_word_ids=False, **kwargs): out = {} - out["hydit_clip"] = self.hydit_clip.tokenize_with_weights(text, return_word_ids) - out["mt5xl"] = self.mt5xl.tokenize_with_weights(text, return_word_ids) + out["hydit_clip"] = self.hydit_clip.tokenize_with_weights(text, return_word_ids, **kwargs) + out["mt5xl"] = self.mt5xl.tokenize_with_weights(text, return_word_ids, **kwargs) return out def untokenize(self, token_weight_pair): diff --git a/comfy/text_encoders/sd3_clip.py b/comfy/text_encoders/sd3_clip.py index 6c2fbeca4..ff5d412db 100644 --- a/comfy/text_encoders/sd3_clip.py +++ b/comfy/text_encoders/sd3_clip.py @@ -45,9 +45,9 @@ class SD3Tokenizer: def tokenize_with_weights(self, text:str, return_word_ids=False, **kwargs): out = {} - out["g"] = self.clip_g.tokenize_with_weights(text, return_word_ids) - out["l"] = self.clip_l.tokenize_with_weights(text, return_word_ids) - out["t5xxl"] = self.t5xxl.tokenize_with_weights(text, return_word_ids) + out["g"] = self.clip_g.tokenize_with_weights(text, return_word_ids, **kwargs) + out["l"] = self.clip_l.tokenize_with_weights(text, return_word_ids, **kwargs) + out["t5xxl"] = self.t5xxl.tokenize_with_weights(text, return_word_ids, **kwargs) return out def untokenize(self, token_weight_pair): diff --git a/comfy/weight_adapter/boft.py b/comfy/weight_adapter/boft.py index c85adc7ab..b2a2f1bd4 100644 --- a/comfy/weight_adapter/boft.py +++ b/comfy/weight_adapter/boft.py @@ -24,7 +24,7 @@ class BOFTAdapter(WeightAdapterBase): ) -> Optional["BOFTAdapter"]: if loaded_keys is None: loaded_keys = set() - blocks_name = "{}.boft_blocks".format(x) + blocks_name = "{}.oft_blocks".format(x) rescale_name = "{}.rescale".format(x) blocks = None @@ -32,17 +32,18 @@ class BOFTAdapter(WeightAdapterBase): blocks = lora[blocks_name] if blocks.ndim == 4: loaded_keys.add(blocks_name) + else: + blocks = None + if blocks is None: + return None rescale = None if rescale_name in lora.keys(): rescale = lora[rescale_name] loaded_keys.add(rescale_name) - if blocks is not None: - weights = (blocks, rescale, alpha, dora_scale) - return cls(loaded_keys, weights) - else: - return None + weights = (blocks, rescale, alpha, dora_scale) + return cls(loaded_keys, weights) def calculate_weight( self, @@ -71,7 +72,7 @@ class BOFTAdapter(WeightAdapterBase): # Get r I = torch.eye(boft_b, device=blocks.device, dtype=blocks.dtype) # for Q = -Q^T - q = blocks - blocks.transpose(1, 2) + q = blocks - blocks.transpose(-1, -2) normed_q = q if alpha > 0: # alpha in boft/bboft is for constraint q_norm = torch.norm(q) + 1e-8 @@ -79,9 +80,8 @@ class BOFTAdapter(WeightAdapterBase): normed_q = q * alpha / q_norm # use float() to prevent unsupported type in .inverse() r = (I + normed_q) @ (I - normed_q).float().inverse() - r = r.to(original_weight) - - inp = org = original_weight + r = r.to(weight) + inp = org = weight r_b = boft_b//2 for i in range(boft_m): @@ -91,14 +91,14 @@ class BOFTAdapter(WeightAdapterBase): if strength != 1: bi = bi * strength + (1-strength) * I inp = ( - inp.unflatten(-1, (-1, g, k)) - .transpose(-2, -1) - .flatten(-3) - .unflatten(-1, (-1, boft_b)) + inp.unflatten(0, (-1, g, k)) + .transpose(1, 2) + .flatten(0, 2) + .unflatten(0, (-1, boft_b)) ) - inp = torch.einsum("b n m, b n ... -> b m ...", inp, bi) + inp = torch.einsum("b i j, b j ...-> b i ...", bi, inp) inp = ( - inp.flatten(-2).unflatten(-1, (-1, k, g)).transpose(-2, -1).flatten(-3) + inp.flatten(0, 1).unflatten(0, (-1, k, g)).transpose(1, 2).flatten(0, 2) ) if rescale is not None: @@ -109,7 +109,7 @@ class BOFTAdapter(WeightAdapterBase): if dora_scale is not None: weight = weight_decompose(dora_scale, weight, lora_diff, alpha, strength, intermediate_dtype, function) else: - weight += function(((strength * alpha) * lora_diff).type(weight.dtype)) + weight += function((strength * lora_diff).type(weight.dtype)) except Exception as e: logging.error("ERROR {} {} {}".format(self.name, key, e)) return weight diff --git a/comfy/weight_adapter/oft.py b/comfy/weight_adapter/oft.py index 0ea229b79..25009eca3 100644 --- a/comfy/weight_adapter/oft.py +++ b/comfy/weight_adapter/oft.py @@ -32,17 +32,18 @@ class OFTAdapter(WeightAdapterBase): blocks = lora[blocks_name] if blocks.ndim == 3: loaded_keys.add(blocks_name) + else: + blocks = None + if blocks is None: + return None rescale = None if rescale_name in lora.keys(): rescale = lora[rescale_name] loaded_keys.add(rescale_name) - if blocks is not None: - weights = (blocks, rescale, alpha, dora_scale) - return cls(loaded_keys, weights) - else: - return None + weights = (blocks, rescale, alpha, dora_scale) + return cls(loaded_keys, weights) def calculate_weight( self, @@ -79,16 +80,17 @@ class OFTAdapter(WeightAdapterBase): normed_q = q * alpha / q_norm # use float() to prevent unsupported type in .inverse() r = (I + normed_q) @ (I - normed_q).float().inverse() - r = r.to(original_weight) + r = r.to(weight) + _, *shape = weight.shape lora_diff = torch.einsum( "k n m, k n ... -> k m ...", (r * strength) - strength * I, - original_weight, - ) + weight.view(block_num, block_size, *shape), + ).view(-1, *shape) if dora_scale is not None: weight = weight_decompose(dora_scale, weight, lora_diff, alpha, strength, intermediate_dtype, function) else: - weight += function(((strength * alpha) * lora_diff).type(weight.dtype)) + weight += function((strength * lora_diff).type(weight.dtype)) except Exception as e: logging.error("ERROR {} {} {}".format(self.name, key, e)) return weight diff --git a/comfy_extras/nodes_cond.py b/comfy_extras/nodes_cond.py index 4c3a1d5bf..574262178 100644 --- a/comfy_extras/nodes_cond.py +++ b/comfy_extras/nodes_cond.py @@ -20,6 +20,29 @@ class CLIPTextEncodeControlnet: c.append(n) return (c, ) +class T5TokenizerOptions: + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "clip": ("CLIP", ), + "min_padding": ("INT", {"default": 0, "min": 0, "max": 10000, "step": 1}), + "min_length": ("INT", {"default": 0, "min": 0, "max": 10000, "step": 1}), + } + } + + RETURN_TYPES = ("CLIP",) + FUNCTION = "set_options" + + def set_options(self, clip, min_padding, min_length): + clip = clip.clone() + for t5_type in ["t5xxl", "pile_t5xl", "t5base", "mt5xl", "umt5xxl"]: + clip.set_tokenizer_option("{}_min_padding".format(t5_type), min_padding) + clip.set_tokenizer_option("{}_min_length".format(t5_type), min_length) + + return (clip, ) + NODE_CLASS_MAPPINGS = { - "CLIPTextEncodeControlnet": CLIPTextEncodeControlnet + "CLIPTextEncodeControlnet": CLIPTextEncodeControlnet, + "T5TokenizerOptions": T5TokenizerOptions, } diff --git a/comfy_extras/nodes_custom_sampler.py b/comfy_extras/nodes_custom_sampler.py index c9689b745..3e5be3d3c 100644 --- a/comfy_extras/nodes_custom_sampler.py +++ b/comfy_extras/nodes_custom_sampler.py @@ -1,3 +1,4 @@ +import math import comfy.samplers import comfy.sample from comfy.k_diffusion import sampling as k_diffusion_sampling @@ -249,6 +250,55 @@ class SetFirstSigma: sigmas[0] = sigma return (sigmas, ) +class ExtendIntermediateSigmas: + @classmethod + def INPUT_TYPES(s): + return {"required": + {"sigmas": ("SIGMAS", ), + "steps": ("INT", {"default": 2, "min": 1, "max": 100}), + "start_at_sigma": ("FLOAT", {"default": -1.0, "min": -1.0, "max": 20000.0, "step": 0.01, "round": False}), + "end_at_sigma": ("FLOAT", {"default": 12.0, "min": 0.0, "max": 20000.0, "step": 0.01, "round": False}), + "spacing": (['linear', 'cosine', 'sine'],), + } + } + RETURN_TYPES = ("SIGMAS",) + CATEGORY = "sampling/custom_sampling/sigmas" + + FUNCTION = "extend" + + def extend(self, sigmas: torch.Tensor, steps: int, start_at_sigma: float, end_at_sigma: float, spacing: str): + if start_at_sigma < 0: + start_at_sigma = float("inf") + + interpolator = { + 'linear': lambda x: x, + 'cosine': lambda x: torch.sin(x*math.pi/2), + 'sine': lambda x: 1 - torch.cos(x*math.pi/2) + }[spacing] + + # linear space for our interpolation function + x = torch.linspace(0, 1, steps + 1, device=sigmas.device)[1:-1] + computed_spacing = interpolator(x) + + extended_sigmas = [] + for i in range(len(sigmas) - 1): + sigma_current = sigmas[i] + sigma_next = sigmas[i+1] + + extended_sigmas.append(sigma_current) + + if end_at_sigma <= sigma_current <= start_at_sigma: + interpolated_steps = computed_spacing * (sigma_next - sigma_current) + sigma_current + extended_sigmas.extend(interpolated_steps.tolist()) + + # Add the last sigma value + if len(sigmas) > 0: + extended_sigmas.append(sigmas[-1]) + + extended_sigmas = torch.FloatTensor(extended_sigmas) + + return (extended_sigmas,) + class KSamplerSelect: @classmethod def INPUT_TYPES(s): @@ -735,6 +785,7 @@ NODE_CLASS_MAPPINGS = { "SplitSigmasDenoise": SplitSigmasDenoise, "FlipSigmas": FlipSigmas, "SetFirstSigma": SetFirstSigma, + "ExtendIntermediateSigmas": ExtendIntermediateSigmas, "CFGGuider": CFGGuider, "DualCFGGuider": DualCFGGuider, diff --git a/comfy_extras/nodes_lt.py b/comfy_extras/nodes_lt.py index ff3fe5cdc..e6dc122ca 100644 --- a/comfy_extras/nodes_lt.py +++ b/comfy_extras/nodes_lt.py @@ -38,6 +38,7 @@ class LTXVImgToVideo: "height": ("INT", {"default": 512, "min": 64, "max": nodes.MAX_RESOLUTION, "step": 32}), "length": ("INT", {"default": 97, "min": 9, "max": nodes.MAX_RESOLUTION, "step": 8}), "batch_size": ("INT", {"default": 1, "min": 1, "max": 4096}), + "strength": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 1.0}), }} RETURN_TYPES = ("CONDITIONING", "CONDITIONING", "LATENT") @@ -46,7 +47,7 @@ class LTXVImgToVideo: CATEGORY = "conditioning/video_models" FUNCTION = "generate" - def generate(self, positive, negative, image, vae, width, height, length, batch_size): + def generate(self, positive, negative, image, vae, width, height, length, batch_size, strength): pixels = comfy.utils.common_upscale(image.movedim(-1, 1), width, height, "bilinear", "center").movedim(1, -1) encode_pixels = pixels[:, :, :, :3] t = vae.encode(encode_pixels) @@ -59,7 +60,7 @@ class LTXVImgToVideo: dtype=torch.float32, device=latent.device, ) - conditioning_latent_frames_mask[:, :, :t.shape[2]] = 0 + conditioning_latent_frames_mask[:, :, :t.shape[2]] = 1.0 - strength return (positive, negative, {"samples": latent, "noise_mask": conditioning_latent_frames_mask}, ) @@ -152,6 +153,15 @@ class LTXVAddGuide: return node_helpers.conditioning_set_values(cond, {"keyframe_idxs": keyframe_idxs}) def append_keyframe(self, positive, negative, frame_idx, latent_image, noise_mask, guiding_latent, strength, scale_factors): + _, latent_idx = self.get_latent_index( + cond=positive, + latent_length=latent_image.shape[2], + guide_length=guiding_latent.shape[2], + frame_idx=frame_idx, + scale_factors=scale_factors, + ) + noise_mask[:, :, latent_idx:latent_idx + guiding_latent.shape[2]] = 1.0 + positive = self.add_keyframe_index(positive, frame_idx, guiding_latent, scale_factors) negative = self.add_keyframe_index(negative, frame_idx, guiding_latent, scale_factors) diff --git a/comfy_extras/nodes_model_merging.py b/comfy_extras/nodes_model_merging.py index ccf601158..f20beab7d 100644 --- a/comfy_extras/nodes_model_merging.py +++ b/comfy_extras/nodes_model_merging.py @@ -209,6 +209,9 @@ def save_checkpoint(model, clip=None, vae=None, clip_vision=None, filename_prefi metadata["modelspec.predict_key"] = "epsilon" elif model.model.model_type == comfy.model_base.ModelType.V_PREDICTION: metadata["modelspec.predict_key"] = "v" + extra_keys["v_pred"] = torch.tensor([]) + if getattr(model_sampling, "zsnr", False): + extra_keys["ztsnr"] = torch.tensor([]) if not args.disable_metadata: metadata["prompt"] = prompt_info @@ -273,7 +276,7 @@ class CLIPSave: comfy.model_management.load_models_gpu([clip.load_model()], force_patch_weights=True) clip_sd = clip.get_sd() - for prefix in ["clip_l.", "clip_g.", ""]: + for prefix in ["clip_l.", "clip_g.", "clip_h.", "t5xxl.", "pile_t5xl.", "mt5xl.", "umt5xxl.", "t5base.", "gemma2_2b.", "llama.", "hydit_clip.", ""]: k = list(filter(lambda a: a.startswith(prefix), clip_sd.keys())) current_clip_sd = {} for x in k: diff --git a/comfy_extras/nodes_optimalsteps.py b/comfy_extras/nodes_optimalsteps.py index f6928199b..e7c851ca2 100644 --- a/comfy_extras/nodes_optimalsteps.py +++ b/comfy_extras/nodes_optimalsteps.py @@ -20,13 +20,14 @@ def loglinear_interp(t_steps, num_steps): NOISE_LEVELS = {"FLUX": [0.9968, 0.9886, 0.9819, 0.975, 0.966, 0.9471, 0.9158, 0.8287, 0.5512, 0.2808, 0.001], "Wan":[1.0, 0.997, 0.995, 0.993, 0.991, 0.989, 0.987, 0.985, 0.98, 0.975, 0.973, 0.968, 0.96, 0.946, 0.927, 0.902, 0.864, 0.776, 0.539, 0.208, 0.001], +"Chroma": [0.992, 0.99, 0.988, 0.985, 0.982, 0.978, 0.973, 0.968, 0.961, 0.953, 0.943, 0.931, 0.917, 0.9, 0.881, 0.858, 0.832, 0.802, 0.769, 0.731, 0.69, 0.646, 0.599, 0.55, 0.501, 0.451, 0.402, 0.355, 0.311, 0.27, 0.232, 0.199, 0.169, 0.143, 0.12, 0.101, 0.084, 0.07, 0.058, 0.048, 0.001], } class OptimalStepsScheduler: @classmethod def INPUT_TYPES(s): return {"required": - {"model_type": (["FLUX", "Wan"], ), + {"model_type": (["FLUX", "Wan", "Chroma"], ), "steps": ("INT", {"default": 20, "min": 3, "max": 1000}), "denoise": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 1.0, "step": 0.01}), } diff --git a/comfy_extras/nodes_post_processing.py b/comfy_extras/nodes_post_processing.py index 5b9542015..cb1a0d883 100644 --- a/comfy_extras/nodes_post_processing.py +++ b/comfy_extras/nodes_post_processing.py @@ -141,6 +141,7 @@ class Quantize: CATEGORY = "image/postprocessing" + @staticmethod def bayer(im, pal_im, order): def normalized_bayer_matrix(n): if n == 0: diff --git a/comfy_extras/nodes_preview_any.py b/comfy_extras/nodes_preview_any.py new file mode 100644 index 000000000..e6805696f --- /dev/null +++ b/comfy_extras/nodes_preview_any.py @@ -0,0 +1,43 @@ +import json +from comfy.comfy_types.node_typing import IO + +# Preview Any - original implement from +# https://github.com/rgthree/rgthree-comfy/blob/main/py/display_any.py +# upstream requested in https://github.com/Kosinkadink/rfcs/blob/main/rfcs/0000-corenodes.md#preview-nodes +class PreviewAny(): + @classmethod + def INPUT_TYPES(cls): + return { + "required": {"source": (IO.ANY, {})}, + } + + RETURN_TYPES = () + FUNCTION = "main" + OUTPUT_NODE = True + + CATEGORY = "utils" + + def main(self, source=None): + value = 'None' + if isinstance(source, str): + value = source + elif isinstance(source, (int, float, bool)): + value = str(source) + elif source is not None: + try: + value = json.dumps(source) + except Exception: + try: + value = str(source) + except Exception: + value = 'source exists, but could not be serialized.' + + return {"ui": {"text": (value,)}} + +NODE_CLASS_MAPPINGS = { + "PreviewAny": PreviewAny, +} + +NODE_DISPLAY_NAME_MAPPINGS = { + "PreviewAny": "Preview Any", +} diff --git a/comfy_extras/nodes_webcam.py b/comfy_extras/nodes_webcam.py index 31eddb2d6..062b15cf8 100644 --- a/comfy_extras/nodes_webcam.py +++ b/comfy_extras/nodes_webcam.py @@ -20,7 +20,7 @@ class WebcamCapture(nodes.LoadImage): CATEGORY = "image" - def load_capture(s, image, **kwargs): + def load_capture(self, image, **kwargs): return super().load_image(folder_paths.get_annotated_filepath(image)) diff --git a/comfyui_version.py b/comfyui_version.py index f9161b37e..8d2068de7 100644 --- a/comfyui_version.py +++ b/comfyui_version.py @@ -1,3 +1,3 @@ # This file is automatically generated by the build process when version is # updated in pyproject.toml. -__version__ = "0.3.29" +__version__ = "0.3.31" diff --git a/hook_breaker_ac10a0.py b/hook_breaker_ac10a0.py new file mode 100644 index 000000000..c3e1c0633 --- /dev/null +++ b/hook_breaker_ac10a0.py @@ -0,0 +1,17 @@ +# Prevent custom nodes from hooking anything important +import comfy.model_management + +HOOK_BREAK = [(comfy.model_management, "cast_to")] + + +SAVED_FUNCTIONS = [] + + +def save_functions(): + for f in HOOK_BREAK: + SAVED_FUNCTIONS.append((f[0], f[1], getattr(f[0], f[1]))) + + +def restore_functions(): + for f in SAVED_FUNCTIONS: + setattr(f[0], f[1], f[2]) diff --git a/main.py b/main.py index ac9d24b7b..5c21542b3 100644 --- a/main.py +++ b/main.py @@ -13,7 +13,7 @@ import logging import sys if __name__ == "__main__": - #NOTE: These do not do anything on core ComfyUI which should already have no communication with the internet, they are for custom nodes. + #NOTE: These do not do anything on core ComfyUI, they are for custom nodes. os.environ['HF_HUB_DISABLE_TELEMETRY'] = '1' os.environ['DO_NOT_TRACK'] = '1' @@ -141,7 +141,7 @@ import nodes import comfy.model_management import comfyui_version import app.logger - +import hook_breaker_ac10a0 def cuda_malloc_warning(): device = comfy.model_management.get_torch_device() @@ -215,6 +215,7 @@ def prompt_worker(q, server_instance): comfy.model_management.soft_empty_cache() last_gc_collect = current_time need_gc = False + hook_breaker_ac10a0.restore_functions() async def run(server_instance, address='', port=8188, verbose=True, call_on_start=None): @@ -268,7 +269,9 @@ def start_comfyui(asyncio_loop=None): prompt_server = server.PromptServer(asyncio_loop) q = execution.PromptQueue(prompt_server) + hook_breaker_ac10a0.save_functions() nodes.init_extra_nodes(init_custom_nodes=not args.disable_all_custom_nodes) + hook_breaker_ac10a0.restore_functions() cuda_malloc_warning() diff --git a/nodes.py b/nodes.py index b79ef730c..d31e0774d 100644 --- a/nodes.py +++ b/nodes.py @@ -917,7 +917,7 @@ class CLIPLoader: @classmethod def INPUT_TYPES(s): return {"required": { "clip_name": (folder_paths.get_filename_list("text_encoders"), ), - "type": (["stable_diffusion", "stable_cascade", "sd3", "stable_audio", "mochi", "ltxv", "pixart", "cosmos", "lumina2", "wan", "hidream"], ), + "type": (["stable_diffusion", "stable_cascade", "sd3", "stable_audio", "mochi", "ltxv", "pixart", "cosmos", "lumina2", "wan", "hidream", "chroma"], ), }, "optional": { "device": (["default", "cpu"], {"advanced": True}), @@ -2258,6 +2258,7 @@ def init_builtin_extra_nodes(): "nodes_optimalsteps.py", "nodes_hidream.py", "nodes_fresca.py", + "nodes_preview_any.py", ] api_nodes_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), "comfy_api_nodes") diff --git a/pyproject.toml b/pyproject.toml index e8fc9555d..8b549a0b6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "ComfyUI" -version = "0.3.29" +version = "0.3.31" readme = "README.md" license = { file = "LICENSE" } requires-python = ">=3.9" @@ -12,6 +12,7 @@ documentation = "https://docs.comfy.org/" [tool.ruff] lint.select = [ + "N805", # invalid-first-argument-name-for-method "S307", # suspicious-eval-usage "S102", # exec "T", # print-usage diff --git a/requirements.txt b/requirements.txt index a0c8fefac..53a0f2345 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -comfyui-frontend-package==1.17.11 +comfyui-frontend-package==1.18.6 comfyui-workflow-templates==0.1.9 torch torchsde diff --git a/tests-unit/comfy_api_test/input_impl_test.py b/tests-unit/comfy_api_test/input_impl_test.py new file mode 100644 index 000000000..5fc21a9a7 --- /dev/null +++ b/tests-unit/comfy_api_test/input_impl_test.py @@ -0,0 +1,91 @@ +import io +from comfy_api.input_impl.video_types import ( + container_to_output_format, + get_open_write_kwargs, +) +from comfy_api.util import VideoContainer + + +def test_container_to_output_format_empty_string(): + """Test that an empty string input returns None. `None` arg allows default auto-detection.""" + assert container_to_output_format("") is None + + +def test_container_to_output_format_none(): + """Test that None input returns None.""" + assert container_to_output_format(None) is None + + +def test_container_to_output_format_comma_separated(): + """Test that a comma-separated list returns a valid singular format from the list.""" + comma_separated_format = "mp4,mov,m4a" + output_format = container_to_output_format(comma_separated_format) + assert output_format in comma_separated_format + + +def test_container_to_output_format_single(): + """Test that a single format string (not comma-separated list) is returned as is.""" + assert container_to_output_format("mp4") == "mp4" + + +def test_get_open_write_kwargs_filepath_no_format(): + """Test that 'format' kwarg is NOT set when dest is a file path.""" + kwargs_auto = get_open_write_kwargs("output.mp4", "mp4", VideoContainer.AUTO) + assert "format" not in kwargs_auto, "Format should not be set for file paths (AUTO)" + + kwargs_specific = get_open_write_kwargs("output.avi", "mp4", "avi") + fail_msg = "Format should not be set for file paths (Specific)" + assert "format" not in kwargs_specific, fail_msg + + +def test_get_open_write_kwargs_base_options_mode(): + """Test basic kwargs for file path: mode and movflags.""" + kwargs = get_open_write_kwargs("output.mp4", "mp4", VideoContainer.AUTO) + assert kwargs["mode"] == "w", "mode should be set to write" + + fail_msg = "movflags should be set to preserve custom metadata tags" + assert "movflags" in kwargs["options"], fail_msg + assert kwargs["options"]["movflags"] == "use_metadata_tags", fail_msg + + +def test_get_open_write_kwargs_bytesio_auto_format(): + """Test kwargs for BytesIO dest with AUTO format.""" + dest = io.BytesIO() + container_fmt = "mov,mp4,m4a" + kwargs = get_open_write_kwargs(dest, container_fmt, VideoContainer.AUTO) + + assert kwargs["mode"] == "w" + assert kwargs["options"]["movflags"] == "use_metadata_tags" + + fail_msg = ( + "Format should be a valid format from the container's format list when AUTO" + ) + assert kwargs["format"] in container_fmt, fail_msg + + +def test_get_open_write_kwargs_bytesio_specific_format(): + """Test kwargs for BytesIO dest with a specific single format.""" + dest = io.BytesIO() + container_fmt = "avi" + to_fmt = VideoContainer.MP4 + kwargs = get_open_write_kwargs(dest, container_fmt, to_fmt) + + assert kwargs["mode"] == "w" + assert kwargs["options"]["movflags"] == "use_metadata_tags" + + fail_msg = "Format should be the specified format (lowercased) when output format is not AUTO" + assert kwargs["format"] == "mp4", fail_msg + + +def test_get_open_write_kwargs_bytesio_specific_format_list(): + """Test kwargs for BytesIO dest with a specific comma-separated format.""" + dest = io.BytesIO() + container_fmt = "avi" + to_fmt = "mov,mp4,m4a" # A format string that is a list + kwargs = get_open_write_kwargs(dest, container_fmt, to_fmt) + + assert kwargs["mode"] == "w" + assert kwargs["options"]["movflags"] == "use_metadata_tags" + + fail_msg = "Format should be a valid format from the specified format list when output format is not AUTO" + assert kwargs["format"] in to_fmt, fail_msg diff --git a/tests-unit/prompt_server_test/user_manager_test.py b/tests-unit/prompt_server_test/user_manager_test.py index 7e523cbf4..b939d8e68 100644 --- a/tests-unit/prompt_server_test/user_manager_test.py +++ b/tests-unit/prompt_server_test/user_manager_test.py @@ -229,3 +229,61 @@ async def test_move_userdata_full_info(aiohttp_client, app, tmp_path): assert not os.path.exists(tmp_path / "source.txt") with open(tmp_path / "dest.txt", "r") as f: assert f.read() == "test content" + + +async def test_listuserdata_v2_empty_root(aiohttp_client, app): + client = await aiohttp_client(app) + resp = await client.get("/v2/userdata") + assert resp.status == 200 + assert await resp.json() == [] + + +async def test_listuserdata_v2_nonexistent_subdirectory(aiohttp_client, app): + client = await aiohttp_client(app) + resp = await client.get("/v2/userdata?path=does_not_exist") + assert resp.status == 404 + + +async def test_listuserdata_v2_default(aiohttp_client, app, tmp_path): + os.makedirs(tmp_path / "test_dir" / "subdir") + (tmp_path / "test_dir" / "file1.txt").write_text("content") + (tmp_path / "test_dir" / "subdir" / "file2.txt").write_text("content") + + client = await aiohttp_client(app) + resp = await client.get("/v2/userdata?path=test_dir") + assert resp.status == 200 + data = await resp.json() + file_paths = {item["path"] for item in data if item["type"] == "file"} + assert file_paths == {"test_dir/file1.txt", "test_dir/subdir/file2.txt"} + + +async def test_listuserdata_v2_normalized_separators(aiohttp_client, app, tmp_path, monkeypatch): + # Force backslash as os separator + monkeypatch.setattr(os, 'sep', '\\') + monkeypatch.setattr(os.path, 'sep', '\\') + os.makedirs(tmp_path / "test_dir" / "subdir") + (tmp_path / "test_dir" / "subdir" / "file1.txt").write_text("x") + + client = await aiohttp_client(app) + resp = await client.get("/v2/userdata?path=test_dir") + assert resp.status == 200 + data = await resp.json() + for item in data: + assert "/" in item["path"] + assert "\\" not in item["path"]\ + +async def test_listuserdata_v2_url_encoded_path(aiohttp_client, app, tmp_path): + # Create a directory with a space in its name and a file inside + os.makedirs(tmp_path / "my dir") + (tmp_path / "my dir" / "file.txt").write_text("content") + + client = await aiohttp_client(app) + # Use URL-encoded space in path parameter + resp = await client.get("/v2/userdata?path=my%20dir&recurse=false") + assert resp.status == 200 + data = await resp.json() + assert len(data) == 1 + entry = data[0] + assert entry["name"] == "file.txt" + # Ensure the path is correctly decoded and uses forward slash + assert entry["path"] == "my dir/file.txt" From adc6067ca76a735446e7d59171adcaeddc872d92 Mon Sep 17 00:00:00 2001 From: Christian Byrne Date: Mon, 5 May 2025 20:37:30 -0700 Subject: [PATCH 111/121] Bump templates version (#154) --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 53a0f2345..59b157158 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ comfyui-frontend-package==1.18.6 -comfyui-workflow-templates==0.1.9 +comfyui-workflow-templates==0.1.10 torch torchsde torchvision From 79f098187f9c835ca4c8fd075380c9fd956916b0 Mon Sep 17 00:00:00 2001 From: Christian Byrne Date: Mon, 5 May 2025 20:39:06 -0700 Subject: [PATCH 112/121] Fix: Kling image gen nodes don't return entire batch when `n` > 1 (#152) --- comfy_api_nodes/nodes_kling.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/comfy_api_nodes/nodes_kling.py b/comfy_api_nodes/nodes_kling.py index 5228e77f1..ae4adc82f 100644 --- a/comfy_api_nodes/nodes_kling.py +++ b/comfy_api_nodes/nodes_kling.py @@ -225,13 +225,16 @@ def video_result_to_node_output( def image_result_to_node_output( - image: KlingImageResult, + images: list[KlingImageResult], ) -> torch.Tensor: """ Converts a KlingImageResult to a tuple containing a [B, H, W, C] tensor. If multiple images are returned, they will be stacked along the batch dimension. """ - return (download_url_to_image_tensor(image.url),) + if len(images) == 1: + return download_url_to_image_tensor(images[0].url) + else: + return torch.cat([download_url_to_image_tensor(image.url) for image in images]) class KlingNodeBase(ComfyNodeABC): @@ -1384,8 +1387,8 @@ class KlingVirtualTryOnNode(KlingImageGenerationBase): final_response = self.get_response(task_id, auth_token) validate_image_result_response(final_response) - image = get_images_from_response(final_response) - return image_result_to_node_output(image) + images = get_images_from_response(final_response) + return image_result_to_node_output(images) class KlingImageGenerationNode(KlingImageGenerationBase): @@ -1513,8 +1516,8 @@ class KlingImageGenerationNode(KlingImageGenerationBase): final_response = self.get_response(task_id, auth_token) validate_image_result_response(final_response) - image = get_images_from_response(final_response) - return image_result_to_node_output(image) + images = get_images_from_response(final_response) + return image_result_to_node_output(images) NODE_CLASS_MAPPINGS = { From 98bfa52ccf568adf98b95ce7d40343540b040817 Mon Sep 17 00:00:00 2001 From: Jedrzej Kosinski Date: Mon, 5 May 2025 22:57:59 -0500 Subject: [PATCH 113/121] Remove pixverse_template from PixVerse Transition Video node (#155) --- comfy_api_nodes/nodes_pixverse.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/comfy_api_nodes/nodes_pixverse.py b/comfy_api_nodes/nodes_pixverse.py index 864bf40e0..dbb90c1dd 100644 --- a/comfy_api_nodes/nodes_pixverse.py +++ b/comfy_api_nodes/nodes_pixverse.py @@ -404,12 +404,6 @@ class PixverseTransitionVideoNode(ComfyNodeABC): "tooltip": "An optional text description of undesired elements on an image.", }, ), - "pixverse_template": ( - PixverseIO.TEMPLATE, - { - "tooltip": "An optional template to influence style of generation, created by the PixVerse Template node." - } - ) }, "hidden": { "auth_token": "AUTH_TOKEN_COMFY_ORG", @@ -426,7 +420,6 @@ class PixverseTransitionVideoNode(ComfyNodeABC): motion_mode: str, seed, negative_prompt: str=None, - pixverse_template: int=None, auth_token=None, **kwargs, ): @@ -457,7 +450,6 @@ class PixverseTransitionVideoNode(ComfyNodeABC): duration=duration_seconds, motion_mode=motion_mode, negative_prompt=negative_prompt if negative_prompt else None, - template_id=pixverse_template, seed=seed, ), auth_token=auth_token, From 79edd2dde7bcbab372d255fb6b54e793dcafb50f Mon Sep 17 00:00:00 2001 From: Jedrzej Kosinski Date: Mon, 5 May 2025 23:09:16 -0500 Subject: [PATCH 114/121] Invert image_weight value on Luma Image to Image node (#156) --- comfy_api_nodes/nodes_luma.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/comfy_api_nodes/nodes_luma.py b/comfy_api_nodes/nodes_luma.py index 6a9d46fdc..0f0d9aa80 100644 --- a/comfy_api_nodes/nodes_luma.py +++ b/comfy_api_nodes/nodes_luma.py @@ -328,11 +328,11 @@ class LumaImageModifyNode(ComfyNodeABC): "image_weight": ( IO.FLOAT, { - "default": 1.0, - "min": 0.02, - "max": 1.0, + "default": 0.1, + "min": 0.0, + "max": 0.98, "step": 0.01, - "tooltip": "Weight of the image; the closer to 0.0, the less the image will be modified.", + "tooltip": "Weight of the image; the closer to 1.0, the less the image will be modified.", }, ), "model": ([model.value for model in LumaImageModel],), @@ -380,7 +380,7 @@ class LumaImageModifyNode(ComfyNodeABC): prompt=prompt, model=model, modify_image_ref=LumaModifyImageRef( - url=image_url, weight=round(image_weight, 2) + url=image_url, weight=round(max(min(1.0-image_weight, 0.98), 0.0), 2) ), ), auth_token=auth_token, From 5b523ce4f4655bde5a4417ac656fc4fd116e547b Mon Sep 17 00:00:00 2001 From: Jedrzej Kosinski Date: Mon, 5 May 2025 23:37:21 -0500 Subject: [PATCH 115/121] Invert and resize mask for Ideogram V3 node to match masking conventions (#158) --- comfy_api_nodes/nodes_ideogram.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/comfy_api_nodes/nodes_ideogram.py b/comfy_api_nodes/nodes_ideogram.py index 3f80a62d7..45c021f4a 100644 --- a/comfy_api_nodes/nodes_ideogram.py +++ b/comfy_api_nodes/nodes_ideogram.py @@ -21,6 +21,7 @@ from comfy_api_nodes.apis.client import ( from comfy_api_nodes.apinode_utils import ( download_url_to_bytesio, bytesio_to_image_tensor, + resize_mask_to_image, ) V1_V1_RES_MAP = { @@ -649,6 +650,10 @@ class IdeogramV3(ComfyNodeABC): # Process image and mask input_tensor = image.squeeze().cpu() + # Resize mask to match image dimension + mask = resize_mask_to_image(mask, image, allow_gradient=False) + # Invert mask, as Ideogram API will edit black areas instead of white areas (opposite of convention). + mask = 1.0 - mask # Validate mask dimensions match image if mask.shape[1:] != image.shape[1:-1]: From 1dc3880ca467445530c2fb658acac57e5b47b2c4 Mon Sep 17 00:00:00 2001 From: Christian Byrne Date: Mon, 5 May 2025 21:58:26 -0700 Subject: [PATCH 116/121] [Kling] Fix: image generation nodes not returning Tuple (#159) --- comfy_api_nodes/nodes_kling.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/comfy_api_nodes/nodes_kling.py b/comfy_api_nodes/nodes_kling.py index ae4adc82f..e60c57b0e 100644 --- a/comfy_api_nodes/nodes_kling.py +++ b/comfy_api_nodes/nodes_kling.py @@ -1388,7 +1388,7 @@ class KlingVirtualTryOnNode(KlingImageGenerationBase): validate_image_result_response(final_response) images = get_images_from_response(final_response) - return image_result_to_node_output(images) + return (image_result_to_node_output(images),) class KlingImageGenerationNode(KlingImageGenerationBase): @@ -1517,7 +1517,7 @@ class KlingImageGenerationNode(KlingImageGenerationBase): validate_image_result_response(final_response) images = get_images_from_response(final_response) - return image_result_to_node_output(images) + return (image_result_to_node_output(images),) NODE_CLASS_MAPPINGS = { From 0cab40b28ffc006bed07511af0ca32cd77bc27c3 Mon Sep 17 00:00:00 2001 From: Christian Byrne Date: Mon, 5 May 2025 22:33:21 -0700 Subject: [PATCH 117/121] [Bug] [Kling] Fix Kling camera control (#161) --- comfy_api_nodes/nodes_kling.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/comfy_api_nodes/nodes_kling.py b/comfy_api_nodes/nodes_kling.py index e60c57b0e..df4e07fdd 100644 --- a/comfy_api_nodes/nodes_kling.py +++ b/comfy_api_nodes/nodes_kling.py @@ -412,10 +412,13 @@ class KlingTextToVideoNode(KlingNodeBase): mode: str, aspect_ratio: str, camera_control: Optional[KlingCameraControl] = None, + model_name: Optional[str] = None, + duration: Optional[str] = None, auth_token: Optional[str] = None, ) -> tuple[VideoFromFile, str, str]: validate_prompts(prompt, negative_prompt, MAX_PROMPT_LENGTH_T2V) - mode, duration, model_name = self.get_mode_string_mapping()[mode] + if model_name is None: + mode, duration, model_name = self.get_mode_string_mapping()[mode] initial_operation = SynchronousOperation( endpoint=ApiEndpoint( path=PATH_TEXT_TO_VIDEO, @@ -502,9 +505,9 @@ class KlingCameraControlT2VNode(KlingTextToVideoNode): auth_token: Optional[str] = None, ): return super().api_call( - model_name=KlingVideoGenModelName.kling_v1_5, + model_name=KlingVideoGenModelName.kling_v1, cfg_scale=cfg_scale, - mode=KlingVideoGenMode.pro, + mode=KlingVideoGenMode.std, aspect_ratio=KlingVideoGenAspectRatio(aspect_ratio), duration=KlingVideoGenDuration.field_5, prompt=prompt, From 5d3e0557531816713b8a3cdf4846d5cb596ffcdd Mon Sep 17 00:00:00 2001 From: Robin Huang Date: Mon, 5 May 2025 22:36:55 -0700 Subject: [PATCH 118/121] Kling Image Gen v2 + improve node descriptions for Flux/OpenAI (#160) --- comfy_api_nodes/apis/__init__.py | 1 + comfy_api_nodes/nodes_bfl.py | 2 +- comfy_api_nodes/nodes_openai.py | 9 --------- 3 files changed, 2 insertions(+), 10 deletions(-) diff --git a/comfy_api_nodes/apis/__init__.py b/comfy_api_nodes/apis/__init__.py index cb448f4fc..aa1c4ce0b 100644 --- a/comfy_api_nodes/apis/__init__.py +++ b/comfy_api_nodes/apis/__init__.py @@ -568,6 +568,7 @@ class KlingImageGenImageReferenceType(str, Enum): class KlingImageGenModelName(str, Enum): kling_v1 = 'kling-v1' kling_v1_5 = 'kling-v1-5' + kling_v2 = 'kling-v2' class KlingImageResult(BaseModel): diff --git a/comfy_api_nodes/nodes_bfl.py b/comfy_api_nodes/nodes_bfl.py index 7f02df88e..122a6ddf8 100644 --- a/comfy_api_nodes/nodes_bfl.py +++ b/comfy_api_nodes/nodes_bfl.py @@ -112,7 +112,7 @@ def convert_image_to_base64(image: torch.Tensor): class FluxProUltraImageNode(ComfyNodeABC): """ - Generates images synchronously based on prompt and resolution. + Generates images using Flux Pro 1.1 Ultra via api based on prompt and resolution. """ MINIMUM_RATIO = 1 / 4 diff --git a/comfy_api_nodes/nodes_openai.py b/comfy_api_nodes/nodes_openai.py index 88db82f09..c18c65d7a 100644 --- a/comfy_api_nodes/nodes_openai.py +++ b/comfy_api_nodes/nodes_openai.py @@ -28,9 +28,6 @@ from comfy_api_nodes.apinode_utils import ( class OpenAIDalle2(ComfyNodeABC): """ Generates images synchronously via OpenAI's DALL·E 2 endpoint. - - Uses the proxy at /proxy/openai/images/generations. Returned URLs are short‑lived, - so download or cache results if you need to keep them. """ def __init__(self): @@ -183,9 +180,6 @@ class OpenAIDalle2(ComfyNodeABC): class OpenAIDalle3(ComfyNodeABC): """ Generates images synchronously via OpenAI's DALL·E 3 endpoint. - - Uses the proxy at /proxy/openai/images/generations. Returned URLs are short‑lived, - so download or cache results if you need to keep them. """ def __init__(self): @@ -291,9 +285,6 @@ class OpenAIDalle3(ComfyNodeABC): class OpenAIGPTImage1(ComfyNodeABC): """ Generates images synchronously via OpenAI's GPT Image 1 endpoint. - - Uses the proxy at /proxy/openai/images/generations. Returned URLs are short‑lived, - so download or cache results if you need to keep them. """ def __init__(self): From 5bb5cc7c5e861d91c5c37314b346e3df54bdabc2 Mon Sep 17 00:00:00 2001 From: Christian Byrne Date: Mon, 5 May 2025 22:57:17 -0700 Subject: [PATCH 119/121] [Kling] Don't return video_id from dual effect video (#162) --- comfy_api_nodes/nodes_kling.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/comfy_api_nodes/nodes_kling.py b/comfy_api_nodes/nodes_kling.py index df4e07fdd..8f94d35fd 100644 --- a/comfy_api_nodes/nodes_kling.py +++ b/comfy_api_nodes/nodes_kling.py @@ -1001,6 +1001,8 @@ class KlingDualCharacterVideoEffectNode(KlingVideoEffectsBase): } DESCRIPTION = "Achieve different special effects when generating a video based on the effect_scene. First image will be positioned on left side, second on right side of the composite." + RETURN_TYPES = ("VIDEO", "STRING") + RETURN_NAMES = ("VIDEO", "duration") def api_call( self, @@ -1012,7 +1014,7 @@ class KlingDualCharacterVideoEffectNode(KlingVideoEffectsBase): duration: KlingVideoGenDuration, auth_token: Optional[str] = None, ): - return super().api_call( + video, _, duration = super().api_call( dual_character=True, effect_scene=effect_scene, model_name=model_name, @@ -1022,7 +1024,7 @@ class KlingDualCharacterVideoEffectNode(KlingVideoEffectsBase): image_2=image_right, auth_token=auth_token, ) - + return video, duration class KlingSingleImageVideoEffectNode(KlingVideoEffectsBase): """Kling Single Image Video Effect Node""" From a75d35e982d7edad868139c11d28ff16561d0cce Mon Sep 17 00:00:00 2001 From: Christian Byrne Date: Mon, 5 May 2025 23:01:18 -0700 Subject: [PATCH 120/121] Bump frontend to 1.18.8 (#163) --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 59b157158..90059407a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -comfyui-frontend-package==1.18.6 +comfyui-frontend-package==1.18.8 comfyui-workflow-templates==0.1.10 torch torchsde From ec89560fad575a27682cb3ee67e61f2a5e4a49d8 Mon Sep 17 00:00:00 2001 From: Christian Byrne Date: Mon, 5 May 2025 23:12:12 -0700 Subject: [PATCH 121/121] Use 3.9 compat syntax (#164) --- comfy_api_nodes/apis/client.py | 17 +++++++++-------- comfy_api_nodes/mapper_utils.py | 27 +++++++++++++-------------- comfy_api_nodes/nodes_kling.py | 1 + 3 files changed, 23 insertions(+), 22 deletions(-) diff --git a/comfy_api_nodes/apis/client.py b/comfy_api_nodes/apis/client.py index 496831ac4..929e386d4 100644 --- a/comfy_api_nodes/apis/client.py +++ b/comfy_api_nodes/apis/client.py @@ -89,6 +89,7 @@ operation = PollingOperation( result = operation.execute(client=api_client) # Returns the final ImageGenerationResult when done """ +from __future__ import annotations import logging import time import io @@ -196,7 +197,6 @@ class ApiClient: "headers": headers, } - def get_headers(self) -> Dict[str, str]: """Get headers for API requests, including authentication if available""" headers = {"Content-Type": "application/json", "Accept": "application/json"} @@ -251,13 +251,14 @@ class ApiClient: logging.debug(f"[DEBUG] Params: {params}") logging.debug(f"[DEBUG] Data: {data}") - match content_type: - case "application/x-www-form-urlencoded": - payload_args = self._create_urlencoded_form_data_args(data, request_headers) - case "multipart/form-data": - payload_args = self._create_form_data_args(data, files, request_headers, multipart_parser) - case _: - payload_args = self._create_json_payload_args(data, request_headers) + if content_type == "application/x-www-form-urlencoded": + payload_args = self._create_urlencoded_form_data_args(data, request_headers) + elif content_type == "multipart/form-data": + payload_args = self._create_form_data_args( + data, files, request_headers, multipart_parser + ) + else: + payload_args = self._create_json_payload_args(data, request_headers) try: response = requests.request( diff --git a/comfy_api_nodes/mapper_utils.py b/comfy_api_nodes/mapper_utils.py index f8fd5632e..6fab8f4bb 100644 --- a/comfy_api_nodes/mapper_utils.py +++ b/comfy_api_nodes/mapper_utils.py @@ -99,19 +99,18 @@ def model_field_to_node_input( field_info: FieldInfo = base_model.model_fields[field_name] result: NodeInput - match input_type: - case IO.IMAGE: - result = _model_field_to_image_input(field_info, **kwargs) - case IO.STRING: - result = _model_field_to_string_input(field_info, **kwargs) - case IO.FLOAT: - result = _model_field_to_float_input(field_info, **kwargs) - case IO.INT: - result = _model_field_to_int_input(field_info, **kwargs) - case IO.COMBO: - result = _model_field_to_combo_input(field_info, **kwargs) - case _: - message = f"Invalid input type: {input_type}" - raise ValueError(message) + if input_type == IO.IMAGE: + result = _model_field_to_image_input(field_info, **kwargs) + elif input_type == IO.STRING: + result = _model_field_to_string_input(field_info, **kwargs) + elif input_type == IO.FLOAT: + result = _model_field_to_float_input(field_info, **kwargs) + elif input_type == IO.INT: + result = _model_field_to_int_input(field_info, **kwargs) + elif input_type == IO.COMBO: + result = _model_field_to_combo_input(field_info, **kwargs) + else: + message = f"Invalid input type: {input_type}" + raise ValueError(message) return result diff --git a/comfy_api_nodes/nodes_kling.py b/comfy_api_nodes/nodes_kling.py index 8f94d35fd..b3be2bac8 100644 --- a/comfy_api_nodes/nodes_kling.py +++ b/comfy_api_nodes/nodes_kling.py @@ -4,6 +4,7 @@ For source of truth on the allowed permutations of request fields, please refere - [Compatibility Table](https://app.klingai.com/global/dev/document-api/apiReference/model/skillsMap) """ +from __future__ import annotations from typing import Optional, TypeVar, Any import math import logging