mirror of
https://git.datalinker.icu/comfyanonymous/ComfyUI
synced 2026-09-06 11:27:04 +08:00
Add Kling Nodes (#12)
This commit is contained in:
Christian Byrne
Jedrzej Kosinski
parent
2828b558a7
commit
723b656af3
File diff suppressed because it is too large
Load Diff
@ -523,7 +523,6 @@ class PollingOperation(Generic[T, R]):
|
||||
|
||||
# Parse response
|
||||
response_obj = self.poll_endpoint.response_model.model_validate(resp)
|
||||
|
||||
# Check if task is complete
|
||||
status = self._check_task_status(response_obj)
|
||||
logging.debug(f"[DEBUG] Task Status: {status}")
|
||||
|
||||
479
comfy_api_nodes/nodes_kling.py
Normal file
479
comfy_api_nodes/nodes_kling.py
Normal file
@ -0,0 +1,479 @@
|
||||
from inspect import cleandoc
|
||||
from typing import Union, Optional
|
||||
import math
|
||||
import logging
|
||||
import torch
|
||||
|
||||
from comfy_api_nodes.apis import (
|
||||
KlingText2VideoRequest,
|
||||
KlingText2VideoResponse,
|
||||
TaskStatus,
|
||||
CameraControl,
|
||||
Config as CameraConfig,
|
||||
Type as CameraType,
|
||||
Duration,
|
||||
Mode,
|
||||
AspectRatio,
|
||||
ModelName,
|
||||
KlingImage2VideoRequest,
|
||||
KlingImage2VideoResponse,
|
||||
)
|
||||
from comfy_api_nodes.apis.client import (
|
||||
ApiEndpoint,
|
||||
HttpMethod,
|
||||
SynchronousOperation,
|
||||
PollingOperation,
|
||||
EmptyRequest,
|
||||
)
|
||||
from comfy_api_nodes.nodes_api import (
|
||||
tensor_to_base64_string,
|
||||
download_url_to_bytesio,
|
||||
)
|
||||
from comfy.comfy_types.node_typing import IO, InputTypeOptions, ComfyNodeABC
|
||||
from comfy_api.input_impl import VideoFromFile
|
||||
from comfy_api_nodes.mapper_utils import model_field_to_node_input
|
||||
|
||||
KLING_API_VERSION = "v1"
|
||||
PATH_TEXT_TO_VIDEO = f"/proxy/kling/{KLING_API_VERSION}/videos/text2video"
|
||||
PATH_IMAGE_TO_VIDEO = f"/proxy/kling/{KLING_API_VERSION}/videos/image2video"
|
||||
PATH_VIDEO_EXTEND = f"/proxy/kling/{KLING_API_VERSION}/videos/video-extend"
|
||||
PATH_LIP_SYNC = f"/proxy/kling/{KLING_API_VERSION}/videos/lip-sync"
|
||||
PATH_VIDEO_EFFECTS = f"/proxy/kling/{KLING_API_VERSION}/videos/effects"
|
||||
PATH_CHARACTER_IMAGE = f"/proxy/kling/{KLING_API_VERSION}/images/generations"
|
||||
PATH_VIRTUAL_TRY_ON = f"/proxy/kling/{KLING_API_VERSION}/images/kolors-virtual-try-on"
|
||||
|
||||
|
||||
class KlingApiError(Exception):
|
||||
"""Base exception for Kling API errors."""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
def is_valid_camera_control_configs(configs: list[float]) -> bool:
|
||||
"""Verifies that at least one camera control configuration is non-zero."""
|
||||
return any(not math.isclose(value, 0.0) for value in configs)
|
||||
|
||||
|
||||
def is_valid_prompt(prompt: str) -> bool:
|
||||
"""Verifies that the prompt is not empty."""
|
||||
return bool(prompt)
|
||||
|
||||
|
||||
def is_valid_initial_response(response: KlingText2VideoResponse) -> bool:
|
||||
"""Verifies that the initial response contains a task ID."""
|
||||
return bool(response.data.task_id)
|
||||
|
||||
|
||||
def is_valid_video_response(response: KlingText2VideoResponse) -> bool:
|
||||
"""Verifies that the response contains a task result with at least one video."""
|
||||
return (
|
||||
response.data.task_result
|
||||
and response.data.task_result.videos
|
||||
and len(response.data.task_result.videos) > 0
|
||||
)
|
||||
|
||||
|
||||
def is_camera_control_supported(model_name: str, duration: str, mode: str) -> bool:
|
||||
"""`camera_control` is only supported in `pro` mode with `5s` duration and `kling-v1-5`"""
|
||||
return model_name == "kling-v1-5" and duration == "5" and mode == "pro"
|
||||
|
||||
|
||||
def get_camera_control_input_config(
|
||||
tooltip: str, default: float = 0.0
|
||||
) -> tuple[IO, InputTypeOptions]:
|
||||
"""Returns common InputTypeOptions for Kling camera control configurations."""
|
||||
input_config = {
|
||||
"default": default,
|
||||
"min": -10.0,
|
||||
"max": 10.0,
|
||||
"step": 0.25,
|
||||
"display": "slider",
|
||||
"tooltip": tooltip,
|
||||
}
|
||||
return IO.FLOAT, input_config
|
||||
|
||||
|
||||
def _get_camera_control_inputs() -> dict[str, tuple[IO, InputTypeOptions]]:
|
||||
"""Returns a dictionary of camera control inputs common to Kling video generation nodes."""
|
||||
return {
|
||||
"camera_control_type": (
|
||||
IO.COMBO,
|
||||
{
|
||||
"options": [
|
||||
camera_control_type.value for camera_control_type in CameraType
|
||||
],
|
||||
"default": "simple",
|
||||
"tooltip": "Predefined camera movements type. simple: Customizable camera movement. down_back: Camera descends and moves backward. forward_up: Camera moves forward and tilts up. right_turn_forward: Rotate right and move forward. left_turn_forward: Rotate left and move forward.",
|
||||
},
|
||||
),
|
||||
"camera_control_horizontal": get_camera_control_input_config(
|
||||
"Controls camera's movement along horizontal axis (x-axis). Negative indicates left, positive indicates right"
|
||||
),
|
||||
"camera_control_vertical": get_camera_control_input_config(
|
||||
"Controls camera's movement along vertical axis (y-axis). Negative indicates downward, positive indicates upward."
|
||||
),
|
||||
"camera_control_pan": get_camera_control_input_config(
|
||||
"Controls camera's rotation in vertical plane (x-axis). Negative indicates downward rotation, positive indicates upward rotation.",
|
||||
default=0.5,
|
||||
),
|
||||
"camera_control_roll": get_camera_control_input_config(
|
||||
"Controls camera's rotation in horizontal plane (y-axis). Negative indicates left rotation, positive indicates right rotation.",
|
||||
),
|
||||
"camera_control_tilt": get_camera_control_input_config(
|
||||
"Controls camera's rolling amount (z-axis). Negative indicates counterclockwise, positive indicates clockwise.",
|
||||
),
|
||||
"camera_control_zoom": get_camera_control_input_config(
|
||||
"Controls change in camera's focal length. Negative indicates narrower field of view, positive indicates wider field of view.",
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def download_url_to_video_output(video_url: str) -> tuple[VideoFromFile]:
|
||||
"""Downloads a video from a URL and returns a VIDEO output."""
|
||||
video_io = download_url_to_bytesio(video_url)
|
||||
if video_io is None:
|
||||
error_msg = f"Failed to download video from {video_url}"
|
||||
logging.error(error_msg)
|
||||
raise KlingApiError(error_msg)
|
||||
return (VideoFromFile(video_io),)
|
||||
|
||||
|
||||
class KlingNodeABC(ComfyNodeABC):
|
||||
"""Base class for Kling nodes."""
|
||||
|
||||
@classmethod
|
||||
def VALIDATE_INPUTS(
|
||||
cls,
|
||||
prompt,
|
||||
negative_prompt,
|
||||
camera_control_horizontal,
|
||||
camera_control_vertical,
|
||||
camera_control_pan,
|
||||
camera_control_roll,
|
||||
camera_control_tilt,
|
||||
camera_control_zoom,
|
||||
) -> Union[str, bool]:
|
||||
if not is_valid_prompt(prompt):
|
||||
return "Prompt is required"
|
||||
if len(prompt) >= 2500:
|
||||
return "Prompt must be less than 2500 characters"
|
||||
if negative_prompt and len(negative_prompt) >= 2500:
|
||||
return "Negative prompt must be less than 2500 characters"
|
||||
if not is_valid_camera_control_configs(
|
||||
[
|
||||
camera_control_horizontal,
|
||||
camera_control_vertical,
|
||||
camera_control_pan,
|
||||
camera_control_roll,
|
||||
camera_control_tilt,
|
||||
camera_control_zoom,
|
||||
]
|
||||
):
|
||||
return "Invalid camera control configs"
|
||||
return True
|
||||
|
||||
DESCRIPTION = cleandoc(__doc__ or "")
|
||||
FUNCTION = "api_call"
|
||||
CATEGORY = "api node/video/kling"
|
||||
API_NODE = True
|
||||
|
||||
|
||||
class KlingTextToVideoNode(KlingNodeABC):
|
||||
"""
|
||||
Kling Text to Video Node.
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def poll_for_task_status(task_id: str, auth_token: str) -> KlingText2VideoResponse:
|
||||
"""Polls the Kling API endpoint until the task reaches a terminal state."""
|
||||
polling_operation = PollingOperation(
|
||||
poll_endpoint=ApiEndpoint(
|
||||
path=f"{PATH_TEXT_TO_VIDEO}/{task_id}",
|
||||
method=HttpMethod.GET,
|
||||
request_model=EmptyRequest,
|
||||
response_model=KlingText2VideoResponse,
|
||||
),
|
||||
completed_statuses=[
|
||||
TaskStatus.succeed.value,
|
||||
],
|
||||
failed_statuses=[TaskStatus.failed.value],
|
||||
status_extractor=lambda response: (
|
||||
response.data.task_status.value
|
||||
if response.data and response.data.task_status
|
||||
else None
|
||||
),
|
||||
auth_token=auth_token,
|
||||
)
|
||||
return polling_operation.execute()
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(s):
|
||||
return {
|
||||
"required": {
|
||||
"prompt": model_field_to_node_input(
|
||||
IO.STRING, KlingText2VideoRequest, "prompt", multiline=True
|
||||
),
|
||||
"negative_prompt": model_field_to_node_input(
|
||||
IO.STRING, KlingText2VideoRequest, "negative_prompt", multiline=True
|
||||
),
|
||||
"cfg_scale": model_field_to_node_input(
|
||||
IO.FLOAT, KlingText2VideoRequest, "cfg_scale"
|
||||
),
|
||||
"mode": model_field_to_node_input(
|
||||
IO.COMBO, KlingText2VideoRequest, "mode", enum_type=Mode
|
||||
),
|
||||
"duration": model_field_to_node_input(
|
||||
IO.COMBO, KlingText2VideoRequest, "duration", enum_type=Duration
|
||||
),
|
||||
"aspect_ratio": model_field_to_node_input(
|
||||
IO.COMBO,
|
||||
KlingText2VideoRequest,
|
||||
"aspect_ratio",
|
||||
enum_type=AspectRatio,
|
||||
),
|
||||
**_get_camera_control_inputs(),
|
||||
},
|
||||
"hidden": {"auth_token": "AUTH_TOKEN_COMFY_ORG"},
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("VIDEO",)
|
||||
|
||||
def api_call(
|
||||
self,
|
||||
prompt: str,
|
||||
negative_prompt: str,
|
||||
duration: int,
|
||||
mode: str,
|
||||
cfg_scale: float,
|
||||
aspect_ratio: str,
|
||||
camera_control_type: str,
|
||||
camera_control_horizontal: float,
|
||||
camera_control_vertical: float,
|
||||
camera_control_pan: float,
|
||||
camera_control_roll: float,
|
||||
camera_control_tilt: float,
|
||||
camera_control_zoom: float,
|
||||
auth_token: Optional[str] = None,
|
||||
) -> tuple[VideoFromFile]:
|
||||
camera_control = None
|
||||
if is_camera_control_supported("kling-v1-6", duration, mode):
|
||||
camera_control = CameraControl(
|
||||
type=CameraType(camera_control_type),
|
||||
config=CameraConfig(
|
||||
horizontal=camera_control_horizontal,
|
||||
vertical=camera_control_vertical,
|
||||
pan=camera_control_pan,
|
||||
roll=camera_control_roll,
|
||||
tilt=camera_control_tilt,
|
||||
zoom=camera_control_zoom,
|
||||
).model_dump(exclude_none=True),
|
||||
)
|
||||
|
||||
initial_operation = SynchronousOperation(
|
||||
endpoint=ApiEndpoint(
|
||||
path=PATH_TEXT_TO_VIDEO,
|
||||
method=HttpMethod.POST,
|
||||
request_model=KlingText2VideoRequest,
|
||||
response_model=KlingText2VideoResponse,
|
||||
),
|
||||
request=KlingText2VideoRequest(
|
||||
prompt=prompt if prompt else None,
|
||||
negative_prompt=negative_prompt if negative_prompt else None,
|
||||
duration=Duration(duration),
|
||||
mode=Mode(mode),
|
||||
cfg_scale=cfg_scale,
|
||||
aspect_ratio=AspectRatio(aspect_ratio),
|
||||
camera_control=camera_control,
|
||||
),
|
||||
auth_token=auth_token,
|
||||
)
|
||||
|
||||
initial_response = initial_operation.execute()
|
||||
if not is_valid_initial_response(initial_response):
|
||||
error_msg = f"Kling initial request failed. Code: {initial_response.code}, Message: {initial_response.message}, Data: {initial_response.data}"
|
||||
logging.error(error_msg)
|
||||
raise KlingApiError(error_msg)
|
||||
|
||||
task_id = initial_response.data.task_id
|
||||
logging.debug("Kling task submitted. Task ID: %s", task_id)
|
||||
|
||||
final_response = self.poll_for_task_status(task_id, auth_token)
|
||||
if not is_valid_video_response(final_response):
|
||||
error_msg = (
|
||||
f"Kling task {task_id} succeeded but no video data found in response."
|
||||
)
|
||||
logging.error(error_msg)
|
||||
raise KlingApiError(error_msg)
|
||||
|
||||
video_url = str(final_response.data.task_result.videos[0].url)
|
||||
logging.debug("Kling task %s succeeded. Video URL: %s", task_id, video_url)
|
||||
|
||||
return download_url_to_video_output(video_url)
|
||||
|
||||
|
||||
class KlingImage2VideoNode(KlingNodeABC):
|
||||
"""
|
||||
Kling Image to Video Node.
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def poll_for_task_status(task_id: str, auth_token: str) -> KlingImage2VideoResponse:
|
||||
"""Polls the Kling API endpoint until the task reaches a terminal state."""
|
||||
polling_operation = PollingOperation(
|
||||
poll_endpoint=ApiEndpoint(
|
||||
path=f"{PATH_IMAGE_TO_VIDEO}/{task_id}",
|
||||
method=HttpMethod.GET,
|
||||
request_model=KlingImage2VideoRequest,
|
||||
response_model=KlingImage2VideoResponse,
|
||||
),
|
||||
completed_statuses=[TaskStatus.succeed.value],
|
||||
failed_statuses=[TaskStatus.failed.value],
|
||||
status_extractor=lambda response: (
|
||||
response.data.task_status.value
|
||||
if response.data and response.data.task_status
|
||||
else None
|
||||
),
|
||||
auth_token=auth_token,
|
||||
)
|
||||
return polling_operation.execute()
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(s):
|
||||
return {
|
||||
"required": {
|
||||
"model_name": model_field_to_node_input(
|
||||
IO.COMBO, KlingImage2VideoRequest, "model_name", enum_type=ModelName
|
||||
),
|
||||
"start_frame": model_field_to_node_input(
|
||||
IO.IMAGE, KlingImage2VideoRequest, "image"
|
||||
),
|
||||
"prompt": model_field_to_node_input(
|
||||
IO.STRING, KlingImage2VideoRequest, "prompt", multiline=True
|
||||
),
|
||||
"negative_prompt": model_field_to_node_input(
|
||||
IO.STRING,
|
||||
KlingImage2VideoRequest,
|
||||
"negative_prompt",
|
||||
multiline=True,
|
||||
),
|
||||
"cfg_scale": model_field_to_node_input(
|
||||
IO.FLOAT, KlingImage2VideoRequest, "cfg_scale"
|
||||
),
|
||||
"mode": model_field_to_node_input(
|
||||
IO.COMBO, KlingImage2VideoRequest, "mode", enum_type=Mode
|
||||
),
|
||||
"aspect_ratio": model_field_to_node_input(
|
||||
IO.COMBO,
|
||||
KlingImage2VideoRequest,
|
||||
"aspect_ratio",
|
||||
enum_type=AspectRatio,
|
||||
),
|
||||
"duration": model_field_to_node_input(
|
||||
IO.COMBO, KlingImage2VideoRequest, "duration", enum_type=Duration
|
||||
),
|
||||
**_get_camera_control_inputs(),
|
||||
},
|
||||
"optional": {
|
||||
"end_frame": model_field_to_node_input(
|
||||
IO.IMAGE, KlingImage2VideoRequest, "image_tail"
|
||||
),
|
||||
},
|
||||
"hidden": {"auth_token": "AUTH_TOKEN_COMFY_ORG"},
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("VIDEO",)
|
||||
|
||||
def api_call(
|
||||
self,
|
||||
model_name: str,
|
||||
start_frame: torch.Tensor,
|
||||
prompt: str,
|
||||
negative_prompt: str,
|
||||
cfg_scale: float,
|
||||
mode: str,
|
||||
aspect_ratio: str,
|
||||
duration: str,
|
||||
camera_control_type: str,
|
||||
camera_control_horizontal: float,
|
||||
camera_control_vertical: float,
|
||||
camera_control_pan: float,
|
||||
camera_control_roll: float,
|
||||
camera_control_tilt: float,
|
||||
camera_control_zoom: float,
|
||||
end_frame: Optional[torch.Tensor] = None,
|
||||
auth_token: Optional[str] = None,
|
||||
) -> tuple[VideoFromFile]:
|
||||
camera_control = None
|
||||
if is_camera_control_supported(model_name, duration, mode):
|
||||
config = None
|
||||
if camera_control_type != "right_turn_forward":
|
||||
config = CameraConfig(
|
||||
horizontal=camera_control_horizontal,
|
||||
vertical=camera_control_vertical,
|
||||
pan=camera_control_pan,
|
||||
roll=camera_control_roll,
|
||||
tilt=camera_control_tilt,
|
||||
zoom=camera_control_zoom,
|
||||
)
|
||||
camera_control = CameraControl(
|
||||
type=CameraType(camera_control_type),
|
||||
config=config if config else None,
|
||||
)
|
||||
|
||||
initial_operation = SynchronousOperation(
|
||||
endpoint=ApiEndpoint(
|
||||
path=PATH_IMAGE_TO_VIDEO,
|
||||
method=HttpMethod.POST,
|
||||
request_model=KlingImage2VideoRequest,
|
||||
response_model=KlingImage2VideoResponse,
|
||||
),
|
||||
request=KlingImage2VideoRequest(
|
||||
model_name=ModelName(model_name),
|
||||
image=tensor_to_base64_string(start_frame),
|
||||
image_tail=(
|
||||
tensor_to_base64_string(end_frame)
|
||||
if end_frame is not None
|
||||
else None
|
||||
),
|
||||
prompt=prompt,
|
||||
negative_prompt=negative_prompt if negative_prompt else None,
|
||||
cfg_scale=cfg_scale,
|
||||
mode=Mode(mode),
|
||||
aspect_ratio=AspectRatio(aspect_ratio),
|
||||
duration=Duration(duration),
|
||||
camera_control=camera_control,
|
||||
),
|
||||
auth_token=auth_token,
|
||||
)
|
||||
initial_response = initial_operation.execute()
|
||||
if not is_valid_initial_response(initial_response):
|
||||
error_msg = f"Kling initial request failed. Code: {initial_response.code}, Message: {initial_response.message}, Data: {initial_response.data}"
|
||||
logging.error(error_msg)
|
||||
raise KlingApiError(error_msg)
|
||||
|
||||
task_id = initial_response.data.task_id
|
||||
logging.debug("Kling task submitted. Task ID: %s", task_id)
|
||||
|
||||
final_response = KlingImage2VideoNode.poll_for_task_status(task_id, auth_token)
|
||||
if not is_valid_video_response(final_response):
|
||||
error_msg = (
|
||||
f"Kling task {task_id} succeeded but no video data found in response."
|
||||
)
|
||||
logging.error(error_msg)
|
||||
raise KlingApiError(error_msg)
|
||||
|
||||
video_url = str(final_response.data.task_result.videos[0].url)
|
||||
logging.info("Attempting to download video from URL: %s", video_url)
|
||||
|
||||
return download_url_to_video_output(video_url)
|
||||
|
||||
|
||||
NODE_CLASS_MAPPINGS = {
|
||||
"KlingTextToVideoNode": KlingTextToVideoNode,
|
||||
"KlingImage2VideoNode": KlingImage2VideoNode,
|
||||
}
|
||||
|
||||
NODE_DISPLAY_NAME_MAPPINGS = {
|
||||
"KlingTextToVideoNode": "Kling Text to Video",
|
||||
"KlingImage2VideoNode": "Kling Image to Video",
|
||||
}
|
||||
Loading…
x
Reference in New Issue
Block a user