xinyun/ComfyUI
1
0
Fork 0
mirror of https://git.datalinker.icu/comfyanonymous/ComfyUI synced 2026-09-04 07:47:05 +08:00
Code Issues Packages Projects Releases Wiki Activity

[Runway] Split I2V nodes into separate gen3 and gen4 nodes (#198)

Browse Source
This commit is contained in:
Christian Byrne 2025-05-18 15:17:09 -07:00 committed by Robin Huang
parent c5884037fc
commit 270cfc1ef2
1 changed files with 112 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

133
comfy_api_nodes/nodes_runway.py
View File

@ -53,9 +53,9 @@ PATH_IMAGE_TO_VIDEO = "/proxy/runway/image_to_video"
PATH_TEXT_TO_IMAGE = "/proxy/runway/text_to_image"
PATH_GET_TASK_STATUS = "/proxy/runway/tasks"
AVERAGE_DURATION_I2V_SECONDS = 128
AVERAGE_DURATION_I2V_SECONDS = 64
AVERAGE_DURATION_FLF_SECONDS = 256
AVERAGE_DURATION_T2I_SECONDS = 32
AVERAGE_DURATION_T2I_SECONDS = 41
class RunwayApiError(Exception):
@ -64,8 +64,8 @@ class RunwayApiError(Exception):
pass
class RunwayBasicAspectRatio(str, Enum):
"""Aspect ratios supported for Image to Video API when not using gen3a_turbo model."""
class RunwayGen4TurboAspectRatio(str, Enum):
"""Aspect ratios supported for Image to Video API when using gen4_turbo model."""
field_1280_720 = "1280:720"
field_720_1280 = "720:1280"
@ -75,6 +75,13 @@ class RunwayBasicAspectRatio(str, Enum):
field_1584_672 = "1584:672"
class RunwayGen3aAspectRatio(str, Enum):
"""Aspect ratios supported for Image to Video API when using gen3a_turbo model."""
field_768_1280 = "768:1280"
field_1280_768 = "1280:768"
# TODO: replace with enum after it's added in comfy-api and generated by code gen
class RunwayTextToImageRatio(str, Enum):
"""Aspect ratios supported for Text to Image API."""
@ -187,7 +194,7 @@ class RunwayVideoGenNode(ComfyNodeABC):
path=f"{PATH_GET_TASK_STATUS}/{task_id}",
method=HttpMethod.GET,
request_model=EmptyRequest,
response_model=RunwayImageToVideoResponse,
response_model=TaskStatusResponse,
),
estimated_duration=AVERAGE_DURATION_FLF_SECONDS,
node_id=node_id,
@ -221,10 +228,10 @@ class RunwayVideoGenNode(ComfyNodeABC):
return (download_url_to_video_output(video_url),)
class RunwayImageToVideoNode(RunwayVideoGenNode):
"""Runway Image to Video Node."""
class RunwayImageToVideoNodeGen3a(RunwayVideoGenNode):
"""Runway Image to Video Node using Gen3a Turbo model."""
DESCRIPTION = "Generate a video from a single starting frame. Before diving in, review these best practices to ensure that your input selections will set your generation up for success: https://help.runwayml.com/hc/en-us/articles/33927968552339-Creating-with-Act-One-on-Gen-3-Alpha-and-Turbo."
DESCRIPTION = "Generate a video from a single starting frame using Gen3a Turbo model. Before diving in, review these best practices to ensure that your input selections will set your generation up for success: https://help.runwayml.com/hc/en-us/articles/33927968552339-Creating-with-Act-One-on-Gen-3-Alpha-and-Turbo."
@classmethod
def INPUT_TYPES(s):
@ -237,9 +244,6 @@ class RunwayImageToVideoNode(RunwayVideoGenNode):
IO.IMAGE,
{"tooltip": "Start frame to be used for the video"},
),
"model": model_field_to_node_input(
IO.COMBO, RunwayImageToVideoRequest, "model", enum_type=Model
),
"duration": model_field_to_node_input(
IO.COMBO, RunwayImageToVideoRequest, "duration", enum_type=Duration
),
@ -247,7 +251,7 @@ class RunwayImageToVideoNode(RunwayVideoGenNode):
IO.COMBO,
RunwayImageToVideoRequest,
"ratio",
enum_type=RunwayBasicAspectRatio,
enum_type=RunwayGen3aAspectRatio,
),
"seed": model_field_to_node_input(
IO.INT,
@ -267,7 +271,6 @@ class RunwayImageToVideoNode(RunwayVideoGenNode):
self,
prompt: str,
start_frame: torch.Tensor,
model: str,
duration: str,
ratio: str,
seed: int,
@ -308,8 +311,91 @@ class RunwayImageToVideoNode(RunwayVideoGenNode):
)
class RunwayStartEndFrameNode(RunwayVideoGenNode):
"""Runway Start End Frame Node."""
class RunwayImageToVideoNodeGen4(RunwayVideoGenNode):
"""Runway Image to Video Node using Gen4 Turbo model."""
DESCRIPTION = "Generate a video from a single starting frame using Gen4 Turbo model. Before diving in, review these best practices to ensure that your input selections will set your generation up for success: https://help.runwayml.com/hc/en-us/articles/37327109429011-Creating-with-Gen-4-Video."
@classmethod
def INPUT_TYPES(s):
return {
"required": {
"prompt": model_field_to_node_input(
IO.STRING, RunwayImageToVideoRequest, "promptText", multiline=True
),
"start_frame": (
IO.IMAGE,
{"tooltip": "Start frame to be used for the video"},
),
"duration": model_field_to_node_input(
IO.COMBO, RunwayImageToVideoRequest, "duration", enum_type=Duration
),
"ratio": model_field_to_node_input(
IO.COMBO,
RunwayImageToVideoRequest,
"ratio",
enum_type=RunwayGen4TurboAspectRatio,
),
"seed": model_field_to_node_input(
IO.INT,
RunwayImageToVideoRequest,
"seed",
control_after_generate=True,
),
},
"hidden": {
"auth_token": "AUTH_TOKEN_COMFY_ORG",
"comfy_api_key": "API_KEY_COMFY_ORG",
"unique_id": "UNIQUE_ID",
},
}
def api_call(
self,
prompt: str,
start_frame: torch.Tensor,
duration: str,
ratio: str,
seed: int,
unique_id: Optional[str] = None,
**kwargs,
) -> tuple[VideoFromFile]:
# Validate inputs
validate_string(prompt, min_length=1)
validate_input_image(start_frame)
# Upload image
download_urls = upload_images_to_comfyapi(
start_frame,
max_images=1,
mime_type="image/png",
auth_kwargs=kwargs,
)
if len(download_urls) != 1:
raise RunwayApiError("Failed to upload one or more images to comfy api.")
return self.generate_video(
RunwayImageToVideoRequest(
promptText=prompt,
seed=seed,
model=Model("gen4_turbo"),
duration=Duration(duration),
ratio=AspectRatio(ratio),
promptImage=RunwayPromptImageObject(
root=[
RunwayPromptImageDetailedObject(
uri=str(download_urls[0]), position="first"
)
]
),
),
auth_kwargs=kwargs,
node_id=unique_id,
)
class RunwayFirstLastFrameNode(RunwayVideoGenNode):
"""Runway First-Last Frame Node."""
DESCRIPTION = "Upload first and last keyframes, draft a prompt, and generate a video. More complex transitions, such as cases where the Last frame is completely different from the First frame, may benefit from the longer 10s duration. This would give the generation more time to smoothly transition between the two inputs. Before diving in, review these best practices to ensure that your input selections will set your generation up for success: https://help.runwayml.com/hc/en-us/articles/34170748696595-Creating-with-Keyframes-on-Gen-3."
@ -322,7 +408,7 @@ class RunwayStartEndFrameNode(RunwayVideoGenNode):
path=f"{PATH_GET_TASK_STATUS}/{task_id}",
method=HttpMethod.GET,
request_model=EmptyRequest,
response_model=RunwayImageToVideoResponse,
response_model=TaskStatusResponse,
),
estimated_duration=AVERAGE_DURATION_FLF_SECONDS,
node_id=node_id,
@ -349,7 +435,10 @@ class RunwayStartEndFrameNode(RunwayVideoGenNode):
IO.COMBO, RunwayImageToVideoRequest, "duration", enum_type=Duration
),
"ratio": model_field_to_node_input(
IO.COMBO, RunwayImageToVideoRequest, "ratio", enum_type=AspectRatio
IO.COMBO,
RunwayImageToVideoRequest,
"ratio",
enum_type=RunwayGen3aAspectRatio,
),
"seed": model_field_to_node_input(
IO.INT,
@ -549,13 +638,15 @@ class RunwayTextToImageNode(ComfyNodeABC):
NODE_CLASS_MAPPINGS = {
"RunwayStartEndFrameNode": RunwayStartEndFrameNode,
"RunwayImageToVideoNode": RunwayImageToVideoNode,
"RunwayFirstLastFrameNode": RunwayFirstLastFrameNode,
"RunwayImageToVideoNodeGen3a": RunwayImageToVideoNodeGen3a,
"RunwayImageToVideoNodeGen4": RunwayImageToVideoNodeGen4,
"RunwayTextToImageNode": RunwayTextToImageNode,
}
NODE_DISPLAY_NAME_MAPPINGS = {
"RunwayStartEndFrameNode": "Runway First-Last-Frame to Video",
"RunwayImageToVideoNode": "Runway Image to Video",
"RunwayFirstLastFrameNode": "Runway First-Last-Frame to Video",
"RunwayImageToVideoNodeGen3a": "Runway Image to Video (Gen3a Turbo)",
"RunwayImageToVideoNodeGen4": "Runway Image to Video (Gen4 Turbo)",
"RunwayTextToImageNode": "Runway Text to Image",
}