ComfyUI/comfy_api_nodes/nodes_runway.py

from inspect import cleandoc
from typing import Union, Optional
import logging

import torch
from comfy_api_nodes.apis import (
    RunwayImageToVideoRequest,
    RunwayImageToVideoResponse,
    RunwayTaskStatusResponse as TaskStatusResponse,
    RunwayTaskStatusEnum as TaskStatus,
    RunwayModelEnum as Model,
    RunwayDurationEnum as Duration,
    RunwayAspectRatioEnum as AspectRatio,
    RunwayPromptImageObject,
    RunwayPromptImageDetailedObject,
)
from comfy_api_nodes.apis.client import (
    ApiEndpoint,
    HttpMethod,
    SynchronousOperation,
    PollingOperation,
    EmptyRequest,
)
from comfy_api_nodes.apinode_utils import (
    download_url_to_bytesio,
    upload_images_to_comfyapi,
)
from comfy.comfy_types.node_typing import IO, ComfyNodeABC
from comfy_api.input_impl import VideoFromFile
from comfy_api_nodes.mapper_utils import model_field_to_node_input

PATH_IMAGE_TO_VIDEO = "/proxy/runway/image-to-video"
PATH_GET_TASK_STATUS = "/proxy/runway/tasks"


class RunwayApiError(Exception):
    """Base exception for Runway API errors."""

    pass


def extract_progress_from_task_status(response: TaskStatusResponse) -> float:
    if hasattr(response, "progress") and response.progress is not None:
        return response.progress * 100
    return None


class RunwayImageToVideoNode(ComfyNodeABC):
    """
    Runway Image to Video Node.
    """

    @staticmethod
    def is_ratio_supported(model: str, ratio: str) -> bool:
        """
        Checks if the chosen aspect ratio is supported by the chosen model.
        """
        if model != "gen3a_turbo" and ratio in [
            "1280:768",
            "768:1280",
        ]:
            return False
        return True

    @staticmethod
    def is_end_frame_supported(model: str) -> bool:
        """
        Checks if the chosen model supports the end frame input.
        """
        return model == "gen3a_turbo"

    @staticmethod
    def is_valid_prompt(prompt: str) -> bool:
        return bool(prompt)

    @staticmethod
    def is_valid_initial_response(response: RunwayImageToVideoResponse) -> bool:
        return bool(response.id)

    @staticmethod
    def is_valid_image(image: torch.Tensor) -> bool:
        """https://docs.dev.runwayml.com/assets/inputs/#common-error-reasons"""
        return image.shape[2] < 8000 and image.shape[1] < 8000

    @staticmethod
    def is_valid_video_response(response: RunwayImageToVideoResponse) -> bool:
        return response.output and len(response.output) > 0

    @staticmethod
    def poll_for_task_status(task_id: str, auth_token: str) -> TaskStatusResponse:
        """
        Polls the Runway API endpoint until the task reaches a terminal state.
        """
        polling_operation = PollingOperation(
            poll_endpoint=ApiEndpoint(
                path=f"{PATH_GET_TASK_STATUS}/{task_id}",
                method=HttpMethod.GET,
                request_model=EmptyRequest,
                response_model=TaskStatusResponse,
            ),
            completed_statuses=[
                TaskStatus.SUCCEEDED.value,
            ],
            failed_statuses=[
                TaskStatus.FAILED.value,
                TaskStatus.CANCELLED.value,
            ],
            progress_extractor=extract_progress_from_task_status,
            status_extractor=lambda response: (response.status.value),
            auth_token=auth_token,
        )
        return polling_operation.execute()

    @classmethod
    def INPUT_TYPES(s):
        return {
            "required": {
                "model": model_field_to_node_input(
                    IO.COMBO, RunwayImageToVideoRequest, "model", enum_type=Model
                ),
                "prompt": model_field_to_node_input(
                    IO.STRING, RunwayImageToVideoRequest, "promptText", multiline=True
                ),
                "duration": model_field_to_node_input(
                    IO.COMBO, RunwayImageToVideoRequest, "duration", enum_type=Duration
                ),
                "ratio": model_field_to_node_input(
                    IO.COMBO, RunwayImageToVideoRequest, "ratio", enum_type=AspectRatio
                ),
                "seed": model_field_to_node_input(
                    IO.INT, RunwayImageToVideoRequest, "seed", control_after_generate=True
                ),
            },
            "optional": {
                "start_frame": (
                    IO.IMAGE,
                    {"tooltip": "Start frame to be used for the video"},
                ),
                "end_frame": (
                    IO.IMAGE,
                    {
                        "tooltip": "End frame to be used for the video. Supported for gen3a_turbo only."
                    },
                ),
            },
            "hidden": {"auth_token": "AUTH_TOKEN_COMFY_ORG"},
        }

    RETURN_TYPES = ("VIDEO",)
    FUNCTION = "api_call"
    CATEGORY = "api node/video/Runway"
    DESCRIPTION = cleandoc(__doc__ or "")
    API_NODE = True

    @classmethod
    def VALIDATE_INPUTS(
        cls,
        model: str,
        ratio: str,
    ) -> Union[str, bool]:
        if not RunwayImageToVideoNode.is_ratio_supported(model, ratio):
            return "Invalid aspect ratio for the chosen model. 1280:768 and 768:1280 are only supported for gen3a_turbo."
        return True

    def api_call(
        self,
        model: str,
        prompt: str,
        duration: str,
        ratio: str,
        seed: int,
        start_frame: Optional[torch.Tensor] = None,
        end_frame: Optional[torch.Tensor] = None,
        auth_token: Optional[str] = None,
    ) -> tuple[VideoFromFile]:
        # Validate manually because optional inputs are not passed to VALIDATE_INPUTS.
        if start_frame is None and end_frame is None:
            message = "Start frame and end frame cannot both be empty."
            raise RunwayApiError(message)
        if end_frame is not None and not RunwayImageToVideoNode.is_end_frame_supported(
            model
        ):
            message = "End frame is only supported for gen3a_turbo model."
            raise RunwayApiError(message)

        prompt_images_tensors: list[torch.Tensor] = []
        if start_frame is not None:
            if not RunwayImageToVideoNode.is_valid_image(start_frame):
                message = "Start frame is not a valid image."
                raise RunwayApiError(message)
            prompt_images_tensors.append(start_frame)

        if end_frame != None:
            if not RunwayImageToVideoNode.is_valid_image(end_frame):
                message = "End frame is not a valid image."
                raise RunwayApiError(message)
            prompt_images_tensors.append(end_frame)

        # stack tensors
        prompt_images_tensor = torch.cat(prompt_images_tensors, dim=0)

        download_urls = upload_images_to_comfyapi(
            prompt_images_tensor,
            max_images=2,
            auth_token=auth_token,
            mime_type="image/png",
        )

        # Create a list of detailed image objects
        prompt_image_details: list[RunwayPromptImageDetailedObject] = [
            RunwayPromptImageDetailedObject(uri=str(download_urls[0]), position="first")
        ]
        if len(download_urls) > 1:
            prompt_image_details.append(
                RunwayPromptImageDetailedObject(
                    uri=str(download_urls[1]), position="last"
                )
            )

        # Wrap the list in the main object if details exist
        prompt_image_object: Optional[RunwayPromptImageObject] = None
        if prompt_image_details:
            prompt_image_object = RunwayPromptImageObject(root=prompt_image_details)

        initial_operation = SynchronousOperation(
            endpoint=ApiEndpoint(
                path=PATH_IMAGE_TO_VIDEO,
                method=HttpMethod.POST,
                request_model=RunwayImageToVideoRequest,
                response_model=RunwayImageToVideoResponse,
            ),
            request=RunwayImageToVideoRequest(
                promptText=prompt,
                seed=seed,
                model=Model(model),
                duration=Duration(duration),
                ratio=AspectRatio(ratio),
                promptImage=prompt_image_object,
            ),
            auth_token=auth_token,
        )

        initial_response = initial_operation.execute()
        if not RunwayImageToVideoNode.is_valid_initial_response(initial_response):
            error_message = "Invalid initial response from Runway API."
            logging.error(error_message)
            raise RunwayApiError(error_message)

        task_id = initial_response.id
        logging.debug("Runway task submitted. Task ID: %s", task_id)

        final_response = self.poll_for_task_status(task_id, auth_token)
        if not RunwayImageToVideoNode.is_valid_video_response(final_response):
            error_message = "Runway task succeeded but no video data found in response."
            logging.error(error_message)
            raise RunwayApiError(error_message)

        video_url = final_response.output[0]
        logging.debug("Attempting to download video from URL: %s", video_url)

        video_io = download_url_to_bytesio(video_url)
        if video_io is None:
            error_msg = f"Failed to download video from {video_url}"
            logging.error(error_msg)
            raise RunwayApiError(error_msg)
        return (VideoFromFile(video_io),)


NODE_CLASS_MAPPINGS = {
    "RunwayImageToVideoNode": RunwayImageToVideoNode,
}

NODE_DISPLAY_NAME_MAPPINGS = {
    "RunwayImageToVideoNode": "Runway Image to Video",
}