Added Pixverse Image to VIdeo node (#77)

comfy_api_nodes/apis/pixverse_api.py

@@ -22,6 +22,7 @@ class PixverseIO:
 class PixverseStatus(int, Enum):
     successful = 1
     generating = 5
+    deleted = 6
     contents_moderation = 7
     failed = 8
 
@@ -60,7 +61,7 @@ class PixverseStyle(str, Enum):
 
 
 # NOTE: forgoing descriptions for now in return for dev speed
-class PixverseDto_V2OpenAPIT2VReq(BaseModel):
+class PixverseTextVideoRequest(BaseModel):
     aspect_ratio: PixverseAspectRatio = Field(...)
     quality: PixverseQuality = Field(...)
     duration: PixverseDuration = Field(...)
@@ -74,23 +75,76 @@ class PixverseDto_V2OpenAPIT2VReq(BaseModel):
     water_mark: Optional[bool] = Field(None)
 
 
-class PixverseController_ResponseData(BaseModel):
+class PixverseImageVideoRequest(BaseModel):
+    quality: PixverseQuality = Field(...)
+    duration: PixverseDuration = Field(...)
+    img_id: int = Field(...)
+    model: Optional[str] = Field("v3.5")
+    motion_mode: Optional[PixverseMotionMode] = Field(PixverseMotionMode.normal)
+    prompt: str = Field(...)
+    negative_prompt: Optional[str] = Field(None)
+    seed: Optional[int] = Field(None)
+    style: Optional[str] = Field(None)
+    template_id: Optional[int] = Field(None)
+    water_mark: Optional[bool] = Field(None)
+
+
+# class PixverseImageVideoRequest(BaseModel):
+#     quality: Optional[PixverseQuality] = Field(None)
+#     duration: Optional[PixverseDuration] = Field(None)
+#     img_id: int = Field(...)
+#     model: Optional[str] = Field("v3.5")
+#     motion_mode: Optional[PixverseMotionMode] = Field(PixverseMotionMode.normal)
+#     prompt: Optional[str ]= Field(None)
+#     negative_prompt: Optional[str] = Field(None)
+#     seed: Optional[int] = Field(None)
+#     style: Optional[str] = Field(None)
+#     template_id: Optional[int] = Field(None)
+#     water_mark: Optional[bool] = Field(None)
+
+
+class PixverseTransitionVideoRequest(BaseModel):
+    quality: PixverseQuality = Field(...)
+    duration: PixverseDuration = Field(...)
+    first_frame_img: int = Field(...)
+    last_frame_img: int = Field(...)
+    model: Optional[str] = Field("v3.5")
+    motion_mode: Optional[PixverseMotionMode] = Field(PixverseMotionMode.normal)
+    prompt: str = Field(...)
+    # negative_prompt: Optional[str] = Field(None)
+    seed: Optional[int] = Field(None)
+    # style: Optional[str] = Field(None)
+    # template_id: Optional[int] = Field(None)
+    # water_mark: Optional[bool] = Field(None)
+
+
+class PixverseImageUploadResponse(BaseModel):
+    ErrCode: Optional[int] = None
+    ErrMsg: Optional[str] = None
+    Resp: Optional[PixverseImgIdResponseObject] = Field(None, alias='Resp')
+
+
+class PixverseImgIdResponseObject(BaseModel):
+    img_id: Optional[int] = None
+
+
+class PixverseVideoResponse(BaseModel):
     ErrCode: Optional[int] = Field(None)
     ErrMsg: Optional[str] = Field(None)
-    Resp: Optional[PixverseDto_V2OpenAPII2VResp] = Field(None)
+    Resp: Optional[PixverseVideoIdResponseObject] = Field(None)
 
 
-class PixverseDto_V2OpenAPII2VResp(BaseModel):
+class PixverseVideoIdResponseObject(BaseModel):
     video_id: int = Field(..., description='Video_id')
 
 
 class PixverseGenerationStatusResponse(BaseModel):
     ErrCode: Optional[int] = Field(None)
     ErrMsg: Optional[str] = Field(None)
-    Resp: Optional[PixverseDto_GetOpenapiMediaDetailResp] = Field(None)
+    Resp: Optional[PixverseGenerationStatusResponseObject] = Field(None)
 
 
-class PixverseDto_GetOpenapiMediaDetailResp(BaseModel):
+class PixverseGenerationStatusResponseObject(BaseModel):
     create_time: Optional[str] = Field(None)
     id: Optional[int] = Field(None)
     modify_time: Optional[str] = Field(None)

comfy_api_nodes/nodes_pixverse.py

@@ -1,8 +1,10 @@
 from inspect import cleandoc
 
 from comfy_api_nodes.apis.pixverse_api import (
-    PixverseDto_V2OpenAPIT2VReq,
+    PixverseTextVideoRequest,
-    PixverseController_ResponseData,
+    PixverseImageVideoRequest,
+    PixverseImageUploadResponse,
+    PixverseVideoResponse,
     PixverseGenerationStatusResponse,
     PixverseAspectRatio,
     PixverseQuality,
@@ -19,9 +21,13 @@ from comfy_api_nodes.apis.client import (
     PollingOperation,
     EmptyRequest,
 )
+from comfy_api_nodes.apinode_utils import (
+    tensor_to_bytesio,
+)
 from comfy.comfy_types.node_typing import IO, ComfyNodeABC
 from comfy_api.input_impl import VideoFromFile
 
+import torch
 import requests
 from io import BytesIO
 
@@ -143,10 +149,10 @@ class PixverseTextToVideoNode(ComfyNodeABC):
             endpoint=ApiEndpoint(
                 path="/proxy/pixverse/video/text/generate",
                 method=HttpMethod.POST,
-                request_model=PixverseDto_V2OpenAPIT2VReq,
+                request_model=PixverseTextVideoRequest,
-                response_model=PixverseController_ResponseData,
+                response_model=PixverseVideoResponse,
             ),
-            request=PixverseDto_V2OpenAPIT2VReq(
+            request=PixverseTextVideoRequest(
                 prompt=prompt,
                 aspect_ratio=aspect_ratio,
                 quality=quality,
@@ -171,7 +177,157 @@ class PixverseTextToVideoNode(ComfyNodeABC):
                 response_model=PixverseGenerationStatusResponse,
             ),
             completed_statuses=[PixverseStatus.successful],
-            failed_statuses=[PixverseStatus.contents_moderation, PixverseStatus.failed],
+            failed_statuses=[PixverseStatus.contents_moderation, PixverseStatus.failed, PixverseStatus.deleted],
+            status_extractor=lambda x: x.Resp.status,
+            auth_token=auth_token,
+        )
+        response_poll = operation.execute()
+
+        vid_response = requests.get(response_poll.Resp.url)
+        return (VideoFromFile(BytesIO(vid_response.content)),)
+
+
+class PixverseImageToVideoNode(ComfyNodeABC):
+    """
+    Generates videos synchronously based on prompt and output_size.
+    """
+
+    RETURN_TYPES = (IO.VIDEO,)
+    DESCRIPTION = cleandoc(__doc__ or "")  # Handle potential None value
+    FUNCTION = "api_call"
+    API_NODE = True
+    CATEGORY = "api node/video/Pixverse"
+
+    @classmethod
+    def INPUT_TYPES(s):
+        return {
+            "required": {
+                "image": (
+                    IO.IMAGE,
+                ),
+                "prompt": (
+                    IO.STRING,
+                    {
+                        "multiline": True,
+                        "default": "",
+                        "tooltip": "Prompt for the video generation",
+                    },
+                ),
+                "quality": (
+                    [resolution.value for resolution in PixverseQuality],
+                    {
+                        "default": PixverseQuality.res_540p,
+                    },
+                ),
+                "duration_seconds": ([dur.value for dur in PixverseDuration],),
+                "motion_mode": ([mode.value for mode in PixverseMotionMode],),
+                "seed": (
+                    IO.INT,
+                    {
+                        "default": 0,
+                        "min": 0,
+                        "max": 2147483647,
+                        "control_after_generate": True,
+                        "tooltip": "Seed for video generation.",
+                    },
+                ),
+            },
+            "optional": {
+                "negative_prompt": (
+                    IO.STRING,
+                    {
+                        "default": "",
+                        "forceInput": True,
+                        "tooltip": "An optional text description of undesired elements on an image.",
+                    },
+                ),
+                "pixverse_template": (
+                    PixverseIO.TEMPLATE,
+                    {
+                        "tooltip": "An optional template to influence style of generation, created by the Pixverse Template node."
+                    }
+                )
+            },
+            "hidden": {
+                "auth_token": "AUTH_TOKEN_COMFY_ORG",
+            },
+        }
+
+    def api_call(
+        self,
+        image: torch.Tensor,
+        prompt: str,
+        quality: str,
+        duration_seconds: int,
+        motion_mode: str,
+        seed,
+        negative_prompt: str=None,
+        pixverse_template: int=None,
+        auth_token=None,
+        **kwargs,
+    ):
+        # first, upload image to Pixverse and get image id to use in actual generation call
+        files = {
+            "image": tensor_to_bytesio(image)
+        }
+        operation = SynchronousOperation(
+            endpoint=ApiEndpoint(
+                path="/proxy/pixverse/image/upload",
+                method=HttpMethod.POST,
+                request_model=EmptyRequest,
+                response_model=PixverseImageUploadResponse,
+            ),
+            request=EmptyRequest(),
+            files=files,
+            content_type="multipart/form-data",
+            auth_token=auth_token,
+        )
+        response_upload: PixverseImageUploadResponse = operation.execute()
+
+        if response_upload.Resp is None:
+            raise Exception(f"Pixverse image upload request failed: '{response_upload.ErrMsg}'")
+
+        # 1080p is limited to 5 seconds duration
+        # only normal motion_mode supported for 1080p or for non-5 second duration
+        if quality == PixverseQuality.res_1080p:
+            motion_mode = PixverseMotionMode.normal
+            duration_seconds = PixverseDuration.dur_5
+        elif duration_seconds != PixverseDuration.dur_5:
+            motion_mode = PixverseMotionMode.normal
+
+        operation = SynchronousOperation(
+            endpoint=ApiEndpoint(
+                path="/proxy/pixverse/video/img/generate",
+                method=HttpMethod.POST,
+                request_model=PixverseImageVideoRequest,
+                response_model=PixverseVideoResponse,
+            ),
+            request=PixverseImageVideoRequest(
+                img_id=response_upload.Resp.img_id,
+                prompt=prompt,
+                quality=quality,
+                duration=duration_seconds,
+                motion_mode=motion_mode,
+                negative_prompt=negative_prompt if negative_prompt else None,
+                template_id=pixverse_template,
+                seed=seed,
+            ),
+            auth_token=auth_token,
+        )
+        response_api = operation.execute()
+
+        if response_api.Resp is None:
+            raise Exception(f"Pixverse request failed: '{response_api.ErrMsg}'")
+
+        operation = PollingOperation(
+            poll_endpoint=ApiEndpoint(
+                path=f"/proxy/pixverse/video/result/{response_api.Resp.video_id}",
+                method=HttpMethod.GET,
+                request_model=EmptyRequest,
+                response_model=PixverseGenerationStatusResponse,
+            ),
+            completed_statuses=[PixverseStatus.successful],
+            failed_statuses=[PixverseStatus.contents_moderation, PixverseStatus.failed, PixverseStatus.deleted],
             status_extractor=lambda x: x.Resp.status,
             auth_token=auth_token,
         )
@@ -183,10 +339,12 @@ class PixverseTextToVideoNode(ComfyNodeABC):
 
 NODE_CLASS_MAPPINGS = {
     "PixverseTextToVideoNode": PixverseTextToVideoNode,
+    "PixverseImageToVideoNode": PixverseImageToVideoNode,
     "PixverseTemplateNode": PixverseTemplateNode,
 }
 
 NODE_DISPLAY_NAME_MAPPINGS = {
     "PixverseTextToVideoNode": "Pixverse Text to Video",
+    "PixverseImageToVideoNode": "Pixverse Image to Video",
     "PixverseTemplateNode": "Pixverse Template",
 }