diff --git a/comfy_api_nodes/apis/pixverse_api.py b/comfy_api_nodes/apis/pixverse_api.py index 85eae2738..9ce488e07 100644 --- a/comfy_api_nodes/apis/pixverse_api.py +++ b/comfy_api_nodes/apis/pixverse_api.py @@ -22,6 +22,7 @@ class PixverseIO: class PixverseStatus(int, Enum): successful = 1 generating = 5 + deleted = 6 contents_moderation = 7 failed = 8 @@ -60,7 +61,7 @@ class PixverseStyle(str, Enum): # NOTE: forgoing descriptions for now in return for dev speed -class PixverseDto_V2OpenAPIT2VReq(BaseModel): +class PixverseTextVideoRequest(BaseModel): aspect_ratio: PixverseAspectRatio = Field(...) quality: PixverseQuality = Field(...) duration: PixverseDuration = Field(...) @@ -74,23 +75,76 @@ class PixverseDto_V2OpenAPIT2VReq(BaseModel): water_mark: Optional[bool] = Field(None) -class PixverseController_ResponseData(BaseModel): +class PixverseImageVideoRequest(BaseModel): + quality: PixverseQuality = Field(...) + duration: PixverseDuration = Field(...) + img_id: int = Field(...) + model: Optional[str] = Field("v3.5") + motion_mode: Optional[PixverseMotionMode] = Field(PixverseMotionMode.normal) + prompt: str = Field(...) + negative_prompt: Optional[str] = Field(None) + seed: Optional[int] = Field(None) + style: Optional[str] = Field(None) + template_id: Optional[int] = Field(None) + water_mark: Optional[bool] = Field(None) + + +# class PixverseImageVideoRequest(BaseModel): +# quality: Optional[PixverseQuality] = Field(None) +# duration: Optional[PixverseDuration] = Field(None) +# img_id: int = Field(...) +# model: Optional[str] = Field("v3.5") +# motion_mode: Optional[PixverseMotionMode] = Field(PixverseMotionMode.normal) +# prompt: Optional[str ]= Field(None) +# negative_prompt: Optional[str] = Field(None) +# seed: Optional[int] = Field(None) +# style: Optional[str] = Field(None) +# template_id: Optional[int] = Field(None) +# water_mark: Optional[bool] = Field(None) + + +class PixverseTransitionVideoRequest(BaseModel): + quality: PixverseQuality = Field(...) + duration: PixverseDuration = Field(...) + first_frame_img: int = Field(...) + last_frame_img: int = Field(...) + model: Optional[str] = Field("v3.5") + motion_mode: Optional[PixverseMotionMode] = Field(PixverseMotionMode.normal) + prompt: str = Field(...) + # negative_prompt: Optional[str] = Field(None) + seed: Optional[int] = Field(None) + # style: Optional[str] = Field(None) + # template_id: Optional[int] = Field(None) + # water_mark: Optional[bool] = Field(None) + + +class PixverseImageUploadResponse(BaseModel): + ErrCode: Optional[int] = None + ErrMsg: Optional[str] = None + Resp: Optional[PixverseImgIdResponseObject] = Field(None, alias='Resp') + + +class PixverseImgIdResponseObject(BaseModel): + img_id: Optional[int] = None + + +class PixverseVideoResponse(BaseModel): ErrCode: Optional[int] = Field(None) ErrMsg: Optional[str] = Field(None) - Resp: Optional[PixverseDto_V2OpenAPII2VResp] = Field(None) + Resp: Optional[PixverseVideoIdResponseObject] = Field(None) -class PixverseDto_V2OpenAPII2VResp(BaseModel): +class PixverseVideoIdResponseObject(BaseModel): video_id: int = Field(..., description='Video_id') class PixverseGenerationStatusResponse(BaseModel): ErrCode: Optional[int] = Field(None) ErrMsg: Optional[str] = Field(None) - Resp: Optional[PixverseDto_GetOpenapiMediaDetailResp] = Field(None) + Resp: Optional[PixverseGenerationStatusResponseObject] = Field(None) -class PixverseDto_GetOpenapiMediaDetailResp(BaseModel): +class PixverseGenerationStatusResponseObject(BaseModel): create_time: Optional[str] = Field(None) id: Optional[int] = Field(None) modify_time: Optional[str] = Field(None) diff --git a/comfy_api_nodes/nodes_pixverse.py b/comfy_api_nodes/nodes_pixverse.py index 70083c2e7..5808575e4 100644 --- a/comfy_api_nodes/nodes_pixverse.py +++ b/comfy_api_nodes/nodes_pixverse.py @@ -1,8 +1,10 @@ from inspect import cleandoc from comfy_api_nodes.apis.pixverse_api import ( - PixverseDto_V2OpenAPIT2VReq, - PixverseController_ResponseData, + PixverseTextVideoRequest, + PixverseImageVideoRequest, + PixverseImageUploadResponse, + PixverseVideoResponse, PixverseGenerationStatusResponse, PixverseAspectRatio, PixverseQuality, @@ -19,9 +21,13 @@ from comfy_api_nodes.apis.client import ( PollingOperation, EmptyRequest, ) +from comfy_api_nodes.apinode_utils import ( + tensor_to_bytesio, +) from comfy.comfy_types.node_typing import IO, ComfyNodeABC from comfy_api.input_impl import VideoFromFile +import torch import requests from io import BytesIO @@ -143,10 +149,10 @@ class PixverseTextToVideoNode(ComfyNodeABC): endpoint=ApiEndpoint( path="/proxy/pixverse/video/text/generate", method=HttpMethod.POST, - request_model=PixverseDto_V2OpenAPIT2VReq, - response_model=PixverseController_ResponseData, + request_model=PixverseTextVideoRequest, + response_model=PixverseVideoResponse, ), - request=PixverseDto_V2OpenAPIT2VReq( + request=PixverseTextVideoRequest( prompt=prompt, aspect_ratio=aspect_ratio, quality=quality, @@ -171,7 +177,157 @@ class PixverseTextToVideoNode(ComfyNodeABC): response_model=PixverseGenerationStatusResponse, ), completed_statuses=[PixverseStatus.successful], - failed_statuses=[PixverseStatus.contents_moderation, PixverseStatus.failed], + failed_statuses=[PixverseStatus.contents_moderation, PixverseStatus.failed, PixverseStatus.deleted], + status_extractor=lambda x: x.Resp.status, + auth_token=auth_token, + ) + response_poll = operation.execute() + + vid_response = requests.get(response_poll.Resp.url) + return (VideoFromFile(BytesIO(vid_response.content)),) + + +class PixverseImageToVideoNode(ComfyNodeABC): + """ + Generates videos synchronously based on prompt and output_size. + """ + + RETURN_TYPES = (IO.VIDEO,) + DESCRIPTION = cleandoc(__doc__ or "") # Handle potential None value + FUNCTION = "api_call" + API_NODE = True + CATEGORY = "api node/video/Pixverse" + + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "image": ( + IO.IMAGE, + ), + "prompt": ( + IO.STRING, + { + "multiline": True, + "default": "", + "tooltip": "Prompt for the video generation", + }, + ), + "quality": ( + [resolution.value for resolution in PixverseQuality], + { + "default": PixverseQuality.res_540p, + }, + ), + "duration_seconds": ([dur.value for dur in PixverseDuration],), + "motion_mode": ([mode.value for mode in PixverseMotionMode],), + "seed": ( + IO.INT, + { + "default": 0, + "min": 0, + "max": 2147483647, + "control_after_generate": True, + "tooltip": "Seed for video generation.", + }, + ), + }, + "optional": { + "negative_prompt": ( + IO.STRING, + { + "default": "", + "forceInput": True, + "tooltip": "An optional text description of undesired elements on an image.", + }, + ), + "pixverse_template": ( + PixverseIO.TEMPLATE, + { + "tooltip": "An optional template to influence style of generation, created by the Pixverse Template node." + } + ) + }, + "hidden": { + "auth_token": "AUTH_TOKEN_COMFY_ORG", + }, + } + + def api_call( + self, + image: torch.Tensor, + prompt: str, + quality: str, + duration_seconds: int, + motion_mode: str, + seed, + negative_prompt: str=None, + pixverse_template: int=None, + auth_token=None, + **kwargs, + ): + # first, upload image to Pixverse and get image id to use in actual generation call + files = { + "image": tensor_to_bytesio(image) + } + operation = SynchronousOperation( + endpoint=ApiEndpoint( + path="/proxy/pixverse/image/upload", + method=HttpMethod.POST, + request_model=EmptyRequest, + response_model=PixverseImageUploadResponse, + ), + request=EmptyRequest(), + files=files, + content_type="multipart/form-data", + auth_token=auth_token, + ) + response_upload: PixverseImageUploadResponse = operation.execute() + + if response_upload.Resp is None: + raise Exception(f"Pixverse image upload request failed: '{response_upload.ErrMsg}'") + + # 1080p is limited to 5 seconds duration + # only normal motion_mode supported for 1080p or for non-5 second duration + if quality == PixverseQuality.res_1080p: + motion_mode = PixverseMotionMode.normal + duration_seconds = PixverseDuration.dur_5 + elif duration_seconds != PixverseDuration.dur_5: + motion_mode = PixverseMotionMode.normal + + operation = SynchronousOperation( + endpoint=ApiEndpoint( + path="/proxy/pixverse/video/img/generate", + method=HttpMethod.POST, + request_model=PixverseImageVideoRequest, + response_model=PixverseVideoResponse, + ), + request=PixverseImageVideoRequest( + img_id=response_upload.Resp.img_id, + prompt=prompt, + quality=quality, + duration=duration_seconds, + motion_mode=motion_mode, + negative_prompt=negative_prompt if negative_prompt else None, + template_id=pixverse_template, + seed=seed, + ), + auth_token=auth_token, + ) + response_api = operation.execute() + + if response_api.Resp is None: + raise Exception(f"Pixverse request failed: '{response_api.ErrMsg}'") + + operation = PollingOperation( + poll_endpoint=ApiEndpoint( + path=f"/proxy/pixverse/video/result/{response_api.Resp.video_id}", + method=HttpMethod.GET, + request_model=EmptyRequest, + response_model=PixverseGenerationStatusResponse, + ), + completed_statuses=[PixverseStatus.successful], + failed_statuses=[PixverseStatus.contents_moderation, PixverseStatus.failed, PixverseStatus.deleted], status_extractor=lambda x: x.Resp.status, auth_token=auth_token, ) @@ -183,10 +339,12 @@ class PixverseTextToVideoNode(ComfyNodeABC): NODE_CLASS_MAPPINGS = { "PixverseTextToVideoNode": PixverseTextToVideoNode, + "PixverseImageToVideoNode": PixverseImageToVideoNode, "PixverseTemplateNode": PixverseTemplateNode, } NODE_DISPLAY_NAME_MAPPINGS = { "PixverseTextToVideoNode": "Pixverse Text to Video", + "PixverseImageToVideoNode": "Pixverse Image to Video", "PixverseTemplateNode": "Pixverse Template", }