diff --git a/comfy_api_nodes/apis/luma_api.py b/comfy_api_nodes/apis/luma_api.py index 1e9350d87..037b2415c 100644 --- a/comfy_api_nodes/apis/luma_api.py +++ b/comfy_api_nodes/apis/luma_api.py @@ -12,6 +12,7 @@ from pydantic import BaseModel, Field, confloat class LumaIO: LUMA_REF = "LUMA_REF" + LUMA_CONCEPTS = "LUMA_CONCEPTS" class LumaReference: @@ -47,6 +48,89 @@ class LumaReferenceChain: return c +class LumaConcept: + def __init__(self, key: str): + self.key = key + + +class LumaConceptChain: + def __init__(self, str_list: list[str] = None): + self.concepts: list[LumaConcept] = [] + if str_list is not None: + for c in str_list: + if c != "None": + self.add(LumaConcept(key=c)) + + def add(self, concept: LumaConcept): + self.concepts.append(concept) + + def create_api_model(self): + if len(self.concepts) == 0: + return None + api_concepts: list[LumaConceptObject] = [] + for concept in self.concepts: + if concept.key == "None": + continue + api_concepts.append(LumaConceptObject(key=concept.key)) + if len(api_concepts) == 0: + return None + return api_concepts + + def clone(self): + c = LumaConceptChain() + for concept in self.concepts: + c.add(concept) + return c + + def clone_and_merge(self, other: LumaConceptChain): + c = self.clone() + for concept in other.concepts: + c.add(concept) + return c + + +def get_luma_concepts(include_none=False): + concepts = [] + if include_none: + concepts.append("None") + return concepts + [ + "truck_left", + "pan_right", + "pedestal_down", + "low_angle", + "pedestal_up", + "selfie", + "pan_left", + "roll_right", + "zoom_in", + "over_the_shoulder", + "orbit_right", + "orbit_left", + "static", + "tiny_planet", + "high_angle", + "bolt_cam", + "dolly_zoom", + "overhead", + "zoom_out", + "handheld", + "roll_left", + "pov", + "aerial_drone", + "push_in", + "crane_down", + "truck_right", + "tilt_down", + "elevator_doors", + "tilt_up", + "ground_level", + "pull_out", + "aerial", + "crane_up", + "eye_level" + ] + + class LumaImageModel(str, Enum): photon_1 = "photon-1" photon_flash_1 = "photon-flash-1" @@ -133,6 +217,10 @@ class LumaKeyframes(BaseModel): frame1: Optional[Union[LumaImageReference, LumaGenerationReference]] = Field(None, description='') +class LumaConceptObject(BaseModel): + key: str = Field(..., description='Camera Concept name') + + class LumaImageGenerationRequest(BaseModel): prompt: str = Field(..., description='The prompt of the generation') model: LumaImageModel = Field(LumaImageModel.photon_1, description='The image model used for the generation') @@ -151,6 +239,7 @@ class LumaGenerationRequest(BaseModel): resolution: Optional[LumaVideoOutputResolution] = Field(None, description='The resolution of the generation') loop: Optional[bool] = Field(None, description='Whether to loop the video') keyframes: Optional[LumaKeyframes] = Field(None, description='The keyframes of the generation') + concepts: Optional[list[LumaConceptObject]] = Field(None, description='Camera Concepts to apply to generation') class LumaGeneration(BaseModel): diff --git a/comfy_api_nodes/nodes_api.py b/comfy_api_nodes/nodes_api.py index f9f6e3a0c..66b464ac5 100644 --- a/comfy_api_nodes/nodes_api.py +++ b/comfy_api_nodes/nodes_api.py @@ -38,7 +38,9 @@ from comfy_api_nodes.apis.luma_api import ( LumaReferenceChain, LumaImageReference, LumaKeyframes, + LumaConceptChain, LumaIO, + get_luma_concepts, ) from comfy_api_nodes.apis.recraft_api import ( RecraftImageGenerationRequest, @@ -965,7 +967,7 @@ class FluxProUltraImageNode(ComfyNodeABC): img.save(img_byte_arr, format='PNG') return base64.b64encode(img_byte_arr.getvalue()).decode() -class LumaReferenceNode: +class LumaReferenceNode(ComfyNodeABC): """ Holds an image and weight for use with Luma Generate Image node. """ @@ -1003,7 +1005,39 @@ class LumaReferenceNode: luma_ref.add(LumaReference(image=image, weight=round(weight, 2))) return (luma_ref, ) -class LumaImageGenerationNode: +class LumaConceptsNode(ComfyNodeABC): + """ + Holds one or more Camera Concepts for use with Luma Text to Video and Luma Image to Video nodes. + """ + RETURN_TYPES = (LumaIO.LUMA_CONCEPTS,) + RETURN_NAMES = ("luma_concepts",) + DESCRIPTION = cleandoc(__doc__ or "") # Handle potential None value + FUNCTION = "create_concepts" + CATEGORY = "api node/Luma" + + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "concept1": (get_luma_concepts(include_none=True), ), + "concept2": (get_luma_concepts(include_none=True), ), + "concept3": (get_luma_concepts(include_none=True), ), + "concept4": (get_luma_concepts(include_none=True), ), + }, + "optional": { + "luma_concepts": (LumaIO.LUMA_CONCEPTS, { + "tooltip": "Optional Camera Concepts to add to the ones chosen here." + }), + } + } + + def create_concepts(self, concept1: str, concept2: str, concept3: str, concept4: str, luma_concepts: LumaConceptChain=None): + chain = LumaConceptChain(str_list=[concept1, concept2, concept3, concept4]) + if luma_concepts is not None: + chain = luma_concepts.clone_and_merge(chain) + return (chain,) + +class LumaImageGenerationNode(ComfyNodeABC): """ Generates images synchronously based on prompt and aspect ratio. """ @@ -1126,7 +1160,7 @@ class LumaImageGenerationNode: chain = LumaReferenceChain(first_ref=LumaReference(image=style_image, weight=weight)) return self._convert_luma_refs(chain, max_refs=1, auth_token=auth_token) -class LumaImageModifyNode: +class LumaImageModifyNode(ComfyNodeABC): """ Modifies images synchronously based on prompt and aspect ratio. """ @@ -1211,7 +1245,7 @@ class LumaImageModifyNode: img = process_image_response(img_response) return (img,) -class LumaTextToVideoGenerationNode: +class LumaTextToVideoGenerationNode(ComfyNodeABC): """ Generates videos synchronously based on prompt and output_size. """ @@ -1254,6 +1288,9 @@ class LumaTextToVideoGenerationNode: }), }, "optional": { + "luma_concepts": (LumaIO.LUMA_CONCEPTS, { + "tooltip": "Optional Camera Concepts to dictate camera motion via the Luma Concepts node." + }), }, "hidden": { "auth_token": "AUTH_TOKEN_COMFY_ORG", @@ -1261,7 +1298,7 @@ class LumaTextToVideoGenerationNode: } def api_call(self, prompt: str, model: str, aspect_ratio: str, resolution: str, duration: str, loop: bool, seed, - auth_token=None, **kwargs): + luma_concepts: LumaConceptChain=None, auth_token=None, **kwargs): operation = SynchronousOperation( endpoint=ApiEndpoint( path="/proxy/luma/generations", @@ -1276,6 +1313,7 @@ class LumaTextToVideoGenerationNode: aspect_ratio=aspect_ratio, duration=duration, loop=loop, + concepts=luma_concepts.create_api_model() if luma_concepts else None ), auth_token=auth_token ) @@ -1298,7 +1336,7 @@ class LumaTextToVideoGenerationNode: vid_response = requests.get(response_poll.assets.video) return (VideoFromFile(BytesIO(vid_response.content)), ) -class LumaImageToVideoGenerationNode: +class LumaImageToVideoGenerationNode(ComfyNodeABC): """ Generates videos synchronously based on prompt, input images, and output_size. """ @@ -1347,6 +1385,9 @@ class LumaImageToVideoGenerationNode: "last_image": (IO.IMAGE, { "tooltip": "Last frame of generated video." }), + "luma_concepts": (LumaIO.LUMA_CONCEPTS, { + "tooltip": "Optional Camera Concepts to dictate camera motion via the Luma Concepts node." + }), }, "hidden": { "auth_token": "AUTH_TOKEN_COMFY_ORG", @@ -1354,7 +1395,7 @@ class LumaImageToVideoGenerationNode: } def api_call(self, prompt: str, model: str, resolution: str, duration: str, loop: bool, seed, - first_image: torch.Tensor=None, last_image: torch.Tensor=None, + first_image: torch.Tensor=None, last_image: torch.Tensor=None, luma_concepts: LumaConceptChain=None, auth_token=None, **kwargs): if first_image is None and last_image is None: raise Exception("At least one of first_image and last_image requires an input.") @@ -1370,11 +1411,12 @@ class LumaImageToVideoGenerationNode: request=LumaGenerationRequest( prompt=prompt, model=model, - aspect_ratio=LumaAspectRatio.ratio_16_9, + aspect_ratio=LumaAspectRatio.ratio_16_9, # ignored, but still needed by the API for some reason resolution=resolution, duration=duration, loop=loop, - keyframes=keyframes + keyframes=keyframes, + concepts=luma_concepts.create_api_model() if luma_concepts else None ), auth_token=auth_token ) @@ -1720,9 +1762,10 @@ NODE_CLASS_MAPPINGS = { "FluxProUltraImageNode": FluxProUltraImageNode, "LumaImageNode": LumaImageGenerationNode, "LumaImageModifyNode": LumaImageModifyNode, - "LumaReferenceNode": LumaReferenceNode, "LumaVideoNode": LumaTextToVideoGenerationNode, "LumaImageToVideoNode": LumaImageToVideoGenerationNode, + "LumaReferenceNode": LumaReferenceNode, + "LumaConceptsNode": LumaConceptsNode, "RecraftTextToImageNode": RecraftTextToImageNode, #"RecraftStyleV3RealisticImage": RecraftStyleV3RealisticImageNode, "RecraftStyleV3DigitalIllustration": RecraftStyleV3DigitalIllustrationNode, @@ -1740,9 +1783,10 @@ NODE_DISPLAY_NAME_MAPPINGS = { "FluxProUltraImageNode": "Flux 1.1 [pro] Ultra Image", "LumaImageNode": "Luma Text to Image", "LumaImageModifyNode": "Luma Image to Image", - "LumaReferenceNode": "Luma Reference", "LumaVideoNode": "Luma Text to Video", "LumaImageToVideoNode": "Luma Image to Video", + "LumaReferenceNode": "Luma Reference", + "LumaConceptsNode": "Luma Concepts", "RecraftTextToImageNode": "Recraft Text to Image", "RecraftStyleV3RealisticImage": "Recraft Style - Realistic Image", "RecraftStyleV3DigitalIllustration": "Recraft Style - Digital Illustration",