Add Camera Concepts (luma_concepts) to Luma Video nodes (#33)

Co-authored-by: Robin Huang <robin.j.huang@gmail.com>

comfy_api_nodes/apis/luma_api.py

@@ -12,6 +12,7 @@ from pydantic import BaseModel, Field, confloat
 
 class LumaIO:
     LUMA_REF = "LUMA_REF"
+    LUMA_CONCEPTS = "LUMA_CONCEPTS"
 
 
 class LumaReference:
@@ -47,6 +48,89 @@ class LumaReferenceChain:
         return c
 
 
+class LumaConcept:
+    def __init__(self, key: str):
+        self.key = key
+
+
+class LumaConceptChain:
+    def __init__(self, str_list: list[str] = None):
+        self.concepts: list[LumaConcept] = []
+        if str_list is not None:
+            for c in str_list:
+                if c != "None":
+                    self.add(LumaConcept(key=c))
+
+    def add(self, concept: LumaConcept):
+        self.concepts.append(concept)
+
+    def create_api_model(self):
+        if len(self.concepts) == 0:
+            return None
+        api_concepts: list[LumaConceptObject] = []
+        for concept in self.concepts:
+            if concept.key == "None":
+                continue
+            api_concepts.append(LumaConceptObject(key=concept.key))
+        if len(api_concepts) == 0:
+            return None
+        return api_concepts
+
+    def clone(self):
+        c = LumaConceptChain()
+        for concept in self.concepts:
+            c.add(concept)
+        return c
+
+    def clone_and_merge(self, other: LumaConceptChain):
+        c = self.clone()
+        for concept in other.concepts:
+            c.add(concept)
+        return c
+
+
+def get_luma_concepts(include_none=False):
+    concepts = []
+    if include_none:
+        concepts.append("None")
+    return concepts + [
+        "truck_left",
+        "pan_right",
+        "pedestal_down",
+        "low_angle",
+        "pedestal_up",
+        "selfie",
+        "pan_left",
+        "roll_right",
+        "zoom_in",
+        "over_the_shoulder",
+        "orbit_right",
+        "orbit_left",
+        "static",
+        "tiny_planet",
+        "high_angle",
+        "bolt_cam",
+        "dolly_zoom",
+        "overhead",
+        "zoom_out",
+        "handheld",
+        "roll_left",
+        "pov",
+        "aerial_drone",
+        "push_in",
+        "crane_down",
+        "truck_right",
+        "tilt_down",
+        "elevator_doors",
+        "tilt_up",
+        "ground_level",
+        "pull_out",
+        "aerial",
+        "crane_up",
+        "eye_level"
+    ]
+
+
 class LumaImageModel(str, Enum):
     photon_1 = "photon-1"
     photon_flash_1 = "photon-flash-1"
@@ -133,6 +217,10 @@ class LumaKeyframes(BaseModel):
     frame1: Optional[Union[LumaImageReference, LumaGenerationReference]] = Field(None, description='')
 
 
+class LumaConceptObject(BaseModel):
+    key: str = Field(..., description='Camera Concept name')
+
+
 class LumaImageGenerationRequest(BaseModel):
     prompt: str = Field(..., description='The prompt of the generation')
     model: LumaImageModel = Field(LumaImageModel.photon_1, description='The image model used for the generation')
@@ -151,6 +239,7 @@ class LumaGenerationRequest(BaseModel):
     resolution: Optional[LumaVideoOutputResolution] = Field(None, description='The resolution of the generation')
     loop: Optional[bool] = Field(None, description='Whether to loop the video')
     keyframes: Optional[LumaKeyframes] = Field(None, description='The keyframes of the generation')
+    concepts: Optional[list[LumaConceptObject]] = Field(None, description='Camera Concepts to apply to generation')
 
 
 class LumaGeneration(BaseModel):

comfy_api_nodes/nodes_api.py

@@ -38,7 +38,9 @@ from comfy_api_nodes.apis.luma_api import (
     LumaReferenceChain,
     LumaImageReference,
     LumaKeyframes,
+    LumaConceptChain,
     LumaIO,
+    get_luma_concepts,
 )
 from comfy_api_nodes.apis.recraft_api import (
     RecraftImageGenerationRequest,
@@ -965,7 +967,7 @@ class FluxProUltraImageNode(ComfyNodeABC):
         img.save(img_byte_arr, format='PNG')
         return base64.b64encode(img_byte_arr.getvalue()).decode()
 
-class LumaReferenceNode:
+class LumaReferenceNode(ComfyNodeABC):
     """
     Holds an image and weight for use with Luma Generate Image node.
     """
@@ -1003,7 +1005,39 @@ class LumaReferenceNode:
         luma_ref.add(LumaReference(image=image, weight=round(weight, 2)))
         return (luma_ref, )
 
-class LumaImageGenerationNode:
+class LumaConceptsNode(ComfyNodeABC):
+    """
+    Holds one or more Camera Concepts for use with Luma Text to Video and Luma Image to Video nodes.
+    """
+    RETURN_TYPES = (LumaIO.LUMA_CONCEPTS,)
+    RETURN_NAMES = ("luma_concepts",)
+    DESCRIPTION = cleandoc(__doc__ or "")  # Handle potential None value
+    FUNCTION = "create_concepts"
+    CATEGORY = "api node/Luma"
+
+    @classmethod
+    def INPUT_TYPES(s):
+        return {
+            "required": {
+                "concept1": (get_luma_concepts(include_none=True), ),
+                "concept2": (get_luma_concepts(include_none=True), ),
+                "concept3": (get_luma_concepts(include_none=True), ),
+                "concept4": (get_luma_concepts(include_none=True), ),
+            },
+            "optional": {
+                "luma_concepts": (LumaIO.LUMA_CONCEPTS, {
+                    "tooltip": "Optional Camera Concepts to add to the ones chosen here."
+                }),
+            }
+        }
+
+    def create_concepts(self, concept1: str, concept2: str, concept3: str, concept4: str, luma_concepts: LumaConceptChain=None):
+        chain = LumaConceptChain(str_list=[concept1, concept2, concept3, concept4])
+        if luma_concepts is not None:
+            chain = luma_concepts.clone_and_merge(chain)
+        return (chain,)
+
+class LumaImageGenerationNode(ComfyNodeABC):
     """
     Generates images synchronously based on prompt and aspect ratio.
     """
@@ -1126,7 +1160,7 @@ class LumaImageGenerationNode:
         chain = LumaReferenceChain(first_ref=LumaReference(image=style_image, weight=weight))
         return self._convert_luma_refs(chain, max_refs=1, auth_token=auth_token)
 
-class LumaImageModifyNode:
+class LumaImageModifyNode(ComfyNodeABC):
     """
     Modifies images synchronously based on prompt and aspect ratio.
     """
@@ -1211,7 +1245,7 @@ class LumaImageModifyNode:
         img = process_image_response(img_response)
         return (img,)
 
-class LumaTextToVideoGenerationNode:
+class LumaTextToVideoGenerationNode(ComfyNodeABC):
     """
     Generates videos synchronously based on prompt and output_size.
     """
@@ -1254,6 +1288,9 @@ class LumaTextToVideoGenerationNode:
                 }),
             },
             "optional": {
+                "luma_concepts": (LumaIO.LUMA_CONCEPTS, {
+                    "tooltip": "Optional Camera Concepts to dictate camera motion via the Luma Concepts node."
+                }),
             },
             "hidden": {
                 "auth_token": "AUTH_TOKEN_COMFY_ORG",
@@ -1261,7 +1298,7 @@ class LumaTextToVideoGenerationNode:
         }
 
     def api_call(self, prompt: str, model: str, aspect_ratio: str, resolution: str, duration: str, loop: bool, seed,
-                 auth_token=None, **kwargs):
+                 luma_concepts: LumaConceptChain=None, auth_token=None, **kwargs):
         operation = SynchronousOperation(
             endpoint=ApiEndpoint(
                 path="/proxy/luma/generations",
@@ -1276,6 +1313,7 @@ class LumaTextToVideoGenerationNode:
                 aspect_ratio=aspect_ratio,
                 duration=duration,
                 loop=loop,
+                concepts=luma_concepts.create_api_model() if luma_concepts else None
             ),
             auth_token=auth_token
         )
@@ -1298,7 +1336,7 @@ class LumaTextToVideoGenerationNode:
         vid_response = requests.get(response_poll.assets.video)
         return (VideoFromFile(BytesIO(vid_response.content)), )
 
-class LumaImageToVideoGenerationNode:
+class LumaImageToVideoGenerationNode(ComfyNodeABC):
     """
     Generates videos synchronously based on prompt, input images, and output_size.
     """
@@ -1347,6 +1385,9 @@ class LumaImageToVideoGenerationNode:
                 "last_image": (IO.IMAGE, {
                     "tooltip": "Last frame of generated video."
                 }),
+                "luma_concepts": (LumaIO.LUMA_CONCEPTS, {
+                    "tooltip": "Optional Camera Concepts to dictate camera motion via the Luma Concepts node."
+                }),
             },
             "hidden": {
                 "auth_token": "AUTH_TOKEN_COMFY_ORG",
@@ -1354,7 +1395,7 @@ class LumaImageToVideoGenerationNode:
         }
 
     def api_call(self, prompt: str, model: str, resolution: str, duration: str, loop: bool, seed,
-                 first_image: torch.Tensor=None, last_image: torch.Tensor=None,
+                 first_image: torch.Tensor=None, last_image: torch.Tensor=None, luma_concepts: LumaConceptChain=None,
                  auth_token=None, **kwargs):
         if first_image is None and last_image is None:
             raise Exception("At least one of first_image and last_image requires an input.")
@@ -1370,11 +1411,12 @@ class LumaImageToVideoGenerationNode:
             request=LumaGenerationRequest(
                 prompt=prompt,
                 model=model,
-                aspect_ratio=LumaAspectRatio.ratio_16_9,
+                aspect_ratio=LumaAspectRatio.ratio_16_9, # ignored, but still needed by the API for some reason
                 resolution=resolution,
                 duration=duration,
                 loop=loop,
-                keyframes=keyframes
+                keyframes=keyframes,
+                concepts=luma_concepts.create_api_model() if luma_concepts else None
             ),
             auth_token=auth_token
         )
@@ -1720,9 +1762,10 @@ NODE_CLASS_MAPPINGS = {
     "FluxProUltraImageNode": FluxProUltraImageNode,
     "LumaImageNode": LumaImageGenerationNode,
     "LumaImageModifyNode": LumaImageModifyNode,
-    "LumaReferenceNode": LumaReferenceNode,
     "LumaVideoNode": LumaTextToVideoGenerationNode,
     "LumaImageToVideoNode": LumaImageToVideoGenerationNode,
+    "LumaReferenceNode": LumaReferenceNode,
+    "LumaConceptsNode": LumaConceptsNode,
     "RecraftTextToImageNode": RecraftTextToImageNode,
     #"RecraftStyleV3RealisticImage": RecraftStyleV3RealisticImageNode,
     "RecraftStyleV3DigitalIllustration": RecraftStyleV3DigitalIllustrationNode,
@@ -1740,9 +1783,10 @@ NODE_DISPLAY_NAME_MAPPINGS = {
     "FluxProUltraImageNode": "Flux 1.1 [pro] Ultra Image",
     "LumaImageNode": "Luma Text to Image",
     "LumaImageModifyNode": "Luma Image to Image",
-    "LumaReferenceNode": "Luma Reference",
     "LumaVideoNode": "Luma Text to Video",
     "LumaImageToVideoNode": "Luma Image to Video",
+    "LumaReferenceNode": "Luma Reference",
+    "LumaConceptsNode": "Luma Concepts",
     "RecraftTextToImageNode": "Recraft Text to Image",
     "RecraftStyleV3RealisticImage": "Recraft Style - Realistic Image",
     "RecraftStyleV3DigitalIllustration": "Recraft Style - Digital Illustration",