Merge branch 'comfyanonymous:master' into add-xla-support

app/user_manager.py

@@ -1,3 +1,4 @@
+from __future__ import annotations
 import json
 import os
 import re

comfy/ldm/aura/mmdit.py

@@ -437,7 +437,8 @@ class MMDiT(nn.Module):
         pos_encoding = pos_encoding[:,from_h:from_h+h,from_w:from_w+w]
         return x + pos_encoding.reshape(1, -1, self.positional_encoding.shape[-1])
 
-    def forward(self, x, timestep, context, **kwargs):
+    def forward(self, x, timestep, context, transformer_options={}, **kwargs):
+        patches_replace = transformer_options.get("patches_replace", {})
         # patchify x, add PE
         b, c, h, w = x.shape
 
@@ -458,15 +459,36 @@ class MMDiT(nn.Module):
 
         global_cond = self.t_embedder(t, x.dtype)  # B, D
 
+        blocks_replace = patches_replace.get("dit", {})
         if len(self.double_layers) > 0:
-            for layer in self.double_layers:
+            for i, layer in enumerate(self.double_layers):
-                c, x = layer(c, x, global_cond, **kwargs)
+                if ("double_block", i) in blocks_replace:
+                    def block_wrap(args):
+                        out = {}
+                        out["txt"], out["img"] = layer(args["txt"],
+                                                       args["img"],
+                                                       args["vec"])
+                        return out
+                    out = blocks_replace[("double_block", i)]({"img": x, "txt": c, "vec": global_cond}, {"original_block": block_wrap})
+                    c = out["txt"]
+                    x = out["img"]
+                else:
+                    c, x = layer(c, x, global_cond, **kwargs)
 
         if len(self.single_layers) > 0:
             c_len = c.size(1)
             cx = torch.cat([c, x], dim=1)
-            for layer in self.single_layers:
+            for i, layer in enumerate(self.single_layers):
-                cx = layer(cx, global_cond, **kwargs)
+                if ("single_block", i) in blocks_replace:
+                    def block_wrap(args):
+                        out = {}
+                        out["img"] = layer(args["img"], args["vec"])
+                        return out
+
+                    out = blocks_replace[("single_block", i)]({"img": cx, "vec": global_cond}, {"original_block": block_wrap})
+                    cx = out["img"]
+                else:
+                    cx = layer(cx, global_cond, **kwargs)
 
             x = cx[:, c_len:]
 

comfy/ldm/genmo/joint_model/asymm_models_joint.py

@@ -494,8 +494,9 @@ class AsymmDiTJoint(nn.Module):
         packed_indices: Dict[str, torch.Tensor] = None,
         rope_cos: torch.Tensor = None,
         rope_sin: torch.Tensor = None,
-        control=None, **kwargs
+        control=None, transformer_options={}, **kwargs
     ):
+        patches_replace = transformer_options.get("patches_replace", {})
         y_feat = context
         y_mask = attention_mask
         sigma = timestep
@@ -515,15 +516,32 @@ class AsymmDiTJoint(nn.Module):
         )
         del y_mask
 
+        blocks_replace = patches_replace.get("dit", {})
         for i, block in enumerate(self.blocks):
-            x, y_feat = block(
+            if ("double_block", i) in blocks_replace:
-                x,
+                def block_wrap(args):
-                c,
+                    out = {}
-                y_feat,
+                    out["img"], out["txt"] = block(
-                rope_cos=rope_cos,
+                                                    args["img"],
-                rope_sin=rope_sin,
+                                                    args["vec"],
-                crop_y=num_tokens,
+                                                    args["txt"],
-            )  # (B, M, D), (B, L, D)
+                                                    rope_cos=args["rope_cos"],
+                                                    rope_sin=args["rope_sin"],
+                                                    crop_y=args["num_tokens"]
+                                                    )
+                    return out
+                out = blocks_replace[("double_block", i)]({"img": x, "txt": y_feat, "vec": c, "rope_cos": rope_cos, "rope_sin": rope_sin, "num_tokens": num_tokens}, {"original_block": block_wrap})
+                y_feat = out["txt"]
+                x = out["img"]
+            else:
+                x, y_feat = block(
+                    x,
+                    c,
+                    y_feat,
+                    rope_cos=rope_cos,
+                    rope_sin=rope_sin,
+                    crop_y=num_tokens,
+                )  # (B, M, D), (B, L, D)
         del y_feat  # Final layers don't use dense text features.
 
         x = self.final_layer(x, c)  # (B, M, patch_size ** 2 * out_channels)