From 9840d2a1a78ec07618dfb77ab0fddccf62360e66 Mon Sep 17 00:00:00 2001 From: JettHu Date: Sat, 20 Apr 2024 01:47:51 +0800 Subject: [PATCH] feat: add attn_function_wrapper in ModelPatcher --- comfy/ldm/modules/attention.py | 141 +++++++++++++++++++++++---------- comfy/model_patcher.py | 24 +++--- 2 files changed, 114 insertions(+), 51 deletions(-) diff --git a/comfy/ldm/modules/attention.py b/comfy/ldm/modules/attention.py index e52466b3b..e7147d0a4 100644 --- a/comfy/ldm/modules/attention.py +++ b/comfy/ldm/modules/attention.py @@ -3,7 +3,7 @@ import torch import torch.nn.functional as F from torch import nn, einsum from einops import rearrange, repeat -from typing import Optional, Any +from typing import Callable, Optional, Any import logging from .diffusionmodules.util import checkpoint, AlphaBlender, timestep_embedding @@ -460,19 +460,20 @@ class BasicTransformerBlock(nn.Module): return checkpoint(self._forward, (x, context, transformer_options), self.parameters(), self.checkpoint) def _forward(self, x, context=None, transformer_options={}): - if "transformer_function" in transformer_options: - return transformer_options["transformer_function"](self, x, context, transformer_options) extra_options = {} block = transformer_options.get("block", None) block_index = transformer_options.get("block_index", 0) transformer_patches = {} transformer_patches_replace = {} + transformer_attn_function_wrapper = {} for k in transformer_options: if k == "patches": transformer_patches = transformer_options[k] elif k == "patches_replace": transformer_patches_replace = transformer_options[k] + elif k == "attn_function_wrapper": + transformer_attn_function_wrapper = transformer_options[k] else: extra_options[k] = transformer_options[k] @@ -485,11 +486,84 @@ class BasicTransformerBlock(nn.Module): if self.is_res: x += x_skip - n = self.norm1(x) - if self.disable_self_attn: - context_attn1 = context + transformer_block = (block[0], block[1], block_index) if block is not None else None + + attn1_replace_patch = transformer_patches_replace.get("attn1", {}) + block_attn1_replace_patch = attn1_replace_patch.get(transformer_block, attn1_replace_patch.get(block, None)) + + attn1_function_wrapper = transformer_attn_function_wrapper.get("attn1", {}) + block_attn1 = transformer_block if transformer_block in attn1_function_wrapper else block + + if block_attn1 in attn1_function_wrapper: + n = attn1_function_wrapper[block_attn1]( + self.apply_attn1, + x, + context, + extra_options=extra_options, + transformer_patches=transformer_patches, + block_attn_replace_patch=block_attn1_replace_patch, + ) else: - context_attn1 = None + n = self.apply_attn1( + x, + context, + extra_options=extra_options, + transformer_patches=transformer_patches, + block_attn_replace_patch=block_attn1_replace_patch, + ) + + x += n + if "middle_patch" in transformer_patches: + patch = transformer_patches["middle_patch"] + for p in patch: + x = p(x, extra_options) + + attn2_replace_patch = transformer_patches_replace.get("attn2", {}) + block_attn2_replace_patch = attn2_replace_patch.get(transformer_block, attn2_replace_patch.get(block, None)) + + attn2_function_wrapper = transformer_attn_function_wrapper.get("attn2", {}) + block_attn2 = transformer_block if transformer_block in attn2_function_wrapper else block + if block_attn2 in attn2_function_wrapper: + n = attn2_function_wrapper[block_attn2]( + self.apply_attn2, + x, + context, + extra_options=extra_options, + transformer_patches=transformer_patches, + block_attn_replace_patch=block_attn2_replace_patch, + n=n, + ) + else: + n = self.apply_attn2( + x, + context, + extra_options=extra_options, + transformer_patches=transformer_patches, + block_attn_replace_patch=block_attn2_replace_patch, + n=n, + ) + + x += n + if self.is_res: + x_skip = x + x = self.ff(self.norm3(x)) + if self.is_res: + x += x_skip + + return x + + def apply_attn1( + self, + x, + context, + extra_options: dict, + transformer_patches: dict, + block_attn_replace_patch: Callable, + **kwargs, + ): + + n = self.norm1(x) + context_attn1 = context if self.disable_self_attn else None value_attn1 = None if "attn1_patch" in transformer_patches: @@ -500,23 +574,14 @@ class BasicTransformerBlock(nn.Module): for p in patch: n, context_attn1, value_attn1 = p(n, context_attn1, value_attn1, extra_options) - if block is not None: - transformer_block = (block[0], block[1], block_index) - else: - transformer_block = None - attn1_replace_patch = transformer_patches_replace.get("attn1", {}) - block_attn1 = transformer_block - if block_attn1 not in attn1_replace_patch: - block_attn1 = block - - if block_attn1 in attn1_replace_patch: + if block_attn_replace_patch is not None: if context_attn1 is None: context_attn1 = n value_attn1 = n n = self.attn1.to_q(n) context_attn1 = self.attn1.to_k(context_attn1) value_attn1 = self.attn1.to_v(value_attn1) - n = attn1_replace_patch[block_attn1](n, context_attn1, value_attn1, extra_options) + n = block_attn_replace_patch(n, context_attn1, value_attn1, extra_options) n = self.attn1.to_out(n) else: n = self.attn1(n, context=context_attn1, value=value_attn1) @@ -526,18 +591,22 @@ class BasicTransformerBlock(nn.Module): for p in patch: n = p(n, extra_options) - x += n - if "middle_patch" in transformer_patches: - patch = transformer_patches["middle_patch"] - for p in patch: - x = p(x, extra_options) + return n + + def apply_attn2( + self, + x, + context, + extra_options: dict, + transformer_patches: dict, + block_attn_replace_patch: Callable, + n, + **kwargs, + ): if self.attn2 is not None: n = self.norm2(x) - if self.switch_temporal_ca_to_sa: - context_attn2 = n - else: - context_attn2 = context + context_attn2 = n if self.switch_temporal_ca_to_sa else context value_attn2 = None if "attn2_patch" in transformer_patches: patch = transformer_patches["attn2_patch"] @@ -545,18 +614,13 @@ class BasicTransformerBlock(nn.Module): for p in patch: n, context_attn2, value_attn2 = p(n, context_attn2, value_attn2, extra_options) - attn2_replace_patch = transformer_patches_replace.get("attn2", {}) - block_attn2 = transformer_block - if block_attn2 not in attn2_replace_patch: - block_attn2 = block - - if block_attn2 in attn2_replace_patch: + if block_attn_replace_patch is not None: if value_attn2 is None: value_attn2 = context_attn2 n = self.attn2.to_q(n) context_attn2 = self.attn2.to_k(context_attn2) value_attn2 = self.attn2.to_v(value_attn2) - n = attn2_replace_patch[block_attn2](n, context_attn2, value_attn2, extra_options) + n = block_attn_replace_patch(n, context_attn2, value_attn2, extra_options) n = self.attn2.to_out(n) else: n = self.attn2(n, context=context_attn2, value=value_attn2) @@ -566,14 +630,7 @@ class BasicTransformerBlock(nn.Module): for p in patch: n = p(n, extra_options) - x += n - if self.is_res: - x_skip = x - x = self.ff(self.norm3(x)) - if self.is_res: - x += x_skip - - return x + return n class SpatialTransformer(nn.Module): diff --git a/comfy/model_patcher.py b/comfy/model_patcher.py index 21d5b1af1..2260ae50e 100644 --- a/comfy/model_patcher.py +++ b/comfy/model_patcher.py @@ -19,23 +19,26 @@ def apply_weight_decompose(dora_scale, weight): return weight * (dora_scale / weight_norm) def set_model_options_patch_replace(model_options, patch, name, block_name, number, transformer_index=None): + return set_model_transformer_options(model_options, patch, name, block_name, number, transformer_index, "patches_replace") + +def set_model_transformer_options(model_options, patch, name, block_name, number, transformer_index=None, opt_key="patches_replace"): to = model_options["transformer_options"].copy() - if "patches_replace" not in to: - to["patches_replace"] = {} + if opt_key not in to: + to[opt_key] = {} else: - to["patches_replace"] = to["patches_replace"].copy() + to[opt_key] = to[opt_key].copy() - if name not in to["patches_replace"]: - to["patches_replace"][name] = {} + if name not in to[opt_key]: + to[opt_key][name] = {} else: - to["patches_replace"][name] = to["patches_replace"][name].copy() + to[opt_key][name] = to[opt_key][name].copy() if transformer_index is not None: block = (block_name, number, transformer_index) else: block = (block_name, number) - to["patches_replace"][name][block] = patch + to[opt_key][name][block] = patch model_options["transformer_options"] = to return model_options @@ -158,8 +161,11 @@ class ModelPatcher: def set_model_output_block_patch(self, patch): self.set_model_patch(patch, "output_block_patch") - def set_model_transformer_function(self, transformer_function): - self.model_options["transformer_options"]["transformer_function"] = transformer_function + def set_model_attn1_function_wrapper(self, patch, block_name, number, transformer_index=None): + set_model_transformer_options(self.model_options, patch, "attn1", block_name, number, transformer_index, "attn_function_wrapper") + + def set_model_attn2_function_wrapper(self, patch, block_name, number, transformer_index=None): + set_model_transformer_options(self.model_options, patch, "attn2", block_name, number, transformer_index, "attn_function_wrapper") def set_model_sampler_cfg_rescaler(self, cfg_rescaler_fn, disable_cfg1_optimization=False): self.model_options["sampler_cfg_rescaler"] = cfg_rescaler_fn