feat: add attn_function_wrapper in ModelPatcher

This commit is contained in:
JettHu 2024-04-20 01:47:51 +08:00
parent cba9c66fd2
commit 9840d2a1a7
2 changed files with 114 additions and 51 deletions

View File

@ -3,7 +3,7 @@ import torch
import torch.nn.functional as F
from torch import nn, einsum
from einops import rearrange, repeat
from typing import Optional, Any
from typing import Callable, Optional, Any
import logging
from .diffusionmodules.util import checkpoint, AlphaBlender, timestep_embedding
@ -460,19 +460,20 @@ class BasicTransformerBlock(nn.Module):
return checkpoint(self._forward, (x, context, transformer_options), self.parameters(), self.checkpoint)
def _forward(self, x, context=None, transformer_options={}):
if "transformer_function" in transformer_options:
return transformer_options["transformer_function"](self, x, context, transformer_options)
extra_options = {}
block = transformer_options.get("block", None)
block_index = transformer_options.get("block_index", 0)
transformer_patches = {}
transformer_patches_replace = {}
transformer_attn_function_wrapper = {}
for k in transformer_options:
if k == "patches":
transformer_patches = transformer_options[k]
elif k == "patches_replace":
transformer_patches_replace = transformer_options[k]
elif k == "attn_function_wrapper":
transformer_attn_function_wrapper = transformer_options[k]
else:
extra_options[k] = transformer_options[k]
@ -485,11 +486,84 @@ class BasicTransformerBlock(nn.Module):
if self.is_res:
x += x_skip
n = self.norm1(x)
if self.disable_self_attn:
context_attn1 = context
transformer_block = (block[0], block[1], block_index) if block is not None else None
attn1_replace_patch = transformer_patches_replace.get("attn1", {})
block_attn1_replace_patch = attn1_replace_patch.get(transformer_block, attn1_replace_patch.get(block, None))
attn1_function_wrapper = transformer_attn_function_wrapper.get("attn1", {})
block_attn1 = transformer_block if transformer_block in attn1_function_wrapper else block
if block_attn1 in attn1_function_wrapper:
n = attn1_function_wrapper[block_attn1](
self.apply_attn1,
x,
context,
extra_options=extra_options,
transformer_patches=transformer_patches,
block_attn_replace_patch=block_attn1_replace_patch,
)
else:
context_attn1 = None
n = self.apply_attn1(
x,
context,
extra_options=extra_options,
transformer_patches=transformer_patches,
block_attn_replace_patch=block_attn1_replace_patch,
)
x += n
if "middle_patch" in transformer_patches:
patch = transformer_patches["middle_patch"]
for p in patch:
x = p(x, extra_options)
attn2_replace_patch = transformer_patches_replace.get("attn2", {})
block_attn2_replace_patch = attn2_replace_patch.get(transformer_block, attn2_replace_patch.get(block, None))
attn2_function_wrapper = transformer_attn_function_wrapper.get("attn2", {})
block_attn2 = transformer_block if transformer_block in attn2_function_wrapper else block
if block_attn2 in attn2_function_wrapper:
n = attn2_function_wrapper[block_attn2](
self.apply_attn2,
x,
context,
extra_options=extra_options,
transformer_patches=transformer_patches,
block_attn_replace_patch=block_attn2_replace_patch,
n=n,
)
else:
n = self.apply_attn2(
x,
context,
extra_options=extra_options,
transformer_patches=transformer_patches,
block_attn_replace_patch=block_attn2_replace_patch,
n=n,
)
x += n
if self.is_res:
x_skip = x
x = self.ff(self.norm3(x))
if self.is_res:
x += x_skip
return x
def apply_attn1(
self,
x,
context,
extra_options: dict,
transformer_patches: dict,
block_attn_replace_patch: Callable,
**kwargs,
):
n = self.norm1(x)
context_attn1 = context if self.disable_self_attn else None
value_attn1 = None
if "attn1_patch" in transformer_patches:
@ -500,23 +574,14 @@ class BasicTransformerBlock(nn.Module):
for p in patch:
n, context_attn1, value_attn1 = p(n, context_attn1, value_attn1, extra_options)
if block is not None:
transformer_block = (block[0], block[1], block_index)
else:
transformer_block = None
attn1_replace_patch = transformer_patches_replace.get("attn1", {})
block_attn1 = transformer_block
if block_attn1 not in attn1_replace_patch:
block_attn1 = block
if block_attn1 in attn1_replace_patch:
if block_attn_replace_patch is not None:
if context_attn1 is None:
context_attn1 = n
value_attn1 = n
n = self.attn1.to_q(n)
context_attn1 = self.attn1.to_k(context_attn1)
value_attn1 = self.attn1.to_v(value_attn1)
n = attn1_replace_patch[block_attn1](n, context_attn1, value_attn1, extra_options)
n = block_attn_replace_patch(n, context_attn1, value_attn1, extra_options)
n = self.attn1.to_out(n)
else:
n = self.attn1(n, context=context_attn1, value=value_attn1)
@ -526,18 +591,22 @@ class BasicTransformerBlock(nn.Module):
for p in patch:
n = p(n, extra_options)
x += n
if "middle_patch" in transformer_patches:
patch = transformer_patches["middle_patch"]
for p in patch:
x = p(x, extra_options)
return n
def apply_attn2(
self,
x,
context,
extra_options: dict,
transformer_patches: dict,
block_attn_replace_patch: Callable,
n,
**kwargs,
):
if self.attn2 is not None:
n = self.norm2(x)
if self.switch_temporal_ca_to_sa:
context_attn2 = n
else:
context_attn2 = context
context_attn2 = n if self.switch_temporal_ca_to_sa else context
value_attn2 = None
if "attn2_patch" in transformer_patches:
patch = transformer_patches["attn2_patch"]
@ -545,18 +614,13 @@ class BasicTransformerBlock(nn.Module):
for p in patch:
n, context_attn2, value_attn2 = p(n, context_attn2, value_attn2, extra_options)
attn2_replace_patch = transformer_patches_replace.get("attn2", {})
block_attn2 = transformer_block
if block_attn2 not in attn2_replace_patch:
block_attn2 = block
if block_attn2 in attn2_replace_patch:
if block_attn_replace_patch is not None:
if value_attn2 is None:
value_attn2 = context_attn2
n = self.attn2.to_q(n)
context_attn2 = self.attn2.to_k(context_attn2)
value_attn2 = self.attn2.to_v(value_attn2)
n = attn2_replace_patch[block_attn2](n, context_attn2, value_attn2, extra_options)
n = block_attn_replace_patch(n, context_attn2, value_attn2, extra_options)
n = self.attn2.to_out(n)
else:
n = self.attn2(n, context=context_attn2, value=value_attn2)
@ -566,14 +630,7 @@ class BasicTransformerBlock(nn.Module):
for p in patch:
n = p(n, extra_options)
x += n
if self.is_res:
x_skip = x
x = self.ff(self.norm3(x))
if self.is_res:
x += x_skip
return x
return n
class SpatialTransformer(nn.Module):

View File

@ -19,23 +19,26 @@ def apply_weight_decompose(dora_scale, weight):
return weight * (dora_scale / weight_norm)
def set_model_options_patch_replace(model_options, patch, name, block_name, number, transformer_index=None):
return set_model_transformer_options(model_options, patch, name, block_name, number, transformer_index, "patches_replace")
def set_model_transformer_options(model_options, patch, name, block_name, number, transformer_index=None, opt_key="patches_replace"):
to = model_options["transformer_options"].copy()
if "patches_replace" not in to:
to["patches_replace"] = {}
if opt_key not in to:
to[opt_key] = {}
else:
to["patches_replace"] = to["patches_replace"].copy()
to[opt_key] = to[opt_key].copy()
if name not in to["patches_replace"]:
to["patches_replace"][name] = {}
if name not in to[opt_key]:
to[opt_key][name] = {}
else:
to["patches_replace"][name] = to["patches_replace"][name].copy()
to[opt_key][name] = to[opt_key][name].copy()
if transformer_index is not None:
block = (block_name, number, transformer_index)
else:
block = (block_name, number)
to["patches_replace"][name][block] = patch
to[opt_key][name][block] = patch
model_options["transformer_options"] = to
return model_options
@ -158,8 +161,11 @@ class ModelPatcher:
def set_model_output_block_patch(self, patch):
self.set_model_patch(patch, "output_block_patch")
def set_model_transformer_function(self, transformer_function):
self.model_options["transformer_options"]["transformer_function"] = transformer_function
def set_model_attn1_function_wrapper(self, patch, block_name, number, transformer_index=None):
set_model_transformer_options(self.model_options, patch, "attn1", block_name, number, transformer_index, "attn_function_wrapper")
def set_model_attn2_function_wrapper(self, patch, block_name, number, transformer_index=None):
set_model_transformer_options(self.model_options, patch, "attn2", block_name, number, transformer_index, "attn_function_wrapper")
def set_model_sampler_cfg_rescaler(self, cfg_rescaler_fn, disable_cfg1_optimization=False):
self.model_options["sampler_cfg_rescaler"] = cfg_rescaler_fn