feat: add attn_function_wrapper in ModelPatcher

This commit is contained in:
JettHu 2024-04-20 01:47:51 +08:00
parent cba9c66fd2
commit 9840d2a1a7
2 changed files with 114 additions and 51 deletions

View File

@ -3,7 +3,7 @@ import torch
import torch.nn.functional as F import torch.nn.functional as F
from torch import nn, einsum from torch import nn, einsum
from einops import rearrange, repeat from einops import rearrange, repeat
from typing import Optional, Any from typing import Callable, Optional, Any
import logging import logging
from .diffusionmodules.util import checkpoint, AlphaBlender, timestep_embedding from .diffusionmodules.util import checkpoint, AlphaBlender, timestep_embedding
@ -460,19 +460,20 @@ class BasicTransformerBlock(nn.Module):
return checkpoint(self._forward, (x, context, transformer_options), self.parameters(), self.checkpoint) return checkpoint(self._forward, (x, context, transformer_options), self.parameters(), self.checkpoint)
def _forward(self, x, context=None, transformer_options={}): def _forward(self, x, context=None, transformer_options={}):
if "transformer_function" in transformer_options:
return transformer_options["transformer_function"](self, x, context, transformer_options)
extra_options = {} extra_options = {}
block = transformer_options.get("block", None) block = transformer_options.get("block", None)
block_index = transformer_options.get("block_index", 0) block_index = transformer_options.get("block_index", 0)
transformer_patches = {} transformer_patches = {}
transformer_patches_replace = {} transformer_patches_replace = {}
transformer_attn_function_wrapper = {}
for k in transformer_options: for k in transformer_options:
if k == "patches": if k == "patches":
transformer_patches = transformer_options[k] transformer_patches = transformer_options[k]
elif k == "patches_replace": elif k == "patches_replace":
transformer_patches_replace = transformer_options[k] transformer_patches_replace = transformer_options[k]
elif k == "attn_function_wrapper":
transformer_attn_function_wrapper = transformer_options[k]
else: else:
extra_options[k] = transformer_options[k] extra_options[k] = transformer_options[k]
@ -485,11 +486,84 @@ class BasicTransformerBlock(nn.Module):
if self.is_res: if self.is_res:
x += x_skip x += x_skip
n = self.norm1(x) transformer_block = (block[0], block[1], block_index) if block is not None else None
if self.disable_self_attn:
context_attn1 = context attn1_replace_patch = transformer_patches_replace.get("attn1", {})
block_attn1_replace_patch = attn1_replace_patch.get(transformer_block, attn1_replace_patch.get(block, None))
attn1_function_wrapper = transformer_attn_function_wrapper.get("attn1", {})
block_attn1 = transformer_block if transformer_block in attn1_function_wrapper else block
if block_attn1 in attn1_function_wrapper:
n = attn1_function_wrapper[block_attn1](
self.apply_attn1,
x,
context,
extra_options=extra_options,
transformer_patches=transformer_patches,
block_attn_replace_patch=block_attn1_replace_patch,
)
else: else:
context_attn1 = None n = self.apply_attn1(
x,
context,
extra_options=extra_options,
transformer_patches=transformer_patches,
block_attn_replace_patch=block_attn1_replace_patch,
)
x += n
if "middle_patch" in transformer_patches:
patch = transformer_patches["middle_patch"]
for p in patch:
x = p(x, extra_options)
attn2_replace_patch = transformer_patches_replace.get("attn2", {})
block_attn2_replace_patch = attn2_replace_patch.get(transformer_block, attn2_replace_patch.get(block, None))
attn2_function_wrapper = transformer_attn_function_wrapper.get("attn2", {})
block_attn2 = transformer_block if transformer_block in attn2_function_wrapper else block
if block_attn2 in attn2_function_wrapper:
n = attn2_function_wrapper[block_attn2](
self.apply_attn2,
x,
context,
extra_options=extra_options,
transformer_patches=transformer_patches,
block_attn_replace_patch=block_attn2_replace_patch,
n=n,
)
else:
n = self.apply_attn2(
x,
context,
extra_options=extra_options,
transformer_patches=transformer_patches,
block_attn_replace_patch=block_attn2_replace_patch,
n=n,
)
x += n
if self.is_res:
x_skip = x
x = self.ff(self.norm3(x))
if self.is_res:
x += x_skip
return x
def apply_attn1(
self,
x,
context,
extra_options: dict,
transformer_patches: dict,
block_attn_replace_patch: Callable,
**kwargs,
):
n = self.norm1(x)
context_attn1 = context if self.disable_self_attn else None
value_attn1 = None value_attn1 = None
if "attn1_patch" in transformer_patches: if "attn1_patch" in transformer_patches:
@ -500,23 +574,14 @@ class BasicTransformerBlock(nn.Module):
for p in patch: for p in patch:
n, context_attn1, value_attn1 = p(n, context_attn1, value_attn1, extra_options) n, context_attn1, value_attn1 = p(n, context_attn1, value_attn1, extra_options)
if block is not None: if block_attn_replace_patch is not None:
transformer_block = (block[0], block[1], block_index)
else:
transformer_block = None
attn1_replace_patch = transformer_patches_replace.get("attn1", {})
block_attn1 = transformer_block
if block_attn1 not in attn1_replace_patch:
block_attn1 = block
if block_attn1 in attn1_replace_patch:
if context_attn1 is None: if context_attn1 is None:
context_attn1 = n context_attn1 = n
value_attn1 = n value_attn1 = n
n = self.attn1.to_q(n) n = self.attn1.to_q(n)
context_attn1 = self.attn1.to_k(context_attn1) context_attn1 = self.attn1.to_k(context_attn1)
value_attn1 = self.attn1.to_v(value_attn1) value_attn1 = self.attn1.to_v(value_attn1)
n = attn1_replace_patch[block_attn1](n, context_attn1, value_attn1, extra_options) n = block_attn_replace_patch(n, context_attn1, value_attn1, extra_options)
n = self.attn1.to_out(n) n = self.attn1.to_out(n)
else: else:
n = self.attn1(n, context=context_attn1, value=value_attn1) n = self.attn1(n, context=context_attn1, value=value_attn1)
@ -526,18 +591,22 @@ class BasicTransformerBlock(nn.Module):
for p in patch: for p in patch:
n = p(n, extra_options) n = p(n, extra_options)
x += n return n
if "middle_patch" in transformer_patches:
patch = transformer_patches["middle_patch"] def apply_attn2(
for p in patch: self,
x = p(x, extra_options) x,
context,
extra_options: dict,
transformer_patches: dict,
block_attn_replace_patch: Callable,
n,
**kwargs,
):
if self.attn2 is not None: if self.attn2 is not None:
n = self.norm2(x) n = self.norm2(x)
if self.switch_temporal_ca_to_sa: context_attn2 = n if self.switch_temporal_ca_to_sa else context
context_attn2 = n
else:
context_attn2 = context
value_attn2 = None value_attn2 = None
if "attn2_patch" in transformer_patches: if "attn2_patch" in transformer_patches:
patch = transformer_patches["attn2_patch"] patch = transformer_patches["attn2_patch"]
@ -545,18 +614,13 @@ class BasicTransformerBlock(nn.Module):
for p in patch: for p in patch:
n, context_attn2, value_attn2 = p(n, context_attn2, value_attn2, extra_options) n, context_attn2, value_attn2 = p(n, context_attn2, value_attn2, extra_options)
attn2_replace_patch = transformer_patches_replace.get("attn2", {}) if block_attn_replace_patch is not None:
block_attn2 = transformer_block
if block_attn2 not in attn2_replace_patch:
block_attn2 = block
if block_attn2 in attn2_replace_patch:
if value_attn2 is None: if value_attn2 is None:
value_attn2 = context_attn2 value_attn2 = context_attn2
n = self.attn2.to_q(n) n = self.attn2.to_q(n)
context_attn2 = self.attn2.to_k(context_attn2) context_attn2 = self.attn2.to_k(context_attn2)
value_attn2 = self.attn2.to_v(value_attn2) value_attn2 = self.attn2.to_v(value_attn2)
n = attn2_replace_patch[block_attn2](n, context_attn2, value_attn2, extra_options) n = block_attn_replace_patch(n, context_attn2, value_attn2, extra_options)
n = self.attn2.to_out(n) n = self.attn2.to_out(n)
else: else:
n = self.attn2(n, context=context_attn2, value=value_attn2) n = self.attn2(n, context=context_attn2, value=value_attn2)
@ -566,14 +630,7 @@ class BasicTransformerBlock(nn.Module):
for p in patch: for p in patch:
n = p(n, extra_options) n = p(n, extra_options)
x += n return n
if self.is_res:
x_skip = x
x = self.ff(self.norm3(x))
if self.is_res:
x += x_skip
return x
class SpatialTransformer(nn.Module): class SpatialTransformer(nn.Module):

View File

@ -19,23 +19,26 @@ def apply_weight_decompose(dora_scale, weight):
return weight * (dora_scale / weight_norm) return weight * (dora_scale / weight_norm)
def set_model_options_patch_replace(model_options, patch, name, block_name, number, transformer_index=None): def set_model_options_patch_replace(model_options, patch, name, block_name, number, transformer_index=None):
return set_model_transformer_options(model_options, patch, name, block_name, number, transformer_index, "patches_replace")
def set_model_transformer_options(model_options, patch, name, block_name, number, transformer_index=None, opt_key="patches_replace"):
to = model_options["transformer_options"].copy() to = model_options["transformer_options"].copy()
if "patches_replace" not in to: if opt_key not in to:
to["patches_replace"] = {} to[opt_key] = {}
else: else:
to["patches_replace"] = to["patches_replace"].copy() to[opt_key] = to[opt_key].copy()
if name not in to["patches_replace"]: if name not in to[opt_key]:
to["patches_replace"][name] = {} to[opt_key][name] = {}
else: else:
to["patches_replace"][name] = to["patches_replace"][name].copy() to[opt_key][name] = to[opt_key][name].copy()
if transformer_index is not None: if transformer_index is not None:
block = (block_name, number, transformer_index) block = (block_name, number, transformer_index)
else: else:
block = (block_name, number) block = (block_name, number)
to["patches_replace"][name][block] = patch to[opt_key][name][block] = patch
model_options["transformer_options"] = to model_options["transformer_options"] = to
return model_options return model_options
@ -158,8 +161,11 @@ class ModelPatcher:
def set_model_output_block_patch(self, patch): def set_model_output_block_patch(self, patch):
self.set_model_patch(patch, "output_block_patch") self.set_model_patch(patch, "output_block_patch")
def set_model_transformer_function(self, transformer_function): def set_model_attn1_function_wrapper(self, patch, block_name, number, transformer_index=None):
self.model_options["transformer_options"]["transformer_function"] = transformer_function set_model_transformer_options(self.model_options, patch, "attn1", block_name, number, transformer_index, "attn_function_wrapper")
def set_model_attn2_function_wrapper(self, patch, block_name, number, transformer_index=None):
set_model_transformer_options(self.model_options, patch, "attn2", block_name, number, transformer_index, "attn_function_wrapper")
def set_model_sampler_cfg_rescaler(self, cfg_rescaler_fn, disable_cfg1_optimization=False): def set_model_sampler_cfg_rescaler(self, cfg_rescaler_fn, disable_cfg1_optimization=False):
self.model_options["sampler_cfg_rescaler"] = cfg_rescaler_fn self.model_options["sampler_cfg_rescaler"] = cfg_rescaler_fn