mirror of
https://git.datalinker.icu/comfyanonymous/ComfyUI
synced 2026-11-23 03:06:59 +08:00
feat: add attn_function_wrapper in ModelPatcher
This commit is contained in:
parent
cba9c66fd2
commit
9840d2a1a7
@ -3,7 +3,7 @@ import torch
|
|||||||
import torch.nn.functional as F
|
import torch.nn.functional as F
|
||||||
from torch import nn, einsum
|
from torch import nn, einsum
|
||||||
from einops import rearrange, repeat
|
from einops import rearrange, repeat
|
||||||
from typing import Optional, Any
|
from typing import Callable, Optional, Any
|
||||||
import logging
|
import logging
|
||||||
|
|
||||||
from .diffusionmodules.util import checkpoint, AlphaBlender, timestep_embedding
|
from .diffusionmodules.util import checkpoint, AlphaBlender, timestep_embedding
|
||||||
@ -460,19 +460,20 @@ class BasicTransformerBlock(nn.Module):
|
|||||||
return checkpoint(self._forward, (x, context, transformer_options), self.parameters(), self.checkpoint)
|
return checkpoint(self._forward, (x, context, transformer_options), self.parameters(), self.checkpoint)
|
||||||
|
|
||||||
def _forward(self, x, context=None, transformer_options={}):
|
def _forward(self, x, context=None, transformer_options={}):
|
||||||
if "transformer_function" in transformer_options:
|
|
||||||
return transformer_options["transformer_function"](self, x, context, transformer_options)
|
|
||||||
extra_options = {}
|
extra_options = {}
|
||||||
block = transformer_options.get("block", None)
|
block = transformer_options.get("block", None)
|
||||||
block_index = transformer_options.get("block_index", 0)
|
block_index = transformer_options.get("block_index", 0)
|
||||||
transformer_patches = {}
|
transformer_patches = {}
|
||||||
transformer_patches_replace = {}
|
transformer_patches_replace = {}
|
||||||
|
transformer_attn_function_wrapper = {}
|
||||||
|
|
||||||
for k in transformer_options:
|
for k in transformer_options:
|
||||||
if k == "patches":
|
if k == "patches":
|
||||||
transformer_patches = transformer_options[k]
|
transformer_patches = transformer_options[k]
|
||||||
elif k == "patches_replace":
|
elif k == "patches_replace":
|
||||||
transformer_patches_replace = transformer_options[k]
|
transformer_patches_replace = transformer_options[k]
|
||||||
|
elif k == "attn_function_wrapper":
|
||||||
|
transformer_attn_function_wrapper = transformer_options[k]
|
||||||
else:
|
else:
|
||||||
extra_options[k] = transformer_options[k]
|
extra_options[k] = transformer_options[k]
|
||||||
|
|
||||||
@ -485,11 +486,84 @@ class BasicTransformerBlock(nn.Module):
|
|||||||
if self.is_res:
|
if self.is_res:
|
||||||
x += x_skip
|
x += x_skip
|
||||||
|
|
||||||
n = self.norm1(x)
|
transformer_block = (block[0], block[1], block_index) if block is not None else None
|
||||||
if self.disable_self_attn:
|
|
||||||
context_attn1 = context
|
attn1_replace_patch = transformer_patches_replace.get("attn1", {})
|
||||||
|
block_attn1_replace_patch = attn1_replace_patch.get(transformer_block, attn1_replace_patch.get(block, None))
|
||||||
|
|
||||||
|
attn1_function_wrapper = transformer_attn_function_wrapper.get("attn1", {})
|
||||||
|
block_attn1 = transformer_block if transformer_block in attn1_function_wrapper else block
|
||||||
|
|
||||||
|
if block_attn1 in attn1_function_wrapper:
|
||||||
|
n = attn1_function_wrapper[block_attn1](
|
||||||
|
self.apply_attn1,
|
||||||
|
x,
|
||||||
|
context,
|
||||||
|
extra_options=extra_options,
|
||||||
|
transformer_patches=transformer_patches,
|
||||||
|
block_attn_replace_patch=block_attn1_replace_patch,
|
||||||
|
)
|
||||||
else:
|
else:
|
||||||
context_attn1 = None
|
n = self.apply_attn1(
|
||||||
|
x,
|
||||||
|
context,
|
||||||
|
extra_options=extra_options,
|
||||||
|
transformer_patches=transformer_patches,
|
||||||
|
block_attn_replace_patch=block_attn1_replace_patch,
|
||||||
|
)
|
||||||
|
|
||||||
|
x += n
|
||||||
|
if "middle_patch" in transformer_patches:
|
||||||
|
patch = transformer_patches["middle_patch"]
|
||||||
|
for p in patch:
|
||||||
|
x = p(x, extra_options)
|
||||||
|
|
||||||
|
attn2_replace_patch = transformer_patches_replace.get("attn2", {})
|
||||||
|
block_attn2_replace_patch = attn2_replace_patch.get(transformer_block, attn2_replace_patch.get(block, None))
|
||||||
|
|
||||||
|
attn2_function_wrapper = transformer_attn_function_wrapper.get("attn2", {})
|
||||||
|
block_attn2 = transformer_block if transformer_block in attn2_function_wrapper else block
|
||||||
|
if block_attn2 in attn2_function_wrapper:
|
||||||
|
n = attn2_function_wrapper[block_attn2](
|
||||||
|
self.apply_attn2,
|
||||||
|
x,
|
||||||
|
context,
|
||||||
|
extra_options=extra_options,
|
||||||
|
transformer_patches=transformer_patches,
|
||||||
|
block_attn_replace_patch=block_attn2_replace_patch,
|
||||||
|
n=n,
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
n = self.apply_attn2(
|
||||||
|
x,
|
||||||
|
context,
|
||||||
|
extra_options=extra_options,
|
||||||
|
transformer_patches=transformer_patches,
|
||||||
|
block_attn_replace_patch=block_attn2_replace_patch,
|
||||||
|
n=n,
|
||||||
|
)
|
||||||
|
|
||||||
|
x += n
|
||||||
|
if self.is_res:
|
||||||
|
x_skip = x
|
||||||
|
x = self.ff(self.norm3(x))
|
||||||
|
if self.is_res:
|
||||||
|
x += x_skip
|
||||||
|
|
||||||
|
return x
|
||||||
|
|
||||||
|
def apply_attn1(
|
||||||
|
self,
|
||||||
|
x,
|
||||||
|
context,
|
||||||
|
extra_options: dict,
|
||||||
|
transformer_patches: dict,
|
||||||
|
block_attn_replace_patch: Callable,
|
||||||
|
**kwargs,
|
||||||
|
):
|
||||||
|
|
||||||
|
n = self.norm1(x)
|
||||||
|
context_attn1 = context if self.disable_self_attn else None
|
||||||
value_attn1 = None
|
value_attn1 = None
|
||||||
|
|
||||||
if "attn1_patch" in transformer_patches:
|
if "attn1_patch" in transformer_patches:
|
||||||
@ -500,23 +574,14 @@ class BasicTransformerBlock(nn.Module):
|
|||||||
for p in patch:
|
for p in patch:
|
||||||
n, context_attn1, value_attn1 = p(n, context_attn1, value_attn1, extra_options)
|
n, context_attn1, value_attn1 = p(n, context_attn1, value_attn1, extra_options)
|
||||||
|
|
||||||
if block is not None:
|
if block_attn_replace_patch is not None:
|
||||||
transformer_block = (block[0], block[1], block_index)
|
|
||||||
else:
|
|
||||||
transformer_block = None
|
|
||||||
attn1_replace_patch = transformer_patches_replace.get("attn1", {})
|
|
||||||
block_attn1 = transformer_block
|
|
||||||
if block_attn1 not in attn1_replace_patch:
|
|
||||||
block_attn1 = block
|
|
||||||
|
|
||||||
if block_attn1 in attn1_replace_patch:
|
|
||||||
if context_attn1 is None:
|
if context_attn1 is None:
|
||||||
context_attn1 = n
|
context_attn1 = n
|
||||||
value_attn1 = n
|
value_attn1 = n
|
||||||
n = self.attn1.to_q(n)
|
n = self.attn1.to_q(n)
|
||||||
context_attn1 = self.attn1.to_k(context_attn1)
|
context_attn1 = self.attn1.to_k(context_attn1)
|
||||||
value_attn1 = self.attn1.to_v(value_attn1)
|
value_attn1 = self.attn1.to_v(value_attn1)
|
||||||
n = attn1_replace_patch[block_attn1](n, context_attn1, value_attn1, extra_options)
|
n = block_attn_replace_patch(n, context_attn1, value_attn1, extra_options)
|
||||||
n = self.attn1.to_out(n)
|
n = self.attn1.to_out(n)
|
||||||
else:
|
else:
|
||||||
n = self.attn1(n, context=context_attn1, value=value_attn1)
|
n = self.attn1(n, context=context_attn1, value=value_attn1)
|
||||||
@ -526,18 +591,22 @@ class BasicTransformerBlock(nn.Module):
|
|||||||
for p in patch:
|
for p in patch:
|
||||||
n = p(n, extra_options)
|
n = p(n, extra_options)
|
||||||
|
|
||||||
x += n
|
return n
|
||||||
if "middle_patch" in transformer_patches:
|
|
||||||
patch = transformer_patches["middle_patch"]
|
def apply_attn2(
|
||||||
for p in patch:
|
self,
|
||||||
x = p(x, extra_options)
|
x,
|
||||||
|
context,
|
||||||
|
extra_options: dict,
|
||||||
|
transformer_patches: dict,
|
||||||
|
block_attn_replace_patch: Callable,
|
||||||
|
n,
|
||||||
|
**kwargs,
|
||||||
|
):
|
||||||
|
|
||||||
if self.attn2 is not None:
|
if self.attn2 is not None:
|
||||||
n = self.norm2(x)
|
n = self.norm2(x)
|
||||||
if self.switch_temporal_ca_to_sa:
|
context_attn2 = n if self.switch_temporal_ca_to_sa else context
|
||||||
context_attn2 = n
|
|
||||||
else:
|
|
||||||
context_attn2 = context
|
|
||||||
value_attn2 = None
|
value_attn2 = None
|
||||||
if "attn2_patch" in transformer_patches:
|
if "attn2_patch" in transformer_patches:
|
||||||
patch = transformer_patches["attn2_patch"]
|
patch = transformer_patches["attn2_patch"]
|
||||||
@ -545,18 +614,13 @@ class BasicTransformerBlock(nn.Module):
|
|||||||
for p in patch:
|
for p in patch:
|
||||||
n, context_attn2, value_attn2 = p(n, context_attn2, value_attn2, extra_options)
|
n, context_attn2, value_attn2 = p(n, context_attn2, value_attn2, extra_options)
|
||||||
|
|
||||||
attn2_replace_patch = transformer_patches_replace.get("attn2", {})
|
if block_attn_replace_patch is not None:
|
||||||
block_attn2 = transformer_block
|
|
||||||
if block_attn2 not in attn2_replace_patch:
|
|
||||||
block_attn2 = block
|
|
||||||
|
|
||||||
if block_attn2 in attn2_replace_patch:
|
|
||||||
if value_attn2 is None:
|
if value_attn2 is None:
|
||||||
value_attn2 = context_attn2
|
value_attn2 = context_attn2
|
||||||
n = self.attn2.to_q(n)
|
n = self.attn2.to_q(n)
|
||||||
context_attn2 = self.attn2.to_k(context_attn2)
|
context_attn2 = self.attn2.to_k(context_attn2)
|
||||||
value_attn2 = self.attn2.to_v(value_attn2)
|
value_attn2 = self.attn2.to_v(value_attn2)
|
||||||
n = attn2_replace_patch[block_attn2](n, context_attn2, value_attn2, extra_options)
|
n = block_attn_replace_patch(n, context_attn2, value_attn2, extra_options)
|
||||||
n = self.attn2.to_out(n)
|
n = self.attn2.to_out(n)
|
||||||
else:
|
else:
|
||||||
n = self.attn2(n, context=context_attn2, value=value_attn2)
|
n = self.attn2(n, context=context_attn2, value=value_attn2)
|
||||||
@ -566,14 +630,7 @@ class BasicTransformerBlock(nn.Module):
|
|||||||
for p in patch:
|
for p in patch:
|
||||||
n = p(n, extra_options)
|
n = p(n, extra_options)
|
||||||
|
|
||||||
x += n
|
return n
|
||||||
if self.is_res:
|
|
||||||
x_skip = x
|
|
||||||
x = self.ff(self.norm3(x))
|
|
||||||
if self.is_res:
|
|
||||||
x += x_skip
|
|
||||||
|
|
||||||
return x
|
|
||||||
|
|
||||||
|
|
||||||
class SpatialTransformer(nn.Module):
|
class SpatialTransformer(nn.Module):
|
||||||
|
|||||||
@ -19,23 +19,26 @@ def apply_weight_decompose(dora_scale, weight):
|
|||||||
return weight * (dora_scale / weight_norm)
|
return weight * (dora_scale / weight_norm)
|
||||||
|
|
||||||
def set_model_options_patch_replace(model_options, patch, name, block_name, number, transformer_index=None):
|
def set_model_options_patch_replace(model_options, patch, name, block_name, number, transformer_index=None):
|
||||||
|
return set_model_transformer_options(model_options, patch, name, block_name, number, transformer_index, "patches_replace")
|
||||||
|
|
||||||
|
def set_model_transformer_options(model_options, patch, name, block_name, number, transformer_index=None, opt_key="patches_replace"):
|
||||||
to = model_options["transformer_options"].copy()
|
to = model_options["transformer_options"].copy()
|
||||||
|
|
||||||
if "patches_replace" not in to:
|
if opt_key not in to:
|
||||||
to["patches_replace"] = {}
|
to[opt_key] = {}
|
||||||
else:
|
else:
|
||||||
to["patches_replace"] = to["patches_replace"].copy()
|
to[opt_key] = to[opt_key].copy()
|
||||||
|
|
||||||
if name not in to["patches_replace"]:
|
if name not in to[opt_key]:
|
||||||
to["patches_replace"][name] = {}
|
to[opt_key][name] = {}
|
||||||
else:
|
else:
|
||||||
to["patches_replace"][name] = to["patches_replace"][name].copy()
|
to[opt_key][name] = to[opt_key][name].copy()
|
||||||
|
|
||||||
if transformer_index is not None:
|
if transformer_index is not None:
|
||||||
block = (block_name, number, transformer_index)
|
block = (block_name, number, transformer_index)
|
||||||
else:
|
else:
|
||||||
block = (block_name, number)
|
block = (block_name, number)
|
||||||
to["patches_replace"][name][block] = patch
|
to[opt_key][name][block] = patch
|
||||||
model_options["transformer_options"] = to
|
model_options["transformer_options"] = to
|
||||||
return model_options
|
return model_options
|
||||||
|
|
||||||
@ -158,8 +161,11 @@ class ModelPatcher:
|
|||||||
def set_model_output_block_patch(self, patch):
|
def set_model_output_block_patch(self, patch):
|
||||||
self.set_model_patch(patch, "output_block_patch")
|
self.set_model_patch(patch, "output_block_patch")
|
||||||
|
|
||||||
def set_model_transformer_function(self, transformer_function):
|
def set_model_attn1_function_wrapper(self, patch, block_name, number, transformer_index=None):
|
||||||
self.model_options["transformer_options"]["transformer_function"] = transformer_function
|
set_model_transformer_options(self.model_options, patch, "attn1", block_name, number, transformer_index, "attn_function_wrapper")
|
||||||
|
|
||||||
|
def set_model_attn2_function_wrapper(self, patch, block_name, number, transformer_index=None):
|
||||||
|
set_model_transformer_options(self.model_options, patch, "attn2", block_name, number, transformer_index, "attn_function_wrapper")
|
||||||
|
|
||||||
def set_model_sampler_cfg_rescaler(self, cfg_rescaler_fn, disable_cfg1_optimization=False):
|
def set_model_sampler_cfg_rescaler(self, cfg_rescaler_fn, disable_cfg1_optimization=False):
|
||||||
self.model_options["sampler_cfg_rescaler"] = cfg_rescaler_fn
|
self.model_options["sampler_cfg_rescaler"] = cfg_rescaler_fn
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user