mirror of
https://git.datalinker.icu/comfyanonymous/ComfyUI
synced 2026-11-22 12:57:02 +08:00
feat: add attn_function_wrapper in ModelPatcher
This commit is contained in:
parent
cba9c66fd2
commit
9840d2a1a7
@ -3,7 +3,7 @@ import torch
|
||||
import torch.nn.functional as F
|
||||
from torch import nn, einsum
|
||||
from einops import rearrange, repeat
|
||||
from typing import Optional, Any
|
||||
from typing import Callable, Optional, Any
|
||||
import logging
|
||||
|
||||
from .diffusionmodules.util import checkpoint, AlphaBlender, timestep_embedding
|
||||
@ -460,19 +460,20 @@ class BasicTransformerBlock(nn.Module):
|
||||
return checkpoint(self._forward, (x, context, transformer_options), self.parameters(), self.checkpoint)
|
||||
|
||||
def _forward(self, x, context=None, transformer_options={}):
|
||||
if "transformer_function" in transformer_options:
|
||||
return transformer_options["transformer_function"](self, x, context, transformer_options)
|
||||
extra_options = {}
|
||||
block = transformer_options.get("block", None)
|
||||
block_index = transformer_options.get("block_index", 0)
|
||||
transformer_patches = {}
|
||||
transformer_patches_replace = {}
|
||||
transformer_attn_function_wrapper = {}
|
||||
|
||||
for k in transformer_options:
|
||||
if k == "patches":
|
||||
transformer_patches = transformer_options[k]
|
||||
elif k == "patches_replace":
|
||||
transformer_patches_replace = transformer_options[k]
|
||||
elif k == "attn_function_wrapper":
|
||||
transformer_attn_function_wrapper = transformer_options[k]
|
||||
else:
|
||||
extra_options[k] = transformer_options[k]
|
||||
|
||||
@ -485,11 +486,84 @@ class BasicTransformerBlock(nn.Module):
|
||||
if self.is_res:
|
||||
x += x_skip
|
||||
|
||||
n = self.norm1(x)
|
||||
if self.disable_self_attn:
|
||||
context_attn1 = context
|
||||
transformer_block = (block[0], block[1], block_index) if block is not None else None
|
||||
|
||||
attn1_replace_patch = transformer_patches_replace.get("attn1", {})
|
||||
block_attn1_replace_patch = attn1_replace_patch.get(transformer_block, attn1_replace_patch.get(block, None))
|
||||
|
||||
attn1_function_wrapper = transformer_attn_function_wrapper.get("attn1", {})
|
||||
block_attn1 = transformer_block if transformer_block in attn1_function_wrapper else block
|
||||
|
||||
if block_attn1 in attn1_function_wrapper:
|
||||
n = attn1_function_wrapper[block_attn1](
|
||||
self.apply_attn1,
|
||||
x,
|
||||
context,
|
||||
extra_options=extra_options,
|
||||
transformer_patches=transformer_patches,
|
||||
block_attn_replace_patch=block_attn1_replace_patch,
|
||||
)
|
||||
else:
|
||||
context_attn1 = None
|
||||
n = self.apply_attn1(
|
||||
x,
|
||||
context,
|
||||
extra_options=extra_options,
|
||||
transformer_patches=transformer_patches,
|
||||
block_attn_replace_patch=block_attn1_replace_patch,
|
||||
)
|
||||
|
||||
x += n
|
||||
if "middle_patch" in transformer_patches:
|
||||
patch = transformer_patches["middle_patch"]
|
||||
for p in patch:
|
||||
x = p(x, extra_options)
|
||||
|
||||
attn2_replace_patch = transformer_patches_replace.get("attn2", {})
|
||||
block_attn2_replace_patch = attn2_replace_patch.get(transformer_block, attn2_replace_patch.get(block, None))
|
||||
|
||||
attn2_function_wrapper = transformer_attn_function_wrapper.get("attn2", {})
|
||||
block_attn2 = transformer_block if transformer_block in attn2_function_wrapper else block
|
||||
if block_attn2 in attn2_function_wrapper:
|
||||
n = attn2_function_wrapper[block_attn2](
|
||||
self.apply_attn2,
|
||||
x,
|
||||
context,
|
||||
extra_options=extra_options,
|
||||
transformer_patches=transformer_patches,
|
||||
block_attn_replace_patch=block_attn2_replace_patch,
|
||||
n=n,
|
||||
)
|
||||
else:
|
||||
n = self.apply_attn2(
|
||||
x,
|
||||
context,
|
||||
extra_options=extra_options,
|
||||
transformer_patches=transformer_patches,
|
||||
block_attn_replace_patch=block_attn2_replace_patch,
|
||||
n=n,
|
||||
)
|
||||
|
||||
x += n
|
||||
if self.is_res:
|
||||
x_skip = x
|
||||
x = self.ff(self.norm3(x))
|
||||
if self.is_res:
|
||||
x += x_skip
|
||||
|
||||
return x
|
||||
|
||||
def apply_attn1(
|
||||
self,
|
||||
x,
|
||||
context,
|
||||
extra_options: dict,
|
||||
transformer_patches: dict,
|
||||
block_attn_replace_patch: Callable,
|
||||
**kwargs,
|
||||
):
|
||||
|
||||
n = self.norm1(x)
|
||||
context_attn1 = context if self.disable_self_attn else None
|
||||
value_attn1 = None
|
||||
|
||||
if "attn1_patch" in transformer_patches:
|
||||
@ -500,23 +574,14 @@ class BasicTransformerBlock(nn.Module):
|
||||
for p in patch:
|
||||
n, context_attn1, value_attn1 = p(n, context_attn1, value_attn1, extra_options)
|
||||
|
||||
if block is not None:
|
||||
transformer_block = (block[0], block[1], block_index)
|
||||
else:
|
||||
transformer_block = None
|
||||
attn1_replace_patch = transformer_patches_replace.get("attn1", {})
|
||||
block_attn1 = transformer_block
|
||||
if block_attn1 not in attn1_replace_patch:
|
||||
block_attn1 = block
|
||||
|
||||
if block_attn1 in attn1_replace_patch:
|
||||
if block_attn_replace_patch is not None:
|
||||
if context_attn1 is None:
|
||||
context_attn1 = n
|
||||
value_attn1 = n
|
||||
n = self.attn1.to_q(n)
|
||||
context_attn1 = self.attn1.to_k(context_attn1)
|
||||
value_attn1 = self.attn1.to_v(value_attn1)
|
||||
n = attn1_replace_patch[block_attn1](n, context_attn1, value_attn1, extra_options)
|
||||
n = block_attn_replace_patch(n, context_attn1, value_attn1, extra_options)
|
||||
n = self.attn1.to_out(n)
|
||||
else:
|
||||
n = self.attn1(n, context=context_attn1, value=value_attn1)
|
||||
@ -526,18 +591,22 @@ class BasicTransformerBlock(nn.Module):
|
||||
for p in patch:
|
||||
n = p(n, extra_options)
|
||||
|
||||
x += n
|
||||
if "middle_patch" in transformer_patches:
|
||||
patch = transformer_patches["middle_patch"]
|
||||
for p in patch:
|
||||
x = p(x, extra_options)
|
||||
return n
|
||||
|
||||
def apply_attn2(
|
||||
self,
|
||||
x,
|
||||
context,
|
||||
extra_options: dict,
|
||||
transformer_patches: dict,
|
||||
block_attn_replace_patch: Callable,
|
||||
n,
|
||||
**kwargs,
|
||||
):
|
||||
|
||||
if self.attn2 is not None:
|
||||
n = self.norm2(x)
|
||||
if self.switch_temporal_ca_to_sa:
|
||||
context_attn2 = n
|
||||
else:
|
||||
context_attn2 = context
|
||||
context_attn2 = n if self.switch_temporal_ca_to_sa else context
|
||||
value_attn2 = None
|
||||
if "attn2_patch" in transformer_patches:
|
||||
patch = transformer_patches["attn2_patch"]
|
||||
@ -545,18 +614,13 @@ class BasicTransformerBlock(nn.Module):
|
||||
for p in patch:
|
||||
n, context_attn2, value_attn2 = p(n, context_attn2, value_attn2, extra_options)
|
||||
|
||||
attn2_replace_patch = transformer_patches_replace.get("attn2", {})
|
||||
block_attn2 = transformer_block
|
||||
if block_attn2 not in attn2_replace_patch:
|
||||
block_attn2 = block
|
||||
|
||||
if block_attn2 in attn2_replace_patch:
|
||||
if block_attn_replace_patch is not None:
|
||||
if value_attn2 is None:
|
||||
value_attn2 = context_attn2
|
||||
n = self.attn2.to_q(n)
|
||||
context_attn2 = self.attn2.to_k(context_attn2)
|
||||
value_attn2 = self.attn2.to_v(value_attn2)
|
||||
n = attn2_replace_patch[block_attn2](n, context_attn2, value_attn2, extra_options)
|
||||
n = block_attn_replace_patch(n, context_attn2, value_attn2, extra_options)
|
||||
n = self.attn2.to_out(n)
|
||||
else:
|
||||
n = self.attn2(n, context=context_attn2, value=value_attn2)
|
||||
@ -566,14 +630,7 @@ class BasicTransformerBlock(nn.Module):
|
||||
for p in patch:
|
||||
n = p(n, extra_options)
|
||||
|
||||
x += n
|
||||
if self.is_res:
|
||||
x_skip = x
|
||||
x = self.ff(self.norm3(x))
|
||||
if self.is_res:
|
||||
x += x_skip
|
||||
|
||||
return x
|
||||
return n
|
||||
|
||||
|
||||
class SpatialTransformer(nn.Module):
|
||||
|
||||
@ -19,23 +19,26 @@ def apply_weight_decompose(dora_scale, weight):
|
||||
return weight * (dora_scale / weight_norm)
|
||||
|
||||
def set_model_options_patch_replace(model_options, patch, name, block_name, number, transformer_index=None):
|
||||
return set_model_transformer_options(model_options, patch, name, block_name, number, transformer_index, "patches_replace")
|
||||
|
||||
def set_model_transformer_options(model_options, patch, name, block_name, number, transformer_index=None, opt_key="patches_replace"):
|
||||
to = model_options["transformer_options"].copy()
|
||||
|
||||
if "patches_replace" not in to:
|
||||
to["patches_replace"] = {}
|
||||
if opt_key not in to:
|
||||
to[opt_key] = {}
|
||||
else:
|
||||
to["patches_replace"] = to["patches_replace"].copy()
|
||||
to[opt_key] = to[opt_key].copy()
|
||||
|
||||
if name not in to["patches_replace"]:
|
||||
to["patches_replace"][name] = {}
|
||||
if name not in to[opt_key]:
|
||||
to[opt_key][name] = {}
|
||||
else:
|
||||
to["patches_replace"][name] = to["patches_replace"][name].copy()
|
||||
to[opt_key][name] = to[opt_key][name].copy()
|
||||
|
||||
if transformer_index is not None:
|
||||
block = (block_name, number, transformer_index)
|
||||
else:
|
||||
block = (block_name, number)
|
||||
to["patches_replace"][name][block] = patch
|
||||
to[opt_key][name][block] = patch
|
||||
model_options["transformer_options"] = to
|
||||
return model_options
|
||||
|
||||
@ -158,8 +161,11 @@ class ModelPatcher:
|
||||
def set_model_output_block_patch(self, patch):
|
||||
self.set_model_patch(patch, "output_block_patch")
|
||||
|
||||
def set_model_transformer_function(self, transformer_function):
|
||||
self.model_options["transformer_options"]["transformer_function"] = transformer_function
|
||||
def set_model_attn1_function_wrapper(self, patch, block_name, number, transformer_index=None):
|
||||
set_model_transformer_options(self.model_options, patch, "attn1", block_name, number, transformer_index, "attn_function_wrapper")
|
||||
|
||||
def set_model_attn2_function_wrapper(self, patch, block_name, number, transformer_index=None):
|
||||
set_model_transformer_options(self.model_options, patch, "attn2", block_name, number, transformer_index, "attn_function_wrapper")
|
||||
|
||||
def set_model_sampler_cfg_rescaler(self, cfg_rescaler_fn, disable_cfg1_optimization=False):
|
||||
self.model_options["sampler_cfg_rescaler"] = cfg_rescaler_fn
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user