From 2f2f723f99b93ed2d2b0b96afc212a3ebb56b22f Mon Sep 17 00:00:00 2001 From: comfyanonymous Date: Thu, 25 Jul 2024 17:06:48 -0400 Subject: [PATCH] Switch to optimized attention in pooler. --- comfy/ldm/hydit/poolers.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/comfy/ldm/hydit/poolers.py b/comfy/ldm/hydit/poolers.py index bcc8ef573..3470041b5 100644 --- a/comfy/ldm/hydit/poolers.py +++ b/comfy/ldm/hydit/poolers.py @@ -1,7 +1,7 @@ import torch import torch.nn as nn import torch.nn.functional as F -# from comfy.ldm.modules.attention import optimized_attention #TODO +from comfy.ldm.modules.attention import optimized_attention #TODO class AttentionPool(nn.Module): @@ -25,13 +25,12 @@ class AttentionPool(nn.Module): v = self.v_proj(x) batch_size = q.shape[1] - q = q.view(1, batch_size * self.num_heads, self.embed_dim // self.num_heads).transpose(0, 1) - k = k.view(k.shape[0], batch_size * self.num_heads, self.embed_dim // self.num_heads).transpose(0, 1) - v = v.view(v.shape[0], batch_size * self.num_heads, self.embed_dim // self.num_heads).transpose(0, 1) + head_dim = self.embed_dim // self.num_heads + q = q.view(1, batch_size * self.num_heads, head_dim).transpose(0, 1).view(batch_size, self.num_heads, -1, head_dim) + k = k.view(k.shape[0], batch_size * self.num_heads, head_dim).transpose(0, 1).view(batch_size, self.num_heads, -1, head_dim) + v = v.view(v.shape[0], batch_size * self.num_heads, head_dim).transpose(0, 1).view(batch_size, self.num_heads, -1, head_dim) - attn_output = F.scaled_dot_product_attention(q, k, v) - - attn_output = attn_output.transpose(0, 1).contiguous().view(1, batch_size, self.embed_dim) + attn_output = optimized_attention(q, k, v, self.num_heads, skip_reshape=True).transpose(0, 1) attn_output = self.c_proj(attn_output) return attn_output.squeeze(0)