From 144ccd1abee14b45065bcc52ac104b1318fbfb41 Mon Sep 17 00:00:00 2001 From: comfyanonymous Date: Thu, 25 Jul 2024 15:32:22 -0400 Subject: [PATCH] Init with right dtype. --- comfy/ldm/hydit/poolers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/comfy/ldm/hydit/poolers.py b/comfy/ldm/hydit/poolers.py index c8b1030b8..bcc8ef573 100644 --- a/comfy/ldm/hydit/poolers.py +++ b/comfy/ldm/hydit/poolers.py @@ -7,7 +7,7 @@ import torch.nn.functional as F class AttentionPool(nn.Module): def __init__(self, spacial_dim: int, embed_dim: int, num_heads: int, output_dim: int = None, dtype=None, device=None, operations=None): super().__init__() - self.positional_embedding = nn.Parameter(torch.randn(spacial_dim + 1, embed_dim) / embed_dim ** 0.5) + self.positional_embedding = nn.Parameter(torch.empty(spacial_dim + 1, embed_dim, dtype=dtype, device=device)) self.k_proj = operations.Linear(embed_dim, embed_dim, dtype=dtype, device=device) self.q_proj = operations.Linear(embed_dim, embed_dim, dtype=dtype, device=device) self.v_proj = operations.Linear(embed_dim, embed_dim, dtype=dtype, device=device)