xinyun/DeepSeek-V3
1
0
Fork 0
mirror of https://git.datalinker.icu/deepseek-ai/DeepSeek-V3.git synced 2026-07-13 03:07:07 +08:00
Code Issues Packages Projects Releases Wiki Activity

fp32 gate bias

Browse Source
This commit is contained in:
Xingkai Yu 2025-08-26 17:39:07 +08:00 committed by GitHub
parent f6e34dd267
commit 4592be48c0
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 1 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
inference/model.py
View File

@ -558,7 +558,7 @@ class Gate(nn.Module):
self.score_func = args.score_func self.score_func = args.score_func
self.route_scale = args.route_scale self.route_scale = args.route_scale
self.weight = nn.Parameter(torch.empty(args.n_routed_experts, args.dim)) self.weight = nn.Parameter(torch.empty(args.n_routed_experts, args.dim))
self.bias = nn.Parameter(torch.empty(args.n_routed_experts)) if self.dim == 7168 else None self.bias = nn.Parameter(torch.empty(args.n_routed_experts, dtype=torch.float32)) if self.dim == 7168 else None
def forward(self, x: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: def forward(self, x: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:
""" """