xinyun/DeepSeek-V3
1
0
Fork 0
mirror of https://git.datalinker.icu/deepseek-ai/DeepSeek-V3.git synced 2025-12-09 21:04:36 +08:00
Code Issues Packages Projects Releases Wiki Activity

Compare commits

base: xinyun:cda950e6c2415f6ca9aad8dadfe1c4d0ca7af6c6
Branches Tags
xinyun:main
xinyun:v1.0.0
..
compare: xinyun:e867d890db1467148e802c9909b4359895f12d9c
Branches Tags
xinyun:main
xinyun:v1.0.0

2 Commits
cda950e6c2 ... e867d890db

Author SHA1 Message Date
Nripesh Niketan
e867d890db
Merge a5336884cfa952634f355549643dcb781eaec872 into 9b4e9788e4a3a731f7567338ed15d3ec549ce03b 2025-09-01 00:53:48 +01:00
GeeeekExplorer
9b4e9788e4 Merge pull request #969 from youkaichao/rmsnorm 
act_quant_kernel
 2025-08-28 11:24:26 +08:00
1 changed files with 1 additions and 6 deletions
Show Stats Expand all files Collapse all files

7
inference/model.py
View File

@ -291,12 +291,7 @@ class RMSNorm(nn.Module):
Returns: Returns:
torch.Tensor: Normalized tensor with the same shape as input. torch.Tensor: Normalized tensor with the same shape as input.
""" """
dtype = x.dtype return F.rms_norm(x, (self.dim,), self.weight, self.eps)
# make sure rms norm is computed in fp32
x = x.to(torch.float32)
var = x.pow(2).mean(-1, keepdim=True)
x = x * torch.rsqrt(var + self.eps)
return (self.weight * x).to(dtype)
def precompute_freqs_cis(args: ModelArgs) -> torch.Tensor: def precompute_freqs_cis(args: ModelArgs) -> torch.Tensor: