xinyun/vllm
1
0
Fork 0
mirror of https://git.datalinker.icu/vllm-project/vllm.git synced 2026-04-17 05:37:03 +08:00
Code Issues Packages Projects Releases Wiki Activity

[MTP] Refactor mtp predictor to avoid d2h operation (#27643)

Browse Source 
Signed-off-by: MengqingCao <cmq0113@163.com>
This commit is contained in:
Mengqing Cao 2025-10-31 01:27:39 +08:00 committed by GitHub
parent ba33e8830d
commit 1004205795
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 1 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
vllm/model_executor/models/deepseek_mtp.py
View File

@ -97,7 +97,7 @@ class DeepSeekMultiTokenPredictorLayer(nn.Module):
) -> torch.Tensor:
assert inputs_embeds is not None
# masking inputs at position 0, as not needed by MTP
inputs_embeds[positions == 0] = 0
inputs_embeds = torch.where(positions.unsqueeze(-1) == 0, 0, inputs_embeds)
inputs_embeds = self.enorm(inputs_embeds)
previous_hidden_states = self.hnorm(previous_hidden_states)