xinyun/vllm
1
0
Fork 0
mirror of https://git.datalinker.icu/vllm-project/vllm.git synced 2025-12-10 05:34:57 +08:00
Code Issues Packages Projects Releases Wiki Activity

[Bugfix] Use random hidden states in dummy sampler run (#18543)

Browse Source 
Signed-off-by: Bowen Wang <abmfy@icloud.com>
This commit is contained in:
Bowen Wang 2025-05-22 06:48:56 -07:00 committed by GitHub
parent 71075029f2
commit 4e04eceb58
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 4 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
vllm/v1/worker/gpu_model_runner.py
View File

@ -1721,6 +1721,10 @@ class GPUModelRunner(LoRAModelRunnerMixin):
self, self,
hidden_states: torch.Tensor, hidden_states: torch.Tensor,
) -> torch.Tensor: ) -> torch.Tensor:
# The dummy hidden states may contain special values,
# like `inf` or `nan`.
# To avoid breaking the sampler, we use a random tensor here instead.
hidden_states = torch.rand_like(hidden_states)
logits = self.model.compute_logits(hidden_states, None) logits = self.model.compute_logits(hidden_states, None)
num_reqs = logits.size(0) num_reqs = logits.size(0)