xinyun/vllm
1
0
Fork 0
mirror of https://git.datalinker.icu/vllm-project/vllm.git synced 2026-04-16 01:17:10 +08:00
Code Issues Packages Projects Releases Wiki Activity

[Bugfix] Check chain_speculative_sampling before calling it (#11673)

Browse Source 
Signed-off-by: Lu Fang <lufang@fb.com>
This commit is contained in:
Lu Fang 2025-01-02 16:58:56 -08:00 committed by GitHub
parent 2f1e8e8f54
commit 07064cb1d4
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 1 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
vllm/model_executor/layers/rejection_sampler.py
View File

@ -118,7 +118,7 @@ class RejectionSampler(SpecDecodeStochasticBaseSampler):
# If use Flashinfer chain_speculative_sampling kernel # If use Flashinfer chain_speculative_sampling kernel
# for rejection sampling # for rejection sampling
if self.use_flashinfer: if self.use_flashinfer and chain_speculative_sampling is not None:
batch_size, k, _ = draft_probs.shape batch_size, k, _ = draft_probs.shape
uniform_samples = self._create_uniform_samples( uniform_samples = self._create_uniform_samples(
seeded_seqs, batch_size, k, draft_probs.device) seeded_seqs, batch_size, k, draft_probs.device)