xinyun/vllm
1
0
Fork 0
mirror of https://git.datalinker.icu/vllm-project/vllm.git synced 2026-05-11 06:25:51 +08:00
Code Issues Packages Projects Releases Wiki Activity

Disable FlashInfer sampler by default (#26859)

Browse Source 
Signed-off-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
Michael Goin 2025-10-14 22:35:18 -04:00 committed by GitHub
parent bfad142e25
commit e66d787bce
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 6 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
vllm/v1/sample/ops/topk_topp_sampler.py
View File

@ -46,23 +46,15 @@ class TopKTopPSampler(nn.Module):
"Falling back to default sampling implementation." "Falling back to default sampling implementation."
) )
self.forward = self.forward_native self.forward = self.forward_native
elif envs.VLLM_USE_FLASHINFER_SAMPLER is not False: elif envs.VLLM_USE_FLASHINFER_SAMPLER:
# NOTE(woosuk): The V0 sampler doesn't use FlashInfer for # Users must opt in explicitly via VLLM_USE_FLASHINFER_SAMPLER=1.
# sampling unless VLLM_USE_FLASHINFER_SAMPLER=1 (i.e., by
# default it is unused). For backward compatibility, we set
# `VLLM_USE_FLASHINFER_SAMPLER` as None by default and
# interpret it differently in V0 and V1 samplers: In V0,
# None means False, while in V1, None means True. This is
# why we use the condition
# `envs.VLLM_USE_FLASHINFER_SAMPLER is not False` here.
logger.info_once("Using FlashInfer for top-p & top-k sampling.") logger.info_once("Using FlashInfer for top-p & top-k sampling.")
self.forward = self.forward_cuda self.forward = self.forward_cuda
else: else:
logger.warning_once( logger.debug_once(
"FlashInfer is available, but it is not enabled. " "FlashInfer top-p/top-k sampling is available but disabled "
"Falling back to the PyTorch-native implementation of " "by default. Set VLLM_USE_FLASHINFER_SAMPLER=1 to opt in "
"top-p & top-k sampling. For the best performance, " "after verifying accuracy for your workloads."
"please set VLLM_USE_FLASHINFER_SAMPLER=1."
) )
self.forward = self.forward_native self.forward = self.forward_native
else: else: