xinyun/vllm
1
0
Fork 0
mirror of https://git.datalinker.icu/vllm-project/vllm.git synced 2026-06-05 09:42:14 +08:00
Code Issues Packages Projects Releases Wiki Activity

config format

Browse Source 
Signed-off-by: Sage Moore <sage@neuralmagic.com>
This commit is contained in:
Sage Moore 2025-06-02 19:13:27 +00:00
parent 92e0cc79a8
commit 44a595f6d6
1 changed files with 12 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
vllm/config.py
View File

@ -1745,7 +1745,7 @@ class ParallelConfig:
"""Enable microbatching for the model executor.""" """Enable microbatching for the model executor."""
always_microbatch_if_enabled: bool = True always_microbatch_if_enabled: bool = True
"""Always microbatch if microbatching is enabled. Easier to sync bewteen """Always microbatch if microbatching is enabled. Easier to sync between
dp workers.""" dp workers."""
microbatching_token_threshold: int = 4 microbatching_token_threshold: int = 4
@ -4325,14 +4325,14 @@ class VllmConfig:
"cascade attention. Disabling cascade attention.") "cascade attention. Disabling cascade attention.")
self.model_config.disable_cascade_attn = True self.model_config.disable_cascade_attn = True
if self.parallel_config.enable_microbatching: if self.parallel_config.enable_microbatching and \
self.compilation_config.level >= CompilationLevel.PIECEWISE:
# Microbatching is not supported with piecewise compilation yet. # Microbatching is not supported with piecewise compilation yet.
# More specifically piecewise cuda-graphs # More specifically piecewise cuda-graphs
if self.compilation_config.level >= CompilationLevel.PIECEWISE: logger.warning_once(
logger.warning_once( "Piecewise compilation is not supported with "
"Piecewise compilation is not supported with " "microbatching. Disabling piecewiseching compilation.")
"microbatching. Disabling piecewiseching compilation.") self.compilation_config.level = CompilationLevel.NO_COMPILATION
self.compilation_config.level = CompilationLevel.NO_COMPILATION
if self.model_config and self.model_config.use_mla and \ if self.model_config and self.model_config.use_mla and \