xinyun/vllm
1
0
Fork 0
mirror of https://git.datalinker.icu/vllm-project/vllm.git synced 2025-12-11 19:27:41 +08:00
Code Issues Packages Projects Releases Wiki Activity

[BugFix] Fix default kv-cache-dtype default for DeepseekV3.2 (#25988)

Browse Source 
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Signed-off-by: simon-mo <simon.mo@hey.com>
This commit is contained in:
Lucas Wilkinson 2025-10-01 00:58:31 -04:00 committed by simon-mo
parent febb688356
commit e4beabd2c8
1 changed files with 14 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
vllm/model_executor/models/config.py
View File

@ -400,7 +400,7 @@ class HybridAttentionMambaModelConfig(VerifyAndUpdateConfig):
"exactly equal.", mamba_padding_pct) "exactly equal.", mamba_padding_pct)
class DeepseekV3ForCausalLM(VerifyAndUpdateConfig): class DeepseekV32ForCausalLM(VerifyAndUpdateConfig):
@classmethod @classmethod
def verify_and_update_config(cls, vllm_config: "VllmConfig") -> None: def verify_and_update_config(cls, vllm_config: "VllmConfig") -> None:
@ -409,20 +409,20 @@ class DeepseekV3ForCausalLM(VerifyAndUpdateConfig):
""" """
hf_config = vllm_config.model_config.hf_config hf_config = vllm_config.model_config.hf_config
# Mirror the check in vllm/model_executor/models/deepseek_v2.py
is_v32 = hasattr(hf_config, "index_topk") is_v32 = hasattr(hf_config, "index_topk")
assert is_v32
if is_v32: # For DeepSeekV3.2, we use a custom fp8 format as default (i.e.
# For DeepSeekV3.2, we use a custom fp8 format as default (i.e. # "auto")
# "auto") cache_config = vllm_config.cache_config
cache_config = vllm_config.cache_config if cache_config.cache_dtype == "auto" or \
if cache_config.cache_dtype == "auto" or \ cache_config.cache_dtype.startswith("fp8"):
cache_config.cache_dtype.startswith("fp8"): cache_config.cache_dtype = "fp8_ds_mla"
cache_config.cache_dtype = "fp8_ds_mla" logger.info("Using custom fp8 kv-cache format for DeepSeekV3.2")
logger.info( if cache_config.cache_dtype == "bfloat16":
"Using custom fp8 kv-cache format for DeepSeekV3.2") cache_config.cache_dtype = "auto"
if cache_config.cache_dtype == "bfloat16": logger.info("Using bfloat16 kv-cache for DeepSeekV3.2")
cache_config.cache_dtype = "auto"
logger.info("Using bfloat16 kv-cache for DeepSeekV3.2")
MODELS_CONFIG_MAP: dict[str, type[VerifyAndUpdateConfig]] = { MODELS_CONFIG_MAP: dict[str, type[VerifyAndUpdateConfig]] = {
@ -441,5 +441,5 @@ MODELS_CONFIG_MAP: dict[str, type[VerifyAndUpdateConfig]] = {
"MambaForCausalLM": MambaModelConfig, "MambaForCausalLM": MambaModelConfig,
"Mamba2ForCausalLM": MambaModelConfig, "Mamba2ForCausalLM": MambaModelConfig,
"FalconMambaForCausalLM": MambaModelConfig, "FalconMambaForCausalLM": MambaModelConfig,
"DeepseekV3ForCausalLM": DeepseekV3ForCausalLM, "DeepseekV32ForCausalLM": DeepseekV32ForCausalLM,
} }