xinyun/vllm
1
0
Fork 0
mirror of https://git.datalinker.icu/vllm-project/vllm.git synced 2025-12-14 09:38:01 +08:00
Code Issues Packages Projects Releases Wiki Activity

[BugFix] Fix default kv-cache-dtype default for DeepseekV3.2 (#25988)

Browse Source 
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Signed-off-by: simon-mo <simon.mo@hey.com>
This commit is contained in:
Lucas Wilkinson 2025-10-01 00:58:31 -04:00 committed by simon-mo
parent febb688356
commit e4beabd2c8
1 changed files with 14 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
vllm/model_executor/models/config.py
View File

@ -400,7 +400,7 @@ class HybridAttentionMambaModelConfig(VerifyAndUpdateConfig):
"exactly equal.", mamba_padding_pct) "exactly equal.", mamba_padding_pct)
class DeepseekV3ForCausalLM(VerifyAndUpdateConfig): class DeepseekV32ForCausalLM(VerifyAndUpdateConfig):
@classmethod @classmethod
def verify_and_update_config(cls, vllm_config: "VllmConfig") -> None: def verify_and_update_config(cls, vllm_config: "VllmConfig") -> None:
@ -409,17 +409,17 @@ class DeepseekV3ForCausalLM(VerifyAndUpdateConfig):
""" """
hf_config = vllm_config.model_config.hf_config hf_config = vllm_config.model_config.hf_config
# Mirror the check in vllm/model_executor/models/deepseek_v2.py
is_v32 = hasattr(hf_config, "index_topk") is_v32 = hasattr(hf_config, "index_topk")
assert is_v32
if is_v32:
# For DeepSeekV3.2, we use a custom fp8 format as default (i.e. # For DeepSeekV3.2, we use a custom fp8 format as default (i.e.
# "auto") # "auto")
cache_config = vllm_config.cache_config cache_config = vllm_config.cache_config
if cache_config.cache_dtype == "auto" or \ if cache_config.cache_dtype == "auto" or \
cache_config.cache_dtype.startswith("fp8"): cache_config.cache_dtype.startswith("fp8"):
cache_config.cache_dtype = "fp8_ds_mla" cache_config.cache_dtype = "fp8_ds_mla"
logger.info( logger.info("Using custom fp8 kv-cache format for DeepSeekV3.2")
"Using custom fp8 kv-cache format for DeepSeekV3.2")
if cache_config.cache_dtype == "bfloat16": if cache_config.cache_dtype == "bfloat16":
cache_config.cache_dtype = "auto" cache_config.cache_dtype = "auto"
logger.info("Using bfloat16 kv-cache for DeepSeekV3.2") logger.info("Using bfloat16 kv-cache for DeepSeekV3.2")
@ -441,5 +441,5 @@ MODELS_CONFIG_MAP: dict[str, type[VerifyAndUpdateConfig]] = {
"MambaForCausalLM": MambaModelConfig, "MambaForCausalLM": MambaModelConfig,
"Mamba2ForCausalLM": MambaModelConfig, "Mamba2ForCausalLM": MambaModelConfig,
"FalconMambaForCausalLM": MambaModelConfig, "FalconMambaForCausalLM": MambaModelConfig,
"DeepseekV3ForCausalLM": DeepseekV3ForCausalLM, "DeepseekV32ForCausalLM": DeepseekV32ForCausalLM,
} }