xinyun/vllm
1
0
Fork 0
mirror of https://git.datalinker.icu/vllm-project/vllm.git synced 2026-01-28 04:27:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

[Bugfix] Enable FP8 KV cache for FlashInfer and Triton backend on non-sm100 GPUs (#24577)

Browse Source 
Signed-off-by: Thien Tran <gau.nernst@yahoo.com.sg>
This commit is contained in:
Thien Tran 2025-09-11 03:33:41 +08:00 committed by GitHub
parent 09e68bce34
commit a0933c3bd6
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 9 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
vllm/platforms/cuda.py
View File

@ -530,6 +530,10 @@ class CudaPlatformBase(Platform):
supported = flash_attn_supports_fp8()
else:
supported = True
elif attention_backend == "FLASHINFER":
supported = True
elif attention_backend == "TRITON_ATTN_VLLM_V1":
supported = cls.supports_fp8()
return supported
@classmethod

6
vllm/v1/attention/backends/flashinfer.py
View File

@ -202,7 +202,11 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
else:
assert self.kv_cache_spec.dtype == self.model_config.dtype
self.kv_cache_dtype = self.kv_cache_spec.dtype
self.q_data_type = self.kv_cache_dtype
if supports_trtllm_attention()[0]:
self.q_data_type = self.kv_cache_dtype
else:
self.q_data_type = self.model_config.dtype
self._cascade_wrapper = None # Wrapper for cascade attention