xinyun/vllm
1
0
Fork 0
mirror of https://git.datalinker.icu/vllm-project/vllm.git synced 2026-09-06 03:16:59 +08:00
Code Issues Packages Projects Releases Wiki Activity

Fix noisy warning for uncalibrated q_scale/p_scale (#17414)

Browse Source 
Signed-off-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
Michael Goin 2025-05-08 15:56:59 -04:00 committed by GitHub
parent 8342e3abd1
commit 4f605a6de5
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 5 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
vllm/model_executor/layers/quantization/kv_cache.py
View File

@ -124,11 +124,12 @@ class BaseKVCacheMethod(QuantizeMethodBase):
# These are used in the final Attention.forward() # These are used in the final Attention.forward()
layer._q_scale.copy_(q_scale) layer._q_scale.copy_(q_scale)
layer._prob_scale.copy_(prob_scale) layer._prob_scale.copy_(prob_scale)
if q_scale == 1.0 or prob_scale == 1.0: if layer.kv_cache_dtype == "fp8" and (q_scale == 1.0
or prob_scale == 1.0):
logger.warning_once( logger.warning_once(
f"Using Q scale {q_scale} and prob scale {prob_scale} " f"Using uncalibrated q_scale {q_scale} and/or prob_scale "
"with fp8 attention. This may cause accuracy issues. " f"{prob_scale} with fp8 attention. This may cause accuracy "
"Please make sure Q/prob scaling factors are " "issues. Please make sure q/prob scaling factors are "
"available in the fp8 checkpoint.") "available in the fp8 checkpoint.")
del layer.k_scale del layer.k_scale