xinyun/vllm
1
0
Fork 0
mirror of https://git.datalinker.icu/vllm-project/vllm.git synced 2026-05-28 18:37:06 +08:00
Code Issues Packages Projects Releases Wiki Activity

[ROCm][FP8] Fix for adjustments needed only for fnuz (#14689)

Browse Source 
Signed-off-by: Gregory Shtrasberg <Gregory.Shtrasberg@amd.com>
This commit is contained in:
Gregory Shtrasberg 2025-03-12 23:14:04 -04:00 committed by GitHub
parent ab426ec9c0
commit a94a699c3f
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 2 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
vllm/model_executor/layers/quantization/kv_cache.py
View File

@ -50,7 +50,7 @@ class BaseKVCacheMethod(QuantizeMethodBase):
# We prefer to use separate k_scale and v_scale if present # We prefer to use separate k_scale and v_scale if present
k_scale = layer.k_scale.to("cpu").tolist() k_scale = layer.k_scale.to("cpu").tolist()
v_scale = layer.v_scale.to("cpu").tolist() v_scale = layer.v_scale.to("cpu").tolist()
if current_platform.is_rocm(): if current_platform.is_fp8_fnuz():
k_scale *= 2 k_scale *= 2
v_scale *= 2 v_scale *= 2
elif layer.k_scale < 0.0 and layer.v_scale < 0.0: elif layer.k_scale < 0.0 and layer.v_scale < 0.0:
@ -66,7 +66,7 @@ class BaseKVCacheMethod(QuantizeMethodBase):
scale_to_duplicate = max(layer.k_scale, layer.v_scale) scale_to_duplicate = max(layer.k_scale, layer.v_scale)
k_scale = scale_to_duplicate.to("cpu").tolist() k_scale = scale_to_duplicate.to("cpu").tolist()
v_scale = scale_to_duplicate.to("cpu").tolist() v_scale = scale_to_duplicate.to("cpu").tolist()
if current_platform.is_rocm(): if current_platform.is_fp8_fnuz():
k_scale *= 2 k_scale *= 2
v_scale *= 2 v_scale *= 2