xinyun/vllm
1
0
Fork 0
mirror of https://git.datalinker.icu/vllm-project/vllm.git synced 2026-08-01 16:47:56 +08:00
Code Issues Packages Projects Releases Wiki Activity

[Bugfix] Fix kv_cache_dtype=fp8 without scales for FP8 checkpoints (#6761)

Browse Source
This commit is contained in:
Michael Goin 2024-07-25 12:46:15 -04:00 committed by GitHub
parent 889da130e7
commit 65b1f121c8
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 12 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
tests/quantization/test_fp8.py
View File

@ -60,12 +60,20 @@ def test_kv_cache_model_load_and_run(vllm_runner, model_id: str):
@pytest.mark.skipif(not is_quant_method_supported("fp8"), @pytest.mark.skipif(not is_quant_method_supported("fp8"),
reason="FP8 is not supported on this GPU type.") reason="FP8 is not supported on this GPU type.")
def test_load_fp16_model(vllm_runner) -> None: @pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8"])
with vllm_runner("facebook/opt-125m", quantization="fp8") as llm: def test_load_fp16_model(vllm_runner, kv_cache_dtype: str) -> None:
with vllm_runner("facebook/opt-125m",
quantization="fp8",
kv_cache_dtype=kv_cache_dtype) as llm:
model = llm.model.llm_engine.model_executor.driver_worker.model_runner.model # noqa: E501 model = llm.model.llm_engine.model_executor.driver_worker.model_runner.model # noqa: E501
fc1 = model.model.decoder.layers[0].fc1 fc1 = model.model.decoder.layers[0].fc1
assert isinstance(fc1.quant_method, Fp8LinearMethod) assert isinstance(fc1.quant_method, Fp8LinearMethod)
if kv_cache_dtype == "fp8":
attn = model.model.decoder.layers[0].self_attn.attn
assert isinstance(attn.quant_method, Fp8KVCacheMethod)
assert attn._k_scale == 1.0
assert attn._v_scale == 1.0
capability = torch.cuda.get_device_capability() capability = torch.cuda.get_device_capability()
capability = capability[0] * 10 + capability[1] capability = capability[0] * 10 + capability[1]

6
vllm/model_executor/layers/quantization/kv_cache.py
View File

@ -46,10 +46,8 @@ class BaseKVCacheMethod(QuantizeMethodBase):
elif layer.k_scale < 0.0 and layer.v_scale < 0.0: elif layer.k_scale < 0.0 and layer.v_scale < 0.0:
# If no scales were loaded (both scales are invalid negative # If no scales were loaded (both scales are invalid negative
# values), use the default value of 1.0 # values), use the default value of 1.0
k_scale = torch.nn.Parameter(torch.tensor(1.0), k_scale = 1.0
requires_grad=False) v_scale = 1.0
v_scale = torch.nn.Parameter(torch.tensor(1.0),
requires_grad=False)
else: else:
# If we find a single kv_scale in the checkpoint, we remap # If we find a single kv_scale in the checkpoint, we remap
# kv_scale to k_scale during weight loading, and duplicate # kv_scale to k_scale during weight loading, and duplicate