xinyun/vllm
1
0
Fork 0
mirror of https://git.datalinker.icu/vllm-project/vllm.git synced 2026-09-09 05:07:01 +08:00
Code Issues Packages Projects Releases Wiki Activity

[Quantization][FP8] Adding support for fp8 gemm layer input in fp8 (#14578)

Browse Source 
Signed-off-by: Gregory Shtrasberg <Gregory.Shtrasberg@amd.com>
Co-authored-by: Luka Govedič <ProExpertProg@users.noreply.github.com>
This commit is contained in:
Gregory Shtrasberg 2025-03-27 22:58:16 -04:00 committed by GitHub
parent e7f720ea56
commit 4d0ec37267
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
5 changed files with 41 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_fp8.py
View File

@ -23,6 +23,7 @@ class CompressedTensorsW8A8Fp8(CompressedTensorsScheme):
def __init__(self, strategy: str, is_static_input_scheme: bool): def __init__(self, strategy: str, is_static_input_scheme: bool):
self.strategy = strategy self.strategy = strategy
self.out_dtype = torch.get_default_dtype()
self.is_static_input_scheme = is_static_input_scheme self.is_static_input_scheme = is_static_input_scheme
self.fp8_linear = Fp8LinearOp(use_per_token_if_dynamic=True) self.fp8_linear = Fp8LinearOp(use_per_token_if_dynamic=True)
@ -143,5 +144,6 @@ class CompressedTensorsW8A8Fp8(CompressedTensorsScheme):
return self.fp8_linear.apply(input=x, return self.fp8_linear.apply(input=x,
weight=layer.weight, weight=layer.weight,
weight_scale=layer.weight_scale, weight_scale=layer.weight_scale,
out_dtype=self.out_dtype,
input_scale=layer.input_scale, input_scale=layer.input_scale,
bias=bias) bias=bias)

2
vllm/model_executor/layers/quantization/fbgemm_fp8.py
View File

@ -73,6 +73,7 @@ class FBGEMMFp8LinearMethod(LinearMethodBase):
def __init__(self, quant_config: FBGEMMFp8Config): def __init__(self, quant_config: FBGEMMFp8Config):
self.quant_config = quant_config self.quant_config = quant_config
self.fp8_linear = Fp8LinearOp(use_per_token_if_dynamic=True) self.fp8_linear = Fp8LinearOp(use_per_token_if_dynamic=True)
self.out_dtype = torch.get_default_dtype()
def create_weights( def create_weights(
self, self,
@ -161,6 +162,7 @@ class FBGEMMFp8LinearMethod(LinearMethodBase):
return self.fp8_linear.apply(input=x, return self.fp8_linear.apply(input=x,
weight=layer.weight, weight=layer.weight,
weight_scale=layer.weight_scale, weight_scale=layer.weight_scale,
out_dtype=self.out_dtype,
input_scale=None, input_scale=None,
input_scale_ub=layer.input_scale_ub, input_scale_ub=layer.input_scale_ub,
bias=bias) bias=bias)

17
vllm/model_executor/layers/quantization/fp8.py
View File

@ -116,6 +116,21 @@ class Fp8Config(QuantizationConfig):
return Fp8KVCacheMethod(self) return Fp8KVCacheMethod(self)
return None return None
def get_cache_scale(self, name: str) -> Optional[str]:
"""
Check whether the param name matches the format for k/v cache scales
in compressed-tensors. If this is the case, return its equivalent
param name expected by vLLM
:param name: param name
:return: matching param name for KV cache scale in vLLM
"""
if name.endswith(".output_scale") and ".k_proj" in name:
return name.replace(".k_proj.output_scale", ".attn.k_scale")
if name.endswith(".output_scale") and ".v_proj" in name:
return name.replace(".v_proj.output_scale", ".attn.v_scale")
return None
class Fp8LinearMethod(LinearMethodBase): class Fp8LinearMethod(LinearMethodBase):
"""Linear method for FP8. """Linear method for FP8.
@ -138,6 +153,7 @@ class Fp8LinearMethod(LinearMethodBase):
def __init__(self, quant_config: Fp8Config): def __init__(self, quant_config: Fp8Config):
self.quant_config = quant_config self.quant_config = quant_config
self.cutlass_block_fp8_supported = cutlass_block_fp8_supported() self.cutlass_block_fp8_supported = cutlass_block_fp8_supported()
self.out_dtype = torch.get_default_dtype()
# For GPUs that lack FP8 hardware support, we can leverage the Marlin # For GPUs that lack FP8 hardware support, we can leverage the Marlin
# kernel for fast weight-only FP8 quantization # kernel for fast weight-only FP8 quantization
@ -386,6 +402,7 @@ class Fp8LinearMethod(LinearMethodBase):
return self.fp8_linear.apply(input=x, return self.fp8_linear.apply(input=x,
weight=layer.weight, weight=layer.weight,
weight_scale=layer.weight_scale, weight_scale=layer.weight_scale,
out_dtype=self.out_dtype,
input_scale=layer.input_scale, input_scale=layer.input_scale,
bias=bias) bias=bias)

2
vllm/model_executor/layers/quantization/quark/schemes/quark_w8a8_fp8.py
View File

@ -22,6 +22,7 @@ class QuarkW8A8Fp8(QuarkScheme):
self.qscheme = qscheme self.qscheme = qscheme
self.is_static_input_scheme = is_static_input_scheme self.is_static_input_scheme = is_static_input_scheme
self.fp8_linear = Fp8LinearOp(use_per_token_if_dynamic=True) self.fp8_linear = Fp8LinearOp(use_per_token_if_dynamic=True)
self.out_dtype = torch.get_default_dtype()
@classmethod @classmethod
def get_min_capability(cls) -> int: def get_min_capability(cls) -> int:
@ -134,5 +135,6 @@ class QuarkW8A8Fp8(QuarkScheme):
return self.fp8_linear.apply(input=x, return self.fp8_linear.apply(input=x,
weight=layer.weight, weight=layer.weight,
weight_scale=layer.weight_scale, weight_scale=layer.weight_scale,
out_dtype=self.out_dtype,
input_scale=layer.input_scale, input_scale=layer.input_scale,
bias=bias) bias=bias)

27
vllm/model_executor/layers/quantization/utils/w8a8_utils.py
View File

@ -163,6 +163,7 @@ class Fp8LinearOp:
input: torch.Tensor, input: torch.Tensor,
weight: torch.Tensor, weight: torch.Tensor,
weight_scale: torch.Tensor, weight_scale: torch.Tensor,
out_dtype: Optional[torch.dtype] = None,
input_scale: Optional[torch.Tensor] = None, input_scale: Optional[torch.Tensor] = None,
input_scale_ub: Optional[torch.Tensor] = None, input_scale_ub: Optional[torch.Tensor] = None,
bias: Optional[torch.Tensor] = None, bias: Optional[torch.Tensor] = None,
@ -182,8 +183,13 @@ class Fp8LinearOp:
if use_per_token_if_dynamic is None: if use_per_token_if_dynamic is None:
use_per_token_if_dynamic = self.use_per_token_if_dynamic use_per_token_if_dynamic = self.use_per_token_if_dynamic
if out_dtype is None:
out_dtype = input.dtype
# cutlass_scaled_mm supports per tensor/channel W and per tensor/token A # cutlass_scaled_mm supports per tensor/channel W and per tensor/token A
if self.cutlass_fp8_supported: if self.cutlass_fp8_supported:
assert input.dtype != current_platform.fp8_dtype(
), "FP8 input to cutlass is not currently implemented"
qinput, x_scale = ops.scaled_fp8_quant( qinput, x_scale = ops.scaled_fp8_quant(
input_2d, input_2d,
input_scale, input_scale,
@ -193,7 +199,7 @@ class Fp8LinearOp:
# Fused GEMM_DQ # Fused GEMM_DQ
output = ops.cutlass_scaled_mm(qinput, output = ops.cutlass_scaled_mm(qinput,
weight, weight,
out_dtype=input.dtype, out_dtype=out_dtype,
scale_a=x_scale, scale_a=x_scale,
scale_b=weight_scale, scale_b=weight_scale,
bias=bias) bias=bias)
@ -202,12 +208,15 @@ class Fp8LinearOp:
# torch.scaled_mm supports per tensor weights + activations only # torch.scaled_mm supports per tensor weights + activations only
# so fallback to naive if per channel or per token # so fallback to naive if per channel or per token
else: else:
# Maybe apply padding to output, see comment in __init__ if input.dtype != current_platform.fp8_dtype():
qinput, x_scale = ops.scaled_fp8_quant( # Maybe apply padding to output, see comment in __init__
input_2d, qinput, x_scale = ops.scaled_fp8_quant(
input_scale, input_2d,
num_token_padding=self.output_padding, input_scale,
use_per_token_if_dynamic=use_per_token_if_dynamic) num_token_padding=self.output_padding,
use_per_token_if_dynamic=use_per_token_if_dynamic)
else:
qinput, x_scale = input_2d, input_scale
per_tensor_weights = (weight_scale.numel() == 1) per_tensor_weights = (weight_scale.numel() == 1)
per_tensor_activations = (x_scale.numel() == 1) per_tensor_activations = (x_scale.numel() == 1)
@ -216,7 +225,7 @@ class Fp8LinearOp:
# Fused GEMM_DQ # Fused GEMM_DQ
output = torch._scaled_mm(qinput, output = torch._scaled_mm(qinput,
weight, weight,
out_dtype=input.dtype, out_dtype=out_dtype,
scale_a=x_scale, scale_a=x_scale,
scale_b=weight_scale, scale_b=weight_scale,
bias=bias) bias=bias)
@ -240,7 +249,7 @@ class Fp8LinearOp:
# Fused GEMM_DQ Rowwise GEMM # Fused GEMM_DQ Rowwise GEMM
output = torch._scaled_mm(qinput, output = torch._scaled_mm(qinput,
weight, weight,
out_dtype=input.dtype, out_dtype=out_dtype,
scale_a=x_scale, scale_a=x_scale,
scale_b=weight_scale.t(), scale_b=weight_scale.t(),
bias=bias) bias=bias)