xinyun/vllm
1
0
Fork 0
mirror of https://git.datalinker.icu/vllm-project/vllm.git synced 2025-12-23 07:25:01 +08:00
Code Issues Packages Projects Releases Wiki Activity

[Bugfix] Enforce contiguous input for dynamic_per_token FP8/INT8 quant (#19452)

Browse Source 
Signed-off-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
Michael Goin 2025-06-12 15:39:15 -04:00 committed by GitHub
parent 9d880f594d
commit a3319f4f04
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 3 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
vllm/_custom_ops.py
View File

@ -1270,7 +1270,7 @@ def scaled_fp8_quant(
device=input.device, device=input.device,
dtype=torch.float32) dtype=torch.float32)
torch.ops._C.dynamic_per_token_scaled_fp8_quant( torch.ops._C.dynamic_per_token_scaled_fp8_quant(
output, input, scale, scale_ub) output, input.contiguous(), scale, scale_ub)
else: else:
scale = torch.zeros(1, device=input.device, dtype=torch.float32) scale = torch.zeros(1, device=input.device, dtype=torch.float32)
torch.ops._C.dynamic_scaled_fp8_quant(output, input, scale) torch.ops._C.dynamic_scaled_fp8_quant(output, input, scale)
@ -1379,8 +1379,8 @@ def scaled_int8_quant(
dtype=torch.float32) dtype=torch.float32)
input_azp = None if symmetric else torch.empty_like(input_scales, input_azp = None if symmetric else torch.empty_like(input_scales,
dtype=torch.int32) dtype=torch.int32)
torch.ops._C.dynamic_scaled_int8_quant(output, input, input_scales, torch.ops._C.dynamic_scaled_int8_quant(output, input.contiguous(),
input_azp) input_scales, input_azp)
return output, input_scales, input_azp return output, input_scales, input_azp