xinyun/vllm
1
0
Fork 0
mirror of https://git.datalinker.icu/vllm-project/vllm.git synced 2026-05-28 12:37:10 +08:00
Code Issues Packages Projects Releases Wiki Activity

[Bugfix] fix for deepseek w4a16 (#8906)

Browse Source 
Co-authored-by: mgoin <michael@neuralmagic.com>
This commit is contained in:
Lucas Wilkinson 2024-09-27 15:12:34 -04:00 committed by GitHub
parent 172d1cd276
commit c5d55356f9
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 5 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
vllm/model_executor/layers/quantization/kernels/marlin.py
View File

@ -38,10 +38,11 @@ class MarlinLinearKernel(MPLinearKernel):
"Marlin, supported group sizes are: "\ "Marlin, supported group sizes are: "\
f"{MARLIN_SUPPORTED_GROUP_SIZES}" f"{MARLIN_SUPPORTED_GROUP_SIZES}"
return check_marlin_supports_shape(c.partition_weight_shape[0], return check_marlin_supports_shape(
c.partition_weight_shape[1], c.partition_weight_shape[1], # out_features
c.full_weight_shape[1], c.partition_weight_shape[0], # in_features
c.group_size) c.full_weight_shape[0], # in_features
c.group_size)
# note assumes that # note assumes that
# `weight_packed` is: {input_dim = 0, output_dim = 1, packed_dim = 0} # `weight_packed` is: {input_dim = 0, output_dim = 1, packed_dim = 0}