xinyun/vllm
1
0
Fork 0
mirror of https://git.datalinker.icu/vllm-project/vllm.git synced 2026-05-11 18:32:31 +08:00
Code Issues Packages Projects Releases Wiki Activity

[Bugfix] Fixes for new marlin moe usage (#18017)

Browse Source 
Signed-off-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
Michael Goin 2025-05-12 23:50:04 -04:00 committed by GitHub
parent d8487ef557
commit 1df491c522
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 5 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe.py
View File

@ -57,9 +57,10 @@ class CompressedTensorsMoEMethod(FusedMoEMethodBase):
"input_activations") "input_activations")
if quant_config._is_wNa16_group_channel(weight_quant, input_quant): if quant_config._is_wNa16_group_channel(weight_quant, input_quant):
# group_size=None means channelwise
group_size = weight_quant.group_size or -1
# Prefer to use the MarlinMoE kernel when it is supported. # Prefer to use the MarlinMoE kernel when it is supported.
if not check_moe_marlin_supports_layer(layer, if not check_moe_marlin_supports_layer(layer, group_size):
weight_quant.group_size):
if (weight_quant.strategy in QuantizationStrategy.GROUP and if (weight_quant.strategy in QuantizationStrategy.GROUP and
weight_quant.actorder in (ActivationOrdering.GROUP, weight_quant.actorder in (ActivationOrdering.GROUP,
ActivationOrdering.DYNAMIC)): ActivationOrdering.DYNAMIC)):

4
vllm/model_executor/layers/quantization/gptq_marlin.py
View File

@ -610,9 +610,9 @@ class GPTQMarlinMoEMethod(FusedMoEMethodBase):
activation: str = "silu", activation: str = "silu",
) -> torch.Tensor: ) -> torch.Tensor:
assert activation == "silu", "Only SiLU activation is supported." assert activation == "silu", "Only SiLU activation is supported."
if apply_router_weight_on_input is not None: if apply_router_weight_on_input:
raise NotImplementedError( raise NotImplementedError(
"Apply router weight on input is not supported for" "Apply router weight on input is not supported for "
"fused Marlin MoE method.") "fused Marlin MoE method.")
topk_weights, topk_ids = FusedMoE.select_experts( topk_weights, topk_ids = FusedMoE.select_experts(