xinyun/vllm
1
0
Fork 0
mirror of https://git.datalinker.icu/vllm-project/vllm.git synced 2026-06-04 11:02:18 +08:00
Code Issues Packages Projects Releases Wiki Activity

[TRTLLM] Remove the MoE GEMM weight name change (#30713)

Browse Source 
Signed-off-by: Ming Yang <minos.future@gmail.com>
This commit is contained in:
Ming Yang 2025-12-16 08:01:38 -08:00 committed by GitHub
parent 59bd5f6a71
commit ce12b407f2
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 16 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe.py
View File

@ -469,16 +469,14 @@ class CompressedTensorsW4A4Nvfp4MoEMethod(CompressedTensorsMoEMethod):
) )
logger.debug_once("Finished shuffling weights for TRT-LLM MOE") logger.debug_once("Finished shuffling weights for TRT-LLM MOE")
layer.gemm1_weights_fp4_shuffled = Parameter( layer.w13_weight = Parameter(
gemm1_weights_fp4_shuffled, requires_grad=False gemm1_weights_fp4_shuffled, requires_grad=False
) )
layer.gemm2_weights_fp4_shuffled = Parameter( layer.w2_weight = Parameter(gemm2_weights_fp4_shuffled, requires_grad=False)
gemm2_weights_fp4_shuffled, requires_grad=False layer.w13_weight_scale = Parameter(
)
layer.gemm1_scales_fp4_shuffled = Parameter(
gemm1_scales_fp4_shuffled, requires_grad=False gemm1_scales_fp4_shuffled, requires_grad=False
) )
layer.gemm2_scales_fp4_shuffled = Parameter( layer.w2_weight_scale = Parameter(
gemm2_scales_fp4_shuffled, requires_grad=False gemm2_scales_fp4_shuffled, requires_grad=False
) )
@ -487,12 +485,6 @@ class CompressedTensorsW4A4Nvfp4MoEMethod(CompressedTensorsMoEMethod):
(layer.w2_input_scale_quant * layer.g1_alphas).to(torch.float32), (layer.w2_input_scale_quant * layer.g1_alphas).to(torch.float32),
requires_grad=False, requires_grad=False,
) )
# Clean up weights that won't be used by TRT-LLM
del layer.w2_weight
del layer.w2_weight_scale
del layer.w13_weight
del layer.w13_weight_scale
else: else:
# swizzle weight scales # swizzle weight scales
layer.w13_weight_scale = torch.nn.Parameter( layer.w13_weight_scale = torch.nn.Parameter(

16
vllm/model_executor/layers/quantization/modelopt.py
View File

@ -1458,16 +1458,14 @@ class ModelOptNvFp4FusedMoE(FusedMoEMethodBase):
) )
logger.debug_once("Finished shuffling weights for TRT-LLM MOE") logger.debug_once("Finished shuffling weights for TRT-LLM MOE")
layer.gemm1_weights_fp4_shuffled = Parameter( layer.w13_weight = Parameter(
gemm1_weights_fp4_shuffled, requires_grad=False gemm1_weights_fp4_shuffled, requires_grad=False
) )
layer.gemm2_weights_fp4_shuffled = Parameter( layer.w2_weight = Parameter(gemm2_weights_fp4_shuffled, requires_grad=False)
gemm2_weights_fp4_shuffled, requires_grad=False layer.w13_weight_scale = Parameter(
)
layer.gemm1_scales_fp4_shuffled = Parameter(
gemm1_scales_fp4_shuffled, requires_grad=False gemm1_scales_fp4_shuffled, requires_grad=False
) )
layer.gemm2_scales_fp4_shuffled = Parameter( layer.w2_weight_scale = Parameter(
gemm2_scales_fp4_shuffled, requires_grad=False gemm2_scales_fp4_shuffled, requires_grad=False
) )
@ -1476,12 +1474,6 @@ class ModelOptNvFp4FusedMoE(FusedMoEMethodBase):
(layer.w2_input_scale_quant * layer.g1_alphas).to(torch.float32), (layer.w2_input_scale_quant * layer.g1_alphas).to(torch.float32),
requires_grad=False, requires_grad=False,
) )
# Clean up weights that won't be used by TRT-LLM
del layer.w2_weight
del layer.w2_weight_scale
del layer.w13_weight
del layer.w13_weight_scale
elif self.use_marlin: elif self.use_marlin:
# Marlin processing # Marlin processing
prepare_moe_fp4_layer_for_marlin(layer) prepare_moe_fp4_layer_for_marlin(layer)

24
vllm/model_executor/layers/quantization/utils/flashinfer_fp4_moe.py
View File

@ -301,18 +301,14 @@ def flashinfer_trtllm_fp4_moe(
hidden_states_scale=hidden_states_scale_linear_fp4.view( hidden_states_scale=hidden_states_scale_linear_fp4.view(
torch.float8_e4m3fn torch.float8_e4m3fn
).flatten(), ).flatten(),
gemm1_weights=layer.gemm1_weights_fp4_shuffled.data, gemm1_weights=layer.w13_weight.data,
gemm1_weights_scale=layer.gemm1_scales_fp4_shuffled.data.view( gemm1_weights_scale=layer.w13_weight_scale.data.view(torch.float8_e4m3fn),
torch.float8_e4m3fn
),
gemm1_bias=None, gemm1_bias=None,
gemm1_alpha=None, gemm1_alpha=None,
gemm1_beta=None, gemm1_beta=None,
gemm1_clamp_limit=None, gemm1_clamp_limit=None,
gemm2_weights=layer.gemm2_weights_fp4_shuffled.data, gemm2_weights=layer.w2_weight.data,
gemm2_weights_scale=layer.gemm2_scales_fp4_shuffled.data.view( gemm2_weights_scale=layer.w2_weight_scale.data.view(torch.float8_e4m3fn),
torch.float8_e4m3fn
),
gemm2_bias=None, gemm2_bias=None,
output1_scale_scalar=layer.g1_scale_c.data, output1_scale_scalar=layer.g1_scale_c.data,
output1_scale_gate_scalar=layer.g1_alphas.data, output1_scale_gate_scalar=layer.g1_alphas.data,
@ -380,18 +376,14 @@ def flashinfer_trtllm_fp4_routed_moe(
hidden_states_scale=hidden_states_scale_linear_fp4.view( hidden_states_scale=hidden_states_scale_linear_fp4.view(
torch.float8_e4m3fn torch.float8_e4m3fn
).flatten(), ).flatten(),
gemm1_weights=layer.gemm1_weights_fp4_shuffled.data, gemm1_weights=layer.w13_weight.data,
gemm1_weights_scale=layer.gemm1_scales_fp4_shuffled.data.view( gemm1_weights_scale=layer.w13_weight_scale.data.view(torch.float8_e4m3fn),
torch.float8_e4m3fn
),
gemm1_bias=None, gemm1_bias=None,
gemm1_alpha=None, gemm1_alpha=None,
gemm1_beta=None, gemm1_beta=None,
gemm1_clamp_limit=None, gemm1_clamp_limit=None,
gemm2_weights=layer.gemm2_weights_fp4_shuffled.data, gemm2_weights=layer.w2_weight.data,
gemm2_weights_scale=layer.gemm2_scales_fp4_shuffled.data.view( gemm2_weights_scale=layer.w2_weight_scale.data.view(torch.float8_e4m3fn),
torch.float8_e4m3fn
),
gemm2_bias=None, gemm2_bias=None,
output1_scale_scalar=layer.g1_scale_c.data, output1_scale_scalar=layer.g1_scale_c.data,
output1_scale_gate_scalar=layer.g1_alphas.data, output1_scale_gate_scalar=layer.g1_alphas.data,