xinyun/vllm
1
0
Fork 0
mirror of https://git.datalinker.icu/vllm-project/vllm.git synced 2025-12-18 03:15:01 +08:00
Code Issues Packages Projects Releases Wiki Activity

Fix routing_bias dtype (#25711)

Browse Source 
Signed-off-by: Shu Wang. <shuw@nvidia.com>
This commit is contained in:
Shu Wang 2025-09-25 18:35:14 -05:00 committed by GitHub
parent 57329a8c01
commit 081b5594a2
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 4 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
vllm/model_executor/layers/quantization/modelopt.py
View File

@ -1454,10 +1454,13 @@ class ModelOptNvFp4FusedMoE(FusedMoEMethodBase):
routing_method_type = flashinfer.RoutingMethodType.DeepSeekV3 routing_method_type = flashinfer.RoutingMethodType.DeepSeekV3
if use_llama4_routing: if use_llama4_routing:
routing_method_type = flashinfer.RoutingMethodType.Llama4 routing_method_type = flashinfer.RoutingMethodType.Llama4
routing_bias = e_score_correction_bias
if routing_bias is not None:
routing_bias = routing_bias.to(torch.bfloat16)
out = flashinfer.fused_moe.trtllm_fp4_block_scale_moe( out = flashinfer.fused_moe.trtllm_fp4_block_scale_moe(
routing_logits=router_logits routing_logits=router_logits
if use_llama4_routing else router_logits.to(torch.float32), if use_llama4_routing else router_logits.to(torch.float32),
routing_bias=e_score_correction_bias, routing_bias=routing_bias,
hidden_states=hidden_states_fp4, hidden_states=hidden_states_fp4,
hidden_states_scale=hidden_states_scale_linear_fp4.view( hidden_states_scale=hidden_states_scale_linear_fp4.view(
torch.float8_e4m3fn).flatten(), torch.float8_e4m3fn).flatten(),