xinyun/vllm
1
0
Fork 0
mirror of https://git.datalinker.icu/vllm-project/vllm.git synced 2025-12-12 16:26:07 +08:00
Code Issues Packages Projects Releases Wiki Activity

[Bugfix][ROCm] Fix ViT rotary embeddings for torch.compile compatibility on ROCm (#27748)

Browse Source 
Signed-off-by: vllmellm <vllm.ellm@embeddedllm.com>
This commit is contained in:
vllmellm 2025-11-04 09:12:19 +08:00 committed by GitHub
parent 7956b0c0bc
commit b13a447546
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 8 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
vllm/model_executor/layers/rotary_embedding/common.py
View File

@ -77,7 +77,11 @@ def dispatch_rotary_emb_function(
if current_platform.is_cuda(): if current_platform.is_cuda():
return apply_rotary_emb return apply_rotary_emb
if current_platform.is_rocm(): # if torch compile is not enabled
# use rotary embedding function from flash_attn package
# otherwise use the naive pytorch embedding implementation
# is faster when torch compile is enabled.
if current_platform.is_rocm() and not torch.compiler.is_compiling():
if find_spec("flash_attn") is not None: if find_spec("flash_attn") is not None:
from flash_attn.ops.triton.rotary import apply_rotary from flash_attn.ops.triton.rotary import apply_rotary
@ -87,10 +91,9 @@ def dispatch_rotary_emb_function(
"flash_attn is not installed. Falling back to PyTorch " "flash_attn is not installed. Falling back to PyTorch "
"implementation for rotary embeddings." "implementation for rotary embeddings."
) )
if default is not None: if default is not None:
return default return default
else:
return apply_rotary_emb_torch return apply_rotary_emb_torch

2
vllm/model_executor/models/glm4_1v.py
View File

@ -370,7 +370,7 @@ class Glm4vVisionAttention(nn.Module):
cu_seqlens_k=cu_seqlens, cu_seqlens_k=cu_seqlens,
max_seqlen_q=max_seqlen, max_seqlen_q=max_seqlen,
max_seqlen_k=max_seqlen, max_seqlen_k=max_seqlen,
dropout_p=0, dropout_p=0.0,
causal=False, causal=False,
) )