xinyun/vllm
1
0
Fork 0
mirror of https://git.datalinker.icu/vllm-project/vllm.git synced 2025-12-19 04:15:01 +08:00
Code Issues Packages Projects Releases Wiki Activity

[Bugfix] Initialize attention bias on the same device as Query/Key/Value for QwenVL Series (#14031)

Browse Source
This commit is contained in:
Yang Liu 2025-02-28 23:36:08 +08:00 committed by GitHub
parent f7bee5c815
commit 9b61dd41e7
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 4 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
vllm/model_executor/models/qwen2_5_vl.py
View File

@ -323,7 +323,8 @@ class Qwen2_5_VisionAttention(nn.Module):
seqlens = (cu_seqlens[1:] - cu_seqlens[:-1]).tolist() seqlens = (cu_seqlens[1:] - cu_seqlens[:-1]).tolist()
attn_bias = BlockDiagonalMask.from_seqlens(q_seqlen=seqlens, attn_bias = BlockDiagonalMask.from_seqlens(q_seqlen=seqlens,
kv_seqlen=None) kv_seqlen=None,
device=q.device)
context_layer = xops.memory_efficient_attention_forward( context_layer = xops.memory_efficient_attention_forward(
q, k, v, attn_bias=attn_bias, p=0, scale=None) q, k, v, attn_bias=attn_bias, p=0, scale=None)

3
vllm/model_executor/models/qwen2_vl.py
View File

@ -367,7 +367,8 @@ class Qwen2VisionAttention(nn.Module):
seqlens = (cu_seqlens[1:] - cu_seqlens[:-1]).tolist() seqlens = (cu_seqlens[1:] - cu_seqlens[:-1]).tolist()
attn_bias = BlockDiagonalMask.from_seqlens(q_seqlen=seqlens, attn_bias = BlockDiagonalMask.from_seqlens(q_seqlen=seqlens,
kv_seqlen=None) kv_seqlen=None,
device=q.device)
context_layer = xops.memory_efficient_attention_forward( context_layer = xops.memory_efficient_attention_forward(
q, k, v, attn_bias=attn_bias, p=0, scale=None) q, k, v, attn_bias=attn_bias, p=0, scale=None)