xinyun/vllm
1
0
Fork 0
mirror of https://git.datalinker.icu/vllm-project/vllm.git synced 2025-12-16 12:35:01 +08:00
Code Issues Packages Projects Releases Wiki Activity

[Bugfix] Add device assertion to TorchSDPA (#5402)

Browse Source
This commit is contained in:
Li, Jiang 2024-06-13 03:58:53 +08:00 committed by GitHub
parent 1a8bfd92d5
commit c3c2903e72
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 3 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
vllm/attention/selector.py
View File

@ -58,6 +58,9 @@ def get_attn_backend(
ROCmFlashAttentionBackend)
return ROCmFlashAttentionBackend
elif backend == _Backend.TORCH_SDPA:
# TODO: make XPU backend available here.
assert is_cpu(), RuntimeError(
"Torch SDPA backend is only used for the CPU device.")
logger.info("Using Torch SDPA backend.")
from vllm.attention.backends.torch_sdpa import TorchSDPABackend
return TorchSDPABackend