[Perf] Remove sync point in vit torch sdpa attn backend (#30232)

Signed-off-by: Dazhi Jiang <dazhi_jiang@163.com>

vllm/attention/ops/vit_attn_wrappers.py

@@ -93,12 +93,12 @@ def torch_sdpa_wrapper(
     cu_seqlens: torch.Tensor,
 ) -> torch.Tensor:
     outputs = []
-    for i in range(1, len(cu_seqlens)):
+
-        start_idx = cu_seqlens[i - 1]
+    lens = (cu_seqlens[1:] - cu_seqlens[:-1]).tolist()
-        end_idx = cu_seqlens[i]
+    q_chunks = torch.split(q, lens, dim=1)
-        q_i = q[:, start_idx:end_idx]
+    k_chunks = torch.split(k, lens, dim=1)
-        k_i = k[:, start_idx:end_idx]
+    v_chunks = torch.split(v, lens, dim=1)
-        v_i = v[:, start_idx:end_idx]
+    for q_i, k_i, v_i in zip(q_chunks, k_chunks, v_chunks):
         q_i, k_i, v_i = (
             einops.rearrange(x, "b s h d -> b h s d") for x in [q_i, k_i, v_i]
         )

vllm/model_executor/models/ernie45_vl.py

@@ -289,12 +289,12 @@ class Ernie4_5_VisionAttention(nn.Module):
         elif self.attn_backend == AttentionBackendEnum.TORCH_SDPA:
             # Execute attention entry by entry for speed & less VRAM.
             outputs = []
-            for i in range(1, len(cu_seqlens)):
+
-                start_idx = cu_seqlens[i - 1]
+            lens = (cu_seqlens[1:] - cu_seqlens[:-1]).tolist()
-                end_idx = cu_seqlens[i]
+            q_chunks = torch.split(q, lens, dim=1)
-                q_i = q[:, start_idx:end_idx]
+            k_chunks = torch.split(k, lens, dim=1)
-                k_i = k[:, start_idx:end_idx]
+            v_chunks = torch.split(v, lens, dim=1)
-                v_i = v[:, start_idx:end_idx]
+            for q_i, k_i, v_i in zip(q_chunks, k_chunks, v_chunks):
                 q_i, k_i, v_i = (
                     rearrange(x, "b s h d -> b h s d") for x in [q_i, k_i, v_i]
                 )

vllm/model_executor/models/glm4_1v.py

@@ -377,12 +377,12 @@ class Glm4vVisionAttention(nn.Module):
         elif self.attn_backend == AttentionBackendEnum.TORCH_SDPA:
             # Execute attention entry by entry for speed & less VRAM.
             outputs = []
-            for i in range(1, len(cu_seqlens)):
+
-                start_idx = cu_seqlens[i - 1]
+            lens = (cu_seqlens[1:] - cu_seqlens[:-1]).tolist()
-                end_idx = cu_seqlens[i]
+            q_chunks = torch.split(q, lens, dim=1)
-                q_i = q[:, start_idx:end_idx]
+            k_chunks = torch.split(k, lens, dim=1)
-                k_i = k[:, start_idx:end_idx]
+            v_chunks = torch.split(v, lens, dim=1)
-                v_i = v[:, start_idx:end_idx]
+            for q_i, k_i, v_i in zip(q_chunks, k_chunks, v_chunks):
                 q_i, k_i, v_i = (
                     rearrange(x, "b s h d -> b h s d") for x in [q_i, k_i, v_i]
                 )

vllm/model_executor/models/qwen2_vl.py

@@ -424,12 +424,12 @@ class Qwen2VisionAttention(nn.Module):
                 k = k.contiguous()
                 v = v.contiguous()
             outputs = []
-            for i in range(1, len(cu_seqlens)):
+
-                start_idx = cu_seqlens[i - 1]
+            lens = (cu_seqlens[1:] - cu_seqlens[:-1]).tolist()
-                end_idx = cu_seqlens[i]
+            q_chunks = torch.split(q, lens, dim=1)
-                q_i = q[:, start_idx:end_idx]
+            k_chunks = torch.split(k, lens, dim=1)
-                k_i = k[:, start_idx:end_idx]
+            v_chunks = torch.split(v, lens, dim=1)
-                v_i = v[:, start_idx:end_idx]
+            for q_i, k_i, v_i in zip(q_chunks, k_chunks, v_chunks):
                 q_i, k_i, v_i = (
                     rearrange(x, "b s h d -> b h s d") for x in [q_i, k_i, v_i]
                 )