xinyun/vllm
1
0
Fork 0
mirror of https://git.datalinker.icu/vllm-project/vllm.git synced 2026-07-20 00:47:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merging PR #12536

Browse Source 
Merged via CLI script
This commit is contained in:
Chen Zhang 2025-02-06 05:24:26 +08:00 committed by GitHub
parent bc1bdecebf
commit 4c3aac51e1
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 22 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
vllm/attention/layer.py
View File

@ -156,9 +156,13 @@ class Attention(nn.Module):
kv_cache: torch.Tensor, kv_cache: torch.Tensor,
attn_metadata: AttentionMetadata, attn_metadata: AttentionMetadata,
) -> torch.Tensor: ) -> torch.Tensor:
if self.calculate_kv_scales and \ # NOTE: please avoid accessing `kv_cache` and `attn_metadata` arguments
attn_metadata.enable_kv_scales_calculation: # directly, use `self.kv_cache` and
self.calc_kv_scales(key, value) # `get_forward_context().attn_metadata` instead.
if self.calculate_kv_scales:
ctx_attn_metadata = get_forward_context().attn_metadata
if ctx_attn_metadata.enable_kv_scales_calculation:
self.calc_kv_scales(key, value)
if self.use_output: if self.use_output:
output = torch.empty_like(query) output = torch.empty_like(query)
hidden_size = query.size(-1) hidden_size = query.size(-1)
@ -172,15 +176,27 @@ class Attention(nn.Module):
if value is not None: if value is not None:
value = value.view(-1, self.num_kv_heads, self.head_size) value = value.view(-1, self.num_kv_heads, self.head_size)
if self.use_direct_call: if self.use_direct_call:
unified_attention_with_output(query, key, value, output, forward_context: ForwardContext = get_forward_context()
self.layer_name) ctx_attn_metadata = forward_context.attn_metadata
self_kv_cache = self.kv_cache[forward_context.virtual_engine]
self.impl.forward(self,
query,
key,
value,
self_kv_cache,
ctx_attn_metadata,
output=output)
else: else:
torch.ops.vllm.unified_attention_with_output( torch.ops.vllm.unified_attention_with_output(
query, key, value, output, self.layer_name) query, key, value, output, self.layer_name)
return output.view(-1, hidden_size) return output.view(-1, hidden_size)
else: else:
if self.use_direct_call: if self.use_direct_call:
return unified_attention(query, key, value, self.layer_name) forward_context = get_forward_context()
ctx_attn_metadata = forward_context.attn_metadata
self_kv_cache = self.kv_cache[forward_context.virtual_engine]
return self.impl.forward(self, query, key, value,
self_kv_cache, ctx_attn_metadata)
else: else:
return torch.ops.vllm.unified_attention( return torch.ops.vllm.unified_attention(
query, key, value, self.layer_name) query, key, value, self.layer_name)