xinyun/vllm
1
0
Fork 0
mirror of https://git.datalinker.icu/vllm-project/vllm.git synced 2026-07-08 19:17:21 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix dp plus full cuda-graph

Browse Source 
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
This commit is contained in:
Lucas Wilkinson 2025-07-27 21:06:56 +00:00
parent a9b2a1d704
commit d80a82f961
1 changed files with 30 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

50
vllm/v1/attention/backends/mla/flashmla.py
View File

@ -67,6 +67,20 @@ class FlashMLAMetadataBuilder(MLACommonMetadataBuilder[FlashMLAMetadata]):
self.cg_buf_tile_scheduler_metadata = None self.cg_buf_tile_scheduler_metadata = None
self.cg_buf_num_splits = None self.cg_buf_num_splits = None
device_properties = torch.cuda.get_device_properties(self.device)
num_sms = device_properties.multi_processor_count
if self.compilation_config.full_cuda_graph:
self.cg_buf_tile_scheduler_metadata = torch.empty(
(num_sms, 8), # TileSchedulerMetaDataSize == 8
device=self.device,
dtype=torch.int32,
)
self.cg_buf_num_splits = torch.empty(
(vllm_config.scheduler_config.max_num_seqs + 1),
device=self.device,
dtype=torch.int32)
def _build_decode(self, block_table_tensor: torch.Tensor, def _build_decode(self, block_table_tensor: torch.Tensor,
seq_lens: torch.Tensor) -> FlashMLADecodeMetadata: seq_lens: torch.Tensor) -> FlashMLADecodeMetadata:
tile_scheduler_metadata, num_splits = \ tile_scheduler_metadata, num_splits = \
@ -77,28 +91,24 @@ class FlashMLAMetadataBuilder(MLACommonMetadataBuilder[FlashMLAMetadata]):
) )
if self.compilation_config.full_cuda_graph: if self.compilation_config.full_cuda_graph:
# First time around (CUDAGraph capture), allocate the static buffer assert self.cg_buf_tile_scheduler_metadata is not None
if self.cg_buf_tile_scheduler_metadata is None: assert self.cg_buf_num_splits is not None
self.cg_buf_tile_scheduler_metadata = tile_scheduler_metadata
self.cg_buf_num_splits = num_splits
else:
assert self.cg_buf_num_splits is not None
# Metadata per-SM, fixed size (#SMs, TileMetadataSize) # Metadata per-SM, fixed size (#SMs, TileMetadataSize)
assert (self.cg_buf_tile_scheduler_metadata.size() == assert (self.cg_buf_tile_scheduler_metadata.size() ==
tile_scheduler_metadata.size()) tile_scheduler_metadata.size())
self.cg_buf_tile_scheduler_metadata.\ self.cg_buf_tile_scheduler_metadata.\
copy_(tile_scheduler_metadata) copy_(tile_scheduler_metadata)
tile_scheduler_metadata = self.cg_buf_tile_scheduler_metadata tile_scheduler_metadata = self.cg_buf_tile_scheduler_metadata
# Num splits is per-batch, varying size (batch_size,) # Num splits is per-batch, varying size (batch_size,)
n = num_splits.size(0) n = num_splits.size(0)
# make sure static buffer is large enough # make sure static buffer is large enough
assert n <= self.cg_buf_num_splits.size(0) assert n <= self.cg_buf_num_splits.size(0)
num_splits_view = self.cg_buf_num_splits[:n] num_splits_view = self.cg_buf_num_splits[:n]
num_splits_view.copy_(num_splits) num_splits_view.copy_(num_splits)
self.cg_buf_num_splits[n:].fill_(0) # fill the rest with 0s self.cg_buf_num_splits[n:].fill_(0) # fill the rest with 0s
num_splits = num_splits_view num_splits = num_splits_view
return FlashMLADecodeMetadata( return FlashMLADecodeMetadata(
block_table=block_table_tensor, block_table=block_table_tensor,