xinyun/vllm
1
0
Fork 0
mirror of https://git.datalinker.icu/vllm-project/vllm.git synced 2026-07-20 08:17:09 +08:00
Code Issues Packages Projects Releases Wiki Activity

[BugFix] Pad input buffers in _dummy_run (#26209)

Browse Source 
Signed-off-by: Varun Sundar Rabindranath <vsundarr@redhat.com>
Co-authored-by: Varun Sundar Rabindranath <vsundarr@redhat.com>
This commit is contained in:
Varun Sundar Rabindranath 2025-10-06 16:07:51 -04:00 committed by GitHub
parent 93540958b8
commit f23b4c04fd
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 10 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
vllm/v1/worker/gpu_model_runner.py
View File

@ -3434,26 +3434,28 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
with self.maybe_dummy_run_with_lora( with self.maybe_dummy_run_with_lora(
self.lora_config, num_scheduled_tokens, remove_lora self.lora_config, num_scheduled_tokens, remove_lora
): ):
model_kwargs = self._init_model_kwargs(num_tokens) # Make sure padding doesn't exceed max_num_tokens
assert num_tokens_after_padding <= self.max_num_tokens
model_kwargs = self._init_model_kwargs(num_tokens_after_padding)
if self.supports_mm_inputs and not self.model_config.is_encoder_decoder: if self.supports_mm_inputs and not self.model_config.is_encoder_decoder:
input_ids = None input_ids = None
inputs_embeds = self.inputs_embeds.gpu[:num_tokens] inputs_embeds = self.inputs_embeds.gpu[:num_tokens_after_padding]
model_kwargs = { model_kwargs = {
**model_kwargs, **model_kwargs,
**self._dummy_mm_kwargs(num_reqs), **self._dummy_mm_kwargs(num_reqs),
} }
elif self.enable_prompt_embeds: elif self.enable_prompt_embeds:
input_ids = None input_ids = None
inputs_embeds = self.inputs_embeds.gpu[:num_tokens] inputs_embeds = self.inputs_embeds.gpu[:num_tokens_after_padding]
model_kwargs = self._init_model_kwargs(num_tokens) model_kwargs = self._init_model_kwargs(num_tokens_after_padding)
else: else:
input_ids = self.input_ids.gpu[:num_tokens] input_ids = self.input_ids.gpu[:num_tokens_after_padding]
inputs_embeds = None inputs_embeds = None
if self.uses_mrope: if self.uses_mrope:
positions = self.mrope_positions.gpu[:, :num_tokens] positions = self.mrope_positions.gpu[:, :num_tokens_after_padding]
else: else:
positions = self.positions.gpu[:num_tokens] positions = self.positions.gpu[:num_tokens_after_padding]
if get_pp_group().is_first_rank: if get_pp_group().is_first_rank:
intermediate_tensors = None intermediate_tensors = None
@ -3468,7 +3470,7 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
) )
intermediate_tensors = self.sync_and_slice_intermediate_tensors( intermediate_tensors = self.sync_and_slice_intermediate_tensors(
num_tokens, None, False num_tokens_after_padding, None, False
) )
# filter out the valid batch descriptor # filter out the valid batch descriptor