xinyun/vllm
1
0
Fork 0
mirror of https://git.datalinker.icu/vllm-project/vllm.git synced 2025-12-09 16:45:52 +08:00
Code Issues Packages Projects Releases Wiki Activity

add truncation

Browse Source 
Signed-off-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>
This commit is contained in:
Woosuk Kwon 2025-10-11 11:20:31 -07:00
parent fbc2cc8217
commit 3a8990743e
1 changed files with 7 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
vllm/v1/worker/gpu_model_runner.py
View File

@ -2056,7 +2056,9 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
pooler_output.append(output)
return ModelRunnerOutput(
req_ids=self.input_batch.req_ids,
# NOTE(woosuk): input_batch.req_ids may include requests that are
# not scheduled in this step. Therefore, we truncate it here.
req_ids=self.input_batch.req_ids[: self.input_batch.num_reqs],
req_id_to_index=self.input_batch.req_id_to_index,
sampled_token_ids=[],
logprobs=None,
@ -2269,7 +2271,10 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
# Copy some objects so they don't get modified after returning.
# This is important when using async scheduling.
req_ids_output_copy = self.input_batch.req_ids.copy()
# NOTE(woosuk): input_batch.req_ids may include requests that are
# not scheduled in this step. Therefore, we truncate it here.
num_reqs = self.input_batch.num_reqs
req_ids_output_copy = self.input_batch.req_ids[:num_reqs].copy()
req_id_to_index_output_copy = self.input_batch.req_id_to_index.copy()
# NOTE: GPU -> CPU Sync happens here.