xinyun/vllm
1
0
Fork 0
mirror of https://git.datalinker.icu/vllm-project/vllm.git synced 2025-12-10 23:45:39 +08:00
Code Issues Packages Projects Releases Wiki Activity

[V1][PP] Do not block engine core when no requests to schedule (#14585)

Browse Source 
Signed-off-by: Cody Yu <hao.yu.cody@gmail.com>
This commit is contained in:
Cody Yu 2025-03-10 19:48:24 -07:00 committed by GitHub
parent c91b64f749
commit 4290b704ff
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 11 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
vllm/v1/engine/core.py
View File

@ -205,23 +205,18 @@ class EngineCore:
self.batch_queue.put_nowait( self.batch_queue.put_nowait(
(future, scheduler_output)) # type: ignore (future, scheduler_output)) # type: ignore
# If all requests are scheduled or the job queue is full, scheduled_batch = (scheduler_output is not None
and scheduler_output.total_num_scheduled_tokens > 0)
# If no more requests can be scheduled and the job queue is not empty,
# block until the first batch in the job queue is finished. # block until the first batch in the job queue is finished.
if (scheduler_output is None if not scheduled_batch and not self.batch_queue.empty():
or scheduler_output.total_num_scheduled_tokens == 0): future, scheduler_output = self.batch_queue.get_nowait()
try: # Blocking until the first result is available.
future, scheduler_output = self.batch_queue.get( model_output = future.result()
timeout=POLLING_TIMEOUT_S) self.batch_queue.task_done()
# Blocking until the first result is available. engine_core_outputs = self.scheduler.update_from_output(
model_output = future.result() scheduler_output, model_output)
self.batch_queue.task_done()
engine_core_outputs = self.scheduler.update_from_output(
scheduler_output, model_output)
except queue.Empty:
# If the queue is empty (timeout at .get), return
# an empty EngineCoreOutputs for logging.
engine_core_outputs = EngineCoreOutputs(
outputs=[], scheduler_stats=self.scheduler.make_stats())
return engine_core_outputs return engine_core_outputs