xinyun/vllm
1
0
Fork 0
mirror of https://git.datalinker.icu/vllm-project/vllm.git synced 2026-05-05 11:11:21 +08:00
Code Issues Packages Projects Releases Wiki Activity

Fix scheduler

Browse Source
This commit is contained in:
Woosuk Kwon 2024-04-25 05:06:22 +00:00
parent 98eda57899
commit b62170e4e3
1 changed files with 4 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
vllm/core/scheduler.py
View File

@ -666,6 +666,10 @@ class Scheduler:
budget.add_num_batched_tokens(seq_group.request_id, num_new_tokens) budget.add_num_batched_tokens(seq_group.request_id, num_new_tokens)
budget.add_num_seqs(seq_group.request_id, num_new_seqs) budget.add_num_seqs(seq_group.request_id, num_new_seqs)
# FIXME(woosuk): For TPUs, we want to schedule only one prompt
# per scheduling step.
break
# Queue requests that couldn't be scheduled. # Queue requests that couldn't be scheduled.
waiting_queue.extendleft(leftover_waiting_sequences) waiting_queue.extendleft(leftover_waiting_sequences)
if len(seq_groups) > 0: if len(seq_groups) > 0: