xinyun/vllm
1
0
Fork 0
mirror of https://git.datalinker.icu/vllm-project/vllm.git synced 2026-07-25 05:27:09 +08:00
Code Issues Packages Projects Releases Wiki Activity

[Bugfix][PD] set max_completion_tokens=1 if req has this value (#21841)

Browse Source 
Signed-off-by: Abirdcfly <fp544037857@gmail.com>
This commit is contained in:
Abirdcfly 2025-08-01 21:08:45 +08:00 committed by GitHub
parent 0a6d305e0f
commit 5c54d9759d
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 4 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
examples/online_serving/disaggregated_serving/disagg_proxy_demo.py
View File

@ -293,6 +293,8 @@ class Proxy:
# add params to request # add params to request
kv_prepare_request = request.copy() kv_prepare_request = request.copy()
kv_prepare_request["max_tokens"] = 1 kv_prepare_request["max_tokens"] = 1
if "max_completion_tokens" in kv_prepare_request:
kv_prepare_request["max_completion_tokens"] = 1
# prefill stage # prefill stage
prefill_instance = self.schedule(self.prefill_cycler) prefill_instance = self.schedule(self.prefill_cycler)

2
examples/online_serving/disaggregated_serving_p2p_nccl_xpyd/disagg_proxy_p2p_nccl_xpyd.py
View File

@ -128,6 +128,8 @@ async def handle_request():
prefill_request = original_request_data.copy() prefill_request = original_request_data.copy()
# change max_tokens = 1 to let it only do prefill # change max_tokens = 1 to let it only do prefill
prefill_request["max_tokens"] = 1 prefill_request["max_tokens"] = 1
if "max_completion_tokens" in prefill_request:
prefill_request["max_completion_tokens"] = 1
global count global count
global prefill_instances global prefill_instances