xinyun/vllm
1
0
Fork 0
mirror of https://git.datalinker.icu/vllm-project/vllm.git synced 2025-12-15 11:36:20 +08:00
Code Issues Packages Projects Releases Wiki Activity

[Misc] Enable multi-step output streaming by default (#9047)

Browse Source
This commit is contained in:
Michael Goin 2024-10-03 22:55:42 -04:00 committed by GitHub
parent aeb37c2a72
commit 303d44790a
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 9 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
vllm/engine/arg_utils.py
View File

@ -145,7 +145,7 @@ class EngineArgs:
max_cpu_loras: Optional[int] = None max_cpu_loras: Optional[int] = None
device: str = 'auto' device: str = 'auto'
num_scheduler_steps: int = 1 num_scheduler_steps: int = 1
multi_step_stream_outputs: bool = False multi_step_stream_outputs: bool = True
ray_workers_use_nsight: bool = False ray_workers_use_nsight: bool = False
num_gpu_blocks_override: Optional[int] = None num_gpu_blocks_override: Optional[int] = None
num_lookahead_slots: int = 0 num_lookahead_slots: int = 0
@ -603,8 +603,12 @@ class EngineArgs:
parser.add_argument( parser.add_argument(
'--multi-step-stream-outputs', '--multi-step-stream-outputs',
action='store_true', action=StoreBoolean,
help='If True, then multi-step will stream outputs for every step') default=EngineArgs.multi_step_stream_outputs,
nargs="?",
const="True",
help='If False, then multi-step will stream outputs at the end '
'of all steps')
parser.add_argument( parser.add_argument(
'--scheduler-delay-factor', '--scheduler-delay-factor',
type=float, type=float,