xinyun/vllm
1
0
Fork 0
mirror of https://git.datalinker.icu/vllm-project/vllm.git synced 2025-12-15 07:57:03 +08:00
Code Issues Packages Projects Releases Wiki Activity

[Misc] Enable multi-step output streaming by default (#9047)

Browse Source
This commit is contained in:
Michael Goin 2024-10-03 22:55:42 -04:00 committed by GitHub
parent aeb37c2a72
commit 303d44790a
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 9 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
vllm/engine/arg_utils.py
View File

@ -145,7 +145,7 @@ class EngineArgs:
max_cpu_loras: Optional[int] = None max_cpu_loras: Optional[int] = None
device: str = 'auto' device: str = 'auto'
num_scheduler_steps: int = 1 num_scheduler_steps: int = 1
multi_step_stream_outputs: bool = False multi_step_stream_outputs: bool = True
ray_workers_use_nsight: bool = False ray_workers_use_nsight: bool = False
num_gpu_blocks_override: Optional[int] = None num_gpu_blocks_override: Optional[int] = None
num_lookahead_slots: int = 0 num_lookahead_slots: int = 0
@ -603,13 +603,17 @@ class EngineArgs:
parser.add_argument( parser.add_argument(
'--multi-step-stream-outputs', '--multi-step-stream-outputs',
action='store_true', action=StoreBoolean,
help='If True, then multi-step will stream outputs for every step') default=EngineArgs.multi_step_stream_outputs,
nargs="?",
const="True",
help='If False, then multi-step will stream outputs at the end '
'of all steps')
parser.add_argument( parser.add_argument(
'--scheduler-delay-factor', '--scheduler-delay-factor',
type=float, type=float,
default=EngineArgs.scheduler_delay_factor, default=EngineArgs.scheduler_delay_factor,
help='Apply a delay (of delay factor multiplied by previous' help='Apply a delay (of delay factor multiplied by previous '
'prompt latency) before scheduling next prompt.') 'prompt latency) before scheduling next prompt.')
parser.add_argument( parser.add_argument(
'--enable-chunked-prefill', '--enable-chunked-prefill',
@ -632,7 +636,7 @@ class EngineArgs:
type=nullable_str, type=nullable_str,
choices=[*QUANTIZATION_METHODS, None], choices=[*QUANTIZATION_METHODS, None],
default=EngineArgs.speculative_model_quantization, default=EngineArgs.speculative_model_quantization,
help='Method used to quantize the weights of speculative model.' help='Method used to quantize the weights of speculative model. '
'If None, we first check the `quantization_config` ' 'If None, we first check the `quantization_config` '
'attribute in the model config file. If that is ' 'attribute in the model config file. If that is '
'None, we assume the model weights are not ' 'None, we assume the model weights are not '