xinyun/vllm
1
0
Fork 0
mirror of https://git.datalinker.icu/vllm-project/vllm.git synced 2025-12-16 16:36:02 +08:00
Code Issues Packages Projects Releases Wiki Activity

Reduce GPU memory utilization to make sure OOM doesn't happen (#153)

Browse Source
This commit is contained in:
Zhuohan Li 2023-06-18 17:33:50 +08:00 committed by GitHub
parent bec7b2dc26
commit bf5f121c02
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 1 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
vllm/engine/arg_utils.py
View File

@ -21,7 +21,7 @@ class EngineArgs:
tensor_parallel_size: int = 1 tensor_parallel_size: int = 1
block_size: int = 16 block_size: int = 16
swap_space: int = 4 # GiB swap_space: int = 4 # GiB
gpu_memory_utilization: float = 0.95 gpu_memory_utilization: float = 0.90
max_num_batched_tokens: int = 2560 max_num_batched_tokens: int = 2560
max_num_seqs: int = 256 max_num_seqs: int = 256
disable_log_stats: bool = False disable_log_stats: bool = False