xinyun/vllm
1
0
Fork 0
mirror of https://git.datalinker.icu/vllm-project/vllm.git synced 2025-12-15 12:36:04 +08:00
Code Issues Packages Projects Releases Wiki Activity

Fix all-reduce memory usage (#2151)

Browse Source
This commit is contained in:
Woosuk Kwon 2023-12-17 01:44:45 -08:00 committed by GitHub
parent 3d1cfbfc74
commit e1d5402238
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 8 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
vllm/worker/worker.py
View File

@ -48,6 +48,14 @@ class Worker:
self.gpu_cache = None self.gpu_cache = None
def init_model(self, cupy_port: Optional[int] = None): def init_model(self, cupy_port: Optional[int] = None):
# torch.distributed.all_reduce does not free the input tensor until
# the synchronization point. This causes the memory usage to grow
# as the number of all_reduce calls increases. This env var disables
# this behavior.
# Related issue:
# https://discuss.pytorch.org/t/cuda-allocation-lifetime-for-inputs-to-distributed-all-reduce/191573
os.environ["TORCH_NCCL_AVOID_RECORD_STREAMS"] = "1"
# This env var set by Ray causes exceptions with graph building. # This env var set by Ray causes exceptions with graph building.
os.environ.pop("NCCL_ASYNC_ERROR_HANDLING", None) os.environ.pop("NCCL_ASYNC_ERROR_HANDLING", None)
# Env vars will be set by Ray. # Env vars will be set by Ray.