xinyun/vllm
1
0
Fork 0
mirror of https://git.datalinker.icu/vllm-project/vllm.git synced 2025-12-27 03:15:16 +08:00
Code Issues Packages Projects Releases Wiki Activity

Run default _AsyncLLMEngine._run_workers_async in threadpool (#1628)

Browse Source
This commit is contained in:
Dominik Schwabe 2023-11-11 23:50:44 +01:00 committed by GitHub
parent 0d578228ca
commit 1b290ace4f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 6 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
vllm/engine/async_llm_engine.py
View File

@ -206,18 +206,17 @@ class _AsyncLLMEngine(LLMEngine):
**kwargs,
) -> Any:
"""Runs the given method on all workers."""
all_outputs = []
coros = []
for worker in self.workers:
if self.parallel_config.worker_use_ray:
executor = partial(worker.execute_method.remote, method)
coros.append(
worker.execute_method.remote(method, *args, **kwargs))
else:
executor = getattr(worker, method)
coros.append(asyncio.get_event_loop().run_in_executor(
None, partial(executor, *args, **kwargs)))
output = executor(*args, **kwargs)
all_outputs.append(output)
if self.parallel_config.worker_use_ray:
all_outputs = await asyncio.gather(*all_outputs)
all_outputs = await asyncio.gather(*coros)
if get_all_outputs:
return all_outputs