From b71c956debf045a9a1545ebfe06961ca5163d91c Mon Sep 17 00:00:00 2001 From: Woosuk Kwon Date: Wed, 11 Sep 2024 20:31:51 -0700 Subject: [PATCH] [TPU] Use Ray for default distributed backend (#8389) --- vllm/config.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/vllm/config.py b/vllm/config.py index 26e4b169587e..8fc8ae6b7dfc 100644 --- a/vllm/config.py +++ b/vllm/config.py @@ -869,6 +869,13 @@ class ParallelConfig: f"distributed executor backend " f"'{self.distributed_executor_backend}'.") + if current_platform.is_tpu() and self.world_size > 1: + if self.distributed_executor_backend is None: + self.distributed_executor_backend = "ray" + if self.distributed_executor_backend != "ray": + raise ValueError( + "TPU backend only supports Ray for distributed inference.") + if self.distributed_executor_backend is None and self.world_size > 1: # We use multiprocessing by default if world_size fits on the # current node and we aren't in a ray placement group.