From dedb1a5424c3bd92f989fff5f4282325134fc6ed Mon Sep 17 00:00:00 2001 From: "rshaw@neuralmagic.com" Date: Mon, 30 Jun 2025 01:30:06 +0000 Subject: [PATCH] updated Signed-off-by: rshaw@neuralmagic.com --- .../kv_transfer/kv_connector/v1/nixl_connector.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/nixl_connector.py b/vllm/distributed/kv_transfer/kv_connector/v1/nixl_connector.py index 46d7c157ef581..47433ae0dabbf 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/nixl_connector.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/nixl_connector.py @@ -330,7 +330,10 @@ class NixlConnectorWorker: self.block_size = vllm_config.cache_config.block_size # Agent. - self.nixl_wrapper = NixlWrapper(str(uuid.uuid4()), None) + self.nixl_wrapper = NixlWrapper(str(uuid.uuid4()), + None, + num_workers=None, + num_shared_workers=16) # Map of engine_id -> {rank0: agent_name0, rank1: agent_name1..}. self._remote_agents: dict[str, dict[int, str]] = defaultdict(dict) @@ -986,9 +989,9 @@ class NixlConnectorWorker: # Begin async xfer. start = time.perf_counter() - # for handle in handles: - # self.nixl_wrapper.transfer(handle) - self.nixl_wrapper.transfer_batched(handles) + for handle in handles: + self.nixl_wrapper.transfer(handle) + # self.nixl_wrapper.transfer_batched(handles) end = time.perf_counter() logger.info("======== LAUNCH TIME: %s ========", end - start)