From 5d8b6653664dbb8585c37fdbd25eb5042523355e Mon Sep 17 00:00:00 2001 From: "rshaw@neuralmagic.com" Date: Mon, 30 Jun 2025 01:59:02 +0000 Subject: [PATCH] updated Signed-off-by: rshaw@neuralmagic.com --- .../kv_transfer/kv_connector/v1/nixl_connector.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/nixl_connector.py b/vllm/distributed/kv_transfer/kv_connector/v1/nixl_connector.py index 82ae98db92165..c1f1db7e567cf 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/nixl_connector.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/nixl_connector.py @@ -340,7 +340,7 @@ class NixlConnectorWorker: self.nixl_wrapper = NixlWrapper(str(uuid.uuid4()), None, num_workers=num_workers, - num_shared_workers=None) + num_shared_workers=None) # Map of engine_id -> {rank0: agent_name0, rank1: agent_name1..}. self._remote_agents: dict[str, dict[int, str]] = defaultdict(dict) @@ -999,9 +999,12 @@ class NixlConnectorWorker: # Begin async xfer. start = time.perf_counter() - for handle in handles: - self.nixl_wrapper.transfer(handle) - # self.nixl_wrapper.transfer_batched(handles) + # IT WORKS WITH THIS: + # for handle in handles: + # self.nixl_wrapper.transfer(handle) + + # IT FAILS WITH THIS: + self.nixl_wrapper.transfer_batched(handles) end = time.perf_counter() logger.info("======== LAUNCH TIME: %s ========", end - start)