From 56939c835d42accacd6cab7c4fbba9b13d62c5fc Mon Sep 17 00:00:00 2001 From: "rshaw@neuralmagic.com" Date: Tue, 1 Jul 2025 01:34:46 +0000 Subject: [PATCH] updated Signed-off-by: rshaw@neuralmagic.com --- .../distributed/kv_transfer/kv_connector/v1/nixl_connector.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/nixl_connector.py b/vllm/distributed/kv_transfer/kv_connector/v1/nixl_connector.py index 0abfa489e1312..54b867e9fead6 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/nixl_connector.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/nixl_connector.py @@ -335,7 +335,7 @@ class NixlConnectorWorker: # Agent. import os - num_workers = 16 + num_workers = 32 # setting num_workers on the prefiller causes no notifs to be recved??? # this is a hack to make sure we set num workers on the prefiller to 1. if os.getenv("VLLM_IS_PREFILL", "0") == "1": @@ -985,7 +985,7 @@ class NixlConnectorWorker: assert len(local_block_descs_ids) == len(remote_block_descs_ids) # Prepare transfer with Nixl. - CHUNK_SIZE = 1000 + CHUNK_SIZE = 500 handles = [] # NOTE: this is a hack to make make_prepped_xfer into threads so that # different workers are allocated for each chuck. Without this change,