xinyun/vllm
1
0
Fork 0
mirror of https://git.datalinker.icu/vllm-project/vllm.git synced 2026-07-08 23:47:12 +08:00
Code Issues Packages Projects Releases Wiki Activity

updated

Browse Source 
Signed-off-by: rshaw@neuralmagic.com <robertgshaw2@gmail.com>
This commit is contained in:
rshaw@neuralmagic.com 2025-07-01 02:49:54 +00:00
parent 56939c835d
commit ff5a0cfa6e
2 changed files with 5 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
pd_justfile/Justfile
View File

@ -12,8 +12,9 @@ port PORT:
prefill: prefill:
VLLM_IS_PREFILL=1 \
VLLM_NIXL_SIDE_CHANNEL_PORT=5557 \ VLLM_NIXL_SIDE_CHANNEL_PORT=5557 \
CUDA_VISIBLE_DEVICES=0 \ CUDA_VISIBLE_DEVICES=7 \
vllm serve {{MODEL}} \ vllm serve {{MODEL}} \
--port $(just port 8100) \ --port $(just port 8100) \
--tensor-parallel-size 1 \ --tensor-parallel-size 1 \
@ -23,8 +24,9 @@ prefill:
--kv-transfer-config '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' --kv-transfer-config '{"kv_connector":"NixlConnector","kv_role":"kv_both"}'
decode: decode:
VLLM_IS_PREFILL=0 \
VLLM_NIXL_SIDE_CHANNEL_PORT=$(just port 5559) \ VLLM_NIXL_SIDE_CHANNEL_PORT=$(just port 5559) \
CUDA_VISIBLE_DEVICES=1 \ CUDA_VISIBLE_DEVICES=6 \
vllm serve {{MODEL}} \ vllm serve {{MODEL}} \
--port $(just port 8300) \ --port $(just port 8300) \
--tensor-parallel-size 1 \ --tensor-parallel-size 1 \

3
vllm/distributed/kv_transfer/kv_connector/v1/nixl_connector.py
View File

@ -7,7 +7,6 @@ import time
import uuid import uuid
from collections import defaultdict from collections import defaultdict
from collections.abc import Iterator from collections.abc import Iterator
from concurrent.futures import ThreadPoolExecutor
from dataclasses import dataclass from dataclasses import dataclass
from typing import TYPE_CHECKING, Any, Optional from typing import TYPE_CHECKING, Any, Optional
@ -335,7 +334,7 @@ class NixlConnectorWorker:
# Agent. # Agent.
import os import os
num_workers = 32 num_workers = 64
# setting num_workers on the prefiller causes no notifs to be recved??? # setting num_workers on the prefiller causes no notifs to be recved???
# this is a hack to make sure we set num workers on the prefiller to 1. # this is a hack to make sure we set num workers on the prefiller to 1.
if os.getenv("VLLM_IS_PREFILL", "0") == "1": if os.getenv("VLLM_IS_PREFILL", "0") == "1":