Signed-off-by: inkcherry <mingzhi.liu@amd.com>
This commit is contained in:
inkcherry 2025-11-21 12:13:24 +00:00
parent 72ccb5d77c
commit 4776e2ddcf

View File

@ -37,7 +37,12 @@ from vllm.distributed.parallel_state import (
) )
from vllm.forward_context import ForwardContext from vllm.forward_context import ForwardContext
from vllm.logger import init_logger from vllm.logger import init_logger
from vllm.utils.network_utils import get_ip, make_zmq_path, make_zmq_socket, get_open_port from vllm.utils.network_utils import (
get_ip,
get_open_port,
make_zmq_path,
make_zmq_socket,
)
from vllm.v1.core.sched.output import SchedulerOutput from vllm.v1.core.sched.output import SchedulerOutput
from vllm.v1.request import RequestStatus from vllm.v1.request import RequestStatus
@ -70,7 +75,8 @@ class MoRIIOConstants:
PING_INTERVAL = 5 PING_INTERVAL = 5
MAX_PING_RETRIES = 100000 MAX_PING_RETRIES = 100000
DEFAULT_HANDSHAKE_PORT = "6301" DEFAULT_HANDSHAKE_PORT = "6301"
DEFAULT_NOTIFY_PORT="61005" DEFAULT_NOTIFY_PORT = "61005"
try: try:
from mori.io import ( from mori.io import (
@ -80,7 +86,7 @@ try:
IOEngineConfig, IOEngineConfig,
MemoryDesc, MemoryDesc,
PollCqMode, PollCqMode,
RdmaBackendConfig RdmaBackendConfig,
) )
logger.info("MoRIIO is available") logger.info("MoRIIO is available")
@ -202,7 +208,6 @@ def get_moriio_mode() -> MoRIIOMode:
return MoRIIOMode.WRITE return MoRIIOMode.WRITE
def get_port_offset(dp_rank: int, tp_rank: int, tp_size: int = 1) -> int: def get_port_offset(dp_rank: int, tp_rank: int, tp_size: int = 1) -> int:
return (dp_rank) * tp_size + tp_rank return (dp_rank) * tp_size + tp_rank
@ -224,7 +229,6 @@ class MoRIIOConfig:
@classmethod @classmethod
def from_vllm_config(cls, vllm_config: VllmConfig) -> "MoRIIOConfig": def from_vllm_config(cls, vllm_config: VllmConfig) -> "MoRIIOConfig":
# Port Configuration: # Port Configuration:
# local_ping_port -> Outgoing heartbeat to proxy # local_ping_port -> Outgoing heartbeat to proxy
# proxy_ping_port -> Remote proxy's heartbeat ingress port # proxy_ping_port -> Remote proxy's heartbeat ingress port
@ -233,7 +237,7 @@ class MoRIIOConfig:
# notify_port -> For synchronizing stages between prefill and decode # notify_port -> For synchronizing stages between prefill and decode
# handshake_port -> For initial handshake between mori engine # handshake_port -> For initial handshake between mori engine
#TODO : merge notify_port and handshake_port to simplify port management, supports non-contiguous ports # TODO : merge notify_port and handshake_port to simplify port management, supports non-contiguous ports
kv_transfer_config = vllm_config.kv_transfer_config kv_transfer_config = vllm_config.kv_transfer_config
extra_config = kv_transfer_config.kv_connector_extra_config extra_config = kv_transfer_config.kv_connector_extra_config
@ -716,8 +720,7 @@ class MoRIIOWrapper:
raise MoRIIOError(f"Unhandled message format: {msg_str}") raise MoRIIOError(f"Unhandled message format: {msg_str}")
def _handle_structured_message(self, data: dict): def _handle_structured_message(self, data: dict):
assert get_role() == ROLE.PRODUCER, "Only prefill can get block messages"
assert get_role()==ROLE.PRODUCER, "Only prefill can get block messages"
req_id = data["req_id"] req_id = data["req_id"]
block_notify_list = data.get("block_notify_list", []) block_notify_list = data.get("block_notify_list", [])
decode_dp_rank = data.get("decode_rank", 0) decode_dp_rank = data.get("decode_rank", 0)
@ -888,7 +891,8 @@ class MoRIIOConnector(KVConnectorBase_V1):
self.connector_worker = MoRIIOConnectorWorker(vllm_config, self.engine_id) self.connector_worker = MoRIIOConnectorWorker(vllm_config, self.engine_id)
logger.info( logger.info(
"Initialized MoRIIO Connector,engine_id:%s,role: %s", "Initialized MoRIIO Connector,engine_id:%s,role: %s",
self.engine_id, role.value self.engine_id,
role.value,
) )
############################################################ ############################################################
@ -905,8 +909,6 @@ class MoRIIOConnector(KVConnectorBase_V1):
if "notify_port" not in extra_config or not extra_config["notify_port"]: if "notify_port" not in extra_config or not extra_config["notify_port"]:
extra_config["notify_port"] = MoRIIOConstants.DEFAULT_NOTIFY_PORT extra_config["notify_port"] = MoRIIOConstants.DEFAULT_NOTIFY_PORT
def get_num_new_matched_tokens( def get_num_new_matched_tokens(
self, request: "Request", num_computed_tokens: int self, request: "Request", num_computed_tokens: int
) -> tuple[int, bool]: ) -> tuple[int, bool]:
@ -1194,7 +1196,7 @@ class MoRIIOConnectorScheduler:
if new_block_ids is not None: if new_block_ids is not None:
block_ids = new_block_ids[0] block_ids = new_block_ids[0]
#TODO : hybrid attn, etc # TODO : hybrid attn, etc
req, existing_blocks = self._reqs_need_pending_save[req_id] req, existing_blocks = self._reqs_need_pending_save[req_id]
updated_blocks = list(existing_blocks) + (block_ids) updated_blocks = list(existing_blocks) + (block_ids)
self._reqs_need_pending_save[req_id] = (req, updated_blocks) self._reqs_need_pending_save[req_id] = (req, updated_blocks)
@ -1356,26 +1358,17 @@ class MoRIIOConnectorWorker:
self._ping_thread = None self._ping_thread = None
self._writer = MoRIIOWriter(self) self._writer = MoRIIOWriter(self)
engine_suffix = ( role = "producer" if self.is_producer else "consumer"
f"{self.moriio_config.local_ip}:{self.moriio_config.handshake_port}" engine_suffix = f"{self.moriio_config.local_ip}:{self.moriio_config.handshake_port}:tp{self.tp_rank}:dp{self.dp_rank}"
f":tp {self.tp_rank}:dp {self.dp_rank}" self.moriio_engine = IOEngine(
f"{role}:{engine_suffix}",
IOEngineConfig(
self.moriio_config.local_ip, self.moriio_config.local_kv_port
),
) )
if not self.is_producer:
self.moriio_engine = IOEngine(
"consumer:" + engine_suffix,
IOEngineConfig(
self.moriio_config.local_ip, self.moriio_config.local_kv_port
),
)
else:
self.moriio_engine = IOEngine(
"producer:" + engine_suffix,
IOEngineConfig(
self.moriio_config.local_ip, self.moriio_config.local_kv_port
),
)
logger.debug( logger.debug(
"build MORI IOEngine %s:%s", "build MORI IOEngine %s (ip=%s port=%s)",
f"{role}:{engine_suffix}",
self.moriio_config.local_ip, self.moriio_config.local_ip,
self.moriio_config.local_kv_port, self.moriio_config.local_kv_port,
) )
@ -1604,18 +1597,6 @@ class MoRIIOConnectorWorker:
if should_break: if should_break:
break break
# def handle_proxy_request(self):
# if self.is_producer:
# raise NotImplementedError(
# "prefill instance doesn't need to send kv cache in pull mode"
# )
# while True:
# socks = dict(self.poller.poll())
# logger.debug("handle_proxy_request: socks = %s", socks)
# if self.metadata_socket not in socks:
# continue
def close(self): def close(self):
if hasattr(self, "_handshake_initiation_executor"): if hasattr(self, "_handshake_initiation_executor"):
self._handshake_initiation_executor.shutdown(wait=False) self._handshake_initiation_executor.shutdown(wait=False)
@ -2264,7 +2245,7 @@ class MoRIIOConnectorWorker:
sess_idx = list(self.layer_name_to_local_kv_cache_metadata.keys()).index( sess_idx = list(self.layer_name_to_local_kv_cache_metadata.keys()).index(
layer_name layer_name
) )
#TODO : apply multi-session batch-read when moriio support it # TODO : apply multi-session batch-read when moriio support it
transfer_status = self.moriio_wrapper.read_remote_data( transfer_status = self.moriio_wrapper.read_remote_data(
offs[2], offs[0], offs[1], sessions[sess_idx] offs[2], offs[0], offs[1], sessions[sess_idx]
) )