xinyun/vllm
1
0
Fork 0
mirror of https://git.datalinker.icu/vllm-project/vllm.git synced 2026-09-09 08:27:03 +08:00
Code Issues Packages Projects Releases Wiki Activity

[Platform] Let EPD work with non-cuda platform (#30225)

Browse Source 
Signed-off-by: wangxiyuan <wangxiyuan1007@gmail.com>
This commit is contained in:
wangxiyuan 2025-12-18 14:45:29 +08:00 committed by GitHub
parent 11a89cf95c
commit a85724bd6e
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 4 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
vllm/distributed/ec_transfer/ec_connector/example_connector.py
View File

@ -73,6 +73,7 @@ class ECExampleConnector(ECConnectorBase):
data hashes (`mm_hash`) to encoder cache tensors. data hashes (`mm_hash`) to encoder cache tensors.
kwargs (dict): Additional keyword arguments for the connector. kwargs (dict): Additional keyword arguments for the connector.
""" """
from vllm.platforms import current_platform
# Get the metadata # Get the metadata
metadata: ECConnectorMetadata = self._get_connector_metadata() metadata: ECConnectorMetadata = self._get_connector_metadata()
@ -91,7 +92,9 @@ class ECExampleConnector(ECConnectorBase):
if mm_data.mm_hash in encoder_cache: if mm_data.mm_hash in encoder_cache:
continue continue
filename = self._generate_filename_debug(mm_data.mm_hash) filename = self._generate_filename_debug(mm_data.mm_hash)
ec_cache = safetensors.torch.load_file(filename)["ec_cache"].cuda() ec_cache = safetensors.torch.load_file(
filename, device=current_platform.device_type
)["ec_cache"]
encoder_cache[mm_data.mm_hash] = ec_cache encoder_cache[mm_data.mm_hash] = ec_cache
logger.debug("Success load encoder cache for hash %s", mm_data.mm_hash) logger.debug("Success load encoder cache for hash %s", mm_data.mm_hash)