xinyun/vllm
1
0
Fork 0
mirror of https://git.datalinker.icu/vllm-project/vllm.git synced 2025-12-14 13:14:59 +08:00
Code Issues Packages Projects Releases Wiki Activity

[Platforms] Refactor xpu code (#10468)

Browse Source 
Signed-off-by: MengqingCao <cmq0113@163.com>
This commit is contained in:
Mengqing Cao 2024-11-20 14:52:13 +08:00 committed by GitHub
parent 09dbf9ff16
commit d5b28447e0
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 21 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
vllm/executor/xpu_executor.py
View File

@ -1,8 +1,5 @@
from typing import Callable, List, Optional, Tuple, Type, Union from typing import Callable, List, Optional, Tuple, Type, Union
import torch
from vllm.config import ModelConfig, ParallelConfig
from vllm.executor.executor_base import ExecutorAsyncBase from vllm.executor.executor_base import ExecutorAsyncBase
from vllm.executor.gpu_executor import GPUExecutor from vllm.executor.gpu_executor import GPUExecutor
from vllm.logger import init_logger from vllm.logger import init_logger
@ -23,7 +20,6 @@ class XPUExecutor(GPUExecutor):
assert self.speculative_config is None, ( assert self.speculative_config is None, (
"Speculative decoding not yet supported for XPU backend") "Speculative decoding not yet supported for XPU backend")
self.model_config = _verify_and_get_model_config(self.model_config)
GPUExecutor._init_executor(self) GPUExecutor._init_executor(self)
def _get_worker_module_and_class( def _get_worker_module_and_class(
@ -53,26 +49,3 @@ class XPUExecutorAsync(XPUExecutor, ExecutorAsyncBase):
output = await make_async(self.driver_worker.execute_model output = await make_async(self.driver_worker.execute_model
)(execute_model_req=execute_model_req) )(execute_model_req=execute_model_req)
return output return output
def _verify_and_get_model_config(config: ModelConfig) -> ModelConfig:
if config.dtype == torch.bfloat16:
logger.warning(
"bfloat16 is not fully supported on XPU, casting to float16.")
config.dtype = torch.float16
if not config.enforce_eager:
logger.warning(
"CUDA graph is not supported on XPU, fallback to the eager "
"mode.")
config.enforce_eager = True
return config
def _verify_and_get_parallel_config(config: ParallelConfig) -> ParallelConfig:
if (config.distributed_executor_backend is not None
and config.distributed_executor_backend != "ray"):
logger.warning(
"%s is not supported on XPU, fallback to ray distributed executor "
"backend.", config.distributed_executor_backend)
config.distributed_executor_backend = "ray"
return config

21
vllm/platforms/xpu.py
View File

@ -1,9 +1,16 @@
from typing import TYPE_CHECKING
import torch import torch
from vllm.logger import init_logger from vllm.logger import init_logger
from .interface import DeviceCapability, Platform, PlatformEnum, _Backend from .interface import DeviceCapability, Platform, PlatformEnum, _Backend
if TYPE_CHECKING:
from vllm.config import VllmConfig
else:
VllmConfig = None
logger = init_logger(__name__) logger = init_logger(__name__)
@ -34,3 +41,17 @@ class XPUPlatform(Platform):
@staticmethod @staticmethod
def inference_mode(): def inference_mode():
return torch.no_grad() return torch.no_grad()
@classmethod
def check_and_update_config(cls, vllm_config: VllmConfig) -> None:
# check and update model config
model_config = vllm_config.model_config
if model_config.dtype == torch.bfloat16:
logger.warning(
"bfloat16 is not fully supported on XPU, casting to float16.")
model_config.dtype = torch.float16
if not model_config.enforce_eager:
logger.warning(
"CUDA graph is not supported on XPU, fallback to the eager "
"mode.")
model_config.enforce_eager = True