mirror of
https://git.datalinker.icu/vllm-project/vllm.git
synced 2026-01-07 00:29:42 +08:00
[Fix] Convert kv_transfer_config from dict to KVTransferConfig (#19262)
This commit is contained in:
parent
861a0a0a39
commit
08500011d3
@ -10,6 +10,7 @@ from typing import (TYPE_CHECKING, Any, Callable, ClassVar, Optional, Union,
|
||||
|
||||
import cloudpickle
|
||||
import torch.nn as nn
|
||||
from pydantic import ValidationError
|
||||
from tqdm.auto import tqdm
|
||||
from typing_extensions import TypeVar, deprecated
|
||||
|
||||
@ -195,6 +196,23 @@ class LLM:
|
||||
if isinstance(worker_cls, type):
|
||||
kwargs["worker_cls"] = cloudpickle.dumps(worker_cls)
|
||||
|
||||
if "kv_transfer_config" in kwargs and isinstance(
|
||||
kwargs["kv_transfer_config"], dict):
|
||||
from vllm.config import KVTransferConfig
|
||||
raw_config_dict = kwargs["kv_transfer_config"]
|
||||
try:
|
||||
kwargs["kv_transfer_config"] = KVTransferConfig(
|
||||
**raw_config_dict)
|
||||
except ValidationError as e:
|
||||
logger.error(
|
||||
"Failed to convert 'kv_transfer_config' dict to "
|
||||
"KVTransferConfig object. Dict: %s. Error: %s",
|
||||
raw_config_dict, e)
|
||||
# Consider re-raising a more specific vLLM error or ValueError
|
||||
# to provide better context to the user.
|
||||
raise ValueError(
|
||||
f"Invalid 'kv_transfer_config' provided: {e}") from e
|
||||
|
||||
if hf_overrides is None:
|
||||
hf_overrides = {}
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user