mirror of
https://git.datalinker.icu/vllm-project/vllm.git
synced 2026-09-06 10:57:00 +08:00
[Fix] Convert kv_transfer_config from dict to KVTransferConfig (#19262)
This commit is contained in:
parent
861a0a0a39
commit
08500011d3
@ -10,6 +10,7 @@ from typing import (TYPE_CHECKING, Any, Callable, ClassVar, Optional, Union,
|
|||||||
|
|
||||||
import cloudpickle
|
import cloudpickle
|
||||||
import torch.nn as nn
|
import torch.nn as nn
|
||||||
|
from pydantic import ValidationError
|
||||||
from tqdm.auto import tqdm
|
from tqdm.auto import tqdm
|
||||||
from typing_extensions import TypeVar, deprecated
|
from typing_extensions import TypeVar, deprecated
|
||||||
|
|
||||||
@ -195,6 +196,23 @@ class LLM:
|
|||||||
if isinstance(worker_cls, type):
|
if isinstance(worker_cls, type):
|
||||||
kwargs["worker_cls"] = cloudpickle.dumps(worker_cls)
|
kwargs["worker_cls"] = cloudpickle.dumps(worker_cls)
|
||||||
|
|
||||||
|
if "kv_transfer_config" in kwargs and isinstance(
|
||||||
|
kwargs["kv_transfer_config"], dict):
|
||||||
|
from vllm.config import KVTransferConfig
|
||||||
|
raw_config_dict = kwargs["kv_transfer_config"]
|
||||||
|
try:
|
||||||
|
kwargs["kv_transfer_config"] = KVTransferConfig(
|
||||||
|
**raw_config_dict)
|
||||||
|
except ValidationError as e:
|
||||||
|
logger.error(
|
||||||
|
"Failed to convert 'kv_transfer_config' dict to "
|
||||||
|
"KVTransferConfig object. Dict: %s. Error: %s",
|
||||||
|
raw_config_dict, e)
|
||||||
|
# Consider re-raising a more specific vLLM error or ValueError
|
||||||
|
# to provide better context to the user.
|
||||||
|
raise ValueError(
|
||||||
|
f"Invalid 'kv_transfer_config' provided: {e}") from e
|
||||||
|
|
||||||
if hf_overrides is None:
|
if hf_overrides is None:
|
||||||
hf_overrides = {}
|
hf_overrides = {}
|
||||||
|
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user