From 8e13d9fe6d486f3bfa096e28d683601d72a5a1cc Mon Sep 17 00:00:00 2001 From: Isotr0py Date: Tue, 12 Aug 2025 00:22:25 +0800 Subject: [PATCH] [Misc] Further clean up some redundant config definitions (#22649) Signed-off-by: Isotr0py <2037008807@qq.com> --- vllm/transformers_utils/config.py | 51 +++++++++++++-------- vllm/transformers_utils/configs/__init__.py | 6 +-- vllm/transformers_utils/configs/mllama.py | 31 ------------- vllm/transformers_utils/configs/nvlm_d.py | 31 ------------- 4 files changed, 34 insertions(+), 85 deletions(-) delete mode 100644 vllm/transformers_utils/configs/mllama.py delete mode 100644 vllm/transformers_utils/configs/nvlm_d.py diff --git a/vllm/transformers_utils/config.py b/vllm/transformers_utils/config.py index 6b70164c8caf..02ea0814ddef 100644 --- a/vllm/transformers_utils/config.py +++ b/vllm/transformers_utils/config.py @@ -32,11 +32,10 @@ from vllm.logger import init_logger from vllm.transformers_utils.configs import (ChatGLMConfig, DeepseekVLV2Config, EAGLEConfig, JAISConfig, KimiVLConfig, MedusaConfig, - MllamaConfig, MLPSpeculatorConfig, + MLPSpeculatorConfig, Nemotron_Nano_VL_Config, - NemotronConfig, NVLM_D_Config, - OvisConfig, RWConfig, - SpeculatorsConfig, + NemotronConfig, OvisConfig, + RWConfig, SpeculatorsConfig, Step3TextConfig, Step3VLConfig, UltravoxConfig) # yapf: enable @@ -68,10 +67,6 @@ def _get_hf_token() -> Optional[str]: return None -_CONFIG_REGISTRY_OVERRIDE_HF: dict[str, type[PretrainedConfig]] = { - "mllama": MllamaConfig -} - _CONFIG_REGISTRY: dict[str, type[PretrainedConfig]] = { "chatglm": ChatGLMConfig, "deepseek_vl_v2": DeepseekVLV2Config, @@ -85,18 +80,30 @@ _CONFIG_REGISTRY: dict[str, type[PretrainedConfig]] = { "eagle": EAGLEConfig, "speculators": SpeculatorsConfig, "nemotron": NemotronConfig, - "NVLM_D": NVLM_D_Config, "ovis": OvisConfig, "ultravox": UltravoxConfig, "step3_vl": Step3VLConfig, "step3_text": Step3TextConfig, - **_CONFIG_REGISTRY_OVERRIDE_HF } _CONFIG_ATTRS_MAPPING: dict[str, str] = { "llm_config": "text_config", } +_AUTO_CONFIG_KWARGS_OVERRIDES: dict[str, dict[str, Any]] = { + "internvl_chat": { + "has_no_defaults_at_init": True + }, + # transformers regards mllama as is_encoder_decoder=False + # vllm needs is_encoder_decoder=True to enable cross-attention + "mllama": { + "is_encoder_decoder": True + }, + "NVLM_D": { + "has_no_defaults_at_init": True + }, +} + class ConfigFormat(str, enum.Enum): AUTO = "auto" @@ -273,11 +280,12 @@ def thinker_uses_mrope(config: PretrainedConfig) -> bool: def is_encoder_decoder(config: PretrainedConfig) -> bool: """Detect if the model with this config is used as an encoder/decoder.""" - text_config = getattr(config, "text_config", None) - if text_config is not None: - return is_encoder_decoder(text_config) - return getattr(config, "is_encoder_decoder", False) + def _is_encoder_decoder(config: PretrainedConfig) -> bool: + return getattr(config, "is_encoder_decoder", False) + + return (_is_encoder_decoder(config) + or _is_encoder_decoder(config.get_text_config())) def is_interleaved(config: PretrainedConfig) -> bool: @@ -291,13 +299,21 @@ def is_interleaved(config: PretrainedConfig) -> bool: return False +def _maybe_update_auto_config_kwargs(kwargs: dict[str, Any], model_type: str): + """ + Update kwargs for AutoConfig initialization based on model_type + """ + if model_type in _AUTO_CONFIG_KWARGS_OVERRIDES: + kwargs.update(_AUTO_CONFIG_KWARGS_OVERRIDES[model_type]) + return kwargs + + def _maybe_remap_hf_config_attrs(config: PretrainedConfig) -> PretrainedConfig: """Remap config attributes to match the expected names.""" for old_attr, new_attr in _CONFIG_ATTRS_MAPPING.items(): if hasattr(config, old_attr): if not hasattr(config, new_attr): config.update({new_attr: getattr(config, old_attr)}) - delattr(config, old_attr) logger.debug("Remapped config attribute '%s' to '%s'", old_attr, new_attr) return config @@ -408,15 +424,14 @@ def get_config( ) else: try: + kwargs = _maybe_update_auto_config_kwargs( + kwargs, model_type=model_type) config = AutoConfig.from_pretrained( model, trust_remote_code=trust_remote_code, revision=revision, code_revision=code_revision, token=_get_hf_token(), - # some old custom model's config needs - # `has_no_defaults_at_init=True` to work. - has_no_defaults_at_init=trust_remote_code, **kwargs, ) except ValueError as e: diff --git a/vllm/transformers_utils/configs/__init__.py b/vllm/transformers_utils/configs/__init__.py index 82d24bb16ba5..8339c55bcf80 100644 --- a/vllm/transformers_utils/configs/__init__.py +++ b/vllm/transformers_utils/configs/__init__.py @@ -17,13 +17,11 @@ from vllm.transformers_utils.configs.falcon import RWConfig from vllm.transformers_utils.configs.jais import JAISConfig from vllm.transformers_utils.configs.kimi_vl import KimiVLConfig from vllm.transformers_utils.configs.medusa import MedusaConfig -from vllm.transformers_utils.configs.mllama import MllamaConfig from vllm.transformers_utils.configs.mlp_speculator import MLPSpeculatorConfig from vllm.transformers_utils.configs.moonvit import MoonViTConfig from vllm.transformers_utils.configs.nemotron import NemotronConfig from vllm.transformers_utils.configs.nemotron_h import NemotronHConfig from vllm.transformers_utils.configs.nemotron_vl import Nemotron_Nano_VL_Config -from vllm.transformers_utils.configs.nvlm_d import NVLM_D_Config from vllm.transformers_utils.configs.ovis import OvisConfig from vllm.transformers_utils.configs.speculators.base import SpeculatorsConfig from vllm.transformers_utils.configs.step3_vl import (Step3TextConfig, @@ -34,18 +32,16 @@ from vllm.transformers_utils.configs.ultravox import UltravoxConfig __all__ = [ "ChatGLMConfig", "DeepseekVLV2Config", + "EAGLEConfig", "RWConfig", "JAISConfig", "MedusaConfig", - "EAGLEConfig", - "MllamaConfig", "MLPSpeculatorConfig", "MoonViTConfig", "KimiVLConfig", "NemotronConfig", "NemotronHConfig", "Nemotron_Nano_VL_Config", - "NVLM_D_Config", "OvisConfig", "SpeculatorsConfig", "UltravoxConfig", diff --git a/vllm/transformers_utils/configs/mllama.py b/vllm/transformers_utils/configs/mllama.py deleted file mode 100644 index f0cd2d52a529..000000000000 --- a/vllm/transformers_utils/configs/mllama.py +++ /dev/null @@ -1,31 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -from transformers.models.mllama import configuration_mllama as mllama_hf_config - - -class MllamaTextConfig(mllama_hf_config.MllamaTextConfig): - ''' - Use this class to override is_encoder_decoder: - - transformers regards mllama as is_encoder_decoder=False - - vllm needs is_encoder_decoder=True to enable cross-attention - ''' - - def __init__( - self, - **kwargs, - ): - super().__init__(**kwargs) - self.is_encoder_decoder = True - - -class MllamaConfig(mllama_hf_config.MllamaConfig): - - def __init__( - self, - text_config=None, - **kwargs, - ): - if isinstance(text_config, dict): - text_config = MllamaTextConfig(**text_config) - super().__init__(text_config=text_config, **kwargs) diff --git a/vllm/transformers_utils/configs/nvlm_d.py b/vllm/transformers_utils/configs/nvlm_d.py deleted file mode 100644 index edfc506882ff..000000000000 --- a/vllm/transformers_utils/configs/nvlm_d.py +++ /dev/null @@ -1,31 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -# Adapted from -# https://huggingface.co/nvidia/NVLM-D-72B/blob/main/configuration_nvlm_d.py -# -------------------------------------------------------- -# NVLM-D -# Copyright (c) 2024 NVIDIA -# Licensed under Apache 2.0 License [see LICENSE for details] -# -------------------------------------------------------- -from transformers import Qwen2Config -from transformers.configuration_utils import PretrainedConfig - - -class NVLM_D_Config(PretrainedConfig): - model_type = 'NVLM_D' - is_composition = True - - def __init__(self, vision_config=None, llm_config=None, **kwargs): - super().__init__(**kwargs) - - # Handle vision_config initialization - if vision_config is None: - vision_config = {} - - # Handle llm_config initialization - if llm_config is None: - llm_config = {} - - self.vision_config = PretrainedConfig(**vision_config) - self.text_config = Qwen2Config(**llm_config)