From b6553be1bc75f046b00046a4ad7576364d03c835 Mon Sep 17 00:00:00 2001 From: Jee Jee Li Date: Tue, 10 Jun 2025 21:51:49 +0800 Subject: [PATCH] [Misc] Slight improvement of the BNB (#19418) Signed-off-by: Jee Jee Li Co-authored-by: Isotr0py <2037008807@qq.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> --- vllm/model_executor/layers/quantization/bitsandbytes.py | 4 +--- vllm/model_executor/model_loader/bitsandbytes_loader.py | 3 ++- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/vllm/model_executor/layers/quantization/bitsandbytes.py b/vllm/model_executor/layers/quantization/bitsandbytes.py index 38935bc967855..53ba84ea8e754 100644 --- a/vllm/model_executor/layers/quantization/bitsandbytes.py +++ b/vllm/model_executor/layers/quantization/bitsandbytes.py @@ -71,9 +71,7 @@ class BitsAndBytesConfig(QuantizationConfig): @staticmethod def get_config_filenames() -> list[str]: - return [ - "adapter_config.json", - ] + return [] @classmethod def from_config(cls, config: dict[str, Any]) -> "BitsAndBytesConfig": diff --git a/vllm/model_executor/model_loader/bitsandbytes_loader.py b/vllm/model_executor/model_loader/bitsandbytes_loader.py index ebbb021cad645..068a4e355ff8d 100644 --- a/vllm/model_executor/model_loader/bitsandbytes_loader.py +++ b/vllm/model_executor/model_loader/bitsandbytes_loader.py @@ -392,7 +392,8 @@ class BitsAndBytesModelLoader(BaseModelLoader): def _get_bnb_target_modules(self, model: nn.Module) -> None: for name, module in model.named_modules(): - if isinstance(module, (LinearBase, )): + if (isinstance(module, LinearBase) and + hasattr(module.quant_method, "quant_config")): if modules_info := self.modules_mapping.get_sub_modules(name): # Map vllm's names to transformers's names. rep_name, sub_modules = modules_info