From 88787bce1d33cacc77a6747d1c8e5e8ff97c8904 Mon Sep 17 00:00:00 2001 From: Kyle Sayers Date: Tue, 18 Feb 2025 00:34:47 -0500 Subject: [PATCH] [Quant] Molmo SupportsQuant (#13336) --- vllm/model_executor/models/molmo.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/vllm/model_executor/models/molmo.py b/vllm/model_executor/models/molmo.py index feb585022317..b2154ef54af3 100644 --- a/vllm/model_executor/models/molmo.py +++ b/vllm/model_executor/models/molmo.py @@ -52,7 +52,8 @@ from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs from vllm.sequence import IntermediateTensors from vllm.utils import JSONTree, json_map_leaves -from .interfaces import SupportsLoRA, SupportsMultiModal, SupportsPP +from .interfaces import (SupportsLoRA, SupportsMultiModal, SupportsPP, + SupportsQuant) from .utils import (AutoWeightsLoader, WeightsMapper, flatten_bn, is_pp_missing_parameter, make_empty_intermediate_tensors_factory, make_layers, @@ -633,7 +634,8 @@ class MolmoDecoderNormAfterLayer(MolmoDecoderLayer): return hidden_states, residual -class MolmoVisionBackbone(nn.Module): +class MolmoVisionBackbone(nn.Module, SupportsQuant): + packed_modules_mapping = {"merged_linear": ["gate_proj", "up_proj"]} def __init__( self, @@ -794,7 +796,7 @@ class MolmoVisionBackbone(nn.Module): @support_torch_compile -class MolmoModel(nn.Module): +class MolmoModel(nn.Module, SupportsQuant): def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): super().__init__() @@ -1402,8 +1404,8 @@ class MolmoMultiModalProcessor(BaseMultiModalProcessor[MolmoProcessingInfo]): @MULTIMODAL_REGISTRY.register_processor(MolmoMultiModalProcessor, info=MolmoProcessingInfo, dummy_inputs=MolmoDummyInputsBuilder) -class MolmoForCausalLM(nn.Module, SupportsMultiModal, SupportsPP, - SupportsLoRA): +class MolmoForCausalLM(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA, + SupportsQuant): hf_to_vllm_mapper = WeightsMapper( orig_to_new_substr={ # vision backbone mapping