xinyun/vllm
1
0
Fork 0
mirror of https://git.datalinker.icu/vllm-project/vllm.git synced 2026-07-10 06:17:09 +08:00
Code Issues Packages Projects Releases Wiki Activity

Support Pixtral-Large HF by using llava multimodal_projector_bias config (#12710)

Browse Source 
Signed-off-by: mgoin <michael@neuralmagic.com>
This commit is contained in:
Michael Goin 2025-02-03 22:55:46 -05:00 committed by GitHub
parent 73b35cca7f
commit 5d98d56089
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
4 changed files with 13 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
vllm/model_executor/models/llava.py
View File

@ -75,19 +75,20 @@ class LlavaMultiModalProjector(nn.Module):
vision_hidden_size: int, vision_hidden_size: int,
text_hidden_size: int, text_hidden_size: int,
projector_hidden_act: str, projector_hidden_act: str,
multimodal_projector_bias: bool,
quant_config: Optional[QuantizationConfig] = None, quant_config: Optional[QuantizationConfig] = None,
prefix: str = ""): prefix: str = ""):
super().__init__() super().__init__()
self.linear_1 = ColumnParallelLinear(vision_hidden_size, self.linear_1 = ColumnParallelLinear(vision_hidden_size,
text_hidden_size, text_hidden_size,
bias=True, bias=multimodal_projector_bias,
quant_config=quant_config, quant_config=quant_config,
prefix=f"{prefix}.linear_1") prefix=f"{prefix}.linear_1")
self.act = get_act_fn(projector_hidden_act) self.act = get_act_fn(projector_hidden_act)
self.linear_2 = RowParallelLinear(text_hidden_size, self.linear_2 = RowParallelLinear(text_hidden_size,
text_hidden_size, text_hidden_size,
bias=True, bias=multimodal_projector_bias,
quant_config=quant_config, quant_config=quant_config,
prefix=f"{prefix}.linear_2") prefix=f"{prefix}.linear_2")
@ -503,6 +504,7 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
vision_hidden_size=config.vision_config.hidden_size, vision_hidden_size=config.vision_config.hidden_size,
text_hidden_size=config.text_config.hidden_size, text_hidden_size=config.text_config.hidden_size,
projector_hidden_act=config.projector_hidden_act, projector_hidden_act=config.projector_hidden_act,
multimodal_projector_bias=config.multimodal_projector_bias,
quant_config=quant_config, quant_config=quant_config,
prefix=maybe_prefix(prefix, "multi_modal_projector")) prefix=maybe_prefix(prefix, "multi_modal_projector"))

3
vllm/model_executor/models/llava_next.py
View File

@ -231,7 +231,8 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsMultiModal,
self.multi_modal_projector = LlavaMultiModalProjector( self.multi_modal_projector = LlavaMultiModalProjector(
vision_hidden_size=vision_hidden_size, vision_hidden_size=vision_hidden_size,
text_hidden_size=config.text_config.hidden_size, text_hidden_size=config.text_config.hidden_size,
projector_hidden_act=config.projector_hidden_act) projector_hidden_act=config.projector_hidden_act,
multimodal_projector_bias=config.multimodal_projector_bias)
self.language_model = init_vllm_registered_model( self.language_model = init_vllm_registered_model(
vllm_config=vllm_config, vllm_config=vllm_config,

9
vllm/model_executor/models/llava_next_video.py
View File

@ -253,16 +253,16 @@ class LlavaNextVideoPooler(nn.Module):
class LlavaNextMultiModalProjector(nn.Module): class LlavaNextMultiModalProjector(nn.Module):
def __init__(self, vision_hidden_size: int, text_hidden_size: int, def __init__(self, vision_hidden_size: int, text_hidden_size: int,
projector_hidden_act: str): projector_hidden_act: str, multimodal_projector_bias: bool):
super().__init__() super().__init__()
self.linear_1 = nn.Linear(vision_hidden_size, self.linear_1 = nn.Linear(vision_hidden_size,
text_hidden_size, text_hidden_size,
bias=True) bias=multimodal_projector_bias)
self.act = get_act_fn(projector_hidden_act) self.act = get_act_fn(projector_hidden_act)
self.linear_2 = nn.Linear(text_hidden_size, self.linear_2 = nn.Linear(text_hidden_size,
text_hidden_size, text_hidden_size,
bias=True) bias=multimodal_projector_bias)
def forward(self, image_features: torch.Tensor) -> torch.Tensor: def forward(self, image_features: torch.Tensor) -> torch.Tensor:
hidden_states = self.linear_1(image_features) hidden_states = self.linear_1(image_features)
@ -298,7 +298,8 @@ class LlavaNextVideoForConditionalGeneration(nn.Module, SupportsMultiModal,
self.multi_modal_projector = LlavaNextMultiModalProjector( self.multi_modal_projector = LlavaNextMultiModalProjector(
vision_hidden_size=config.vision_config.hidden_size, vision_hidden_size=config.vision_config.hidden_size,
text_hidden_size=config.text_config.hidden_size, text_hidden_size=config.text_config.hidden_size,
projector_hidden_act=config.projector_hidden_act) projector_hidden_act=config.projector_hidden_act,
multimodal_projector_bias=config.multimodal_projector_bias)
self.language_model = init_vllm_registered_model( self.language_model = init_vllm_registered_model(
vllm_config=vllm_config, vllm_config=vllm_config,
hf_config=config.text_config, hf_config=config.text_config,

4
vllm/model_executor/models/llava_onevision.py
View File

@ -372,11 +372,11 @@ class LlavaOnevisionMultiModalProjector(nn.Module):
self.linear_1 = nn.Linear(config.vision_config.hidden_size, self.linear_1 = nn.Linear(config.vision_config.hidden_size,
config.text_config.hidden_size, config.text_config.hidden_size,
bias=True) bias=config.multimodal_projector_bias)
self.act = get_act_fn(config.projector_hidden_act) self.act = get_act_fn(config.projector_hidden_act)
self.linear_2 = nn.Linear(config.text_config.hidden_size, self.linear_2 = nn.Linear(config.text_config.hidden_size,
config.text_config.hidden_size, config.text_config.hidden_size,
bias=True) bias=config.multimodal_projector_bias)
def forward(self, image_features: torch.Tensor) -> torch.Tensor: def forward(self, image_features: torch.Tensor) -> torch.Tensor:
hidden_states = self.linear_1(image_features) hidden_states = self.linear_1(image_features)