xinyun/vllm
1
0
Fork 0
mirror of https://git.datalinker.icu/vllm-project/vllm.git synced 2025-12-10 03:35:17 +08:00
Code Issues Packages Projects Releases Wiki Activity

[Bugfix] Fix _init_vision_model in NVLM_D model (#9611)

Browse Source 
Co-authored-by: Isotr0py <2037008807@qq.com>
This commit is contained in:
Cyrus Leung 2024-10-23 22:09:04 +08:00 committed by GitHub
parent 31a08f5bd2
commit e7116c017c
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 28 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
vllm/model_executor/models/nvlm_d.py
View File

@ -58,12 +58,31 @@ class NVLM_D_Model(InternVLChatModel):
nn.Linear(llm_intermediate_size, llm_hidden_size, bias=False), nn.Linear(llm_intermediate_size, llm_hidden_size, bias=False),
) )
def _init_vision_model(self, config: PretrainedConfig, def _init_vision_model(
quant_config: Optional[QuantizationConfig], self,
num_hidden_layers: int): config: PretrainedConfig,
# We added additional dummy heads to the original num of heads to make quant_config: Optional[QuantizationConfig],
# the number of heads divisible by 8. *,
return InternVisionModel(config.vision_config, is_mono: bool,
quant_config=quant_config, prefix: str,
num_hidden_layers_override=num_hidden_layers, ):
num_dummy_heads=7) if not is_mono:
vision_feature_layer = config.select_layer
if vision_feature_layer < 0:
num_hidden_layers = config.vision_config.num_hidden_layers \
+ vision_feature_layer + 1
else:
num_hidden_layers = vision_feature_layer + 1
# We added additional dummy heads to the original num of heads to
# make the number of heads divisible by 8.
return InternVisionModel(
config.vision_config,
quant_config=quant_config,
num_hidden_layers_override=num_hidden_layers,
num_dummy_heads=7,
prefix=prefix,
)
else:
msg = "Monolith mode is not applicable to NVLM_D"
raise NotImplementedError(msg)