xinyun/vllm
1
0
Fork 0
mirror of https://git.datalinker.icu/vllm-project/vllm.git synced 2026-01-25 09:24:26 +08:00
Code Issues Packages Projects Releases Wiki Activity

[Model] Siglip2 Model Support (#27566)

Browse Source 
Signed-off-by: piood <2477084691@qq.com>
This commit is contained in:
Yu Jiaqi 2025-10-27 21:57:37 +08:00 committed by GitHub
parent 9273754222
commit 4f882be4a0
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
4 changed files with 18 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
docs/models/supported_models.md
View File

@ -775,7 +775,7 @@ The following table lists those that are tested in vLLM.
| `CLIPModel` | CLIP | T / I | `openai/clip-vit-base-patch32`, `openai/clip-vit-large-patch14`, etc. | | |
| `LlavaNextForConditionalGeneration`<sup>C</sup> | LLaVA-NeXT-based | T / I | `royokong/e5-v` | | ✅︎ |
| `Phi3VForCausalLM`<sup>C</sup> | Phi-3-Vision-based | T + I | `TIGER-Lab/VLM2Vec-Full` | | ✅︎ |
| `SiglipModel` | SigLIP | T / I | `google/siglip-base-patch16-224` | | |
| `SiglipModel` | SigLIP, SigLIP2 | T / I | `google/siglip-base-patch16-224`, `google/siglip2-base-patch16-224` | | |
| `*ForConditionalGeneration`<sup>C</sup>, `*ForCausalLM`<sup>C</sup>, etc. | Generative models | \* | N/A | \* | \* |
<sup>C</sup> Automatically converted into an embedding model via `--convert embed`. ([details](./pooling_models.md#model-conversion))

2
tests/models/multimodal/pooling/test_siglip.py
View File

@ -19,7 +19,7 @@ HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts(
}
)
MODELS = ["google/siglip-base-patch16-224"]
MODELS = ["google/siglip-base-patch16-224", "google/siglip2-base-patch16-224"]
def _run_test(

8
vllm/model_executor/models/siglip.py
View File

@ -174,9 +174,11 @@ class SiglipMultiModalProcessor(BaseMultiModalProcessor[SiglipProcessingInfo]):
@cached_property
def image_token_id(self) -> int:
tokenizer = self.info.get_tokenizer()
dummy_token_id = 0
assert dummy_token_id not in tokenizer.all_special_ids
dummy_token_id = next(
token_id
for token_id in range(tokenizer.vocab_size)
if token_id not in tokenizer.all_special_ids
)
return dummy_token_id

12
vllm/transformers_utils/config.py
View File

@ -26,7 +26,10 @@ from huggingface_hub.utils import (
)
from transformers import GenerationConfig, PretrainedConfig
from transformers.models.auto.image_processing_auto import get_image_processor_config
from transformers.models.auto.modeling_auto import MODEL_FOR_CAUSAL_LM_MAPPING_NAMES
from transformers.models.auto.modeling_auto import (
MODEL_FOR_CAUSAL_LM_MAPPING_NAMES,
MODEL_MAPPING_NAMES,
)
from transformers.models.auto.tokenization_auto import get_tokenizer_config
from transformers.utils import CONFIG_NAME as HF_CONFIG_NAME
@ -616,6 +619,13 @@ def get_config(
model_type = MODEL_FOR_CAUSAL_LM_MAPPING_NAMES[config.model_type]
config.update({"architectures": [model_type]})
# Architecture mapping for models without explicit architectures field
if not config.architectures:
if config.model_type not in MODEL_MAPPING_NAMES:
raise ValueError(f"Cannot find architecture name for {config.model_type}")
model_type = MODEL_MAPPING_NAMES[config.model_type]
config.update({"architectures": [model_type]})
# ModelOpt 0.31.0 and after saves the quantization config in the model
# config file.
quantization_config = config_dict.get("quantization_config", None)