xinyun/vllm
1
0
Fork 0
mirror of https://git.datalinker.icu/vllm-project/vllm.git synced 2026-04-03 06:37:02 +08:00
Code Issues Packages Projects Releases Wiki Activity

[Bugfix] Set enforce_eager automatically for mllama (#12127)

Browse Source 
Signed-off-by: Chen Zhang <zhangch99@outlook.com>
This commit is contained in:
Chen Zhang 2025-01-17 04:30:08 +08:00 committed by GitHub
parent 62b06ba23d
commit d06e824006
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 5 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
examples/offline_inference/vision_language.py
View File

@ -325,7 +325,6 @@ def run_mllama(question: str, modality: str):
model=model_name,
max_model_len=4096,
max_num_seqs=16,
enforce_eager=True,
disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache,
)

1
examples/offline_inference/vision_language_multi_image.py
View File

@ -186,7 +186,6 @@ def load_mllama(question, image_urls: List[str]) -> ModelRequestData:
model=model_name,
max_model_len=4096,
max_num_seqs=16,
enforce_eager=True,
limit_mm_per_prompt={"image": len(image_urls)},
)

8
vllm/config.py
View File

@ -607,10 +607,12 @@ class ModelConfig:
self.max_seq_len_to_capture = min(self.max_seq_len_to_capture,
self.max_model_len)
if (self.hf_config.model_type == 'deepseek_v3'
MODEL_NOT_SUPPORT_CUDA_GRAPH = ['deepseek_v3', 'mllama']
if (self.hf_config.model_type in MODEL_NOT_SUPPORT_CUDA_GRAPH
and not self.enforce_eager):
logger.warning("CUDA graph is not supported for Deepseek V3 yet, "
"fallback to the eager mode.")
logger.warning(
"CUDA graph is not supported for %s yet, fallback to the eager "
"mode.", self.hf_config.model_type)
self.enforce_eager = True
def _verify_bnb_config(self) -> None: