From 4ee4826ede9706ca33492ead69fef99d0b0b0667 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B1=AA=E5=BF=97=E9=B9=8F?= Date: Fri, 16 May 2025 21:20:13 -0700 Subject: [PATCH] [BugFix] Correct max_model_len derivation from config.json for Mistral format (#17937) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: 汪志鹏 Co-authored-by: tracelogfb <48808670+tracelogfb@users.noreply.github.com> Co-authored-by: Stephen Chen --- vllm/transformers_utils/config.py | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/vllm/transformers_utils/config.py b/vllm/transformers_utils/config.py index 02034bf02ec9e..5f45ff133855c 100644 --- a/vllm/transformers_utils/config.py +++ b/vllm/transformers_utils/config.py @@ -686,9 +686,24 @@ def load_params_config(model: Union[str, Path], revision: Optional[str], config_dict["hidden_act"] = config_dict.get("activation", "silu") config_dict["tie_word_embeddings"] = config_dict.get( "tie_embeddings", False) - config_dict["max_seq_len"] = config_dict.get("max_seq_len", 128_000) - config_dict["max_position_embeddings"] = config_dict.get( - "max_position_embeddings", 128_000) + + if config_dict.get("max_position_embeddings") is None: + max_position_embeddings = 128_000 + try: + trust_remote_code_val = kwargs.get("trust_remote_code", False) + hf_config = get_config(model=model, + trust_remote_code=trust_remote_code_val, + revision=revision, + config_format=ConfigFormat.HF) + if hf_value := hf_config.get_text_config().max_position_embeddings: + max_position_embeddings = hf_value + except Exception as e: + logger.warning( + "The params.json file is missing 'max_position_embeddings'" + " and could not get a value from the HF config." + " Defaulting to 128000", + exc_info=e) + config_dict["max_position_embeddings"] = max_position_embeddings if config_dict.get("quantization") is not None: quantization = config_dict.get("quantization", {})