diff --git a/vllm/config/model.py b/vllm/config/model.py
index 3fb448ebbf36..28af82de7722 100644
--- a/vllm/config/model.py
+++ b/vllm/config/model.py
@@ -1334,11 +1334,13 @@ class ModelConfig:
                 self.hf_config_path or self.model,
                 trust_remote_code=self.trust_remote_code,
                 revision=self.revision,
+                config_format=self.config_format,
             )
         else:
             config = try_get_generation_config(
                 self.generation_config,
                 trust_remote_code=self.trust_remote_code,
+                config_format=self.config_format,
             )
 
         if config is None:
diff --git a/vllm/transformers_utils/config.py b/vllm/transformers_utils/config.py
index b0816cfb0702..4f5e5c01e5cb 100644
--- a/vllm/transformers_utils/config.py
+++ b/vllm/transformers_utils/config.py
@@ -949,6 +949,7 @@ def try_get_generation_config(
     model: str,
     trust_remote_code: bool,
     revision: Optional[str] = None,
+    config_format: Union[str, ConfigFormat] = "auto",
 ) -> Optional[GenerationConfig]:
     try:
         return GenerationConfig.from_pretrained(
@@ -961,6 +962,7 @@ def try_get_generation_config(
                 model,
                 trust_remote_code=trust_remote_code,
                 revision=revision,
+                config_format=config_format,
             )
             return GenerationConfig.from_model_config(config)
         except OSError:  # Not found