xinyun/vllm
1
0
Fork 0
mirror of https://git.datalinker.icu/vllm-project/vllm.git synced 2026-01-18 16:14:34 +08:00
Code Issues Packages Projects Releases Wiki Activity

[Bugfix] Fix using -O[0,3] with LLM entrypoint (#10677)

Browse Source 
Signed-off-by: mgoin <michael@neuralmagic.com>
This commit is contained in:
Michael Goin 2024-11-26 13:44:01 -05:00 committed by GitHub
parent f5792c7c4a
commit 9a99273b48
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 12 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
vllm/engine/arg_utils.py
View File

@ -206,7 +206,10 @@ class EngineArgs:
# support `EngineArgs(compilation_config={...})`
# without having to manually construct a
# CompilationConfig object
if isinstance(self.compilation_config, (int, dict)):
if isinstance(self.compilation_config, (int)):
self.compilation_config = CompilationConfig.from_cli(
str(self.compilation_config))
elif isinstance(self.compilation_config, (dict)):
self.compilation_config = CompilationConfig.from_cli(
json.dumps(self.compilation_config))

10
vllm/entrypoints/llm.py
View File

@ -185,8 +185,14 @@ class LLM:
kwargs["disable_log_stats"] = True
if compilation_config is not None:
compilation_config_instance = CompilationConfig.from_cli(
json.dumps(compilation_config))
if isinstance(compilation_config, (int)):
compilation_config_instance = CompilationConfig.from_cli(
str(compilation_config))
elif isinstance(compilation_config, (dict)):
compilation_config_instance = CompilationConfig.from_cli(
json.dumps(compilation_config))
else:
compilation_config_instance = compilation_config
else:
compilation_config_instance = None