xinyun/vllm
1
0
Fork 0
mirror of https://git.datalinker.icu/vllm-project/vllm.git synced 2025-12-10 04:15:01 +08:00
Code Issues Packages Projects Releases Wiki Activity

Temporarily enforce eager mode for GPTQ models (#2154)

Browse Source
This commit is contained in:
Woosuk Kwon 2023-12-17 01:51:12 -08:00 committed by GitHub
parent 26c52a5ea6
commit 3a765bd5e1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 5 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
vllm/config.py
View File

@ -185,6 +185,11 @@ class ModelConfig:
self.max_context_len_to_capture = self.max_model_len
self.max_context_len_to_capture = min(self.max_context_len_to_capture,
self.max_model_len)
if self.quantization == "gptq" and not self.enforce_eager:
# Related issue: https://github.com/vllm-project/vllm/issues/2147
logger.warning("GPTQ does not support CUDA graph yet. Disabling "
"CUDA graph.")
self.enforce_eager = True
def verify_with_parallel_config(
self,