xinyun/vllm
1
0
Fork 0
mirror of https://git.datalinker.icu/vllm-project/vllm.git synced 2026-07-18 17:47:12 +08:00
Code Issues Packages Projects Releases Wiki Activity

[Misc] Improve the readability of BNB error messages (#12320)

Browse Source 
Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>
This commit is contained in:
Jee Jee Li 2025-01-23 00:56:54 +08:00 committed by GitHub
parent fc66dee76d
commit 84bee4bd5c
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 2 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
vllm/model_executor/model_loader/loader.py
View File

@ -1076,8 +1076,8 @@ class BitsAndBytesModelLoader(BaseModelLoader):
# weight tensor. So TP does not work with pre_quantized bnb models. # weight tensor. So TP does not work with pre_quantized bnb models.
if pre_quant and get_tensor_model_parallel_world_size() > 1: if pre_quant and get_tensor_model_parallel_world_size() > 1:
raise ValueError( raise ValueError(
"Prequant BitsAndBytes models with TP is not supported." "Prequant BitsAndBytes models with tensor parallelism is not "
"Please try with PP.") "supported. Please try with pipeline parallelism.")
load_8bit = False load_8bit = False
if pre_quant: if pre_quant: