xinyun/vllm
1
0
Fork 0
mirror of https://git.datalinker.icu/vllm-project/vllm.git synced 2026-09-09 03:17:02 +08:00
Code Issues Packages Projects Releases Wiki Activity

[Misc] Upgrade BNB version (#15183)

Browse Source
This commit is contained in:
Jee Jee Li 2025-03-24 13:51:42 +08:00 committed by GitHub
parent d20e261199
commit 3892e58ad7
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
4 changed files with 10 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Dockerfile
View File

@ -286,7 +286,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \
if [ "$TARGETPLATFORM" = "linux/arm64" ]; then \ if [ "$TARGETPLATFORM" = "linux/arm64" ]; then \
uv pip install accelerate hf_transfer 'modelscope!=1.15.0' 'bitsandbytes>=0.42.0' 'timm==0.9.10' boto3 runai-model-streamer runai-model-streamer[s3]; \ uv pip install accelerate hf_transfer 'modelscope!=1.15.0' 'bitsandbytes>=0.42.0' 'timm==0.9.10' boto3 runai-model-streamer runai-model-streamer[s3]; \
else \ else \
uv pip install accelerate hf_transfer 'modelscope!=1.15.0' 'bitsandbytes>=0.45.0' 'timm==0.9.10' boto3 runai-model-streamer runai-model-streamer[s3]; \ uv pip install accelerate hf_transfer 'modelscope!=1.15.0' 'bitsandbytes>=0.45.3' 'timm==0.9.10' boto3 runai-model-streamer runai-model-streamer[s3]; \
fi fi
ENV VLLM_USAGE_SOURCE production-docker-image ENV VLLM_USAGE_SOURCE production-docker-image

2
docs/source/features/quantization/bnb.md
View File

@ -9,7 +9,7 @@ Compared to other quantization methods, BitsAndBytes eliminates the need for cal
Below are the steps to utilize BitsAndBytes with vLLM. Below are the steps to utilize BitsAndBytes with vLLM.
```console ```console
pip install bitsandbytes>=0.45.0 pip install bitsandbytes>=0.45.3
``` ```
vLLM reads the model's config file and supports both in-flight quantization and pre-quantized checkpoint. vLLM reads the model's config file and supports both in-flight quantization and pre-quantized checkpoint.

8
vllm/model_executor/layers/quantization/bitsandbytes.py
View File

@ -155,12 +155,12 @@ class BitsAndBytesLinearMethod(LinearMethodBase):
def __init__(self, quant_config: BitsAndBytesConfig): def __init__(self, quant_config: BitsAndBytesConfig):
try: try:
import bitsandbytes import bitsandbytes
if bitsandbytes.__version__ < "0.45.0": if bitsandbytes.__version__ < "0.45.3":
raise ImportError("bitsandbytes version is wrong. Please " raise ImportError("bitsandbytes version is wrong. Please "
"install bitsandbytes>=0.45.0.") "install bitsandbytes>=0.45.3.")
except ImportError as err: except ImportError as err:
raise ImportError("Please install bitsandbytes>=0.45.0 via " raise ImportError("Please install bitsandbytes>=0.45.3 via "
"`pip install bitsandbytes>=0.45.0` to use " "`pip install bitsandbytes>=0.45.3` to use "
"bitsandbytes quantizer.") from err "bitsandbytes quantizer.") from err
self.quant_config = quant_config self.quant_config = quant_config

8
vllm/model_executor/model_loader/loader.py
View File

@ -862,12 +862,12 @@ class BitsAndBytesModelLoader(BaseModelLoader):
try: try:
import bitsandbytes import bitsandbytes
if bitsandbytes.__version__ < "0.45.0": if bitsandbytes.__version__ < "0.45.3":
raise ImportError("bitsandbytes version is wrong. Please " raise ImportError("bitsandbytes version is wrong. Please "
"install bitsandbytes>=0.45.0.") "install bitsandbytes>=0.45.3.")
except ImportError as err: except ImportError as err:
raise ImportError("Please install bitsandbytes>=0.45.0 via " raise ImportError("Please install bitsandbytes>=0.45.3 via "
"`pip install bitsandbytes>=0.45.0` to use " "`pip install bitsandbytes>=0.45.3` to use "
"bitsandbytes quantizer.") from err "bitsandbytes quantizer.") from err
hf_weights_files, use_safetensors = self._prepare_weights( hf_weights_files, use_safetensors = self._prepare_weights(