xinyun/vllm
1
0
Fork 0
mirror of https://git.datalinker.icu/vllm-project/vllm.git synced 2025-12-10 14:54:57 +08:00
Code Issues Packages Projects Releases Wiki Activity

[Model] Fix Baichuan BNB online quantization (#10572)

Browse Source 
Signed-off-by: Chen Wu <cntryroa@gmail.com>
This commit is contained in:
Chen Wu 2024-11-23 13:13:59 +08:00 committed by GitHub
parent 7d8ffb344f
commit cfea9c04ef
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 15 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
vllm/model_executor/models/baichuan.py
View File

@ -350,6 +350,21 @@ class BaiChuanBaseForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
embedding_modules = {}
embedding_padding_modules = []
# BitandBytes specific attributes
default_bitsandbytes_target_modules = [
".W_pack.",
".o_proj.",
".down_proj.",
".up_proj.",
".gate_proj.",
".up_proj.",
]
bitsandbytes_stacked_params_mapping = {
# shard_name, weight_name, index
"gate_proj": ("gate_up_proj", 0),
"up_proj": ("gate_up_proj", 1),
}
def __init__(
self,
*,