xinyun/vllm
1
0
Fork 0
mirror of https://git.datalinker.icu/vllm-project/vllm.git synced 2026-01-05 03:24:03 +08:00
Code Issues Packages Projects Releases Wiki Activity

[Model] support bitsandbytes quantization with minicpm3 model (#10682)

Browse Source 
Signed-off-by: Ubuntu <zixuanzhang@bytedance.com>
This commit is contained in:
zixuanzhang226 2024-11-27 23:58:02 -08:00 committed by GitHub
parent cb4e1c3f3a
commit 70dc14fbd0
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 6 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
vllm/model_executor/models/minicpm3.py
View File

@ -241,6 +241,12 @@ class MiniCPM3ForCausalLM(MiniCPMForCausalLM):
# `embedding_modules` and `embedding_padding_modules`
# are inherited from MiniCPMForCausalLM
bitsandbytes_stacked_params_mapping = {
# shard_name, weight_name, index
"gate_proj": ("gate_up_proj", 0),
"up_proj": ("gate_up_proj", 1),
}
def _init_model(self, *, vllm_config: VllmConfig, prefix: str = ""):
self.model = MiniCPM3Model(vllm_config=vllm_config,
prefix=maybe_prefix(prefix, "model"))