xinyun/vllm
1
0
Fork 0
mirror of https://git.datalinker.icu/vllm-project/vllm.git synced 2026-05-26 20:28:04 +08:00
Code Issues Packages Projects Releases Wiki Activity

[ Misc ] Enable Quantizing All Layers of DeekSeekv2 (#6423)

Browse Source
This commit is contained in:
Robert Shaw 2024-07-14 17:38:42 -04:00 committed by GitHub
parent ccd3c04571
commit 73030b7dae
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 6 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.buildkite/lm-eval-harness/run-lm-eval-gsm-vllm-baseline.sh
View File

@ -46,6 +46,6 @@ while getopts "m:b:l:f:t:" OPT; do
done done
lm_eval --model vllm \ lm_eval --model vllm \
--model_args pretrained=$MODEL,tensor_parallel_size=$TP_SIZE,add_bos_token=true,distributed_executor_backend="ray",trust_remote_code=true \ --model_args pretrained=$MODEL,tensor_parallel_size=$TP_SIZE,add_bos_token=true,distributed_executor_backend="ray",trust_remote_code=true,max_model_len=4096 \
--tasks gsm8k --num_fewshot $FEWSHOT --limit $LIMIT \ --tasks gsm8k --num_fewshot $FEWSHOT --limit $LIMIT \
--batch_size $BATCH_SIZE --batch_size $BATCH_SIZE

5
vllm/model_executor/model_loader/weight_utils.py
View File

@ -431,6 +431,11 @@ def convert_pyslice_to_tensor(x: Any) -> torch.Tensor:
def default_weight_loader(param: torch.Tensor, def default_weight_loader(param: torch.Tensor,
loaded_weight: torch.Tensor) -> None: loaded_weight: torch.Tensor) -> None:
"""Default weight loader.""" """Default weight loader."""
# If the weight on disk does not have a shape, give it one
# (such scales for AutoFp8).
if len(loaded_weight.shape) == 0:
loaded_weight = loaded_weight.reshape(1)
assert param.size() == loaded_weight.size() assert param.size() == loaded_weight.size()
param.data.copy_(loaded_weight) param.data.copy_(loaded_weight)