xinyun/vllm
1
0
Fork 0
mirror of https://git.datalinker.icu/vllm-project/vllm.git synced 2025-12-10 00:25:01 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix: incorrect bigcode attention heads num (#676)

Browse Source
This commit is contained in:
Wen Sun 2023-08-05 01:35:22 +08:00 committed by GitHub
parent aa84c92ef6
commit 621980bdc0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 5 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
vllm/config.py
View File

@ -98,9 +98,11 @@ class ModelConfig:
# Note: for falcon, when new_decoder_architecture is True, the
# multi_query flag is ignored and we use n_head_kv for the number of
# KV heads.
if (getattr(self.hf_config, "multi_query", False) and
(self.hf_config.model_type == "falcon" and
not getattr(self.hf_config, "new_decoder_architecture", False))):
new_decoder_arch_falcon = (
self.hf_config.model_type == "falcon"
and getattr(self.hf_config, "new_decoder_architecture", False))
if not new_decoder_arch_falcon and getattr(self.hf_config,
"multi_query", False):
# Multi-query attention, only one KV head.
return 1
# For Falcon: