xinyun/vllm
1
0
Fork 0
mirror of https://git.datalinker.icu/vllm-project/vllm.git synced 2025-12-11 04:44:59 +08:00
Code Issues Packages Projects Releases Wiki Activity

[Bugfix] Fix Llama GGUF initialization (#18717)

Browse Source 
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
Cyrus Leung 2025-05-26 22:49:22 +08:00 committed by GitHub
parent 82e2339b06
commit a869baca73
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 1 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
vllm/model_executor/models/llama.py
View File

@ -208,7 +208,7 @@ class LlamaAttention(nn.Module):
quant_config: Optional[QuantizationConfig]) -> None: quant_config: Optional[QuantizationConfig]) -> None:
is_neox_style = True is_neox_style = True
is_gguf = quant_config and quant_config.get_name() == "gguf" is_gguf = quant_config and quant_config.get_name() == "gguf"
if is_gguf and self.config.model_type == "llama": if is_gguf and config.model_type == "llama":
is_neox_style = False is_neox_style = False
self.rotary_emb = get_rope( self.rotary_emb = get_rope(