xinyun/vllm
1
0
Fork 0
mirror of https://git.datalinker.icu/vllm-project/vllm.git synced 2026-05-28 20:47:08 +08:00
Code Issues Packages Projects Releases Wiki Activity

[Model] Ignore rotary embed load for Cohere model (#17319)

Browse Source
This commit is contained in:
Ekagra Ranjan 2025-04-29 03:30:40 -04:00 committed by GitHub
parent 4464109219
commit 97cc8729f0
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 4 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
vllm/model_executor/models/commandr.py
View File

@ -418,6 +418,10 @@ class CohereForCausalLM(nn.Module, SupportsLoRA, SupportsPP, SupportsQuant):
loaded_params: Set[str] = set() loaded_params: Set[str] = set()
for name, loaded_weight in weights: for name, loaded_weight in weights:
# Skip loading rotary embeddings since vLLM has its own
if "rotary_emb.inv_freq" in name:
continue
if (self.quant_config is not None and if (self.quant_config is not None and
(scale_name := self.quant_config.get_cache_scale(name))): (scale_name := self.quant_config.get_cache_scale(name))):
# Loading kv cache quantization scales # Loading kv cache quantization scales