xinyun/vllm
1
0
Fork 0
mirror of https://git.datalinker.icu/vllm-project/vllm.git synced 2026-07-09 23:27:20 +08:00
Code Issues Packages Projects Releases Wiki Activity

[Bugfix] Fix gpt2 GGUF inference (#12467)

Browse Source 
Signed-off-by: Isotr0py <2037008807@qq.com>
This commit is contained in:
Isotr0py 2025-01-27 18:31:49 +08:00 committed by GitHub
parent 624a1e4711
commit ce69f7f754
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 8 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
vllm/model_executor/models/gpt2.py
View File

@ -258,13 +258,13 @@ class GPT2LMHeadModel(nn.Module, SupportsPP):
self.transformer = GPT2Model(vllm_config=vllm_config, self.transformer = GPT2Model(vllm_config=vllm_config,
prefix=maybe_prefix( prefix=maybe_prefix(
prefix, "transformer")) prefix, "transformer"))
self.lm_head = ParallelLMHead(self.config.vocab_size,
self.config.hidden_size,
quant_config=quant_config,
prefix=f"{prefix}.lm_head")
if self.config.tie_word_embeddings: if self.config.tie_word_embeddings:
self.lm_head = self.transformer.wte self.lm_head = self.lm_head.tie_weights(self.transformer.wte)
else:
self.lm_head = ParallelLMHead(self.config.vocab_size,
self.config.hidden_size,
quant_config=quant_config,
prefix=f"{prefix}.lm_head")
self.logits_processor = LogitsProcessor(config.vocab_size) self.logits_processor = LogitsProcessor(config.vocab_size)
self.sampler = get_sampler() self.sampler = get_sampler()
self.make_empty_intermediate_tensors = ( self.make_empty_intermediate_tensors = (
@ -309,15 +309,12 @@ class GPT2LMHeadModel(nn.Module, SupportsPP):
params_dict = dict(self.named_parameters(remove_duplicate=False)) params_dict = dict(self.named_parameters(remove_duplicate=False))
loaded_params: Set[str] = set() loaded_params: Set[str] = set()
for name, loaded_weight in weights: for name, loaded_weight in weights:
if name.startswith("lm_head"):
# GPT-2 ties the weights of the embedding layer and the final
# linear layer.
continue
if ".attn.bias" in name or ".attn.masked_bias" in name: if ".attn.bias" in name or ".attn.masked_bias" in name:
# Skip attention mask. # Skip attention mask.
# NOTE: "c_attn.bias" should not be skipped. # NOTE: "c_attn.bias" should not be skipped.
continue continue
if not name.startswith("transformer."): if not name.startswith("transformer.") and not name.startswith(
"lm_head"):
name = "transformer." + name name = "transformer." + name
if is_pp_missing_parameter(name, self): if is_pp_missing_parameter(name, self):