xinyun/vllm
1
0
Fork 0
mirror of https://git.datalinker.icu/vllm-project/vllm.git synced 2025-12-11 13:05:01 +08:00
Code Issues Packages Projects Releases Wiki Activity

[Bugfix][gpt-oss] passing the cache config in gpt-oss (#23613)

Browse Source 
Signed-off-by: Wei Wei <wwei6@meta.com>
This commit is contained in:
Wei 2025-08-26 19:54:23 -07:00 committed by GitHub
parent 6dab89b8ec
commit fecbb7c782
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 6 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
vllm/model_executor/models/gpt_oss.py
View File

@ -174,12 +174,15 @@ class TransformerBlock(torch.nn.Module):
def __init__( def __init__(
self, self,
config: GptOssConfig, config: GptOssConfig,
cache_config: CacheConfig,
quant_config: QuantizationConfig, quant_config: QuantizationConfig,
prefix: str = "", prefix: str = "",
): ):
super().__init__() super().__init__()
self.layer_idx = extract_layer_index(prefix) self.layer_idx = extract_layer_index(prefix)
self.attn = OAIAttention(config, prefix=f"{prefix}.attn") self.attn = OAIAttention(config,
prefix=f"{prefix}.attn",
cache_config=cache_config)
self.mlp = MLPBlock(config, self.mlp = MLPBlock(config,
self.layer_idx, self.layer_idx,
quant_config=quant_config, quant_config=quant_config,
@ -203,6 +206,7 @@ class GptOssModel(nn.Module):
): ):
super().__init__() super().__init__()
self.config = vllm_config.model_config.hf_config self.config = vllm_config.model_config.hf_config
self.cache_config = vllm_config.cache_config
self.quant_config = vllm_config.quant_config self.quant_config = vllm_config.quant_config
self.parallel_config = vllm_config.parallel_config self.parallel_config = vllm_config.parallel_config
self.config.hidden_size = self.config.hidden_size self.config.hidden_size = self.config.hidden_size
@ -213,6 +217,7 @@ class GptOssModel(nn.Module):
self.layers = torch.nn.ModuleList([ self.layers = torch.nn.ModuleList([
TransformerBlock( TransformerBlock(
self.config, self.config,
cache_config=self.cache_config,
quant_config=self.quant_config, quant_config=self.quant_config,
prefix=maybe_prefix(prefix, f"block.{layer_idx}"), prefix=maybe_prefix(prefix, f"block.{layer_idx}"),
) for layer_idx in range(self.config.num_hidden_layers) ) for layer_idx in range(self.config.num_hidden_layers)