xinyun/vllm
1
0
Fork 0
mirror of https://git.datalinker.icu/vllm-project/vllm.git synced 2025-12-19 06:45:01 +08:00
Code Issues Packages Projects Releases Wiki Activity

[Model] Support Mistral-Nemo (#6548)

Browse Source
This commit is contained in:
Michael Goin 2024-07-18 16:31:50 -04:00 committed by GitHub
parent ecdb462c24
commit 15c6a079b1
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 5 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
vllm/model_executor/models/llama.py
View File

@ -89,6 +89,7 @@ class LlamaAttention(nn.Module):
def __init__( def __init__(
self, self,
config: LlamaConfig,
hidden_size: int, hidden_size: int,
num_heads: int, num_heads: int,
num_kv_heads: int, num_kv_heads: int,
@ -115,7 +116,9 @@ class LlamaAttention(nn.Module):
# the KV heads across multiple tensor parallel GPUs. # the KV heads across multiple tensor parallel GPUs.
assert tp_size % self.total_num_kv_heads == 0 assert tp_size % self.total_num_kv_heads == 0
self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size) self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
self.head_dim = hidden_size // self.total_num_heads # MistralConfig has an optional head_dim introduced by Mistral-Nemo
self.head_dim = getattr(config, "head_dim",
self.hidden_size // self.total_num_heads)
self.q_size = self.num_heads * self.head_dim self.q_size = self.num_heads * self.head_dim
self.kv_size = self.num_kv_heads * self.head_dim self.kv_size = self.num_kv_heads * self.head_dim
self.scaling = self.head_dim**-0.5 self.scaling = self.head_dim**-0.5
@ -189,6 +192,7 @@ class LlamaDecoderLayer(nn.Module):
attention_bias = getattr(config, "attention_bias", False) or getattr( attention_bias = getattr(config, "attention_bias", False) or getattr(
config, "bias", False) config, "bias", False)
self.self_attn = LlamaAttention( self.self_attn = LlamaAttention(
config=config,
hidden_size=self.hidden_size, hidden_size=self.hidden_size,
num_heads=config.num_attention_heads, num_heads=config.num_attention_heads,
num_kv_heads=getattr(config, "num_key_value_heads", num_kv_heads=getattr(config, "num_key_value_heads",