xinyun/vllm
1
0
Fork 0
mirror of https://git.datalinker.icu/vllm-project/vllm.git synced 2026-07-10 07:07:10 +08:00
Code Issues Packages Projects Releases Wiki Activity

support falcon_mamba

Browse Source 
Signed-off-by: Xingyu Liu <charlotteliu12x@gmail.com>
This commit is contained in:
Xingyu Liu 2025-12-09 13:56:46 -08:00
parent 48eeb1ffba
commit 5870d362be
3 changed files with 38 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

57
tests/config/model_arch_groundtruth.json
View File

@ -13,7 +13,7 @@
"total_num_kv_heads": 0, "total_num_kv_heads": 0,
"num_experts": 0, "num_experts": 0,
"is_deepseek_mla": false, "is_deepseek_mla": false,
"supports_multimodal": false, "is_multimodal_model": false,
"dtype": "torch.float32" "dtype": "torch.float32"
}, },
"mistralai/Mamba-Codestral-7B-v0.1": { "mistralai/Mamba-Codestral-7B-v0.1": {
@ -30,7 +30,7 @@
"total_num_kv_heads": 0, "total_num_kv_heads": 0,
"num_experts": 0, "num_experts": 0,
"is_deepseek_mla": false, "is_deepseek_mla": false,
"supports_multimodal": false, "is_multimodal_model": false,
"dtype": "torch.bfloat16" "dtype": "torch.bfloat16"
}, },
"ibm-nasa-geospatial/Prithvi-EO-2.0-300M-TL-Sen1Floods11": { "ibm-nasa-geospatial/Prithvi-EO-2.0-300M-TL-Sen1Floods11": {
@ -47,9 +47,26 @@
"total_num_kv_heads": 0, "total_num_kv_heads": 0,
"num_experts": 0, "num_experts": 0,
"is_deepseek_mla": false, "is_deepseek_mla": false,
"supports_multimodal": true, "is_multimodal_model": true,
"dtype": "torch.float32" "dtype": "torch.float32"
}, },
"tiiuae/falcon-mamba-7b-instruct": {
"architectures": [
"FalconMambaForCausalLM"
],
"model_type": "falcon_mamba",
"text_model_type": "falcon_mamba",
"hidden_size": 4096,
"total_num_hidden_layers": 64,
"total_num_attention_heads": 0,
"head_size": 0,
"vocab_size": 65024,
"total_num_kv_heads": 0,
"num_experts": 0,
"is_deepseek_mla": false,
"is_multimodal_model": false,
"dtype": "torch.bfloat16"
},
"Zyphra/Zamba2-7B-instruct": { "Zyphra/Zamba2-7B-instruct": {
"architectures": [ "architectures": [
"Zamba2ForCausalLM" "Zamba2ForCausalLM"
@ -64,7 +81,7 @@
"total_num_kv_heads": 32, "total_num_kv_heads": 32,
"num_experts": 0, "num_experts": 0,
"is_deepseek_mla": false, "is_deepseek_mla": false,
"supports_multimodal": false, "is_multimodal_model": false,
"dtype": "torch.bfloat16" "dtype": "torch.bfloat16"
}, },
"mosaicml/mpt-7b": { "mosaicml/mpt-7b": {
@ -81,7 +98,7 @@
"total_num_kv_heads": 32, "total_num_kv_heads": 32,
"num_experts": 0, "num_experts": 0,
"is_deepseek_mla": false, "is_deepseek_mla": false,
"supports_multimodal": false, "is_multimodal_model": false,
"dtype": "torch.bfloat16" "dtype": "torch.bfloat16"
}, },
"databricks/dbrx-instruct": { "databricks/dbrx-instruct": {
@ -98,7 +115,7 @@
"total_num_kv_heads": 8, "total_num_kv_heads": 8,
"num_experts": 0, "num_experts": 0,
"is_deepseek_mla": false, "is_deepseek_mla": false,
"supports_multimodal": false, "is_multimodal_model": false,
"dtype": "torch.bfloat16" "dtype": "torch.bfloat16"
}, },
"tiiuae/falcon-7b": { "tiiuae/falcon-7b": {
@ -115,7 +132,7 @@
"total_num_kv_heads": 1, "total_num_kv_heads": 1,
"num_experts": 0, "num_experts": 0,
"is_deepseek_mla": false, "is_deepseek_mla": false,
"supports_multimodal": false, "is_multimodal_model": false,
"dtype": "torch.bfloat16" "dtype": "torch.bfloat16"
}, },
"tiiuae/falcon-40b": { "tiiuae/falcon-40b": {
@ -132,7 +149,7 @@
"total_num_kv_heads": 8, "total_num_kv_heads": 8,
"num_experts": 0, "num_experts": 0,
"is_deepseek_mla": false, "is_deepseek_mla": false,
"supports_multimodal": false, "is_multimodal_model": false,
"dtype": "torch.bfloat16" "dtype": "torch.bfloat16"
}, },
"luccafong/deepseek_mtp_main_random": { "luccafong/deepseek_mtp_main_random": {
@ -149,7 +166,7 @@
"total_num_kv_heads": 32, "total_num_kv_heads": 32,
"num_experts": 72, "num_experts": 72,
"is_deepseek_mla": true, "is_deepseek_mla": true,
"supports_multimodal": false, "is_multimodal_model": false,
"dtype": "torch.bfloat16" "dtype": "torch.bfloat16"
}, },
"luccafong/deepseek_mtp_draft_random": { "luccafong/deepseek_mtp_draft_random": {
@ -166,7 +183,7 @@
"total_num_kv_heads": 32, "total_num_kv_heads": 32,
"num_experts": 72, "num_experts": 72,
"is_deepseek_mla": true, "is_deepseek_mla": true,
"supports_multimodal": false, "is_multimodal_model": false,
"dtype": "torch.bfloat16" "dtype": "torch.bfloat16"
}, },
"Qwen/Qwen3-Next-80B-A3B-Instruct": { "Qwen/Qwen3-Next-80B-A3B-Instruct": {
@ -183,7 +200,7 @@
"total_num_kv_heads": 2, "total_num_kv_heads": 2,
"num_experts": 512, "num_experts": 512,
"is_deepseek_mla": false, "is_deepseek_mla": false,
"supports_multimodal": false, "is_multimodal_model": false,
"dtype": "torch.bfloat16" "dtype": "torch.bfloat16"
}, },
"tiny-random/qwen3-next-moe": { "tiny-random/qwen3-next-moe": {
@ -200,7 +217,7 @@
"total_num_kv_heads": 8, "total_num_kv_heads": 8,
"num_experts": 32, "num_experts": 32,
"is_deepseek_mla": false, "is_deepseek_mla": false,
"supports_multimodal": false, "is_multimodal_model": false,
"dtype": "torch.bfloat16" "dtype": "torch.bfloat16"
}, },
"zai-org/GLM-4.5": { "zai-org/GLM-4.5": {
@ -217,7 +234,7 @@
"total_num_kv_heads": 8, "total_num_kv_heads": 8,
"num_experts": 160, "num_experts": 160,
"is_deepseek_mla": false, "is_deepseek_mla": false,
"supports_multimodal": false, "is_multimodal_model": false,
"dtype": "torch.bfloat16" "dtype": "torch.bfloat16"
}, },
"baidu/ERNIE-4.5-21B-A3B-PT": { "baidu/ERNIE-4.5-21B-A3B-PT": {
@ -234,7 +251,7 @@
"total_num_kv_heads": 4, "total_num_kv_heads": 4,
"num_experts": 64, "num_experts": 64,
"is_deepseek_mla": false, "is_deepseek_mla": false,
"supports_multimodal": false, "is_multimodal_model": false,
"dtype": "torch.bfloat16" "dtype": "torch.bfloat16"
}, },
"lmsys/gpt-oss-20b-bf16": { "lmsys/gpt-oss-20b-bf16": {
@ -251,7 +268,7 @@
"total_num_kv_heads": 8, "total_num_kv_heads": 8,
"num_experts": 32, "num_experts": 32,
"is_deepseek_mla": false, "is_deepseek_mla": false,
"supports_multimodal": false, "is_multimodal_model": false,
"dtype": "torch.bfloat16" "dtype": "torch.bfloat16"
}, },
"deepseek-ai/DeepSeek-V3.2-Exp": { "deepseek-ai/DeepSeek-V3.2-Exp": {
@ -268,7 +285,7 @@
"total_num_kv_heads": 128, "total_num_kv_heads": 128,
"num_experts": 256, "num_experts": 256,
"is_deepseek_mla": true, "is_deepseek_mla": true,
"supports_multimodal": false, "is_multimodal_model": false,
"dtype": "torch.bfloat16" "dtype": "torch.bfloat16"
}, },
"meta-llama/Llama-4-Scout-17B-16E-Instruct": { "meta-llama/Llama-4-Scout-17B-16E-Instruct": {
@ -285,7 +302,7 @@
"total_num_kv_heads": 8, "total_num_kv_heads": 8,
"num_experts": 16, "num_experts": 16,
"is_deepseek_mla": false, "is_deepseek_mla": false,
"supports_multimodal": true, "is_multimodal_model": true,
"dtype": "torch.bfloat16" "dtype": "torch.bfloat16"
}, },
"nvidia/Llama-3_3-Nemotron-Super-49B-v1": { "nvidia/Llama-3_3-Nemotron-Super-49B-v1": {
@ -302,7 +319,7 @@
"total_num_kv_heads": 8, "total_num_kv_heads": 8,
"num_experts": 0, "num_experts": 0,
"is_deepseek_mla": false, "is_deepseek_mla": false,
"supports_multimodal": false, "is_multimodal_model": false,
"dtype": "torch.bfloat16" "dtype": "torch.bfloat16"
}, },
"XiaomiMiMo/MiMo-7B-RL": { "XiaomiMiMo/MiMo-7B-RL": {
@ -319,7 +336,7 @@
"total_num_kv_heads": 8, "total_num_kv_heads": 8,
"num_experts": 0, "num_experts": 0,
"is_deepseek_mla": false, "is_deepseek_mla": false,
"supports_multimodal": false, "is_multimodal_model": false,
"dtype": "torch.bfloat16" "dtype": "torch.bfloat16"
}, },
"meituan-longcat/LongCat-Flash-Chat": { "meituan-longcat/LongCat-Flash-Chat": {
@ -336,7 +353,7 @@
"total_num_kv_heads": 64, "total_num_kv_heads": 64,
"num_experts": 512, "num_experts": 512,
"is_deepseek_mla": true, "is_deepseek_mla": true,
"supports_multimodal": false, "is_multimodal_model": false,
"dtype": "torch.float32" "dtype": "torch.float32"
} }
} }

5
vllm/config/model.py
View File

@ -1188,15 +1188,10 @@ class ModelConfig:
return self.hf_config.model_type in MM_PREFIX_LM_MODELS return self.hf_config.model_type in MM_PREFIX_LM_MODELS
def get_head_size(self) -> int: def get_head_size(self) -> int:
if self.is_attention_free:
return 0
return self.model_arch_config.head_size return self.model_arch_config.head_size
def get_total_num_kv_heads(self) -> int: def get_total_num_kv_heads(self) -> int:
"""Returns the total number of KV heads.""" """Returns the total number of KV heads."""
if self.is_attention_free:
return 0
return self.model_arch_config.total_num_kv_heads return self.model_arch_config.total_num_kv_heads
def get_num_kv_heads(self, parallel_config: ParallelConfig) -> int: def get_num_kv_heads(self, parallel_config: ParallelConfig) -> int:

1
vllm/transformers_utils/model_arch_config_convertor.py
View File

@ -365,6 +365,7 @@ class LongCatFlashMTPModelArchConfigConvertor(ModelArchConfigConvertorBase):
# hf_config.model_type -> convertor class # hf_config.model_type -> convertor class
MODEL_ARCH_CONFIG_CONVERTORS = { MODEL_ARCH_CONFIG_CONVERTORS = {
"mamba": MambaModelArchConfigConvertor, "mamba": MambaModelArchConfigConvertor,
"falcon_mamba": MambaModelArchConfigConvertor,
"timm_wrapper": TerratorchModelArchConfigConvertor, "timm_wrapper": TerratorchModelArchConfigConvertor,
"zamba2": Zamba2ModelArchConfigConvertor, "zamba2": Zamba2ModelArchConfigConvertor,
"mpt": MPTModelArchConfigConvertor, "mpt": MPTModelArchConfigConvertor,