From 06311e295666916d3456a357cdd91dd2a03c34e2 Mon Sep 17 00:00:00 2001 From: Jee Jee Li Date: Fri, 13 Sep 2024 15:58:28 +0800 Subject: [PATCH] [Misc] Skip loading extra bias for Qwen2-VL GPTQ-Int8 (#8442) --- vllm/model_executor/models/qwen2_vl.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/vllm/model_executor/models/qwen2_vl.py b/vllm/model_executor/models/qwen2_vl.py index 3f8c590a39b00..179399a12a3d5 100644 --- a/vllm/model_executor/models/qwen2_vl.py +++ b/vllm/model_executor/models/qwen2_vl.py @@ -1055,6 +1055,9 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal): if weight_name not in name: continue name = name.replace(weight_name, param_name) + # Skip loading extra bias for GPTQ models. + if name.endswith(".bias") and name not in params_dict: + continue param = params_dict[name] weight_loader = param.weight_loader weight_loader(param, loaded_weight, shard_id) @@ -1078,6 +1081,9 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal): loaded_weight = loaded_weight.transpose(0, 1) loaded_weight = loaded_weight.reshape(-1) try: + # Skip loading extra bias for GPTQ models. + if name.endswith(".bias") and name not in params_dict: + continue param = params_dict[name] except KeyError: print(params_dict.keys())