xinyun/vllm
1
0
Fork 0
mirror of https://git.datalinker.icu/vllm-project/vllm.git synced 2025-12-16 07:15:01 +08:00
Code Issues Packages Projects Releases Wiki Activity

Fix ShardedStateLoader for vllm fp8 quantization (#7708)

Browse Source
This commit is contained in:
Flex Wang 2024-08-22 05:25:04 -07:00 committed by GitHub
parent a3fce56b88
commit 4f419c00a6
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 4 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
vllm/model_executor/model_loader/loader.py
View File

@ -579,6 +579,10 @@ class ShardedStateLoader(BaseModelLoader):
with torch.device(device_config.device): with torch.device(device_config.device):
model = _initialize_model(model_config, self.load_config, model = _initialize_model(model_config, self.load_config,
lora_config, cache_config) lora_config, cache_config)
for _, module in model.named_modules():
quant_method = getattr(module, "quant_method", None)
if quant_method is not None:
quant_method.process_weights_after_loading(module)
rank = get_tensor_model_parallel_rank() rank = get_tensor_model_parallel_rank()
pattern = os.path.join( pattern = os.path.join(
local_model_path, local_model_path,