xinyun/vllm
1
0
Fork 0
mirror of https://git.datalinker.icu/vllm-project/vllm.git synced 2026-01-05 10:04:00 +08:00
Code Issues Packages Projects Releases Wiki Activity

[Bugfix] Fix lora loading for Compressed Tensors in #9120 (#9179)

Browse Source
This commit is contained in:
Ahmad Fahadh Ilyas 2024-10-09 05:10:44 -07:00 committed by GitHub
parent dc4aea677a
commit 21906a6f50
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 3 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
vllm/lora/layers.py
View File

@ -39,6 +39,9 @@ def _get_lora_device(base_layer: nn.Module) -> torch.device:
# unquantizedLinear
if hasattr(base_layer, "weight"):
return base_layer.weight.device
# Compressed Tensor
elif hasattr(base_layer, "weight_packed"):
return base_layer.weight_packed.device
# GPTQ/AWQ
elif hasattr(base_layer, "qweight"):
return base_layer.qweight.device