xinyun/vllm
1
0
Fork 0
mirror of https://git.datalinker.icu/vllm-project/vllm.git synced 2025-12-13 13:25:30 +08:00
Code Issues Packages Projects Releases Wiki Activity

[Bugfix] Fix _get_lora_device for HQQ marlin (#12090)

Browse Source 
Signed-off-by: Varun Sundar Rabindranath <varun@neuralmagic.com>
Co-authored-by: Varun Sundar Rabindranath <varun@neuralmagic.com>
This commit is contained in:
Varun Sundar Rabindranath 2025-01-16 01:29:42 +05:30 committed by GitHub
parent 70755e819e
commit ebd8c669ef
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 3 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
vllm/lora/layers.py
View File

@ -51,6 +51,9 @@ def _get_lora_device(base_layer: nn.Module) -> torch.device:
# marlin # marlin
elif hasattr(base_layer, "B"): elif hasattr(base_layer, "B"):
return base_layer.B.device return base_layer.B.device
# HQQ marlin
elif hasattr(base_layer, "W_q"):
return base_layer.W_q.device
else: else:
raise ValueError(f"Unsupported base layer: {base_layer}") raise ValueError(f"Unsupported base layer: {base_layer}")