xinyun/vllm
1
0
Fork 0
mirror of https://git.datalinker.icu/vllm-project/vllm.git synced 2025-12-14 22:15:54 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(gguf_loader): accept HF repo paths & URLs for GGUF (#20793)

Browse Source 
Signed-off-by: Hardik <hardikgupta1999@gmail.com>
Signed-off-by: Isotr0py <2037008807@qq.com>
Co-authored-by: Isotr0py <2037008807@qq.com>
Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
This commit is contained in:
Hardik Gupta 2025-07-23 20:21:02 -07:00 committed by GitHub
parent f3137cdd81
commit 11599b0e1f
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 12 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
vllm/model_executor/model_loader/gguf_loader.py
View File

@ -6,6 +6,7 @@ from collections.abc import Generator
import gguf import gguf
import torch import torch
import torch.nn as nn import torch.nn as nn
from huggingface_hub import hf_hub_download
from transformers import AutoModelForCausalLM from transformers import AutoModelForCausalLM
from vllm.config import LoadConfig, ModelConfig, VllmConfig from vllm.config import LoadConfig, ModelConfig, VllmConfig
@ -32,8 +33,18 @@ class GGUFModelLoader(BaseModelLoader):
def _prepare_weights(self, model_name_or_path: str): def _prepare_weights(self, model_name_or_path: str):
if os.path.isfile(model_name_or_path): if os.path.isfile(model_name_or_path):
return model_name_or_path return model_name_or_path
# for raw HTTPS link
if model_name_or_path.startswith(
("http://", "https://")) and model_name_or_path.endswith(".gguf"):
return hf_hub_download(url=model_name_or_path)
# repo id/filename.gguf
if "/" in model_name_or_path and model_name_or_path.endswith(".gguf"):
repo_id, filename = model_name_or_path.rsplit("/", 1)
return hf_hub_download(repo_id=repo_id, filename=filename)
else: else:
raise ValueError(f"{model_name_or_path} is not a file.") raise ValueError(
f"Unrecognised GGUF reference: {model_name_or_path} "
"(expected local file, raw URL, or <repo_id>/<filename>.gguf)")
def _get_gguf_weights_map(self, model_config: ModelConfig): def _get_gguf_weights_map(self, model_config: ModelConfig):
""" """