mirror of
https://git.datalinker.icu/ltdrdata/ComfyUI-Manager
synced 2025-12-09 22:24:23 +08:00
feat: support huggingface snapshot downloader
fixed: An issue where JS did not properly handle model download errors. fixed: better security message for model downloading
This commit is contained in:
Dr.Lt.Data
parent
411c0633a3
commit
559c011420
@ -42,7 +42,7 @@ import manager_downloader
|
||||
from node_package import InstalledNodePackage
|
||||
|
||||
|
||||
version_code = [3, 17, 11]
|
||||
version_code = [3, 18]
|
||||
version_str = f"V{version_code[0]}.{version_code[1]}" + (f'.{version_code[2]}' if len(version_code) > 2 else '')
|
||||
|
||||
|
||||
|
||||
@ -3,6 +3,11 @@ from urllib.parse import urlparse
|
||||
import urllib
|
||||
import sys
|
||||
import logging
|
||||
import requests
|
||||
from huggingface_hub import HfApi
|
||||
from tqdm.auto import tqdm
|
||||
|
||||
|
||||
aria2 = os.getenv('COMFYUI_MANAGER_ARIA2_SERVER')
|
||||
HF_ENDPOINT = os.getenv('HF_ENDPOINT')
|
||||
|
||||
@ -117,3 +122,37 @@ def download_url_with_agent(url, save_path):
|
||||
|
||||
print("Installation was successful.")
|
||||
return True
|
||||
|
||||
# NOTE: snapshot_download doesn't provide file size tqdm.
|
||||
def download_repo_in_bytes(repo_id, local_dir):
|
||||
api = HfApi()
|
||||
repo_info = api.repo_info(repo_id=repo_id, files_metadata=True)
|
||||
|
||||
os.makedirs(local_dir, exist_ok=True)
|
||||
|
||||
total_size = 0
|
||||
for file_info in repo_info.siblings:
|
||||
if file_info.size is not None:
|
||||
total_size += file_info.size
|
||||
|
||||
pbar = tqdm(total=total_size, unit="B", unit_scale=True, desc="Downloading")
|
||||
|
||||
for file_info in repo_info.siblings:
|
||||
out_path = os.path.join(local_dir, file_info.rfilename)
|
||||
os.makedirs(os.path.dirname(out_path), exist_ok=True)
|
||||
|
||||
if file_info.size is None:
|
||||
continue
|
||||
|
||||
download_url = f"https://huggingface.co/{repo_id}/resolve/main/{file_info.rfilename}"
|
||||
|
||||
with requests.get(download_url, stream=True) as r, open(out_path, "wb") as f:
|
||||
r.raise_for_status()
|
||||
for chunk in r.iter_content(chunk_size=65536):
|
||||
if chunk:
|
||||
f.write(chunk)
|
||||
pbar.update(len(chunk))
|
||||
|
||||
pbar.close()
|
||||
|
||||
|
||||
|
||||
@ -21,6 +21,8 @@ import logging
|
||||
import asyncio
|
||||
import queue
|
||||
|
||||
import manager_downloader
|
||||
|
||||
|
||||
logging.info(f"### Loading: ComfyUI-Manager ({core.version_str})")
|
||||
|
||||
@ -30,6 +32,7 @@ comfyui_tag = None
|
||||
SECURITY_MESSAGE_MIDDLE_OR_BELOW = "ERROR: To use this action, a security_level of `middle or below` is required. Please contact the administrator.\nReference: https://github.com/ltdrdata/ComfyUI-Manager#security-policy"
|
||||
SECURITY_MESSAGE_NORMAL_MINUS = "ERROR: To use this feature, you must either set '--listen' to a local IP and set the security level to 'normal-' or lower, or set the security level to 'middle' or 'weak'. Please contact the administrator.\nReference: https://github.com/ltdrdata/ComfyUI-Manager#security-policy"
|
||||
SECURITY_MESSAGE_GENERAL = "ERROR: This installation is not allowed in this security_level. Please contact the administrator.\nReference: https://github.com/ltdrdata/ComfyUI-Manager#security-policy"
|
||||
SECURITY_MESSAGE_NORMAL_MINUS_MODEL = "ERROR: Downloading models that are not in '.safetensors' format is only allowed for models registered in the 'default' channel at this security level. If you want to download this model, set the security level to 'normal-' or lower."
|
||||
|
||||
routes = PromptServer.instance.routes
|
||||
|
||||
@ -304,6 +307,9 @@ def get_model_path(data, show_log=False):
|
||||
base_model = get_model_dir(data, show_log)
|
||||
if base_model is None:
|
||||
return None
|
||||
else:
|
||||
if data['filename'] == '<huggingface>':
|
||||
return os.path.join(base_model, os.path.basename(data['url']))
|
||||
else:
|
||||
return os.path.join(base_model, data['filename'])
|
||||
|
||||
@ -477,7 +483,18 @@ async def task_worker():
|
||||
try:
|
||||
if model_path is not None:
|
||||
logging.info(f"Install model '{json_data['name']}' from '{model_url}' into '{model_path}'")
|
||||
if not core.get_config()['model_download_by_agent'] and (
|
||||
|
||||
if json_data['filename'] == '<huggingface>':
|
||||
if os.path.exists(os.path.join(model_path, os.path.dirname(json_data['url']))):
|
||||
logging.error(f"[ComfyUI-Manager] the model path already exists: {model_path}")
|
||||
return f"The model path already exists: {model_path}"
|
||||
|
||||
logging.info(f"[ComfyUI-Manager] Downloading '{model_url}' into '{model_path}'")
|
||||
manager_downloader.download_repo_in_bytes(repo_id=model_url, local_dir=model_path)
|
||||
|
||||
return 'success'
|
||||
|
||||
elif not core.get_config()['model_download_by_agent'] and (
|
||||
model_url.startswith('https://github.com') or model_url.startswith('https://huggingface.co') or model_url.startswith('https://heibox.uni-heidelberg.de')):
|
||||
model_dir = get_model_dir(json_data, True)
|
||||
download_url(model_url, model_dir, filename=json_data['filename'])
|
||||
@ -493,13 +510,13 @@ async def task_worker():
|
||||
if res and model_path.endswith('.zip'):
|
||||
res = core.unzip(model_path)
|
||||
else:
|
||||
logging.error(f"Model installation error: invalid model type - {json_data['type']}")
|
||||
logging.error(f"[ComfyUI-Manager] Model installation error: invalid model type - {json_data['type']}")
|
||||
|
||||
if res:
|
||||
return 'success'
|
||||
|
||||
except Exception as e:
|
||||
logging.error(f"[ERROR] {e}", file=sys.stderr)
|
||||
logging.error(f"[ComfyUI-Manager] ERROR: {e}", file=sys.stderr)
|
||||
|
||||
return f"Model installation error: {model_url}"
|
||||
|
||||
@ -786,15 +803,18 @@ async def fetch_customnode_alternatives(request):
|
||||
|
||||
|
||||
def check_model_installed(json_obj):
|
||||
def is_exists(model_dir_name, file_name):
|
||||
def is_exists(model_dir_name, filename, url):
|
||||
if filename == '<huggingface>':
|
||||
filename = os.path.basename(url)
|
||||
|
||||
dirs = folder_paths.get_folder_paths(model_dir_name)
|
||||
|
||||
for x in dirs:
|
||||
if os.path.exists(os.path.join(x, file_name)):
|
||||
if os.path.exists(os.path.join(x, filename)):
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
|
||||
model_dir_names = ['checkpoints', 'loras', 'vae', 'text_encoders', 'diffusion_models', 'clip_vision', 'embeddings',
|
||||
'diffusers', 'vae_approx', 'controlnet', 'gligen', 'upscale_models', 'hypernetworks',
|
||||
'photomaker', 'classifiers']
|
||||
@ -814,23 +834,30 @@ def check_model_installed(json_obj):
|
||||
if item['save_path'] == 'default':
|
||||
model_dir_name = model_dir_name_map.get(item['type'].lower())
|
||||
if model_dir_name is not None:
|
||||
item['installed'] = str(is_exists(model_dir_name, item['filename']))
|
||||
item['installed'] = str(is_exists(model_dir_name, item['filename'], item['url']))
|
||||
else:
|
||||
item['installed'] = 'False'
|
||||
else:
|
||||
model_dir_name = item['save_path'].split('/')[0]
|
||||
if model_dir_name in folder_paths.folder_names_and_paths:
|
||||
if is_exists(model_dir_name, item['filename']):
|
||||
if is_exists(model_dir_name, item['filename'], item['url']):
|
||||
item['installed'] = 'True'
|
||||
|
||||
if 'installed' not in item:
|
||||
fullpath = os.path.join(folder_paths.models_dir, item['save_path'], item['filename'])
|
||||
if item['filename'] == '<huggingface>':
|
||||
filename = os.path.basename(item['url'])
|
||||
else:
|
||||
filename = item['filename']
|
||||
|
||||
fullpath = os.path.join(folder_paths.models_dir, item['save_path'], filename)
|
||||
|
||||
item['installed'] = 'True' if os.path.exists(fullpath) else 'False'
|
||||
|
||||
with concurrent.futures.ThreadPoolExecutor(8) as executor:
|
||||
for item in json_obj['models']:
|
||||
executor.submit(process_model_phase, item)
|
||||
|
||||
|
||||
@routes.get("/externalmodel/getlist")
|
||||
async def fetch_externalmodel_list(request):
|
||||
json_obj = await core.get_data_by_mode(request.rel_url.query["mode"], 'model-list.json')
|
||||
@ -1337,10 +1364,10 @@ async def install_model(request):
|
||||
|
||||
if not is_allowed_security_level('middle'):
|
||||
logging.error(SECURITY_MESSAGE_MIDDLE_OR_BELOW)
|
||||
return web.Response(status=403)
|
||||
return web.Response(status=403, text="A security error has occurred. Please check the terminal logs")
|
||||
|
||||
if not json_data['filename'].endswith('.safetensors') and not is_allowed_security_level('high'):
|
||||
models_json = await core.get_data_by_mode('cache', 'model-list.json')
|
||||
models_json = await core.get_data_by_mode('cache', 'model-list.json', 'default')
|
||||
|
||||
is_belongs_to_whitelist = False
|
||||
for x in models_json['models']:
|
||||
@ -1349,8 +1376,8 @@ async def install_model(request):
|
||||
break
|
||||
|
||||
if not is_belongs_to_whitelist:
|
||||
logging.error(SECURITY_MESSAGE_NORMAL_MINUS)
|
||||
return web.Response(status=403)
|
||||
logging.error(SECURITY_MESSAGE_NORMAL_MINUS_MODEL)
|
||||
return web.Response(status=403, text="A security error has occurred. Please check the terminal logs")
|
||||
|
||||
install_item = json_data.get('ui_id'), json_data
|
||||
task_queue.put(("install-model", install_item))
|
||||
|
||||
@ -1369,14 +1369,14 @@ export class CustomNodesManager {
|
||||
});
|
||||
|
||||
if (res.status != 200) {
|
||||
errorMsg = `${item.title} ${mode} failed: `;
|
||||
errorMsg = `'${item.title}': `;
|
||||
|
||||
if(res.status == 403) {
|
||||
errorMsg += `This action is not allowed with this security level configuration.`;
|
||||
errorMsg += `This action is not allowed with this security level configuration.\n`;
|
||||
} else if(res.status == 404) {
|
||||
errorMsg += `With the current security level configuration, only custom nodes from the <B>"default channel"</B> can be installed.`;
|
||||
errorMsg += `With the current security level configuration, only custom nodes from the <B>"default channel"</B> can be installed.\n`;
|
||||
} else {
|
||||
errorMsg += await res.text();
|
||||
errorMsg += await res.text() + '\n';
|
||||
}
|
||||
|
||||
break;
|
||||
@ -1387,11 +1387,11 @@ export class CustomNodesManager {
|
||||
|
||||
if(errorMsg) {
|
||||
this.showError(errorMsg);
|
||||
show_message("Installation Error:\n"+errorMsg);
|
||||
show_message("[Installation Errors]\n"+errorMsg);
|
||||
|
||||
// reset
|
||||
for(let k in target_items) {
|
||||
let item = this.install_context.targets[k];
|
||||
const item = target_items[k];
|
||||
this.grid.updateCell(item, "action");
|
||||
}
|
||||
}
|
||||
|
||||
@ -640,7 +640,6 @@ export class ModelManager {
|
||||
}
|
||||
|
||||
btn.classList.add("cmm-btn-loading");
|
||||
this.showLoading();
|
||||
this.showError("");
|
||||
|
||||
let needRefresh = false;
|
||||
@ -671,7 +670,14 @@ export class ModelManager {
|
||||
});
|
||||
|
||||
if (res.status != 200) {
|
||||
errorMsg = `Install failed: ${item.name} ${res.error.message}`;
|
||||
errorMsg = `'${item.name}': `;
|
||||
|
||||
if(res.status == 403) {
|
||||
errorMsg += `This action is not allowed with this security level configuration.\n`;
|
||||
} else {
|
||||
errorMsg += await res.text() + '\n';
|
||||
}
|
||||
|
||||
break;
|
||||
}
|
||||
}
|
||||
@ -680,11 +686,11 @@ export class ModelManager {
|
||||
|
||||
if(errorMsg) {
|
||||
this.showError(errorMsg);
|
||||
show_message("Installation Error:\n"+errorMsg);
|
||||
show_message("[Installation Errors]\n"+errorMsg);
|
||||
|
||||
// reset
|
||||
for (const hash of list) {
|
||||
const item = this.grid.getRowItemBy("hash", hash);
|
||||
for(let k in target_items) {
|
||||
const item = target_items[k];
|
||||
this.grid.updateCell(item, "installed");
|
||||
}
|
||||
}
|
||||
|
||||
@ -4662,6 +4662,29 @@
|
||||
"filename": "customnet_inpaint_v1.pt",
|
||||
"url": "https://huggingface.co/TencentARC/CustomNet/resolve/main/customnet_inpaint_v1.pt",
|
||||
"size": "5.71GB"
|
||||
},
|
||||
|
||||
{
|
||||
"name": "deepseek-ai/Janus-Pro-1B",
|
||||
"type": "Janus-Pro",
|
||||
"base": "Janus-Pro",
|
||||
"save_path": "Janus-Pro",
|
||||
"description": "[SNAPSHOT] Janus-Pro-1B model.[w/You cannot download this item on ComfyUI-Manager versions below V3.18]",
|
||||
"reference": "https://huggingface.co/deepseek-ai/Janus-Pro-1B",
|
||||
"filename": "<huggingface>",
|
||||
"url": "deepseek-ai/Janus-Pro-1B",
|
||||
"size": "7.8GB"
|
||||
},
|
||||
{
|
||||
"name": "deepseek-ai/Janus-Pro-7B",
|
||||
"type": "Janus-Pro",
|
||||
"base": "Janus-Pro",
|
||||
"save_path": "Janus-Pro",
|
||||
"description": "[SNAPSHOT] Janus-Pro-7B model.[w/You cannot download this item on ComfyUI-Manager versions below V3.18]",
|
||||
"reference": "https://huggingface.co/deepseek-ai/Janus-Pro-7B",
|
||||
"filename": "<huggingface>",
|
||||
"url": "deepseek-ai/Janus-Pro-7B",
|
||||
"size": "14.85GB"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
@ -1,5 +1,28 @@
|
||||
{
|
||||
"models": [
|
||||
{
|
||||
"name": "deepseek-ai/Janus-Pro-1B",
|
||||
"type": "Janus-Pro",
|
||||
"base": "Janus-Pro",
|
||||
"save_path": "Janus-Pro",
|
||||
"description": "[SNAPSHOT] Janus-Pro-1B model.[w/You cannot download this item on ComfyUI-Manager versions below V3.18]",
|
||||
"reference": "https://huggingface.co/deepseek-ai/Janus-Pro-1B",
|
||||
"filename": "<huggingface>",
|
||||
"url": "deepseek-ai/Janus-Pro-1B",
|
||||
"size": "7.8GB"
|
||||
},
|
||||
{
|
||||
"name": "deepseek-ai/Janus-Pro-7B",
|
||||
"type": "Janus-Pro",
|
||||
"base": "Janus-Pro",
|
||||
"save_path": "Janus-Pro",
|
||||
"description": "[SNAPSHOT] Janus-Pro-7B model.[w/You cannot download this item on ComfyUI-Manager versions below V3.18]",
|
||||
"reference": "https://huggingface.co/deepseek-ai/Janus-Pro-7B",
|
||||
"filename": "<huggingface>",
|
||||
"url": "deepseek-ai/Janus-Pro-7B",
|
||||
"size": "14.85GB"
|
||||
},
|
||||
|
||||
{
|
||||
"name": "Leoxing/pia.ckpt",
|
||||
"type": "animatediff-pia",
|
||||
|
||||
@ -1,7 +1,7 @@
|
||||
[project]
|
||||
name = "comfyui-manager"
|
||||
description = "ComfyUI-Manager provides features to install and manage custom nodes for ComfyUI, as well as various functionalities to assist with ComfyUI."
|
||||
version = "3.17.11"
|
||||
version = "3.18"
|
||||
license = { file = "LICENSE.txt" }
|
||||
dependencies = ["GitPython", "PyGithub", "matrix-client==0.4.0", "transformers", "huggingface-hub>0.20", "typer", "rich", "typing-extensions"]
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user