diff --git a/glob/manager_core.py b/glob/manager_core.py index b7bfa1f7..c8ce863f 100644 --- a/glob/manager_core.py +++ b/glob/manager_core.py @@ -42,7 +42,7 @@ import manager_downloader from node_package import InstalledNodePackage -version_code = [3, 17, 11] +version_code = [3, 18] version_str = f"V{version_code[0]}.{version_code[1]}" + (f'.{version_code[2]}' if len(version_code) > 2 else '') @@ -2098,7 +2098,7 @@ async def get_data_by_mode(mode, filename, channel_url=None): cache_uri = os.path.join(manager_util.cache_dir, cache_uri) if mode == "cache" and manager_util.is_file_created_within_one_day(cache_uri): - json_obj = await manager_util.get_data(cache_uri) + json_obj = await manager_util.get_data(cache_uri) else: json_obj = await manager_util.get_data(uri) with manager_util.cache_lock: diff --git a/glob/manager_downloader.py b/glob/manager_downloader.py index 41ba7649..cce62e50 100644 --- a/glob/manager_downloader.py +++ b/glob/manager_downloader.py @@ -3,6 +3,11 @@ from urllib.parse import urlparse import urllib import sys import logging +import requests +from huggingface_hub import HfApi +from tqdm.auto import tqdm + + aria2 = os.getenv('COMFYUI_MANAGER_ARIA2_SERVER') HF_ENDPOINT = os.getenv('HF_ENDPOINT') @@ -117,3 +122,37 @@ def download_url_with_agent(url, save_path): print("Installation was successful.") return True + +# NOTE: snapshot_download doesn't provide file size tqdm. +def download_repo_in_bytes(repo_id, local_dir): + api = HfApi() + repo_info = api.repo_info(repo_id=repo_id, files_metadata=True) + + os.makedirs(local_dir, exist_ok=True) + + total_size = 0 + for file_info in repo_info.siblings: + if file_info.size is not None: + total_size += file_info.size + + pbar = tqdm(total=total_size, unit="B", unit_scale=True, desc="Downloading") + + for file_info in repo_info.siblings: + out_path = os.path.join(local_dir, file_info.rfilename) + os.makedirs(os.path.dirname(out_path), exist_ok=True) + + if file_info.size is None: + continue + + download_url = f"https://huggingface.co/{repo_id}/resolve/main/{file_info.rfilename}" + + with requests.get(download_url, stream=True) as r, open(out_path, "wb") as f: + r.raise_for_status() + for chunk in r.iter_content(chunk_size=65536): + if chunk: + f.write(chunk) + pbar.update(len(chunk)) + + pbar.close() + + diff --git a/glob/manager_server.py b/glob/manager_server.py index f47a2a6f..c8089b46 100644 --- a/glob/manager_server.py +++ b/glob/manager_server.py @@ -21,6 +21,8 @@ import logging import asyncio import queue +import manager_downloader + logging.info(f"### Loading: ComfyUI-Manager ({core.version_str})") @@ -30,6 +32,7 @@ comfyui_tag = None SECURITY_MESSAGE_MIDDLE_OR_BELOW = "ERROR: To use this action, a security_level of `middle or below` is required. Please contact the administrator.\nReference: https://github.com/ltdrdata/ComfyUI-Manager#security-policy" SECURITY_MESSAGE_NORMAL_MINUS = "ERROR: To use this feature, you must either set '--listen' to a local IP and set the security level to 'normal-' or lower, or set the security level to 'middle' or 'weak'. Please contact the administrator.\nReference: https://github.com/ltdrdata/ComfyUI-Manager#security-policy" SECURITY_MESSAGE_GENERAL = "ERROR: This installation is not allowed in this security_level. Please contact the administrator.\nReference: https://github.com/ltdrdata/ComfyUI-Manager#security-policy" +SECURITY_MESSAGE_NORMAL_MINUS_MODEL = "ERROR: Downloading models that are not in '.safetensors' format is only allowed for models registered in the 'default' channel at this security level. If you want to download this model, set the security level to 'normal-' or lower." routes = PromptServer.instance.routes @@ -305,7 +308,10 @@ def get_model_path(data, show_log=False): if base_model is None: return None else: - return os.path.join(base_model, data['filename']) + if data['filename'] == '': + return os.path.join(base_model, os.path.basename(data['url'])) + else: + return os.path.join(base_model, data['filename']) def check_state_of_git_node_pack(node_packs, do_fetch=False, do_update_check=True, do_update=False): @@ -477,7 +483,18 @@ async def task_worker(): try: if model_path is not None: logging.info(f"Install model '{json_data['name']}' from '{model_url}' into '{model_path}'") - if not core.get_config()['model_download_by_agent'] and ( + + if json_data['filename'] == '': + if os.path.exists(os.path.join(model_path, os.path.dirname(json_data['url']))): + logging.error(f"[ComfyUI-Manager] the model path already exists: {model_path}") + return f"The model path already exists: {model_path}" + + logging.info(f"[ComfyUI-Manager] Downloading '{model_url}' into '{model_path}'") + manager_downloader.download_repo_in_bytes(repo_id=model_url, local_dir=model_path) + + return 'success' + + elif not core.get_config()['model_download_by_agent'] and ( model_url.startswith('https://github.com') or model_url.startswith('https://huggingface.co') or model_url.startswith('https://heibox.uni-heidelberg.de')): model_dir = get_model_dir(json_data, True) download_url(model_url, model_dir, filename=json_data['filename']) @@ -493,13 +510,13 @@ async def task_worker(): if res and model_path.endswith('.zip'): res = core.unzip(model_path) else: - logging.error(f"Model installation error: invalid model type - {json_data['type']}") + logging.error(f"[ComfyUI-Manager] Model installation error: invalid model type - {json_data['type']}") if res: return 'success' except Exception as e: - logging.error(f"[ERROR] {e}", file=sys.stderr) + logging.error(f"[ComfyUI-Manager] ERROR: {e}", file=sys.stderr) return f"Model installation error: {model_url}" @@ -786,15 +803,18 @@ async def fetch_customnode_alternatives(request): def check_model_installed(json_obj): - def is_exists(model_dir_name, file_name): + def is_exists(model_dir_name, filename, url): + if filename == '': + filename = os.path.basename(url) + dirs = folder_paths.get_folder_paths(model_dir_name) + for x in dirs: - if os.path.exists(os.path.join(x, file_name)): + if os.path.exists(os.path.join(x, filename)): return True return False - model_dir_names = ['checkpoints', 'loras', 'vae', 'text_encoders', 'diffusion_models', 'clip_vision', 'embeddings', 'diffusers', 'vae_approx', 'controlnet', 'gligen', 'upscale_models', 'hypernetworks', 'photomaker', 'classifiers'] @@ -814,23 +834,30 @@ def check_model_installed(json_obj): if item['save_path'] == 'default': model_dir_name = model_dir_name_map.get(item['type'].lower()) if model_dir_name is not None: - item['installed'] = str(is_exists(model_dir_name, item['filename'])) + item['installed'] = str(is_exists(model_dir_name, item['filename'], item['url'])) else: item['installed'] = 'False' else: model_dir_name = item['save_path'].split('/')[0] if model_dir_name in folder_paths.folder_names_and_paths: - if is_exists(model_dir_name, item['filename']): + if is_exists(model_dir_name, item['filename'], item['url']): item['installed'] = 'True' if 'installed' not in item: - fullpath = os.path.join(folder_paths.models_dir, item['save_path'], item['filename']) + if item['filename'] == '': + filename = os.path.basename(item['url']) + else: + filename = item['filename'] + + fullpath = os.path.join(folder_paths.models_dir, item['save_path'], filename) + item['installed'] = 'True' if os.path.exists(fullpath) else 'False' with concurrent.futures.ThreadPoolExecutor(8) as executor: for item in json_obj['models']: executor.submit(process_model_phase, item) + @routes.get("/externalmodel/getlist") async def fetch_externalmodel_list(request): json_obj = await core.get_data_by_mode(request.rel_url.query["mode"], 'model-list.json') @@ -1337,10 +1364,10 @@ async def install_model(request): if not is_allowed_security_level('middle'): logging.error(SECURITY_MESSAGE_MIDDLE_OR_BELOW) - return web.Response(status=403) + return web.Response(status=403, text="A security error has occurred. Please check the terminal logs") if not json_data['filename'].endswith('.safetensors') and not is_allowed_security_level('high'): - models_json = await core.get_data_by_mode('cache', 'model-list.json') + models_json = await core.get_data_by_mode('cache', 'model-list.json', 'default') is_belongs_to_whitelist = False for x in models_json['models']: @@ -1349,8 +1376,8 @@ async def install_model(request): break if not is_belongs_to_whitelist: - logging.error(SECURITY_MESSAGE_NORMAL_MINUS) - return web.Response(status=403) + logging.error(SECURITY_MESSAGE_NORMAL_MINUS_MODEL) + return web.Response(status=403, text="A security error has occurred. Please check the terminal logs") install_item = json_data.get('ui_id'), json_data task_queue.put(("install-model", install_item)) diff --git a/js/custom-nodes-manager.js b/js/custom-nodes-manager.js index dd0a975f..6ad026d4 100644 --- a/js/custom-nodes-manager.js +++ b/js/custom-nodes-manager.js @@ -1369,14 +1369,14 @@ export class CustomNodesManager { }); if (res.status != 200) { - errorMsg = `${item.title} ${mode} failed: `; + errorMsg = `'${item.title}': `; if(res.status == 403) { - errorMsg += `This action is not allowed with this security level configuration.`; + errorMsg += `This action is not allowed with this security level configuration.\n`; } else if(res.status == 404) { - errorMsg += `With the current security level configuration, only custom nodes from the "default channel" can be installed.`; + errorMsg += `With the current security level configuration, only custom nodes from the "default channel" can be installed.\n`; } else { - errorMsg += await res.text(); + errorMsg += await res.text() + '\n'; } break; @@ -1387,11 +1387,11 @@ export class CustomNodesManager { if(errorMsg) { this.showError(errorMsg); - show_message("Installation Error:\n"+errorMsg); + show_message("[Installation Errors]\n"+errorMsg); // reset for(let k in target_items) { - let item = this.install_context.targets[k]; + const item = target_items[k]; this.grid.updateCell(item, "action"); } } diff --git a/js/model-manager.js b/js/model-manager.js index b86f1219..c46d71c4 100644 --- a/js/model-manager.js +++ b/js/model-manager.js @@ -640,7 +640,6 @@ export class ModelManager { } btn.classList.add("cmm-btn-loading"); - this.showLoading(); this.showError(""); let needRefresh = false; @@ -671,7 +670,14 @@ export class ModelManager { }); if (res.status != 200) { - errorMsg = `Install failed: ${item.name} ${res.error.message}`; + errorMsg = `'${item.name}': `; + + if(res.status == 403) { + errorMsg += `This action is not allowed with this security level configuration.\n`; + } else { + errorMsg += await res.text() + '\n'; + } + break; } } @@ -680,11 +686,11 @@ export class ModelManager { if(errorMsg) { this.showError(errorMsg); - show_message("Installation Error:\n"+errorMsg); + show_message("[Installation Errors]\n"+errorMsg); // reset - for (const hash of list) { - const item = this.grid.getRowItemBy("hash", hash); + for(let k in target_items) { + const item = target_items[k]; this.grid.updateCell(item, "installed"); } } diff --git a/model-list.json b/model-list.json index 4e3796e6..5dc5c59e 100644 --- a/model-list.json +++ b/model-list.json @@ -4662,6 +4662,29 @@ "filename": "customnet_inpaint_v1.pt", "url": "https://huggingface.co/TencentARC/CustomNet/resolve/main/customnet_inpaint_v1.pt", "size": "5.71GB" + }, + + { + "name": "deepseek-ai/Janus-Pro-1B", + "type": "Janus-Pro", + "base": "Janus-Pro", + "save_path": "Janus-Pro", + "description": "[SNAPSHOT] Janus-Pro-1B model.[w/You cannot download this item on ComfyUI-Manager versions below V3.18]", + "reference": "https://huggingface.co/deepseek-ai/Janus-Pro-1B", + "filename": "", + "url": "deepseek-ai/Janus-Pro-1B", + "size": "7.8GB" + }, + { + "name": "deepseek-ai/Janus-Pro-7B", + "type": "Janus-Pro", + "base": "Janus-Pro", + "save_path": "Janus-Pro", + "description": "[SNAPSHOT] Janus-Pro-7B model.[w/You cannot download this item on ComfyUI-Manager versions below V3.18]", + "reference": "https://huggingface.co/deepseek-ai/Janus-Pro-7B", + "filename": "", + "url": "deepseek-ai/Janus-Pro-7B", + "size": "14.85GB" } ] } diff --git a/node_db/new/model-list.json b/node_db/new/model-list.json index b16f8c72..0d024062 100644 --- a/node_db/new/model-list.json +++ b/node_db/new/model-list.json @@ -1,5 +1,28 @@ { "models": [ + { + "name": "deepseek-ai/Janus-Pro-1B", + "type": "Janus-Pro", + "base": "Janus-Pro", + "save_path": "Janus-Pro", + "description": "[SNAPSHOT] Janus-Pro-1B model.[w/You cannot download this item on ComfyUI-Manager versions below V3.18]", + "reference": "https://huggingface.co/deepseek-ai/Janus-Pro-1B", + "filename": "", + "url": "deepseek-ai/Janus-Pro-1B", + "size": "7.8GB" + }, + { + "name": "deepseek-ai/Janus-Pro-7B", + "type": "Janus-Pro", + "base": "Janus-Pro", + "save_path": "Janus-Pro", + "description": "[SNAPSHOT] Janus-Pro-7B model.[w/You cannot download this item on ComfyUI-Manager versions below V3.18]", + "reference": "https://huggingface.co/deepseek-ai/Janus-Pro-7B", + "filename": "", + "url": "deepseek-ai/Janus-Pro-7B", + "size": "14.85GB" + }, + { "name": "Leoxing/pia.ckpt", "type": "animatediff-pia", diff --git a/pyproject.toml b/pyproject.toml index a6acdbd0..00d7d6a8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "comfyui-manager" description = "ComfyUI-Manager provides features to install and manage custom nodes for ComfyUI, as well as various functionalities to assist with ComfyUI." -version = "3.17.11" +version = "3.18" license = { file = "LICENSE.txt" } dependencies = ["GitPython", "PyGithub", "matrix-client==0.4.0", "transformers", "huggingface-hub>0.20", "typer", "rich", "typing-extensions"]