diff --git a/README.md b/README.md index f4ac73ea..b8e6cc60 100644 --- a/README.md +++ b/README.md @@ -146,7 +146,15 @@ In `ComfyUI-Manager` V3.0 and later, configuration files and dynamically generat * Saved snapshot files: `/default/ComfyUI-Manager/snapshots` * Startup script files: `/default/ComfyUI-Manager/startup-scripts` * Component files: `/default/ComfyUI-Manager/components` - + + +## `extra_model_paths.yaml` Configuration +The following settings are applied based on the section marked as `is_default`. + +* `custom_nodes`: Path for installing custom nodes + * Importing does not need to adhere to the path set as `is_default`, but this is the path where custom nodes are installed by the `ComfyUI Nodes Manager`. +* `download_model_base`: Path for downloading models + ## Snapshot-Manager * When you press `Save snapshot` or use `Update All` on `Manager Menu`, the current installation status snapshot is saved. diff --git a/extras.json b/extras.json index e6332abc..22949306 100644 --- a/extras.json +++ b/extras.json @@ -9,7 +9,7 @@ "comfyui-layerdiffuse", "comfyui-liveportraitkj", "aigodlike-comfyui-translation", - "comfyui-reactor-node", + "comfyui-reactor", "comfyui_instantid", "sd-dynamic-thresholding", "pr-was-node-suite-comfyui-47064894", diff --git a/glob/cnr_utils.py b/glob/cnr_utils.py index 12287c36..7ee6a906 100644 --- a/glob/cnr_utils.py +++ b/glob/cnr_utils.py @@ -5,6 +5,7 @@ import manager_util import toml import os import asyncio +import json base_url = "https://api.comfy.org" @@ -13,33 +14,55 @@ lock = asyncio.Lock() is_cache_loading = False -async def get_cnr_data(page=1, limit=1000, cache_mode=True, dont_wait=True): +async def get_cnr_data(cache_mode=True, dont_wait=True): global is_cache_loading - uri = f'{base_url}/nodes?page={page}&limit={limit}' + uri = f'{base_url}/nodes' - def touch(json_obj): - for v in json_obj['nodes']: + async def fetch_all(): + remained = True + page = 1 + + full_nodes = {} + while remained: + sub_uri = f'{base_url}/nodes?page={page}&limit=30' + sub_json_obj = await manager_util.get_data_with_cache(sub_uri, cache_mode=False, silent=True) + remained = page < sub_json_obj['totalPages'] + + for x in sub_json_obj['nodes']: + full_nodes[x['id']] = x + + if page % 5 == 0: + print(f"FETCH ComfyRegistry Data: {page}/{sub_json_obj['totalPages']}") + page += 1 + + print("FETCH ComfyRegistry Data [DONE]") + + for v in full_nodes.values(): if 'latest_version' not in v: v['latest_version'] = dict(version='nightly') + return {'nodes': list(full_nodes.values())} if cache_mode: - if dont_wait: - json_obj = await manager_util.get_data_with_cache(uri, cache_mode=cache_mode, dont_wait=True) # fallback - - if 'nodes' in json_obj: - touch(json_obj) - return json_obj['nodes'] - else: - return {} - is_cache_loading = True + cache_state = manager_util.get_cache_state(uri) + + if dont_wait: + if cache_state == 'not-cached': + return {} + else: + print("[ComfyUI-Manager] The ComfyRegistry cache update is still in progress, so an outdated cache is being used.") + with open(manager_util.get_cache_path(uri), 'r', encoding="UTF-8", errors="ignore") as json_file: + return json.load(json_file)['nodes'] + + if cache_state == 'cached': + with open(manager_util.get_cache_path(uri), 'r', encoding="UTF-8", errors="ignore") as json_file: + return json.load(json_file)['nodes'] try: - json_obj = await manager_util.get_data_with_cache(uri, cache_mode=cache_mode) - touch(json_obj) - + json_obj = await fetch_all() + manager_util.save_to_cache(uri, json_obj) return json_obj['nodes'] except: res = {} diff --git a/glob/manager_core.py b/glob/manager_core.py index 0fb70fb5..d6d18046 100644 --- a/glob/manager_core.py +++ b/glob/manager_core.py @@ -41,7 +41,7 @@ import manager_downloader from node_package import InstalledNodePackage -version_code = [3, 8, 1] +version_code = [3, 9] version_str = f"V{version_code[0]}.{version_code[1]}" + (f'.{version_code[2]}' if len(version_code) > 2 else '') @@ -682,11 +682,10 @@ class UnifiedManager: self.active_nodes = {} # node_id -> node_version * fullpath # reload 'cnr_map' and 'repo_cnr_map' - cnrs = await cnr_utils.get_cnr_data(cache_mode=cache_mode, dont_wait=dont_wait) + cnrs = await cnr_utils.get_cnr_data(cache_mode=cache_mode=='cache', dont_wait=dont_wait) for x in cnrs: self.cnr_map[x['id']] = x - if 'repository' in x: normalized_url = git_utils.normalize_url(x['repository']) self.repo_cnr_map[normalized_url] = x @@ -2696,8 +2695,8 @@ def map_to_unified_keys(json_obj): return res -async def get_unified_total_nodes(channel, mode): - await unified_manager.reload(mode) +async def get_unified_total_nodes(channel, mode, regsitry_cache_mode='cache'): + await unified_manager.reload(regsitry_cache_mode) res = await unified_manager.get_custom_nodes(channel, mode) @@ -2780,6 +2779,7 @@ async def get_unified_total_nodes(channel, mode): author = cnr['publisher']['name'] title = cnr['name'] reference = f"https://registry.comfy.org/nodes/{cnr['id']}" + repository = cnr.get('repository', '') install_type = "cnr" description = cnr.get('description', '') @@ -2811,7 +2811,7 @@ async def get_unified_total_nodes(channel, mode): if ver is None: ver = cnr['latest_version']['version'] - item = dict(author=author, title=title, reference=reference, install_type=install_type, + item = dict(author=author, title=title, reference=reference, repository=repository, install_type=install_type, description=description, state=state, updatable=updatable, version=ver) if active_version: diff --git a/glob/manager_server.py b/glob/manager_server.py index c420e4d9..a686f9c4 100644 --- a/glob/manager_server.py +++ b/glob/manager_server.py @@ -566,7 +566,7 @@ async def fetch_customnode_list(request): else: channel = core.get_config()['channel_url'] - node_packs = await core.get_unified_total_nodes(channel, request.rel_url.query["mode"]) + node_packs = await core.get_unified_total_nodes(channel, request.rel_url.query["mode"], 'cache') json_obj_github = core.get_data_by_mode(request.rel_url.query["mode"], 'github-stats.json', 'default') json_obj_extras = core.get_data_by_mode(request.rel_url.query["mode"], 'extras.json', 'default') @@ -1434,8 +1434,8 @@ async def default_cache_update(): await asyncio.gather(a, b, c, d, e) # load at least once - await core.unified_manager.reload('cache', dont_wait=False) - await core.unified_manager.get_custom_nodes('default', 'cache') + await core.unified_manager.reload('remote', dont_wait=False) + await core.unified_manager.get_custom_nodes('default', 'remote') # NOTE: hide migration button temporarily. # if not core.get_config()['skip_migration_check']: diff --git a/glob/manager_util.py b/glob/manager_util.py index 7ab6b625..48a2007f 100644 --- a/glob/manager_util.py +++ b/glob/manager_util.py @@ -130,9 +130,34 @@ async def get_data(uri, silent=False): return json_obj -async def get_data_with_cache(uri, silent=False, cache_mode=True, dont_wait=False): +def get_cache_path(uri): cache_uri = str(simple_hash(uri)) + '_' + os.path.basename(uri).replace('&', "_").replace('?', "_").replace('=', "_") - cache_uri = os.path.join(cache_dir, cache_uri+'.json') + return os.path.join(cache_dir, cache_uri+'.json') + + +def get_cache_state(uri): + cache_uri = get_cache_path(uri) + + if not os.path.exists(cache_uri): + return "not-cached" + elif is_file_created_within_one_day(cache_uri): + return "cached" + + return "expired" + + +def save_to_cache(uri, json_obj, silent=False): + cache_uri = get_cache_path(uri) + + with cache_lock: + with open(cache_uri, "w", encoding='utf-8') as file: + json.dump(json_obj, file, indent=4, sort_keys=True) + if not silent: + logging.info(f"[ComfyUI-Manager] default cache updated: {uri}") + + +async def get_data_with_cache(uri, silent=False, cache_mode=True, dont_wait=False): + cache_uri = get_cache_path(uri) if cache_mode and dont_wait: # NOTE: return the cache if possible, even if it is expired, so do not cache diff --git a/js/custom-nodes-manager.js b/js/custom-nodes-manager.js index e147c34a..2996be1c 100644 --- a/js/custom-nodes-manager.js +++ b/js/custom-nodes-manager.js @@ -975,7 +975,10 @@ export class CustomNodesManager { } const link = document.createElement('a'); - link.href = rowItem.reference; + if(rowItem.originalData.repository) + link.href = rowItem.originalData.repository; + else + link.href = rowItem.reference; link.target = '_blank'; link.innerHTML = `${title}`; container.appendChild(link); diff --git a/pyproject.toml b/pyproject.toml index 06a31313..972311c5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "comfyui-manager" description = "ComfyUI-Manager provides features to install and manage custom nodes for ComfyUI, as well as various functionalities to assist with ComfyUI." -version = "3.8.1" +version = "3.9" license = { file = "LICENSE.txt" } dependencies = ["GitPython", "PyGithub", "matrix-client==0.4.0", "transformers", "huggingface-hub>0.20", "typer", "rich", "typing-extensions"]