From 897debb1066b5e03f21286816d0707992fae3ceb Mon Sep 17 00:00:00 2001 From: "Dr.Lt.Data" Date: Mon, 13 Jan 2025 06:10:59 +0900 Subject: [PATCH] improved: prevent hang UI while CNR loading fixed: normalize id for pyproject.toml --- cm-cli.py | 2 +- glob/cnr_utils.py | 38 ++++++++++++++++++++++++++++++++------ glob/manager_core.py | 8 ++++---- glob/manager_server.py | 2 +- glob/manager_util.py | 17 +++++++++++++++-- pyproject.toml | 2 +- 6 files changed, 54 insertions(+), 15 deletions(-) diff --git a/cm-cli.py b/cm-cli.py index 80dde26e..85781089 100644 --- a/cm-cli.py +++ b/cm-cli.py @@ -118,7 +118,7 @@ class Ctx: if channel is not None: self.channel = channel - asyncio.run(unified_manager.reload(cache_mode=self.mode == 'cache')) + asyncio.run(unified_manager.reload(cache_mode=self.mode == 'cache', dont_wait=False)) asyncio.run(unified_manager.load_nightly(self.channel, self.mode)) def set_no_deps(self, no_deps): diff --git a/glob/cnr_utils.py b/glob/cnr_utils.py index 3c6431e8..12287c36 100644 --- a/glob/cnr_utils.py +++ b/glob/cnr_utils.py @@ -4,23 +4,49 @@ from typing import List import manager_util import toml import os +import asyncio base_url = "https://api.comfy.org" -async def get_cnr_data(page=1, limit=1000, cache_mode=True): - try: - uri = f'{base_url}/nodes?page={page}&limit={limit}' - json_obj = await manager_util.get_data_with_cache(uri, cache_mode=cache_mode) +lock = asyncio.Lock() +is_cache_loading = False + +async def get_cnr_data(page=1, limit=1000, cache_mode=True, dont_wait=True): + global is_cache_loading + + uri = f'{base_url}/nodes?page={page}&limit={limit}' + + def touch(json_obj): for v in json_obj['nodes']: if 'latest_version' not in v: v['latest_version'] = dict(version='nightly') + + if cache_mode: + if dont_wait: + json_obj = await manager_util.get_data_with_cache(uri, cache_mode=cache_mode, dont_wait=True) # fallback + + if 'nodes' in json_obj: + touch(json_obj) + return json_obj['nodes'] + else: + return {} + + is_cache_loading = True + + try: + json_obj = await manager_util.get_data_with_cache(uri, cache_mode=cache_mode) + touch(json_obj) + return json_obj['nodes'] except: res = {} print("Cannot connect to comfyregistry.") + finally: + if cache_mode: + is_cache_loading = False return res @@ -92,7 +118,7 @@ def install_node(node_id, version=None): def all_versions_of_node(node_id): - url = f"https://api.comfy.org/nodes/{node_id}/versions?statuses=NodeVersionStatusActive&statuses=NodeVersionStatusPending" + url = f"{base_url}/nodes/{node_id}/versions?statuses=NodeVersionStatusActive&statuses=NodeVersionStatusPending" response = requests.get(url) if response.status_code == 200: @@ -113,7 +139,7 @@ def read_cnr_info(fullpath): data = toml.load(f) project = data.get('project', {}) - name = project.get('name') + name = project.get('name').strip().lower() version = project.get('version') urls = project.get('urls', {}) diff --git a/glob/manager_core.py b/glob/manager_core.py index 8590a21f..e290548d 100644 --- a/glob/manager_core.py +++ b/glob/manager_core.py @@ -41,7 +41,7 @@ import manager_downloader from node_package import InstalledNodePackage -version_code = [3, 6, 5] +version_code = [3, 7] version_str = f"V{version_code[0]}.{version_code[1]}" + (f'.{version_code[2]}' if len(version_code) > 2 else '') @@ -507,7 +507,7 @@ class UnifiedManager: if node_package.is_disabled and node_package.is_nightly: self.nightly_inactive_nodes[node_package.id] = node_package.fullpath - if node_package.is_enabled: + if node_package.is_enabled and not node_package.is_unknown: self.active_nodes[node_package.id] = node_package.version, node_package.fullpath if node_package.is_enabled and node_package.is_unknown: @@ -664,7 +664,7 @@ class UnifiedManager: return latest - async def reload(self, cache_mode): + async def reload(self, cache_mode, dont_wait=True): self.custom_node_map_cache = {} self.cnr_inactive_nodes = {} # node_id -> node_version -> fullpath self.nightly_inactive_nodes = {} # node_id -> fullpath @@ -673,7 +673,7 @@ class UnifiedManager: self.active_nodes = {} # node_id -> node_version * fullpath # reload 'cnr_map' and 'repo_cnr_map' - cnrs = await cnr_utils.get_cnr_data(cache_mode=cache_mode) + cnrs = await cnr_utils.get_cnr_data(cache_mode=cache_mode, dont_wait=dont_wait) for x in cnrs: self.cnr_map[x['id']] = x diff --git a/glob/manager_server.py b/glob/manager_server.py index a53688f6..781b68b4 100644 --- a/glob/manager_server.py +++ b/glob/manager_server.py @@ -1415,7 +1415,7 @@ async def default_cache_update(): await asyncio.gather(a, b, c, d, e) # load at least once - await core.unified_manager.reload('cache') + await core.unified_manager.reload('cache', dont_wait=False) await core.unified_manager.get_custom_nodes('default', 'cache') # NOTE: hide migration button temporarily. diff --git a/glob/manager_util.py b/glob/manager_util.py index e9369ef5..9fb6a914 100644 --- a/glob/manager_util.py +++ b/glob/manager_util.py @@ -11,6 +11,8 @@ from datetime import datetime import subprocess import sys import re +import logging + cache_lock = threading.Lock() @@ -128,15 +130,26 @@ async def get_data(uri, silent=False): return json_obj -async def get_data_with_cache(uri, silent=False, cache_mode=True): +async def get_data_with_cache(uri, silent=False, cache_mode=True, dont_wait=False): cache_uri = str(simple_hash(uri)) + '_' + os.path.basename(uri).replace('&', "_").replace('?', "_").replace('=', "_") cache_uri = os.path.join(cache_dir, cache_uri+'.json') + if cache_mode and dont_wait: + # NOTE: return the cache if possible, even if it is expired, so do not cache + if not os.path.exists(cache_uri): + logging.error(f"[ComfyUI-Manager] The network connection is unstable, so it is operating in fallback mode: {uri}") + + return {} + else: + if not is_file_created_within_one_day(cache_uri): + logging.error(f"[ComfyUI-Manager] The network connection is unstable, so it is operating in outdated cache mode: {uri}") + + return await get_data(cache_uri, silent=silent) + if cache_mode and is_file_created_within_one_day(cache_uri): json_obj = await get_data(cache_uri, silent=silent) else: json_obj = await get_data(uri, silent=silent) - with cache_lock: with open(cache_uri, "w", encoding='utf-8') as file: json.dump(json_obj, file, indent=4, sort_keys=True) diff --git a/pyproject.toml b/pyproject.toml index 73280443..408ebf3e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "comfyui-manager" description = "ComfyUI-Manager provides features to install and manage custom nodes for ComfyUI, as well as various functionalities to assist with ComfyUI." -version = "3.6.5" +version = "3.7" license = { file = "LICENSE.txt" } dependencies = ["GitPython", "PyGithub", "matrix-client==0.4.0", "transformers", "huggingface-hub>0.20", "typer", "rich", "typing-extensions"]