From f374ea714d8c32c2f214e818146b06754c963028 Mon Sep 17 00:00:00 2001 From: pythongosssss <125205205+pythongosssss@users.noreply.github.com> Date: Fri, 10 May 2024 22:07:46 +0100 Subject: [PATCH 1/9] Setting for saving and restoring canvas position and zoom level (#3437) --- web/scripts/app.js | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/web/scripts/app.js b/web/scripts/app.js index a3105c275..a0fe8b290 100644 --- a/web/scripts/app.js +++ b/web/scripts/app.js @@ -262,6 +262,36 @@ export class ComfyApp { }) ); } + + #addRestoreWorkflowView() { + const serialize = LGraph.prototype.serialize; + const self = this; + LGraph.prototype.serialize = function() { + const workflow = serialize.apply(this, arguments); + + // Store the drag & scale info in the serialized workflow if the setting is enabled + if (self.enableWorkflowViewRestore.value) { + if (!workflow.extra) { + workflow.extra = {}; + } + workflow.extra.ds = { + scale: self.canvas.ds.scale, + offset: self.canvas.ds.offset, + }; + } else if (workflow.extra?.ds) { + // Clear any old view data + delete workflow.extra.ds; + } + + return workflow; + } + this.enableWorkflowViewRestore = this.ui.settings.addSetting({ + id: "Comfy.EnableWorkflowViewRestore", + name: "Save and restore canvas position and zoom level in workflows", + type: "boolean", + defaultValue: true + }); + } /** * Adds special context menu handling for nodes @@ -1505,6 +1535,7 @@ export class ComfyApp { this.#addProcessKeyHandler(); this.#addConfigureHandler(); this.#addApiUpdateHandlers(); + this.#addRestoreWorkflowView(); this.graph = new LGraph(); @@ -1805,6 +1836,10 @@ export class ComfyApp { try { this.graph.configure(graphData); + if (this.enableWorkflowViewRestore.value && graphData.extra?.ds) { + this.canvas.ds.offset = graphData.extra.ds.offset; + this.canvas.ds.scale = graphData.extra.ds.scale; + } } catch (error) { let errorHint = []; // Try extracting filename to see if it was caused by an extension script From 4f63ee99f185db316a017520570310d094efffba Mon Sep 17 00:00:00 2001 From: comfyanonymous Date: Fri, 10 May 2024 17:30:52 -0400 Subject: [PATCH 2/9] Add a button to reset the view. --- web/scripts/app.js | 6 ++++++ web/scripts/ui.js | 7 +++++++ 2 files changed, 13 insertions(+) diff --git a/web/scripts/app.js b/web/scripts/app.js index a0fe8b290..7ed262cb2 100644 --- a/web/scripts/app.js +++ b/web/scripts/app.js @@ -2313,6 +2313,12 @@ export class ComfyApp { await this.#invokeExtensionsAsync("refreshComboInNodes", defs); } + resetView() { + app.canvas.ds.scale = 1; + app.canvas.ds.offset = [0, 0] + app.graph.setDirtyCanvas(true, true); + } + /** * Clean current state */ diff --git a/web/scripts/ui.js b/web/scripts/ui.js index d0fa46efb..36fed3238 100644 --- a/web/scripts/ui.js +++ b/web/scripts/ui.js @@ -597,16 +597,23 @@ export class ComfyUI { if (!confirmClear.value || confirm("Clear workflow?")) { app.clean(); app.graph.clear(); + app.resetView(); } } }), $el("button", { id: "comfy-load-default-button", textContent: "Load Default", onclick: async () => { if (!confirmClear.value || confirm("Load default workflow?")) { + app.resetView(); await app.loadGraphData() } } }), + $el("button", { + id: "comfy-reset-view-button", textContent: "Reset View", onclick: async () => { + app.resetView(); + } + }), ]); const devMode = this.settings.addSetting({ From e1489ad2576651a1384e9b82fd1991ac2f8764e0 Mon Sep 17 00:00:00 2001 From: comfyanonymous Date: Sat, 11 May 2024 21:46:05 -0400 Subject: [PATCH 3/9] Fix issue with lowvram mode breaking model saving. --- comfy/model_management.py | 8 ++++---- comfy/model_patcher.py | 12 +++++++++--- comfy/sd.py | 2 +- comfy_extras/nodes_model_merging.py | 2 +- 4 files changed, 15 insertions(+), 9 deletions(-) diff --git a/comfy/model_management.py b/comfy/model_management.py index 913b6844f..15dd73a6b 100644 --- a/comfy/model_management.py +++ b/comfy/model_management.py @@ -285,7 +285,7 @@ class LoadedModel: else: return self.model_memory() - def model_load(self, lowvram_model_memory=0): + def model_load(self, lowvram_model_memory=0, force_patch_weights=False): patch_model_to = self.device self.model.model_patches_to(self.device) @@ -295,7 +295,7 @@ class LoadedModel: try: if lowvram_model_memory > 0 and load_weights: - self.real_model = self.model.patch_model_lowvram(device_to=patch_model_to, lowvram_model_memory=lowvram_model_memory) + self.real_model = self.model.patch_model_lowvram(device_to=patch_model_to, lowvram_model_memory=lowvram_model_memory, force_patch_weights=force_patch_weights) else: self.real_model = self.model.patch_model(device_to=patch_model_to, patch_weights=load_weights) except Exception as e: @@ -379,7 +379,7 @@ def free_memory(memory_required, device, keep_loaded=[]): if mem_free_torch > mem_free_total * 0.25: soft_empty_cache() -def load_models_gpu(models, memory_required=0): +def load_models_gpu(models, memory_required=0, force_patch_weights=False): global vram_state inference_memory = minimum_inference_memory() @@ -444,7 +444,7 @@ def load_models_gpu(models, memory_required=0): if vram_set_state == VRAMState.NO_VRAM: lowvram_model_memory = 64 * 1024 * 1024 - cur_loaded_model = loaded_model.model_load(lowvram_model_memory) + cur_loaded_model = loaded_model.model_load(lowvram_model_memory, force_patch_weights=force_patch_weights) current_loaded_models.insert(0, loaded_model) return diff --git a/comfy/model_patcher.py b/comfy/model_patcher.py index cf51c4ad8..48e5be31e 100644 --- a/comfy/model_patcher.py +++ b/comfy/model_patcher.py @@ -272,7 +272,7 @@ class ModelPatcher: return self.model - def patch_model_lowvram(self, device_to=None, lowvram_model_memory=0): + def patch_model_lowvram(self, device_to=None, lowvram_model_memory=0, force_patch_weights=False): self.patch_model(device_to, patch_weights=False) logging.info("loading in lowvram mode {}".format(lowvram_model_memory/(1024 * 1024))) @@ -296,9 +296,15 @@ class ModelPatcher: if lowvram_weight: if weight_key in self.patches: - m.weight_function = LowVramPatch(weight_key, self) + if force_patch_weights: + self.patch_weight_to_device(weight_key) + else: + m.weight_function = LowVramPatch(weight_key, self) if bias_key in self.patches: - m.bias_function = LowVramPatch(bias_key, self) + if force_patch_weights: + self.patch_weight_to_device(bias_key) + else: + m.bias_function = LowVramPatch(bias_key, self) m.prev_comfy_cast_weights = m.comfy_cast_weights m.comfy_cast_weights = True diff --git a/comfy/sd.py b/comfy/sd.py index 9671e4aee..8044c184f 100644 --- a/comfy/sd.py +++ b/comfy/sd.py @@ -562,7 +562,7 @@ def save_checkpoint(output_path, model, clip=None, vae=None, clip_vision=None, m load_models.append(clip.load_model()) clip_sd = clip.get_sd() - model_management.load_models_gpu(load_models) + model_management.load_models_gpu(load_models, force_patch_weights=True) clip_vision_sd = clip_vision.get_sd() if clip_vision is not None else None sd = model.model.state_dict_for_saving(clip_sd, vae.get_sd(), clip_vision_sd) for k in extra_keys: diff --git a/comfy_extras/nodes_model_merging.py b/comfy_extras/nodes_model_merging.py index 2a431f65d..8c5dc9859 100644 --- a/comfy_extras/nodes_model_merging.py +++ b/comfy_extras/nodes_model_merging.py @@ -262,7 +262,7 @@ class CLIPSave: for x in extra_pnginfo: metadata[x] = json.dumps(extra_pnginfo[x]) - comfy.model_management.load_models_gpu([clip.load_model()]) + comfy.model_management.load_models_gpu([clip.load_model()], force_patch_weights=True) clip_sd = clip.get_sd() for prefix in ["clip_l.", "clip_g.", ""]: From 49c20cdc70149b2f18586d7de5c99b1013044ab5 Mon Sep 17 00:00:00 2001 From: comfyanonymous Date: Sun, 12 May 2024 05:34:43 -0400 Subject: [PATCH 4/9] No longer necessary. --- comfy/model_management.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/comfy/model_management.py b/comfy/model_management.py index 15dd73a6b..5a66a3836 100644 --- a/comfy/model_management.py +++ b/comfy/model_management.py @@ -393,8 +393,6 @@ def load_models_gpu(models, memory_required=0, force_patch_weights=False): loaded_model = LoadedModel(x) if loaded_model in current_loaded_models: - index = current_loaded_models.index(loaded_model) - current_loaded_models.insert(0, current_loaded_models.pop(index)) models_already_loaded.append(loaded_model) else: if hasattr(x, "model"): From fa6dd7e5bbee031defa640534b0924313757676f Mon Sep 17 00:00:00 2001 From: comfyanonymous Date: Sun, 12 May 2024 06:13:45 -0400 Subject: [PATCH 5/9] Fix lowvram issue with saving checkpoints. The previous fix didn't cover the case where the model was loaded in lowvram mode right before. --- comfy/model_management.py | 23 ++++++++++++++++++++--- comfy/model_patcher.py | 6 ++++++ 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/comfy/model_management.py b/comfy/model_management.py index 5a66a3836..3d01e8a23 100644 --- a/comfy/model_management.py +++ b/comfy/model_management.py @@ -309,6 +309,11 @@ class LoadedModel: self.weights_loaded = True return self.real_model + def should_reload_model(self, force_patch_weights=False): + if force_patch_weights and self.model.lowvram_patch_counter > 0: + return True + return False + def model_unload(self, unpatch_weights=True): self.model.unpatch_model(self.model.offload_device, unpatch_weights=unpatch_weights) self.model.model_patches_to(self.model.offload_device) @@ -391,10 +396,22 @@ def load_models_gpu(models, memory_required=0, force_patch_weights=False): models_already_loaded = [] for x in models: loaded_model = LoadedModel(x) + loaded = None - if loaded_model in current_loaded_models: - models_already_loaded.append(loaded_model) - else: + try: + loaded_model_index = current_loaded_models.index(loaded_model) + except: + loaded_model_index = None + + if loaded_model_index is not None: + loaded = current_loaded_models[loaded_model_index] + if loaded.should_reload_model(force_patch_weights=force_patch_weights): #TODO: cleanup this model reload logic + current_loaded_models.pop(loaded_model_index).model_unload(unpatch_weights=True) + loaded = None + else: + models_already_loaded.append(loaded) + + if loaded is None: if hasattr(x, "model"): logging.info(f"Requested to load {x.model.__class__.__name__}") models_to_load.append(loaded_model) diff --git a/comfy/model_patcher.py b/comfy/model_patcher.py index 48e5be31e..c38b2f79b 100644 --- a/comfy/model_patcher.py +++ b/comfy/model_patcher.py @@ -58,6 +58,7 @@ class ModelPatcher: self.weight_inplace_update = weight_inplace_update self.model_lowvram = False + self.lowvram_patch_counter = 0 self.patches_uuid = uuid.uuid4() def model_size(self): @@ -284,6 +285,7 @@ class ModelPatcher: return self.model_patcher.calculate_weight(self.model_patcher.patches[self.key], weight, self.key) mem_counter = 0 + patch_counter = 0 for n, m in self.model.named_modules(): lowvram_weight = False if hasattr(m, "comfy_cast_weights"): @@ -300,11 +302,13 @@ class ModelPatcher: self.patch_weight_to_device(weight_key) else: m.weight_function = LowVramPatch(weight_key, self) + patch_counter += 1 if bias_key in self.patches: if force_patch_weights: self.patch_weight_to_device(bias_key) else: m.bias_function = LowVramPatch(bias_key, self) + patch_counter += 1 m.prev_comfy_cast_weights = m.comfy_cast_weights m.comfy_cast_weights = True @@ -317,6 +321,7 @@ class ModelPatcher: logging.debug("lowvram: loaded module regularly {}".format(m)) self.model_lowvram = True + self.lowvram_patch_counter = patch_counter return self.model def calculate_weight(self, patches, weight, key): @@ -468,6 +473,7 @@ class ModelPatcher: m.bias_function = None self.model_lowvram = False + self.lowvram_patch_counter = 0 keys = list(self.backup.keys()) From f509c6fe21179db585372ec4443c80179fa6c659 Mon Sep 17 00:00:00 2001 From: Simon Lui <502929+simonlui@users.noreply.github.com> Date: Sun, 12 May 2024 03:36:30 -0700 Subject: [PATCH 6/9] Fix Intel GPU memory allocation accuracy and documentation update. (#3459) * Change calculation of memory total to be more accurate, allocated is actually smaller than reserved. * Update README.md install documentation for Intel GPUs. --- README.md | 11 ++++++++++- comfy/model_management.py | 6 +++--- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 2636ce140..312468a98 100644 --- a/README.md +++ b/README.md @@ -136,7 +136,16 @@ After this you should have everything installed and can proceed to running Comfy ### Others: -#### [Intel Arc](https://github.com/comfyanonymous/ComfyUI/discussions/476) +#### Intel GPUs + +Intel GPU support is available for all Intel GPUs supported by Intel's Extension for Pytorch (IPEX) with the support requirements listed in the [Installation](https://intel.github.io/intel-extension-for-pytorch/index.html#installation?platform=gpu) page. Choose your platform and method of install and follow the instructions. The steps are as follows: + +1. Start by installing the drivers or kernel listed or newer in the Installation page of IPEX linked above for Windows and Linux if needed. +1. Follow the instructions to install [Intel's oneAPI Basekit](https://www.intel.com/content/www/us/en/developer/tools/oneapi/base-toolkit-download.html) for your platform. +1. Install the packages for IPEX using the instructions provided in the Installation page for your platform. +1. Follow the [ComfyUI manual installation](#manual-install-windows-linux) instructions for Windows and Linux and run ComfyUI normally as described above after everything is installed. + +Additional discussion and help can be found [here](https://github.com/comfyanonymous/ComfyUI/discussions/476). #### Apple Mac silicon diff --git a/comfy/model_management.py b/comfy/model_management.py index 3d01e8a23..7b54b2567 100644 --- a/comfy/model_management.py +++ b/comfy/model_management.py @@ -102,8 +102,8 @@ def get_total_memory(dev=None, torch_total_too=False): elif is_intel_xpu(): stats = torch.xpu.memory_stats(dev) mem_reserved = stats['reserved_bytes.all.current'] - mem_total = torch.xpu.get_device_properties(dev).total_memory mem_total_torch = mem_reserved + mem_total = torch.xpu.get_device_properties(dev).total_memory else: stats = torch.cuda.memory_stats(dev) mem_reserved = stats['reserved_bytes.all.current'] @@ -701,10 +701,10 @@ def get_free_memory(dev=None, torch_free_too=False): elif is_intel_xpu(): stats = torch.xpu.memory_stats(dev) mem_active = stats['active_bytes.all.current'] - mem_allocated = stats['allocated_bytes.all.current'] mem_reserved = stats['reserved_bytes.all.current'] mem_free_torch = mem_reserved - mem_active - mem_free_total = torch.xpu.get_device_properties(dev).total_memory - mem_allocated + mem_free_xpu = torch.xpu.get_device_properties(dev).total_memory - mem_reserved + mem_free_total = mem_free_xpu + mem_free_torch else: stats = torch.cuda.memory_stats(dev) mem_active = stats['active_bytes.all.current'] From 22edd3add541cdb956b50a3c6412ce2bb36c1090 Mon Sep 17 00:00:00 2001 From: shawnington <88048838+shawnington@users.noreply.github.com> Date: Sun, 12 May 2024 04:07:38 -0700 Subject: [PATCH 7/9] =?UTF-8?q?Fix=20to=20LoadImage=20Node=20for=20#3416?= =?UTF-8?q?=20HDR=20images=20loading=20additional=20smaller=E2=80=A6=20(#3?= =?UTF-8?q?454)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Fix to LoadImage Node for #3416 HDR images loading additional smaller images. Added a blocking if statement in the ImageSequence.Iterator that checks if subsequent images after the first match dimensionally, and prevent them from being appended to output_images if they do not match. This does not fix or change current behavior for PIL 10.2.0 where the images are loaded at the same size, but it does for 10.3.0 where they are loaded at their correct smaller sizes. * added list of excluded formats that should return 1 image added an explicit check for the image format so that additional formats can be added to the list that have problematic behavior. --- nodes.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/nodes.py b/nodes.py index 488afd577..37e7d7345 100644 --- a/nodes.py +++ b/nodes.py @@ -1461,12 +1461,24 @@ class LoadImage: output_images = [] output_masks = [] + w, h = None, None + + excluded_formats = ['MPO'] + for i in ImageSequence.Iterator(img): i = node_helpers.pillow(ImageOps.exif_transpose, i) if i.mode == 'I': i = i.point(lambda i: i * (1 / 255)) image = i.convert("RGB") + + if len(output_images) == 0: + w = image.size[0] + h = image.size[1] + + if image.size[0] != w or image.size[1] != h: + continue + image = np.array(image).astype(np.float32) / 255.0 image = torch.from_numpy(image)[None,] if 'A' in i.getbands(): @@ -1477,7 +1489,7 @@ class LoadImage: output_images.append(image) output_masks.append(mask.unsqueeze(0)) - if len(output_images) > 1: + if len(output_images) > 1 and img.format not in excluded_formats: output_image = torch.cat(output_images, dim=0) output_mask = torch.cat(output_masks, dim=0) else: From 794a357f7a4bd40c6e2dafab9d93142be9933a56 Mon Sep 17 00:00:00 2001 From: comfyanonymous Date: Sun, 12 May 2024 07:24:12 -0400 Subject: [PATCH 8/9] Update the nightly workflow. --- .github/workflows/windows_release_nightly_pytorch.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/windows_release_nightly_pytorch.yml b/.github/workflows/windows_release_nightly_pytorch.yml index 672a7f220..1434b0a23 100644 --- a/.github/workflows/windows_release_nightly_pytorch.yml +++ b/.github/workflows/windows_release_nightly_pytorch.yml @@ -7,7 +7,7 @@ on: description: 'cuda version' required: true type: string - default: "121" + default: "124" python_minor: description: 'python minor version' @@ -19,7 +19,7 @@ on: description: 'python patch version' required: true type: string - default: "2" + default: "3" # push: # branches: # - master From ece5acb8e8025d8ca26aa880f604d971d245475d Mon Sep 17 00:00:00 2001 From: comfyanonymous Date: Sun, 12 May 2024 16:05:10 -0400 Subject: [PATCH 9/9] Fix nightly package workflow. --- .github/workflows/windows_release_nightly_pytorch.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/windows_release_nightly_pytorch.yml b/.github/workflows/windows_release_nightly_pytorch.yml index 1434b0a23..fa24a985c 100644 --- a/.github/workflows/windows_release_nightly_pytorch.yml +++ b/.github/workflows/windows_release_nightly_pytorch.yml @@ -49,7 +49,7 @@ jobs: echo 'import site' >> ./python3${{ inputs.python_minor }}._pth curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py ./python.exe get-pip.py - python -m pip wheel torch torchvision torchaudio mpmath==1.3.0 --pre --extra-index-url https://download.pytorch.org/whl/nightly/cu${{ inputs.cu }} -r ../ComfyUI/requirements.txt pygit2 -w ../temp_wheel_dir + python -m pip wheel torch torchvision torchaudio mpmath==1.3.0 numpy==1.26.4 --pre --extra-index-url https://download.pytorch.org/whl/nightly/cu${{ inputs.cu }} -r ../ComfyUI/requirements.txt pygit2 -w ../temp_wheel_dir ls ../temp_wheel_dir ./python.exe -s -m pip install --pre ../temp_wheel_dir/* sed -i '1i../ComfyUI' ./python3${{ inputs.python_minor }}._pth