From 57ff206bba45e9dca3b91ddb4236dac426e363df Mon Sep 17 00:00:00 2001 From: storyicon Date: Thu, 13 Jun 2024 09:53:38 +0000 Subject: [PATCH] feat: support node smart offload, reduce peak VRAM/RAM usage Signed-off-by: storyicon --- comfy/cli_args.py | 1 + execution.py | 61 +++++++++++++++++++++++++++++++++++++++++------ 2 files changed, 55 insertions(+), 7 deletions(-) diff --git a/comfy/cli_args.py b/comfy/cli_args.py index b8ac9bc69..670f91e10 100644 --- a/comfy/cli_args.py +++ b/comfy/cli_args.py @@ -122,6 +122,7 @@ parser.add_argument("--multi-user", action="store_true", help="Enables per-user parser.add_argument("--verbose", action="store_true", help="Enables more debug prints.") +parser.add_argument("--node-smart-offload-level", type=int, default=0, choices=[0, 1, 2], help="automatically release unreferenced node outputs, helping to reduce peak VRAM/RAM usage during execution and mitigate out-of-memory issues. 0: Means disabling this feature; 1: Only release outputs that were never referenced; 2: Release all currently unreferenced outputs.") if comfy.options.args_parsing: args = parser.parse_args() diff --git a/execution.py b/execution.py index 76225a962..480e6284b 100644 --- a/execution.py +++ b/execution.py @@ -9,7 +9,7 @@ from typing import List, Literal, NamedTuple, Optional import torch import nodes - +from comfy.cli_args import args import comfy.model_management def get_input_data(inputs, class_def, unique_id, outputs={}, prompt={}, extra_data={}): @@ -116,22 +116,66 @@ def format_value(x): else: return str(x) -def recursive_execute(server, prompt, outputs, current_item, extra_data, executed, prompt_id, outputs_ui, object_storage): +class InvalidCache(list): + pass + +def build_node_reference_map(prompt): + reference_map = {} + for node_id, node in prompt.items(): + for intput_label, input_data in node['inputs'].items(): + if isinstance(input_data, list): + input_unique_id = input_data[0] + output_index = input_data[1] + key = (input_unique_id, output_index) + if key not in reference_map: + reference_map[key] = [] + reference_map[key].append(node_id) + return reference_map + +def release_outputs_from_reference_map(outputs, reference_map, executed_node_id): + offload_level = args.node_smart_offload_level + if offload_level == 0: + return + for unique_key, output_reference in reference_map.items(): + if executed_node_id in output_reference: + output_reference.remove(executed_node_id) + for node_id, node_outputs in outputs.items(): + for i in range(len(node_outputs)): + key = (node_id, i) + releasable = key not in reference_map + if not releasable and offload_level == 2: + releasable = not reference_map[key] + if (not isinstance(node_outputs[i], InvalidCache)) and releasable: + node_outputs[i] = InvalidCache() + +def is_output_cached(outputs, node_unique_id, output_index=None): + if node_unique_id not in outputs: + return False + if output_index is not None: + if isinstance(outputs[node_unique_id][output_index], InvalidCache): + return False + else: + for item in outputs[node_unique_id]: + if isinstance(item, InvalidCache): + return False + return True + +def recursive_execute(server, prompt, outputs, current_item, extra_data, executed, prompt_id, outputs_ui, object_storage, reference_map): unique_id = current_item inputs = prompt[unique_id]['inputs'] class_type = prompt[unique_id]['class_type'] class_def = nodes.NODE_CLASS_MAPPINGS[class_type] - if unique_id in outputs: + if is_output_cached(outputs, unique_id): + release_outputs_from_reference_map(outputs, reference_map, unique_id) return (True, None, None) for x in inputs: input_data = inputs[x] - if isinstance(input_data, list): input_unique_id = input_data[0] output_index = input_data[1] - if input_unique_id not in outputs: - result = recursive_execute(server, prompt, outputs, input_unique_id, extra_data, executed, prompt_id, outputs_ui, object_storage) + if not is_output_cached(outputs, input_unique_id, output_index): + result = recursive_execute(server, prompt, outputs, input_unique_id, extra_data, executed, prompt_id, outputs_ui, object_storage, reference_map) if result[0] is not True: # Another node failed further upstream return result @@ -191,6 +235,7 @@ def recursive_execute(server, prompt, outputs, current_item, extra_data, execute executed.add(unique_id) + release_outputs_from_reference_map(outputs, reference_map, unique_id) return (True, None, None) def recursive_will_execute(prompt, outputs, current_item, memo={}): @@ -379,6 +424,8 @@ class PromptExecutor: for node_id in list(execute_outputs): to_execute += [(0, node_id)] + # key: (node_id, output_index), value: [node_id, ...] + reference_map = build_node_reference_map(prompt) while len(to_execute) > 0: #always execute the output that depends on the least amount of unexecuted nodes first memo = {} @@ -388,7 +435,7 @@ class PromptExecutor: # This call shouldn't raise anything if there's an error deep in # the actual SD code, instead it will report the node where the # error was raised - self.success, error, ex = recursive_execute(self.server, prompt, self.outputs, output_node_id, extra_data, executed, prompt_id, self.outputs_ui, self.object_storage) + self.success, error, ex = recursive_execute(self.server, prompt, self.outputs, output_node_id, extra_data, executed, prompt_id, self.outputs_ui, self.object_storage, reference_map) if self.success is not True: self.handle_execution_error(prompt_id, prompt, current_outputs, executed, error, ex) break