From e99d97a9d943c3601ccdf411608bffb771db5bf1 Mon Sep 17 00:00:00 2001 From: Chenlei Hu Date: Wed, 26 Jun 2024 01:23:55 -0400 Subject: [PATCH 01/24] Remove duplicated Reset View button (#3865) * Remove duplicated Reset View button * Disable flaky test --- .github/workflows/test-browser.yml | 2 +- web/scripts/ui.js | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/test-browser.yml b/.github/workflows/test-browser.yml index 049b1cde2..323c96cf3 100644 --- a/.github/workflows/test-browser.yml +++ b/.github/workflows/test-browser.yml @@ -24,7 +24,7 @@ jobs: with: repository: "huchenlei/ComfyUI_frontend" path: "ComfyUI_frontend" - ref: "23a4ee83144d9bf22e7218b9c78c71f8544c8b27" + ref: "577aea9dfb680a14b3ae4fa626a85098c3207030" - uses: actions/setup-node@v3 with: node-version: lts/* diff --git a/web/scripts/ui.js b/web/scripts/ui.js index 87b4887eb..bb094d8b4 100644 --- a/web/scripts/ui.js +++ b/web/scripts/ui.js @@ -603,7 +603,6 @@ export class ComfyUI { onclick: () => app.refreshComboInNodes() }), $el("button", {id: "comfy-clipspace-button", textContent: "Clipspace", onclick: () => app.openClipspace()}), - $el("button", {id: "comfy-reset-view-button", textContent: "Reset View", onclick: () => app.resetView()}), $el("button", { id: "comfy-clear-button", textContent: "Clear", onclick: () => { if (!confirmClear.value || confirm("Clear workflow?")) { From edfce78c86c7615039dd8d74e89957a0eebbdc03 Mon Sep 17 00:00:00 2001 From: "Alex \"mcmonkey\" Goodwin" <4000772+mcmonkey4eva@users.noreply.github.com> Date: Tue, 25 Jun 2024 22:37:27 -0700 Subject: [PATCH 02/24] add issue templates for ComfyUI Issues Page (#3868) --- .github/ISSUE_TEMPLATE/bug-report.yml | 45 ++++++++++++++++++++++ .github/ISSUE_TEMPLATE/config.yml | 8 ++++ .github/ISSUE_TEMPLATE/feature-request.yml | 32 +++++++++++++++ .github/ISSUE_TEMPLATE/user-support.yml | 32 +++++++++++++++ 4 files changed, 117 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE/bug-report.yml create mode 100644 .github/ISSUE_TEMPLATE/config.yml create mode 100644 .github/ISSUE_TEMPLATE/feature-request.yml create mode 100644 .github/ISSUE_TEMPLATE/user-support.yml diff --git a/.github/ISSUE_TEMPLATE/bug-report.yml b/.github/ISSUE_TEMPLATE/bug-report.yml new file mode 100644 index 000000000..87bef3681 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug-report.yml @@ -0,0 +1,45 @@ +name: Bug Report +description: "Something is broken inside of ComfyUI. (Do not use this if you're just having issues and need help, or if the issue relates to a custom node)" +labels: [ "Potential Bug" ] +body: + - type: markdown + attributes: + value: | + Before submitting a **Bug Report**, please ensure the following: + + **1:** You are running the latest version of ComfyUI. + **2:** You have looked at the existing bug reports and made sure this isn't already reported. + **3:** This is an actual bug in ComfyUI, not just a support question and not caused by an custom node. A bug is when you can specify exact steps to replicate what went wrong and others will be able to repeat your steps and see the same issue happen. + + If unsure, ask on the [ComfyUI Matrix Space](https://app.element.io/#/room/%23comfyui_space%3Amatrix.org) or the [Comfy Org Discord](https://discord.gg/comfyorg) first. + - type: textarea + attributes: + label: Expected Behavior + description: "What you expected to happen." + validations: + required: true + - type: textarea + attributes: + label: Actual Behavior + description: "What actually happened. Please include a screenshot of the issue if possible." + validations: + required: true + - type: textarea + attributes: + label: Steps to Reproduce + description: "Describe how to reproduce the issue. Please be sure to attach a workflow JSON or PNG, ideally one that doesn't require custom nodes to test. If the bug open happens when certain custom nodes are used, most likely that custom node is what has the bug rather than ComfyUI, in which case it should be reported to the node's author." + validations: + required: true + - type: textarea + attributes: + label: Debug Logs + description: "Please copy the output from your terminal logs here." + render: powershell + validations: + required: true + - type: textarea + attributes: + label: Other + description: "Any other additional information you think might be helpful." + validations: + required: false diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 000000000..2c519ede5 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,8 @@ +blank_issues_enabled: true +contact_links: + - name: ComfyUI Matrix Space + url: https://app.element.io/#/room/%23comfyui_space%3Amatrix.org + about: The ComfyUI Matrix Space is available for support and general discussion related to ComfyUI (Matrix is like Discord but open source). + - name: Comfy Org Discord + url: https://discord.gg/comfyorg + about: The Comfy Org Discord is available for support and general discussion related to ComfyUI. diff --git a/.github/ISSUE_TEMPLATE/feature-request.yml b/.github/ISSUE_TEMPLATE/feature-request.yml new file mode 100644 index 000000000..419721b63 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature-request.yml @@ -0,0 +1,32 @@ +name: Feature Request +description: "You have an idea for something new you would like to see added to ComfyUI's core." +labels: [ "Feature" ] +body: + - type: markdown + attributes: + value: | + Before submitting a **Feature Request**, please ensure the following: + + **1:** You are running the latest version of ComfyUI. + **2:** You have looked to make sure there is not already a feature that does what you need, and there is not already a Feature Request listed for the same idea. + **3:** This is something that makes sense to add to ComfyUI Core, and wouldn't make more sense as a custom node. + + If unsure, ask on the [ComfyUI Matrix Space](https://app.element.io/#/room/%23comfyui_space%3Amatrix.org) or the [Comfy Org Discord](https://discord.gg/comfyorg) first. + - type: textarea + attributes: + label: Feature Idea + description: "Describe the feature you want to see." + validations: + required: true + - type: textarea + attributes: + label: Existing Solutions + description: "Please search through available custom nodes / extensions to see if there are existing custom solutions for this. If so, please link the options you found here as a reference." + validations: + required: false + - type: textarea + attributes: + label: Other + description: "Any other additional information you think might be helpful." + validations: + required: false diff --git a/.github/ISSUE_TEMPLATE/user-support.yml b/.github/ISSUE_TEMPLATE/user-support.yml new file mode 100644 index 000000000..df28804c6 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/user-support.yml @@ -0,0 +1,32 @@ +name: User Support +description: "Use this if you need help with something, or you're experiencing an issue." +labels: [ "User Support" ] +body: + - type: markdown + attributes: + value: | + Before submitting a **User Report** issue, please ensure the following: + + **1:** You are running the latest version of ComfyUI. + **2:** You have made an effort to find public answers to your question before asking here. In other words, you googled it first, and scrolled through recent help topics. + + If unsure, ask on the [ComfyUI Matrix Space](https://app.element.io/#/room/%23comfyui_space%3Amatrix.org) or the [Comfy Org Discord](https://discord.gg/comfyorg) first. + - type: textarea + attributes: + label: Your question + description: "Post your question here. Please be as detailed as possible." + validations: + required: true + - type: textarea + attributes: + label: Logs + description: "If your question relates to an issue you're experiencing, please go to `Server` -> `Logs` -> potentially set `View Type` to `Debug` as well, then copypaste all the text into here." + render: powershell + validations: + required: false + - type: textarea + attributes: + label: Other + description: "Any other additional information you think might be helpful." + validations: + required: false From e3579f33600fea742611cc62503b446833d45735 Mon Sep 17 00:00:00 2001 From: pythongosssss <125205205+pythongosssss@users.noreply.github.com> Date: Wed, 26 Jun 2024 14:08:48 +0100 Subject: [PATCH 03/24] Fix merge issue breaking api json loading (#3876) --- web/scripts/app.js | 1 - 1 file changed, 1 deletion(-) diff --git a/web/scripts/app.js b/web/scripts/app.js index 725538563..a354ff805 100644 --- a/web/scripts/app.js +++ b/web/scripts/app.js @@ -2321,7 +2321,6 @@ export class ComfyApp { node.id = isNaN(+id) ? id : +id; node.title = data._meta?.title ?? node.title app.graph.add(node); - graph.add(node); } this.changeWorkflow(() => { From f12fa1d8d7b42888b6491aa6aa4956aefca3aba5 Mon Sep 17 00:00:00 2001 From: Chenlei Hu Date: Wed, 26 Jun 2024 09:09:21 -0400 Subject: [PATCH 04/24] Enable browser tests on push (#3878) --- .github/workflows/test-browser.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/test-browser.yml b/.github/workflows/test-browser.yml index 323c96cf3..987a72ae3 100644 --- a/.github/workflows/test-browser.yml +++ b/.github/workflows/test-browser.yml @@ -7,6 +7,8 @@ name: Playwright Browser Tests CI on: + push: + branches: [ main, master ] pull_request: branches: [ main, master ] From a3e83f695d6588bc06fdd40e039cd1e5b5e169dd Mon Sep 17 00:00:00 2001 From: Chenlei Hu Date: Wed, 26 Jun 2024 19:58:56 -0400 Subject: [PATCH 05/24] Update test ref (#3882) * Update ref * Disable some tests --- .github/workflows/test-browser.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-browser.yml b/.github/workflows/test-browser.yml index 987a72ae3..65483b00f 100644 --- a/.github/workflows/test-browser.yml +++ b/.github/workflows/test-browser.yml @@ -26,7 +26,7 @@ jobs: with: repository: "huchenlei/ComfyUI_frontend" path: "ComfyUI_frontend" - ref: "577aea9dfb680a14b3ae4fa626a85098c3207030" + ref: "fcc54d803e5b6a9b08a462a1d94899318c96dcbb" - uses: actions/setup-node@v3 with: node-version: lts/* From bc5a0f10db34a5c942fa3c531864da9665faaaa5 Mon Sep 17 00:00:00 2001 From: Chenlei Hu Date: Wed, 26 Jun 2024 19:59:09 -0400 Subject: [PATCH 06/24] Ignore *.log (#3880) --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 9f0389241..9473f43ab 100644 --- a/.gitignore +++ b/.gitignore @@ -15,4 +15,5 @@ venv/ !/web/extensions/logging.js.example !/web/extensions/core/ /tests-ui/data/object_info.json -/user/ \ No newline at end of file +/user/ +*.log \ No newline at end of file From 175fe02522751c9f5e9a8309b3aaa5d07a542296 Mon Sep 17 00:00:00 2001 From: Chenlei Hu Date: Wed, 26 Jun 2024 19:59:19 -0400 Subject: [PATCH 07/24] Ignore .vscode/ (#3879) --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 9473f43ab..a9beebe73 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,7 @@ __pycache__/ !custom_nodes/example_node.py.example extra_model_paths.yaml /.vs +.vscode/ .idea/ venv/ /web/extensions/* From 44947e7ad489eaadb38007ca7aabd71aef7fa10e Mon Sep 17 00:00:00 2001 From: comfyanonymous Date: Wed, 26 Jun 2024 22:40:05 -0400 Subject: [PATCH 08/24] Add DEIS order 3 sampler. Order 4 seems to give bad results. --- comfy/k_diffusion/deis.py | 122 ++++++++++++++++++++++++++++++++++ comfy/k_diffusion/sampling.py | 50 ++++++++++++++ comfy/samplers.py | 2 +- 3 files changed, 173 insertions(+), 1 deletion(-) create mode 100644 comfy/k_diffusion/deis.py diff --git a/comfy/k_diffusion/deis.py b/comfy/k_diffusion/deis.py new file mode 100644 index 000000000..0be2036c3 --- /dev/null +++ b/comfy/k_diffusion/deis.py @@ -0,0 +1,122 @@ +#Taken from: https://github.com/zju-pi/diff-sampler/blob/main/gits-main/solver_utils.py +#under Apache 2 license +import torch +import numpy as np + +# A pytorch reimplementation of DEIS (https://github.com/qsh-zh/deis). +############################# +### Utils for DEIS solver ### +############################# +#---------------------------------------------------------------------------- +# Transfer from the input time (sigma) used in EDM to that (t) used in DEIS. + +def edm2t(edm_steps, epsilon_s=1e-3, sigma_min=0.002, sigma_max=80): + vp_sigma = lambda beta_d, beta_min: lambda t: (np.e ** (0.5 * beta_d * (t ** 2) + beta_min * t) - 1) ** 0.5 + vp_sigma_inv = lambda beta_d, beta_min: lambda sigma: ((beta_min ** 2 + 2 * beta_d * (sigma ** 2 + 1).log()).sqrt() - beta_min) / beta_d + vp_beta_d = 2 * (np.log(torch.tensor(sigma_min).cpu() ** 2 + 1) / epsilon_s - np.log(torch.tensor(sigma_max).cpu() ** 2 + 1)) / (epsilon_s - 1) + vp_beta_min = np.log(torch.tensor(sigma_max).cpu() ** 2 + 1) - 0.5 * vp_beta_d + t_steps = vp_sigma_inv(vp_beta_d.clone().detach().cpu(), vp_beta_min.clone().detach().cpu())(edm_steps.clone().detach().cpu()) + return t_steps, vp_beta_min, vp_beta_d + vp_beta_min + +#---------------------------------------------------------------------------- + +def cal_poly(prev_t, j, taus): + poly = 1 + for k in range(prev_t.shape[0]): + if k == j: + continue + poly *= (taus - prev_t[k]) / (prev_t[j] - prev_t[k]) + return poly + +#---------------------------------------------------------------------------- +# Transfer from t to alpha_t. + +def t2alpha_fn(beta_0, beta_1, t): + return torch.exp(-0.5 * t ** 2 * (beta_1 - beta_0) - t * beta_0) + +#---------------------------------------------------------------------------- + +def cal_intergrand(beta_0, beta_1, taus): + with torch.inference_mode(mode=False): + taus = taus.clone() + beta_0 = beta_0.clone() + beta_1 = beta_1.clone() + with torch.enable_grad(): + taus.requires_grad_(True) + alpha = t2alpha_fn(beta_0, beta_1, taus) + log_alpha = alpha.log() + log_alpha.sum().backward() + d_log_alpha_dtau = taus.grad + integrand = -0.5 * d_log_alpha_dtau / torch.sqrt(alpha * (1 - alpha)) + return integrand + +#---------------------------------------------------------------------------- + +def get_deis_coeff_list(t_steps, max_order, N=10000, deis_mode='tab'): + """ + Get the coefficient list for DEIS sampling. + + Args: + t_steps: A pytorch tensor. The time steps for sampling. + max_order: A `int`. Maximum order of the solver. 1 <= max_order <= 4 + N: A `int`. Use how many points to perform the numerical integration when deis_mode=='tab'. + deis_mode: A `str`. Select between 'tab' and 'rhoab'. Type of DEIS. + Returns: + A pytorch tensor. A batch of generated samples or sampling trajectories if return_inters=True. + """ + if deis_mode == 'tab': + t_steps, beta_0, beta_1 = edm2t(t_steps) + C = [] + for i, (t_cur, t_next) in enumerate(zip(t_steps[:-1], t_steps[1:])): + order = min(i+1, max_order) + if order == 1: + C.append([]) + else: + taus = torch.linspace(t_cur, t_next, N) # split the interval for integral appximation + dtau = (t_next - t_cur) / N + prev_t = t_steps[[i - k for k in range(order)]] + coeff_temp = [] + integrand = cal_intergrand(beta_0, beta_1, taus) + for j in range(order): + poly = cal_poly(prev_t, j, taus) + coeff_temp.append(torch.sum(integrand * poly) * dtau) + C.append(coeff_temp) + + elif deis_mode == 'rhoab': + # Analytical solution, second order + def get_def_intergral_2(a, b, start, end, c): + coeff = (end**3 - start**3) / 3 - (end**2 - start**2) * (a + b) / 2 + (end - start) * a * b + return coeff / ((c - a) * (c - b)) + + # Analytical solution, third order + def get_def_intergral_3(a, b, c, start, end, d): + coeff = (end**4 - start**4) / 4 - (end**3 - start**3) * (a + b + c) / 3 \ + + (end**2 - start**2) * (a*b + a*c + b*c) / 2 - (end - start) * a * b * c + return coeff / ((d - a) * (d - b) * (d - c)) + + C = [] + for i, (t_cur, t_next) in enumerate(zip(t_steps[:-1], t_steps[1:])): + order = min(i, max_order) + if order == 0: + C.append([]) + else: + prev_t = t_steps[[i - k for k in range(order+1)]] + if order == 1: + coeff_cur = ((t_next - prev_t[1])**2 - (t_cur - prev_t[1])**2) / (2 * (t_cur - prev_t[1])) + coeff_prev1 = (t_next - t_cur)**2 / (2 * (prev_t[1] - t_cur)) + coeff_temp = [coeff_cur, coeff_prev1] + elif order == 2: + coeff_cur = get_def_intergral_2(prev_t[1], prev_t[2], t_cur, t_next, t_cur) + coeff_prev1 = get_def_intergral_2(t_cur, prev_t[2], t_cur, t_next, prev_t[1]) + coeff_prev2 = get_def_intergral_2(t_cur, prev_t[1], t_cur, t_next, prev_t[2]) + coeff_temp = [coeff_cur, coeff_prev1, coeff_prev2] + elif order == 3: + coeff_cur = get_def_intergral_3(prev_t[1], prev_t[2], prev_t[3], t_cur, t_next, t_cur) + coeff_prev1 = get_def_intergral_3(t_cur, prev_t[2], prev_t[3], t_cur, t_next, prev_t[1]) + coeff_prev2 = get_def_intergral_3(t_cur, prev_t[1], prev_t[3], t_cur, t_next, prev_t[2]) + coeff_prev3 = get_def_intergral_3(t_cur, prev_t[1], prev_t[2], t_cur, t_next, prev_t[3]) + coeff_temp = [coeff_cur, coeff_prev1, coeff_prev2, coeff_prev3] + C.append(coeff_temp) + print(C) + return C + diff --git a/comfy/k_diffusion/sampling.py b/comfy/k_diffusion/sampling.py index 9b58cb1dc..f8091bb3f 100644 --- a/comfy/k_diffusion/sampling.py +++ b/comfy/k_diffusion/sampling.py @@ -7,6 +7,7 @@ import torchsde from tqdm.auto import trange, tqdm from . import utils +from . import deis import comfy.model_patcher def append_zero(x): @@ -946,6 +947,55 @@ def sample_ipndm_v(model, x, sigmas, extra_args=None, callback=None, disable=Non return x_next +#From https://github.com/zju-pi/diff-sampler/blob/main/diff-solvers-main/solvers.py +#under Apache 2 license +@torch.no_grad() +def sample_deis(model, x, sigmas, extra_args=None, callback=None, disable=None, max_order=3, deis_mode='tab'): + extra_args = {} if extra_args is None else extra_args + s_in = x.new_ones([x.shape[0]]) + + x_next = x + t_steps = sigmas + + coeff_list = deis.get_deis_coeff_list(t_steps, max_order, deis_mode=deis_mode) + + buffer_model = [] + for i in trange(len(sigmas) - 1, disable=disable): + t_cur = sigmas[i] + t_next = sigmas[i + 1] + + x_cur = x_next + + denoised = model(x_cur, t_cur * s_in, **extra_args) + if callback is not None: + callback({'x': x, 'i': i, 'sigma': sigmas[i], 'sigma_hat': sigmas[i], 'denoised': denoised}) + + d_cur = (x_cur - denoised) / t_cur + + order = min(max_order, i+1) + if t_next <= 0: + order = 1 + + if order == 1: # First Euler step. + x_next = x_cur + (t_next - t_cur) * d_cur + elif order == 2: # Use one history point. + coeff_cur, coeff_prev1 = coeff_list[i] + x_next = x_cur + coeff_cur * d_cur + coeff_prev1 * buffer_model[-1] + elif order == 3: # Use two history points. + coeff_cur, coeff_prev1, coeff_prev2 = coeff_list[i] + x_next = x_cur + coeff_cur * d_cur + coeff_prev1 * buffer_model[-1] + coeff_prev2 * buffer_model[-2] + elif order == 4: # Use three history points. + coeff_cur, coeff_prev1, coeff_prev2, coeff_prev3 = coeff_list[i] + x_next = x_cur + coeff_cur * d_cur + coeff_prev1 * buffer_model[-1] + coeff_prev2 * buffer_model[-2] + coeff_prev3 * buffer_model[-3] + + if len(buffer_model) == max_order - 1: + for k in range(max_order - 2): + buffer_model[k] = buffer_model[k+1] + buffer_model[-1] = d_cur.detach() + else: + buffer_model.append(d_cur.detach()) + + return x_next @torch.no_grad() def sample_euler_pp(model, x, sigmas, extra_args=None, callback=None, disable=None): diff --git a/comfy/samplers.py b/comfy/samplers.py index c3a80dde0..7f7114dbb 100644 --- a/comfy/samplers.py +++ b/comfy/samplers.py @@ -540,7 +540,7 @@ class Sampler: KSAMPLER_NAMES = ["euler", "euler_pp", "euler_ancestral", "euler_ancestral_pp", "heun", "heunpp2","dpm_2", "dpm_2_ancestral", "lms", "dpm_fast", "dpm_adaptive", "dpmpp_2s_ancestral", "dpmpp_sde", "dpmpp_sde_gpu", "dpmpp_2m", "dpmpp_2m_sde", "dpmpp_2m_sde_gpu", "dpmpp_3m_sde", "dpmpp_3m_sde_gpu", "ddpm", "lcm", - "ipndm", "ipndm_v"] + "ipndm", "ipndm_v", "deis"] class KSAMPLER(Sampler): def __init__(self, sampler_function, extra_options={}, inpaint_options={}): From 3b423afccab877b57fc24141690426ca1660e4bb Mon Sep 17 00:00:00 2001 From: Chenlei Hu Date: Thu, 27 Jun 2024 00:22:55 -0400 Subject: [PATCH 09/24] Add audio widget (#3863) * Add audio widget * Fix audio bugs * Add CSS * Populate audio widget when load history --- comfy_extras/nodes_audio.py | 11 +- web/extensions/core/uploadAudio.js | 176 +++++++++++++++++++++++++++++ web/scripts/app.js | 11 +- web/style.css | 4 + 4 files changed, 199 insertions(+), 3 deletions(-) create mode 100644 web/extensions/core/uploadAudio.js diff --git a/comfy_extras/nodes_audio.py b/comfy_extras/nodes_audio.py index 57d0a20ab..8efa7c0a8 100644 --- a/comfy_extras/nodes_audio.py +++ b/comfy_extras/nodes_audio.py @@ -93,11 +93,18 @@ class SaveAudio: return { "ui": { "audio": results } } class LoadAudio: + SUPPORTED_FORMATS = ('.wav', '.mp3', '.ogg', '.flac', '.aiff', '.aif') + @classmethod def INPUT_TYPES(s): input_dir = folder_paths.get_input_directory() - files = [f for f in os.listdir(input_dir) if os.path.isfile(os.path.join(input_dir, f))] - return {"required": {"audio": [sorted(files), ]}, } + files = [ + f for f in os.listdir(input_dir) + if (os.path.isfile(os.path.join(input_dir, f)) + and f.endswith(LoadAudio.SUPPORTED_FORMATS) + ) + ] + return {"required": {"audio": (sorted(files), {"audio_upload": True})}} CATEGORY = "_for_testing/audio" diff --git a/web/extensions/core/uploadAudio.js b/web/extensions/core/uploadAudio.js new file mode 100644 index 000000000..d3dfef4a7 --- /dev/null +++ b/web/extensions/core/uploadAudio.js @@ -0,0 +1,176 @@ +import { app } from "../../scripts/app.js" +import { api } from "../../scripts/api.js" + +function splitFilePath(path) { + const folder_separator = path.lastIndexOf("/") + if (folder_separator === -1) { + return ["", path] + } + return [ + path.substring(0, folder_separator), + path.substring(folder_separator + 1) + ] +} + +function getResourceURL(subfolder, filename, type = "input") { + const params = [ + "filename=" + encodeURIComponent(filename), + "type=" + type, + "subfolder=" + subfolder, + app.getPreviewFormatParam().substring(1), + app.getRandParam().substring(1) + ].join("&") + + return `/view?${params}` +} + +async function uploadFile( + audioWidget, + audioUIWidget, + file, + updateNode, + pasted = false +) { + try { + // Wrap file in formdata so it includes filename + const body = new FormData() + body.append("image", file) + if (pasted) body.append("subfolder", "pasted") + const resp = await api.fetchApi("/upload/image", { + method: "POST", + body + }) + + if (resp.status === 200) { + const data = await resp.json() + // Add the file to the dropdown list and update the widget value + let path = data.name + if (data.subfolder) path = data.subfolder + "/" + path + + if (!audioWidget.options.values.includes(path)) { + audioWidget.options.values.push(path) + } + + if (updateNode) { + audioUIWidget.element.src = api.apiURL( + getResourceURL(...splitFilePath(path)) + ) + audioWidget.value = path + } + } else { + alert(resp.status + " - " + resp.statusText) + } + } catch (error) { + alert(error) + } +} + +// AudioWidget MUST be registered first, as AUDIOUPLOAD depends on AUDIO_UI to be +// present. +app.registerExtension({ + name: "Comfy.AudioWidget", + async beforeRegisterNodeDef(nodeType, nodeData) { + if (["LoadAudio", "SaveAudio"].includes(nodeType.comfyClass)) { + nodeData.input.required.audioUI = ["AUDIO_UI"] + } + }, + getCustomWidgets() { + return { + AUDIO_UI(node, inputName) { + const audio = document.createElement("audio") + audio.controls = true + audio.classList.add("comfy-audio") + audio.setAttribute("name", "media") + + const audioUIWidget = node.addDOMWidget( + inputName, + /* name=*/ "audioUI", + audio + ) + // @ts-ignore + // TODO: Sort out the DOMWidget type. + audioUIWidget.serialize = false + + const isOutputNode = node.constructor.nodeData.output_node + if (isOutputNode) { + // Hide the audio widget when there is no audio initially. + audioUIWidget.element.classList.add("empty-audio-widget") + // Populate the audio widget UI on node execution. + const onExecuted = node.onExecuted + node.onExecuted = function(message) { + onExecuted?.apply(this, arguments) + const audios = message.audio + if (!audios) return + const audio = audios[0] + audioUIWidget.element.src = api.apiURL( + getResourceURL(audio.subfolder, audio.filename, "output") + ) + audioUIWidget.element.classList.remove("empty-audio-widget") + } + } + return { widget: audioUIWidget } + } + } + }, + onNodeOutputsUpdated(nodeOutputs) { + for (const [nodeId, output] of Object.entries(nodeOutputs)) { + const node = app.graph.getNodeById(Number.parseInt(nodeId)); + if ("audio" in output) { + const audioUIWidget = node.widgets.find((w) => w.name === "audioUI"); + const audio = output.audio[0]; + audioUIWidget.element.src = api.apiURL(getResourceURL(audio.subfolder, audio.filename, "output")); + audioUIWidget.element.classList.remove("empty-audio-widget"); + } + } + }, +}) + +app.registerExtension({ + name: "Comfy.UploadAudio", + async beforeRegisterNodeDef(nodeType, nodeData) { + if (nodeData?.input?.required?.audio?.[1]?.audio_upload === true) { + nodeData.input.required.upload = ["AUDIOUPLOAD"] + } + }, + getCustomWidgets() { + return { + AUDIOUPLOAD(node, inputName) { + // The widget that allows user to select file. + const audioWidget = node.widgets.find(w => w.name === "audio") + const audioUIWidget = node.widgets.find(w => w.name === "audioUI") + + const onAudioWidgetUpdate = () => { + audioUIWidget.element.src = api.apiURL( + getResourceURL(...splitFilePath(audioWidget.value)) + ) + } + // Initially load default audio file to audioUIWidget. + onAudioWidgetUpdate() + audioWidget.callback = onAudioWidgetUpdate + + const fileInput = document.createElement("input") + fileInput.type = "file" + fileInput.accept = "audio/*" + fileInput.style.display = "none" + fileInput.onchange = () => { + if (fileInput.files.length) { + uploadFile(audioWidget, audioUIWidget, fileInput.files[0], true) + } + } + // The widget to pop up the upload dialog. + const uploadWidget = node.addWidget( + "button", + inputName, + /* value=*/ "", + () => { + fileInput.click() + } + ) + uploadWidget.label = "choose file to upload" + uploadWidget.serialize = false + + return { widget: uploadWidget } + } + } + } +}) diff --git a/web/scripts/app.js b/web/scripts/app.js index a354ff805..5fc9b9c6c 100644 --- a/web/scripts/app.js +++ b/web/scripts/app.js @@ -71,7 +71,7 @@ export class ComfyApp { * Stores the execution output data for each node * @type {Record} */ - this.nodeOutputs = {}; + this._nodeOutputs = {}; /** * Stores the preview image data for each node @@ -86,6 +86,15 @@ export class ComfyApp { this.shiftDown = false; } + get nodeOutputs() { + return this._nodeOutputs; + } + + set nodeOutputs(value) { + this._nodeOutputs = value; + this.#invokeExtensions("onNodeOutputsUpdated", value); + } + getPreviewFormatParam() { let preview_format = this.ui.settings.getSettingValue("Comfy.PreviewFormat"); if(preview_format) diff --git a/web/style.css b/web/style.css index 8091a489f..e983b652a 100644 --- a/web/style.css +++ b/web/style.css @@ -632,3 +632,7 @@ dialog::backdrop { border-top: none; } } + +audio.comfy-audio.empty-audio-widget { + display: none; +} From 4650e7d6e9e846160906b94d575705151d59fdd1 Mon Sep 17 00:00:00 2001 From: comfyanonymous Date: Thu, 27 Jun 2024 01:28:59 -0400 Subject: [PATCH 10/24] Save and load workflow from the flac files output by SaveAudio. --- comfy_extras/nodes_audio.py | 79 ++++++++++++++++++++++++++++++++++++- web/scripts/app.js | 15 ++++++- web/scripts/pnginfo.js | 72 +++++++++++++++++++++++++++++++++ web/scripts/ui.js | 2 +- 4 files changed, 164 insertions(+), 4 deletions(-) diff --git a/comfy_extras/nodes_audio.py b/comfy_extras/nodes_audio.py index 8efa7c0a8..34bcfa96d 100644 --- a/comfy_extras/nodes_audio.py +++ b/comfy_extras/nodes_audio.py @@ -3,6 +3,10 @@ import torch import comfy.model_management import folder_paths import os +import io +import json +import struct +from comfy.cli_args import args class EmptyLatentAudio: def __init__(self): @@ -53,6 +57,61 @@ class VAEDecodeAudio: audio = vae.decode(samples["samples"]).movedim(-1, 1) return ({"waveform": audio, "sample_rate": 44100}, ) + +def create_vorbis_comment_block(comment_dict, last_block): + vendor_string = b'ComfyUI' + vendor_length = len(vendor_string) + + comments = [] + for key, value in comment_dict.items(): + comment = f"{key}={value}".encode('utf-8') + comments.append(struct.pack('I', len(comment_data))[1:] + comment_data + + return comment_block + +def insert_or_replace_vorbis_comment(flac_io, comment_dict): + if len(comment_dict) == 0: + return flac_io + + flac_io.seek(4) + + blocks = [] + last_block = False + + while not last_block: + header = flac_io.read(4) + last_block = (header[0] & 0x80) != 0 + block_type = header[0] & 0x7F + block_length = struct.unpack('>I', b'\x00' + header[1:])[0] + block_data = flac_io.read(block_length) + + if block_type == 4 or block_type == 1: + pass + else: + header = bytes([(header[0] & (~0x80))]) + header[1:] + blocks.append(header + block_data) + + blocks.append(create_vorbis_comment_block(comment_dict, last_block=True)) + + new_flac_io = io.BytesIO() + new_flac_io.write(b'fLaC') + for block in blocks: + new_flac_io.write(block) + + new_flac_io.write(flac_io.read()) + return new_flac_io + + class SaveAudio: def __init__(self): self.output_dir = folder_paths.get_output_directory() @@ -78,11 +137,27 @@ class SaveAudio: filename_prefix += self.prefix_append full_output_folder, filename, counter, subfolder, filename_prefix = folder_paths.get_save_image_path(filename_prefix, self.output_dir) results = list() + + metadata = {} + if not args.disable_metadata: + if prompt is not None: + metadata["prompt"] = json.dumps(prompt) + if extra_pnginfo is not None: + for x in extra_pnginfo: + metadata[x] = json.dumps(extra_pnginfo[x]) + for (batch_number, waveform) in enumerate(audio["waveform"]): - #TODO: metadata filename_with_batch_num = filename.replace("%batch_num%", str(batch_number)) file = f"{filename_with_batch_num}_{counter:05}_.flac" - torchaudio.save(os.path.join(full_output_folder, file), waveform, audio["sample_rate"], format="FLAC") + + buff = io.BytesIO() + torchaudio.save(buff, waveform, audio["sample_rate"], format="FLAC") + + buff = insert_or_replace_vorbis_comment(buff, metadata) + + with open(os.path.join(full_output_folder, file), 'wb') as f: + f.write(buff.getbuffer()) + results.append({ "filename": file, "subfolder": subfolder, diff --git a/web/scripts/app.js b/web/scripts/app.js index 5fc9b9c6c..43df61065 100644 --- a/web/scripts/app.js +++ b/web/scripts/app.js @@ -3,7 +3,7 @@ import { ComfyWidgets, initWidgets } from "./widgets.js"; import { ComfyUI, $el } from "./ui.js"; import { api } from "./api.js"; import { defaultGraph } from "./defaultGraph.js"; -import { getPngMetadata, getWebpMetadata, importA1111, getLatentMetadata } from "./pnginfo.js"; +import { getPngMetadata, getWebpMetadata, getFlacMetadata, importA1111, getLatentMetadata } from "./pnginfo.js"; import { addDomClippingSetting } from "./domWidget.js"; import { createImageHost, calculateImageGrid } from "./ui/imagePreview.js"; import { ComfyAppMenu } from "./ui/menu/index.js"; @@ -2277,6 +2277,19 @@ export class ComfyApp { const workflow = pngInfo?.workflow || pngInfo?.Workflow; const prompt = pngInfo?.prompt || pngInfo?.Prompt; + if (workflow) { + this.loadGraphData(JSON.parse(workflow), true, true, fileName); + } else if (prompt) { + this.loadApiJson(JSON.parse(prompt), fileName); + } else { + this.showErrorOnFileLoad(file); + } + } else if (file.type === "audio/flac") { + const pngInfo = await getFlacMetadata(file); + // Support loading workflows from that webp custom node. + const workflow = pngInfo?.workflow; + const prompt = pngInfo?.prompt; + if (workflow) { this.loadGraphData(JSON.parse(workflow), true, true, fileName); } else if (prompt) { diff --git a/web/scripts/pnginfo.js b/web/scripts/pnginfo.js index 7132fb60f..2c03cf74a 100644 --- a/web/scripts/pnginfo.js +++ b/web/scripts/pnginfo.js @@ -163,6 +163,78 @@ export function getLatentMetadata(file) { }); } + +function getString(dataView, offset, length) { + let string = ''; + for (let i = 0; i < length; i++) { + string += String.fromCharCode(dataView.getUint8(offset + i)); + } + return string; +} + +// Function to parse the Vorbis Comment block +function parseVorbisComment(dataView) { + let offset = 0; + const vendorLength = dataView.getUint32(offset, true); + offset += 4; + const vendorString = getString(dataView, offset, vendorLength); + offset += vendorLength; + + const userCommentListLength = dataView.getUint32(offset, true); + offset += 4; + const comments = {}; + for (let i = 0; i < userCommentListLength; i++) { + const commentLength = dataView.getUint32(offset, true); + offset += 4; + const comment = getString(dataView, offset, commentLength); + offset += commentLength; + + const [key, value] = comment.split('='); + + comments[key] = value; + } + + return comments; +} + +// Function to read a FLAC file and parse Vorbis comments +export function getFlacMetadata(file) { + return new Promise((r) => { + const reader = new FileReader(); + reader.onload = function(event) { + const arrayBuffer = event.target.result; + const dataView = new DataView(arrayBuffer); + + // Verify the FLAC signature + const signature = String.fromCharCode(...new Uint8Array(arrayBuffer, 0, 4)); + if (signature !== 'fLaC') { + console.error('Not a valid FLAC file'); + return; + } + + // Parse metadata blocks + let offset = 4; + let vorbisComment = null; + while (offset < dataView.byteLength) { + const isLastBlock = dataView.getUint8(offset) & 0x80; + const blockType = dataView.getUint8(offset) & 0x7F; + const blockSize = dataView.getUint32(offset, false) & 0xFFFFFF; + offset += 4; + + if (blockType === 4) { // Vorbis Comment block type + vorbisComment = parseVorbisComment(new DataView(arrayBuffer, offset, blockSize)); + } + + offset += blockSize; + if (isLastBlock) break; + } + + r(vorbisComment); + }; + reader.readAsArrayBuffer(file); + }); +} + export async function importA1111(graph, parameters) { const p = parameters.lastIndexOf("\nSteps:"); if (p > -1) { diff --git a/web/scripts/ui.js b/web/scripts/ui.js index bb094d8b4..2c47412c9 100644 --- a/web/scripts/ui.js +++ b/web/scripts/ui.js @@ -373,7 +373,7 @@ export class ComfyUI { const fileInput = $el("input", { id: "comfy-file-input", type: "file", - accept: ".json,image/png,.latent,.safetensors,image/webp", + accept: ".json,image/png,.latent,.safetensors,image/webp,audio/flac", style: {display: "none"}, parent: document.body, onchange: () => { From 4f9d2b057c97be22aa46b91ed8954a3cee424fb4 Mon Sep 17 00:00:00 2001 From: comfyanonymous Date: Thu, 27 Jun 2024 02:54:15 -0400 Subject: [PATCH 11/24] Remove print. --- comfy/k_diffusion/deis.py | 1 - 1 file changed, 1 deletion(-) diff --git a/comfy/k_diffusion/deis.py b/comfy/k_diffusion/deis.py index 0be2036c3..607410656 100644 --- a/comfy/k_diffusion/deis.py +++ b/comfy/k_diffusion/deis.py @@ -117,6 +117,5 @@ def get_deis_coeff_list(t_steps, max_order, N=10000, deis_mode='tab'): coeff_prev3 = get_def_intergral_3(t_cur, prev_t[1], prev_t[2], t_cur, t_next, prev_t[3]) coeff_temp = [coeff_cur, coeff_prev1, coeff_prev2, coeff_prev3] C.append(coeff_temp) - print(C) return C From 5ff3d4eb3a91d1e66d35a919dd38c02b4652e96a Mon Sep 17 00:00:00 2001 From: Chenlei Hu Date: Thu, 27 Jun 2024 09:13:52 -0400 Subject: [PATCH 12/24] Fix audio upload when no audio in input dir (#3891) --- web/extensions/core/uploadAudio.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/web/extensions/core/uploadAudio.js b/web/extensions/core/uploadAudio.js index d3dfef4a7..12a2a196b 100644 --- a/web/extensions/core/uploadAudio.js +++ b/web/extensions/core/uploadAudio.js @@ -145,7 +145,9 @@ app.registerExtension({ ) } // Initially load default audio file to audioUIWidget. - onAudioWidgetUpdate() + if (audioWidget.value) { + onAudioWidgetUpdate() + } audioWidget.callback = onAudioWidgetUpdate const fileInput = document.createElement("input") From 8ceb5a02a3182e6551690d9e56934bdca1cebcca Mon Sep 17 00:00:00 2001 From: comfyanonymous Date: Thu, 27 Jun 2024 11:06:52 -0400 Subject: [PATCH 13/24] Support saving stable audio checkpoint that can be loaded back. --- comfy/model_base.py | 9 +++++++++ comfy/sd.py | 2 +- comfy/supported_models.py | 5 ++++- 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/comfy/model_base.py b/comfy/model_base.py index f45b375de..80f6667ec 100644 --- a/comfy/model_base.py +++ b/comfy/model_base.py @@ -627,3 +627,12 @@ class StableAudio1(BaseModel): cross_attn = torch.cat([cross_attn.to(device), seconds_start_embed.repeat((cross_attn.shape[0], 1, 1)), seconds_total_embed.repeat((cross_attn.shape[0], 1, 1))], dim=1) out['c_crossattn'] = comfy.conds.CONDRegular(cross_attn) return out + + def state_dict_for_saving(self, clip_state_dict=None, vae_state_dict=None, clip_vision_state_dict=None): + sd = super().state_dict_for_saving(clip_state_dict=clip_state_dict, vae_state_dict=vae_state_dict, clip_vision_state_dict=clip_vision_state_dict) + d = {"conditioner.conditioners.seconds_start.": self.seconds_start_embedder.state_dict(), "conditioner.conditioners.seconds_total.": self.seconds_total_embedder.state_dict()} + for k in d: + s = d[k] + for l in s: + sd["{}{}".format(k, l)] = s[l] + return sd diff --git a/comfy/sd.py b/comfy/sd.py index ea6e9b663..dda0887ba 100644 --- a/comfy/sd.py +++ b/comfy/sd.py @@ -236,7 +236,7 @@ class VAE: self.first_stage_model = AutoencodingEngine(regularizer_config={'target': "comfy.ldm.models.autoencoder.DiagonalGaussianRegularizer"}, encoder_config={'target': "comfy.ldm.modules.diffusionmodules.model.Encoder", 'params': ddconfig}, decoder_config={'target': "comfy.ldm.modules.diffusionmodules.model.Decoder", 'params': ddconfig}) - elif "decoder.layers.0.weight_v" in sd: + elif "decoder.layers.1.layers.0.beta" in sd: self.first_stage_model = AudioOobleckVAE() self.memory_used_encode = lambda shape, dtype: (1000 * shape[2]) * model_management.dtype_size(dtype) self.memory_used_decode = lambda shape, dtype: (1000 * shape[2] * 2048) * model_management.dtype_size(dtype) diff --git a/comfy/supported_models.py b/comfy/supported_models.py index 761498dbc..21fdb7ec7 100644 --- a/comfy/supported_models.py +++ b/comfy/supported_models.py @@ -543,13 +543,16 @@ class StableAudio(supported_models_base.BASE): seconds_total_sd = utils.state_dict_prefix_replace(state_dict, {"conditioner.conditioners.seconds_total.": ""}, filter_keys=True) return model_base.StableAudio1(self, seconds_start_embedder_weights=seconds_start_sd, seconds_total_embedder_weights=seconds_total_sd, device=device) - def process_unet_state_dict(self, state_dict): for k in list(state_dict.keys()): if k.endswith(".cross_attend_norm.beta") or k.endswith(".ff_norm.beta") or k.endswith(".pre_norm.beta"): #These weights are all zero state_dict.pop(k) return state_dict + def process_unet_state_dict_for_saving(self, state_dict): + replace_prefix = {"": "model.model."} + return utils.state_dict_prefix_replace(state_dict, replace_prefix) + def clip_target(self, state_dict={}): return supported_models_base.ClipTarget(sa_t5.SAT5Tokenizer, sa_t5.SAT5Model) From 449bf52923896236ed645da2ff05aadbf2b0d536 Mon Sep 17 00:00:00 2001 From: Robin Huang Date: Thu, 27 Jun 2024 13:08:26 -0700 Subject: [PATCH 14/24] Add integration test for Linux with Nvidia GPU. #3884 (#3895) * Add linux integration test. * Fix directory path. * Add paths ignore. * Fix conda env directory path. --- .../conda-environments/linux-environment.yml | 28 +++++++++++ .github/workflows/linux-integration-test.yml | 50 +++++++++++++++++++ 2 files changed, 78 insertions(+) create mode 100644 .github/conda-environments/linux-environment.yml create mode 100644 .github/workflows/linux-integration-test.yml diff --git a/.github/conda-environments/linux-environment.yml b/.github/conda-environments/linux-environment.yml new file mode 100644 index 000000000..c9ebd6406 --- /dev/null +++ b/.github/conda-environments/linux-environment.yml @@ -0,0 +1,28 @@ +name: comfyui +channels: + - pytorch + - nvidia + - defaults +dependencies: + - python>=3.9 + - pip + - pytorch + - torchvision + - torchaudio + - pytorch-cuda=12.1 + - pip: + # comfyui requirements + - einops + - transformers>=4.25.1 + - safetensors>=0.4.2 + - aiohttp + - pyyaml + - Pillow + - scipy + - tqdm + - psutil + # comfy-action requirements + - requests + - google-cloud-storage + - comfy-cli + - charset-normalizer \ No newline at end of file diff --git a/.github/workflows/linux-integration-test.yml b/.github/workflows/linux-integration-test.yml new file mode 100644 index 000000000..eed84a77c --- /dev/null +++ b/.github/workflows/linux-integration-test.yml @@ -0,0 +1,50 @@ +name: (Linux) ComfyUI Integration Tests +on: + push: + branches: + - master + paths-ignore: + - 'app/**' + - 'input/**' + - 'output/**' + - 'model/**' + - 'notebook/**' + - 'script_example/**' + - 'tests/**' + - 'tests-ui/**' + - '.github/**' + - '.ci/**' + - 'web/**' + workflow_dispatch: + pull_request: + branches: + - master + paths-ignore: + - 'app/**' + - 'input/**' + - 'output/**' + - 'model/**' + - 'notebook/**' + - 'script_example/**' + - 'tests/**' + - 'tests-ui/**' + - '.github/**' + - '.ci/**' + - 'web/**' + +jobs: + test-workflows: + runs-on: [self-hosted, Linux, t4] + steps: + - name: Test ComfyUI Workflows + uses: comfy-org/comfy-action@main + with: + os: linux + cuda_version: 12.1 + models-json: '{"v1-5-pruned-emaonly.ckpt": {"url": "https://huggingface.co/runwayml/stable-diffusion-v1-5/resolve/main/v1-5-pruned-emaonly.ckpt", "directory": "checkpoints"}}' + workflow_filenames: "default.json" + gcs_bucket_name: 'comfy-ci-results' + google_credentials: ${{ secrets.GCS_SERVICE_ACCOUNT_JSON }} + output_prefix: 'ComfyUI' + conda_env_file: '.github/conda-environments/linux-environment.yml' + timeout: 50 From 97b409cd486b1a3b1114ae204802c81f3727d1a2 Mon Sep 17 00:00:00 2001 From: Robin Huang Date: Thu, 27 Jun 2024 13:10:16 -0700 Subject: [PATCH 15/24] Add macOs integration test for default workflow. (#3898) --- .../conda-environments/mac-environment.yml | 26 ++++++++++ .github/workflows/mac-integration-test.yml | 50 +++++++++++++++++++ 2 files changed, 76 insertions(+) create mode 100644 .github/conda-environments/mac-environment.yml create mode 100644 .github/workflows/mac-integration-test.yml diff --git a/.github/conda-environments/mac-environment.yml b/.github/conda-environments/mac-environment.yml new file mode 100644 index 000000000..33832b854 --- /dev/null +++ b/.github/conda-environments/mac-environment.yml @@ -0,0 +1,26 @@ +name: comfyui +channels: + - pytorch-nightly + - defaults +dependencies: + - python>=3.9 + - pytorch-nightly::pytorch + - torchvision + - torchaudio + - pip: + - pip + # comfyui requirements + - einops + - transformers>=4.25.1 + - safetensors>=0.4.2 + - aiohttp + - pyyaml + - Pillow + - scipy + - tqdm + - psutil + # comfy-action requirements + - requests + - google-cloud-storage + - comfy-cli + - charset-normalizer diff --git a/.github/workflows/mac-integration-test.yml b/.github/workflows/mac-integration-test.yml new file mode 100644 index 000000000..6a0482b1b --- /dev/null +++ b/.github/workflows/mac-integration-test.yml @@ -0,0 +1,50 @@ +name: (macOS) ComfyUI Integration Tests +on: + push: + branches: + - master + paths-ignore: + - 'app/**' + - 'input/**' + - 'output/**' + - 'model/**' + - 'notebook/**' + - 'script_example/**' + - 'tests/**' + - 'tests-ui/**' + - '.github/**' + - '.ci/**' + - 'web/**' + workflow_dispatch: + pull_request: + branches: + - master + paths-ignore: + - 'app/**' + - 'input/**' + - 'output/**' + - 'model/**' + - 'notebook/**' + - 'script_example/**' + - 'tests/**' + - 'tests-ui/**' + - '.github/**' + - '.ci/**' + - 'web/**' + +jobs: + test-workflows: + runs-on: [self-hosted, m2] + steps: + - name: Test ComfyUI Workflows + uses: comfy-org/comfy-action@main + with: + os: mac + models-json: '{"v1-5-pruned-emaonly.ckpt": {"url": "https://huggingface.co/runwayml/stable-diffusion-v1-5/resolve/main/v1-5-pruned-emaonly.ckpt", "directory": "checkpoints"}}' + workflow_filenames: "default.json" + gcs_bucket_name: 'comfy-ci-results' + google_credentials: ${{ secrets.GCS_SERVICE_ACCOUNT_JSON }} + output_prefix: 'ComfyUI' + conda_env_file: '.github/conda-environments/mac-environment.yml' + timeout: 50 + From 66aaa14001be9f7cf2b52f84c0dff588e36aabbf Mon Sep 17 00:00:00 2001 From: comfyanonymous Date: Tue, 25 Jun 2024 17:02:05 -0400 Subject: [PATCH 16/24] Controlnet refactor. --- comfy/cldm/cldm.py | 9 +++++---- comfy/controlnet.py | 35 ++++++++++----------------------- comfy/ldm/cascade/controlnet.py | 2 +- comfy/t2i_adapter/adapter.py | 10 ++++++++-- 4 files changed, 24 insertions(+), 32 deletions(-) diff --git a/comfy/cldm/cldm.py b/comfy/cldm/cldm.py index 28076dd92..f8a161594 100644 --- a/comfy/cldm/cldm.py +++ b/comfy/cldm/cldm.py @@ -289,7 +289,8 @@ class ControlNet(nn.Module): guided_hint = self.input_hint_block(hint, emb, context) - outs = [] + out_output = [] + out_middle = [] hs = [] if self.num_classes is not None: @@ -304,10 +305,10 @@ class ControlNet(nn.Module): guided_hint = None else: h = module(h, emb, context) - outs.append(zero_conv(h, emb, context)) + out_output.append(zero_conv(h, emb, context)) h = self.middle_block(h, emb, context) - outs.append(self.middle_block_out(h, emb, context)) + out_middle.append(self.middle_block_out(h, emb, context)) - return outs + return {"middle": out_middle, "output": out_output} diff --git a/comfy/controlnet.py b/comfy/controlnet.py index 8cf4a61a6..f50df6835 100644 --- a/comfy/controlnet.py +++ b/comfy/controlnet.py @@ -89,27 +89,12 @@ class ControlBase: return self.previous_controlnet.inference_memory_requirements(dtype) return 0 - def control_merge(self, control_input, control_output, control_prev, output_dtype): + def control_merge(self, control, control_prev, output_dtype): out = {'input':[], 'middle':[], 'output': []} - if control_input is not None: - for i in range(len(control_input)): - key = 'input' - x = control_input[i] - if x is not None: - x *= self.strength - if x.dtype != output_dtype: - x = x.to(output_dtype) - out[key].insert(0, x) - - if control_output is not None: + for key in control: + control_output = control[key] for i in range(len(control_output)): - if i == (len(control_output) - 1): - key = 'middle' - index = 0 - else: - key = 'output' - index = i x = control_output[i] if x is not None: if self.global_average_pooling: @@ -120,6 +105,7 @@ class ControlBase: x = x.to(output_dtype) out[key].append(x) + if control_prev is not None: for x in ['input', 'middle', 'output']: o = out[x] @@ -182,7 +168,7 @@ class ControlNet(ControlBase): x_noisy = self.model_sampling_current.calculate_input(t, x_noisy) control = self.control_model(x=x_noisy.to(dtype), hint=self.cond_hint, timesteps=timestep.float(), context=context.to(dtype), y=y) - return self.control_merge(None, control, control_prev, output_dtype) + return self.control_merge(control, control_prev, output_dtype) def copy(self): c = ControlNet(None, global_average_pooling=self.global_average_pooling, load_device=self.load_device, manual_cast_dtype=self.manual_cast_dtype) @@ -490,12 +476,11 @@ class T2IAdapter(ControlBase): self.control_input = self.t2i_model(self.cond_hint.to(x_noisy.dtype)) self.t2i_model.cpu() - control_input = list(map(lambda a: None if a is None else a.clone(), self.control_input)) - mid = None - if self.t2i_model.xl == True: - mid = control_input[-1:] - control_input = control_input[:-1] - return self.control_merge(control_input, mid, control_prev, x_noisy.dtype) + control_input = {} + for k in self.control_input: + control_input[k] = list(map(lambda a: None if a is None else a.clone(), self.control_input[k])) + + return self.control_merge(control_input, control_prev, x_noisy.dtype) def copy(self): c = T2IAdapter(self.t2i_model, self.channels_in, self.compression_ratio, self.upscale_algorithm) diff --git a/comfy/ldm/cascade/controlnet.py b/comfy/ldm/cascade/controlnet.py index 5dac59394..7a52c3c26 100644 --- a/comfy/ldm/cascade/controlnet.py +++ b/comfy/ldm/cascade/controlnet.py @@ -90,4 +90,4 @@ class ControlNet(nn.Module): proj_outputs = [None for _ in range(max(self.proj_blocks) + 1)] for i, idx in enumerate(self.proj_blocks): proj_outputs[idx] = self.projections[i](x) - return proj_outputs + return {"input": proj_outputs[::-1]} diff --git a/comfy/t2i_adapter/adapter.py b/comfy/t2i_adapter/adapter.py index e9a606b1c..10ea18e32 100644 --- a/comfy/t2i_adapter/adapter.py +++ b/comfy/t2i_adapter/adapter.py @@ -153,7 +153,13 @@ class Adapter(nn.Module): features.append(None) features.append(x) - return features + features = features[::-1] + + if self.xl: + return {"input": features[1:], "middle": features[:1]} + else: + return {"input": features} + class LayerNorm(nn.LayerNorm): @@ -290,4 +296,4 @@ class Adapter_light(nn.Module): features.append(None) features.append(x) - return features + return {"input": features[::-1]} From f8f7568d031a61f5d21ca0ed56ff56d7d83c9115 Mon Sep 17 00:00:00 2001 From: comfyanonymous Date: Tue, 25 Jun 2024 23:40:44 -0400 Subject: [PATCH 17/24] Basic SD3 controlnet implementation. Still missing the node to properly use it. --- comfy/cldm/mmdit.py | 91 +++++++++++++++++++++ comfy/controlnet.py | 45 +++++++++- comfy/ldm/modules/diffusionmodules/mmdit.py | 30 +++++-- comfy/model_detection.py | 11 ++- comfy/utils.py | 3 +- 5 files changed, 165 insertions(+), 15 deletions(-) create mode 100644 comfy/cldm/mmdit.py diff --git a/comfy/cldm/mmdit.py b/comfy/cldm/mmdit.py new file mode 100644 index 000000000..6e72474ce --- /dev/null +++ b/comfy/cldm/mmdit.py @@ -0,0 +1,91 @@ +import torch +from typing import Dict, Optional +import comfy.ldm.modules.diffusionmodules.mmdit +import comfy.latent_formats + +class ControlNet(comfy.ldm.modules.diffusionmodules.mmdit.MMDiT): + def __init__( + self, + num_blocks = None, + dtype = None, + device = None, + operations = None, + **kwargs, + ): + super().__init__(dtype=dtype, device=device, operations=operations, final_layer=False, num_blocks=num_blocks, **kwargs) + # controlnet_blocks + self.controlnet_blocks = torch.nn.ModuleList([]) + for _ in range(len(self.joint_blocks)): + self.controlnet_blocks.append(operations.Linear(self.hidden_size, self.hidden_size, device=device, dtype=dtype)) + + self.pos_embed_input = comfy.ldm.modules.diffusionmodules.mmdit.PatchEmbed( + None, + self.patch_size, + self.in_channels, + self.hidden_size, + bias=True, + strict_img_size=False, + dtype=dtype, + device=device, + operations=operations + ) + + self.latent_format = comfy.latent_formats.SD3() + + def forward( + self, + x: torch.Tensor, + timesteps: torch.Tensor, + y: Optional[torch.Tensor] = None, + context: Optional[torch.Tensor] = None, + hint = None, + ) -> torch.Tensor: + + #weird sd3 controlnet specific stuff + hint = hint * self.latent_format.scale_factor # self.latent_format.process_in(hint) + y = torch.zeros_like(y) + + + if self.context_processor is not None: + context = self.context_processor(context) + + hw = x.shape[-2:] + x = self.x_embedder(x) + self.cropped_pos_embed(hw, device=x.device).to(dtype=x.dtype, device=x.device) + x += self.pos_embed_input(hint) + + c = self.t_embedder(timesteps, dtype=x.dtype) + if y is not None and self.y_embedder is not None: + y = self.y_embedder(y) + c = c + y + + if context is not None: + context = self.context_embedder(context) + + if self.register_length > 0: + context = torch.cat( + ( + repeat(self.register, "1 ... -> b ...", b=x.shape[0]), + default(context, torch.Tensor([]).type_as(x)), + ), + 1, + ) + + output = [] + + blocks = len(self.joint_blocks) + for i in range(blocks): + context, x = self.joint_blocks[i]( + context, + x, + c=c, + use_checkpoint=self.use_checkpoint, + ) + + out = self.controlnet_blocks[i](x) + count = self.depth // blocks + if i == blocks - 1: + count -= 1 + for j in range(count): + output.append(out) + + return {"output": output} diff --git a/comfy/controlnet.py b/comfy/controlnet.py index f50df6835..9202c3194 100644 --- a/comfy/controlnet.py +++ b/comfy/controlnet.py @@ -11,6 +11,7 @@ import comfy.ops import comfy.cldm.cldm import comfy.t2i_adapter.adapter import comfy.ldm.cascade.controlnet +import comfy.cldm.mmdit def broadcast_image_to(tensor, target_batch_size, batched_number): @@ -94,13 +95,17 @@ class ControlBase: for key in control: control_output = control[key] + applied_to = set() for i in range(len(control_output)): x = control_output[i] if x is not None: if self.global_average_pooling: x = torch.mean(x, dim=(2, 3), keepdim=True).repeat(1, 1, x.shape[2], x.shape[3]) - x *= self.strength + if x not in applied_to: #memory saving strategy, allow shared tensors and only apply strength to shared tensors once + applied_to.add(x) + x *= self.strength + if x.dtype != output_dtype: x = x.to(output_dtype) @@ -120,17 +125,18 @@ class ControlBase: if o[i].shape[0] < prev_val.shape[0]: o[i] = prev_val + o[i] else: - o[i] += prev_val + o[i] = prev_val + o[i] #TODO: change back to inplace add if shared tensors stop being an issue return out class ControlNet(ControlBase): - def __init__(self, control_model=None, global_average_pooling=False, device=None, load_device=None, manual_cast_dtype=None): + def __init__(self, control_model=None, global_average_pooling=False, compression_ratio=8, device=None, load_device=None, manual_cast_dtype=None): super().__init__(device) self.control_model = control_model self.load_device = load_device if control_model is not None: self.control_model_wrapped = comfy.model_patcher.ModelPatcher(self.control_model, load_device=load_device, offload_device=comfy.model_management.unet_offload_device()) + self.compression_ratio = compression_ratio self.global_average_pooling = global_average_pooling self.model_sampling_current = None self.manual_cast_dtype = manual_cast_dtype @@ -308,6 +314,37 @@ class ControlLora(ControlNet): def inference_memory_requirements(self, dtype): return comfy.utils.calculate_parameters(self.control_weights) * comfy.model_management.dtype_size(dtype) + ControlBase.inference_memory_requirements(self, dtype) +def load_controlnet_mmdit(sd): + new_sd = comfy.model_detection.convert_diffusers_mmdit(sd, "") + model_config = comfy.model_detection.model_config_from_unet(new_sd, "", True) + num_blocks = comfy.model_detection.count_blocks(new_sd, 'joint_blocks.{}.') + for k in sd: + new_sd[k] = sd[k] + + supported_inference_dtypes = model_config.supported_inference_dtypes + + controlnet_config = model_config.unet_config + unet_dtype = comfy.model_management.unet_dtype(supported_dtypes=supported_inference_dtypes) + load_device = comfy.model_management.get_torch_device() + manual_cast_dtype = comfy.model_management.unet_manual_cast(unet_dtype, load_device) + if manual_cast_dtype is not None: + operations = comfy.ops.manual_cast + else: + operations = comfy.ops.disable_weight_init + + control_model = comfy.cldm.mmdit.ControlNet(num_blocks=num_blocks, operations=operations, device=load_device, dtype=unet_dtype, **controlnet_config) + missing, unexpected = control_model.load_state_dict(new_sd, strict=False) + + if len(missing) > 0: + logging.warning("missing controlnet keys: {}".format(missing)) + + if len(unexpected) > 0: + logging.debug("unexpected controlnet keys: {}".format(unexpected)) + + control = ControlNet(control_model, compression_ratio=1, load_device=load_device, manual_cast_dtype=manual_cast_dtype) + return control + + def load_controlnet(ckpt_path, model=None): controlnet_data = comfy.utils.load_torch_file(ckpt_path, safe_load=True) if "lora_controlnet" in controlnet_data: @@ -360,6 +397,8 @@ def load_controlnet(ckpt_path, model=None): if len(leftover_keys) > 0: logging.warning("leftover keys: {}".format(leftover_keys)) controlnet_data = new_sd + elif "controlnet_blocks.0.weight" in controlnet_data: #SD3 diffusers format + return load_controlnet_mmdit(controlnet_data) pth_key = 'control_model.zero_convs.0.0.weight' pth = False diff --git a/comfy/ldm/modules/diffusionmodules/mmdit.py b/comfy/ldm/modules/diffusionmodules/mmdit.py index 20d3a321a..927451534 100644 --- a/comfy/ldm/modules/diffusionmodules/mmdit.py +++ b/comfy/ldm/modules/diffusionmodules/mmdit.py @@ -745,6 +745,8 @@ class MMDiT(nn.Module): qkv_bias: bool = True, context_processor_layers = None, context_size = 4096, + num_blocks = None, + final_layer = True, dtype = None, #TODO device = None, operations = None, @@ -766,7 +768,10 @@ class MMDiT(nn.Module): # apply magic --> this defines a head_size of 64 self.hidden_size = 64 * depth num_heads = depth + if num_blocks is None: + num_blocks = depth + self.depth = depth self.num_heads = num_heads self.x_embedder = PatchEmbed( @@ -821,7 +826,7 @@ class MMDiT(nn.Module): mlp_ratio=mlp_ratio, qkv_bias=qkv_bias, attn_mode=attn_mode, - pre_only=i == depth - 1, + pre_only=(i == num_blocks - 1) and final_layer, rmsnorm=rmsnorm, scale_mod_only=scale_mod_only, swiglu=swiglu, @@ -830,11 +835,12 @@ class MMDiT(nn.Module): device=device, operations=operations ) - for i in range(depth) + for i in range(num_blocks) ] ) - self.final_layer = FinalLayer(self.hidden_size, patch_size, self.out_channels, dtype=dtype, device=device, operations=operations) + if final_layer: + self.final_layer = FinalLayer(self.hidden_size, patch_size, self.out_channels, dtype=dtype, device=device, operations=operations) if compile_core: assert False @@ -893,6 +899,7 @@ class MMDiT(nn.Module): x: torch.Tensor, c_mod: torch.Tensor, context: Optional[torch.Tensor] = None, + control = None, ) -> torch.Tensor: if self.register_length > 0: context = torch.cat( @@ -905,13 +912,20 @@ class MMDiT(nn.Module): # context is B, L', D # x is B, L, D - for block in self.joint_blocks: - context, x = block( + blocks = len(self.joint_blocks) + for i in range(blocks): + context, x = self.joint_blocks[i]( context, x, c=c_mod, use_checkpoint=self.use_checkpoint, ) + if control is not None: + control_o = control.get("output") + if i < len(control_o): + add = control_o[i] + if add is not None: + x += add x = self.final_layer(x, c_mod) # (N, T, patch_size ** 2 * out_channels) return x @@ -922,6 +936,7 @@ class MMDiT(nn.Module): t: torch.Tensor, y: Optional[torch.Tensor] = None, context: Optional[torch.Tensor] = None, + control = None, ) -> torch.Tensor: """ Forward pass of DiT. @@ -943,7 +958,7 @@ class MMDiT(nn.Module): if context is not None: context = self.context_embedder(context) - x = self.forward_core_with_concat(x, c, context) + x = self.forward_core_with_concat(x, c, context, control) x = self.unpatchify(x, hw=hw) # (N, out_channels, H, W) return x[:,:,:hw[-2],:hw[-1]] @@ -956,7 +971,8 @@ class OpenAISignatureMMDITWrapper(MMDiT): timesteps: torch.Tensor, context: Optional[torch.Tensor] = None, y: Optional[torch.Tensor] = None, + control = None, **kwargs, ) -> torch.Tensor: - return super().forward(x, timesteps, context=context, y=y) + return super().forward(x, timesteps, context=context, y=y, control=control) diff --git a/comfy/model_detection.py b/comfy/model_detection.py index e09dd381a..0b678480f 100644 --- a/comfy/model_detection.py +++ b/comfy/model_detection.py @@ -41,7 +41,9 @@ def detect_unet_config(state_dict, key_prefix): unet_config["in_channels"] = state_dict['{}x_embedder.proj.weight'.format(key_prefix)].shape[1] patch_size = state_dict['{}x_embedder.proj.weight'.format(key_prefix)].shape[2] unet_config["patch_size"] = patch_size - unet_config["out_channels"] = state_dict['{}final_layer.linear.weight'.format(key_prefix)].shape[0] // (patch_size * patch_size) + final_layer = '{}final_layer.linear.weight'.format(key_prefix) + if final_layer in state_dict: + unet_config["out_channels"] = state_dict[final_layer].shape[0] // (patch_size * patch_size) unet_config["depth"] = state_dict['{}x_embedder.proj.weight'.format(key_prefix)].shape[0] // 64 unet_config["input_size"] = None @@ -435,10 +437,11 @@ def model_config_from_diffusers_unet(state_dict): return None def convert_diffusers_mmdit(state_dict, output_prefix=""): - depth = count_blocks(state_dict, 'transformer_blocks.{}.') - if depth > 0: + num_blocks = count_blocks(state_dict, 'transformer_blocks.{}.') + if num_blocks > 0: + depth = state_dict["pos_embed.proj.weight"].shape[0] // 64 out_sd = {} - sd_map = comfy.utils.mmdit_to_diffusers({"depth": depth}, output_prefix=output_prefix) + sd_map = comfy.utils.mmdit_to_diffusers({"depth": depth, "num_blocks": num_blocks}, output_prefix=output_prefix) for k in sd_map: weight = state_dict.get(k, None) if weight is not None: diff --git a/comfy/utils.py b/comfy/utils.py index ed6c58a64..48618e076 100644 --- a/comfy/utils.py +++ b/comfy/utils.py @@ -298,7 +298,8 @@ def mmdit_to_diffusers(mmdit_config, output_prefix=""): key_map = {} depth = mmdit_config.get("depth", 0) - for i in range(depth): + num_blocks = mmdit_config.get("num_blocks", depth) + for i in range(num_blocks): block_from = "transformer_blocks.{}".format(i) block_to = "{}joint_blocks.{}".format(output_prefix, i) From 264caca20e1e051f3bcbec59d7209513ea99d4ed Mon Sep 17 00:00:00 2001 From: comfyanonymous Date: Wed, 26 Jun 2024 16:14:47 -0400 Subject: [PATCH 18/24] ControlNetApplySD3 node can now be used to use SD3 controlnets. --- comfy/cldm/mmdit.py | 5 ----- comfy/controlnet.py | 28 ++++++++++++++++++++++++---- comfy_extras/nodes_sd3.py | 15 +++++++++++++++ nodes.py | 4 ++-- 4 files changed, 41 insertions(+), 11 deletions(-) diff --git a/comfy/cldm/mmdit.py b/comfy/cldm/mmdit.py index 6e72474ce..ee0282bc7 100644 --- a/comfy/cldm/mmdit.py +++ b/comfy/cldm/mmdit.py @@ -1,7 +1,6 @@ import torch from typing import Dict, Optional import comfy.ldm.modules.diffusionmodules.mmdit -import comfy.latent_formats class ControlNet(comfy.ldm.modules.diffusionmodules.mmdit.MMDiT): def __init__( @@ -30,8 +29,6 @@ class ControlNet(comfy.ldm.modules.diffusionmodules.mmdit.MMDiT): operations=operations ) - self.latent_format = comfy.latent_formats.SD3() - def forward( self, x: torch.Tensor, @@ -42,10 +39,8 @@ class ControlNet(comfy.ldm.modules.diffusionmodules.mmdit.MMDiT): ) -> torch.Tensor: #weird sd3 controlnet specific stuff - hint = hint * self.latent_format.scale_factor # self.latent_format.process_in(hint) y = torch.zeros_like(y) - if self.context_processor is not None: context = self.context_processor(context) diff --git a/comfy/controlnet.py b/comfy/controlnet.py index 9202c3194..d00395139 100644 --- a/comfy/controlnet.py +++ b/comfy/controlnet.py @@ -7,6 +7,7 @@ import comfy.model_management import comfy.model_detection import comfy.model_patcher import comfy.ops +import comfy.latent_formats import comfy.cldm.cldm import comfy.t2i_adapter.adapter @@ -38,6 +39,8 @@ class ControlBase: self.cond_hint = None self.strength = 1.0 self.timestep_percent_range = (0.0, 1.0) + self.latent_format = None + self.vae = None self.global_average_pooling = False self.timestep_range = None self.compression_ratio = 8 @@ -48,10 +51,12 @@ class ControlBase: self.device = device self.previous_controlnet = None - def set_cond_hint(self, cond_hint, strength=1.0, timestep_percent_range=(0.0, 1.0)): + def set_cond_hint(self, cond_hint, strength=1.0, timestep_percent_range=(0.0, 1.0), vae=None): self.cond_hint_original = cond_hint self.strength = strength self.timestep_percent_range = timestep_percent_range + if self.latent_format is not None: + self.vae = vae return self def pre_run(self, model, percent_to_timestep_function): @@ -84,6 +89,8 @@ class ControlBase: c.global_average_pooling = self.global_average_pooling c.compression_ratio = self.compression_ratio c.upscale_algorithm = self.upscale_algorithm + c.latent_format = self.latent_format + c.vae = self.vae def inference_memory_requirements(self, dtype): if self.previous_controlnet is not None: @@ -129,7 +136,7 @@ class ControlBase: return out class ControlNet(ControlBase): - def __init__(self, control_model=None, global_average_pooling=False, compression_ratio=8, device=None, load_device=None, manual_cast_dtype=None): + def __init__(self, control_model=None, global_average_pooling=False, compression_ratio=8, latent_format=None, device=None, load_device=None, manual_cast_dtype=None): super().__init__(device) self.control_model = control_model self.load_device = load_device @@ -140,6 +147,7 @@ class ControlNet(ControlBase): self.global_average_pooling = global_average_pooling self.model_sampling_current = None self.manual_cast_dtype = manual_cast_dtype + self.latent_format = latent_format def get_control(self, x_noisy, t, cond, batched_number): control_prev = None @@ -162,7 +170,17 @@ class ControlNet(ControlBase): if self.cond_hint is not None: del self.cond_hint self.cond_hint = None - self.cond_hint = comfy.utils.common_upscale(self.cond_hint_original, x_noisy.shape[3] * self.compression_ratio, x_noisy.shape[2] * self.compression_ratio, self.upscale_algorithm, "center").to(dtype).to(self.device) + compression_ratio = self.compression_ratio + if self.vae is not None: + compression_ratio *= self.vae.downscale_ratio + self.cond_hint = comfy.utils.common_upscale(self.cond_hint_original, x_noisy.shape[3] * compression_ratio, x_noisy.shape[2] * compression_ratio, self.upscale_algorithm, "center") + if self.vae is not None: + loaded_models = comfy.model_management.loaded_models(only_currently_used=True) + self.cond_hint = self.vae.encode(self.cond_hint.movedim(1, -1)) + comfy.model_management.load_models_gpu(loaded_models) + if self.latent_format is not None: + self.cond_hint = self.latent_format.process_in(self.cond_hint) + self.cond_hint = self.cond_hint.to(device=self.device, dtype=dtype) if x_noisy.shape[0] != self.cond_hint.shape[0]: self.cond_hint = broadcast_image_to(self.cond_hint, x_noisy.shape[0], batched_number) @@ -341,7 +359,9 @@ def load_controlnet_mmdit(sd): if len(unexpected) > 0: logging.debug("unexpected controlnet keys: {}".format(unexpected)) - control = ControlNet(control_model, compression_ratio=1, load_device=load_device, manual_cast_dtype=manual_cast_dtype) + latent_format = comfy.latent_formats.SD3() + latent_format.shift_factor = 0 #SD3 controlnet weirdness + control = ControlNet(control_model, compression_ratio=1, latent_format=latent_format, load_device=load_device, manual_cast_dtype=manual_cast_dtype) return control diff --git a/comfy_extras/nodes_sd3.py b/comfy_extras/nodes_sd3.py index d0303aec5..548b1ad6a 100644 --- a/comfy_extras/nodes_sd3.py +++ b/comfy_extras/nodes_sd3.py @@ -80,8 +80,23 @@ class CLIPTextEncodeSD3: return ([[cond, {"pooled_output": pooled}]], ) +class ControlNetApplySD3(nodes.ControlNetApplyAdvanced): + @classmethod + def INPUT_TYPES(s): + return {"required": {"positive": ("CONDITIONING", ), + "negative": ("CONDITIONING", ), + "control_net": ("CONTROL_NET", ), + "vae": ("VAE", ), + "image": ("IMAGE", ), + "strength": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 10.0, "step": 0.01}), + "start_percent": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 1.0, "step": 0.001}), + "end_percent": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 1.0, "step": 0.001}) + }} + CATEGORY = "_for_testing/sd3" + NODE_CLASS_MAPPINGS = { "TripleCLIPLoader": TripleCLIPLoader, "EmptySD3LatentImage": EmptySD3LatentImage, "CLIPTextEncodeSD3": CLIPTextEncodeSD3, + "ControlNetApplySD3": ControlNetApplySD3, } diff --git a/nodes.py b/nodes.py index 99645b81c..04775d290 100644 --- a/nodes.py +++ b/nodes.py @@ -783,7 +783,7 @@ class ControlNetApplyAdvanced: CATEGORY = "conditioning" - def apply_controlnet(self, positive, negative, control_net, image, strength, start_percent, end_percent): + def apply_controlnet(self, positive, negative, control_net, image, strength, start_percent, end_percent, vae=None): if strength == 0: return (positive, negative) @@ -800,7 +800,7 @@ class ControlNetApplyAdvanced: if prev_cnet in cnets: c_net = cnets[prev_cnet] else: - c_net = control_net.copy().set_cond_hint(control_hint, strength, (start_percent, end_percent)) + c_net = control_net.copy().set_cond_hint(control_hint, strength, (start_percent, end_percent), vae) c_net.set_previous_controlnet(prev_cnet) cnets[prev_cnet] = c_net From 0d9009c96ea5a1922a1c187deef27c7f133ee946 Mon Sep 17 00:00:00 2001 From: pythongosssss <125205205+pythongosssss@users.noreply.github.com> Date: Fri, 28 Jun 2024 06:07:19 +0100 Subject: [PATCH 19/24] New menu/workflows fixes (#3900) * Fix auto queue * Detect added nodes via search * Fix loading workflows * Add button click style --- web/scripts/changeTracker.js | 12 +++++++++++- web/scripts/ui/menu/menu.css | 4 ++++ web/scripts/ui/menu/queueButton.js | 4 ++-- web/scripts/workflows.js | 4 ++-- 4 files changed, 19 insertions(+), 5 deletions(-) diff --git a/web/scripts/changeTracker.js b/web/scripts/changeTracker.js index 59901d5fc..041c83122 100644 --- a/web/scripts/changeTracker.js +++ b/web/scripts/changeTracker.js @@ -3,7 +3,6 @@ import { api } from "./api.js"; import { clone } from "./utils.js"; - export class ChangeTracker { static MAX_HISTORY = 50; #app; @@ -170,6 +169,17 @@ export class ChangeTracker { return v; }; + // Detects nodes being added via the node search dialog + const onNodeAdded = LiteGraph.LGraph.prototype.onNodeAdded; + LiteGraph.LGraph.prototype.onNodeAdded = function () { + const v = onNodeAdded?.apply(this, arguments); + const ct = changeTracker(); + if (!ct.isOurLoad) { + ct.checkState(); + } + return v; + }; + // Store node outputs api.addEventListener("executed", ({ detail }) => { const prompt = app.workflowManager.queuedPrompts[detail.prompt_id]; diff --git a/web/scripts/ui/menu/menu.css b/web/scripts/ui/menu/menu.css index 20eeab2cf..afaed3fb0 100644 --- a/web/scripts/ui/menu/menu.css +++ b/web/scripts/ui/menu/menu.css @@ -19,8 +19,12 @@ padding: 4px 8px; box-sizing: border-box; margin: 0; + transition: box-shadow 0.1s; } +.comfyui-button:active { + box-shadow: inset 1px 1px 10px rgba(0, 0, 0, 0.5); +} .comfyui-button:disabled { opacity: 0.5; cursor: not-allowed; diff --git a/web/scripts/ui/menu/queueButton.js b/web/scripts/ui/menu/queueButton.js index 3c29ab090..608f4cc9b 100644 --- a/web/scripts/ui/menu/queueButton.js +++ b/web/scripts/ui/menu/queueButton.js @@ -13,8 +13,8 @@ export class ComfyQueueButton { queuePrompt = async (e) => { this.#internalQueueSize += this.queueOptions.batchCount; - // Hold shift to queue front - await this.app.queuePrompt(-e.shiftKey, this.queueOptions.batchCount); + // Hold shift to queue front, event is undefined when auto-queue is enabled + await this.app.queuePrompt(e?.shiftKey ? -1 : 0, this.queueOptions.batchCount); }; constructor(app) { diff --git a/web/scripts/workflows.js b/web/scripts/workflows.js index 16bbc9976..d38b6f5fc 100644 --- a/web/scripts/workflows.js +++ b/web/scripts/workflows.js @@ -301,11 +301,11 @@ export class ComfyWorkflow { load = async () => { if (this.isOpen) { - await this.manager.app.loadGraphData(this.changeTracker.activeState, true, this); + await this.manager.app.loadGraphData(this.changeTracker.activeState, true, true, this); } else { const data = await this.getWorkflowData(); if (!data) return; - await this.manager.app.loadGraphData(data, true, this); + await this.manager.app.loadGraphData(data, true, true, this); } }; From 7ecb2ec1699dc988f93188c04733eeb8a41e8196 Mon Sep 17 00:00:00 2001 From: comfyanonymous Date: Fri, 28 Jun 2024 02:55:36 -0400 Subject: [PATCH 20/24] Audio second setting in EmptyLatentAudio. --- comfy_extras/nodes_audio.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/comfy_extras/nodes_audio.py b/comfy_extras/nodes_audio.py index 34bcfa96d..6481ed3ba 100644 --- a/comfy_extras/nodes_audio.py +++ b/comfy_extras/nodes_audio.py @@ -14,15 +14,16 @@ class EmptyLatentAudio: @classmethod def INPUT_TYPES(s): - return {"required": {}} + return {"required": {"seconds": ("FLOAT", {"default": 47.6, "min": 1.0, "max": 1000.0, "step": 0.1})}} RETURN_TYPES = ("LATENT",) FUNCTION = "generate" CATEGORY = "_for_testing/audio" - def generate(self): + def generate(self, seconds): batch_size = 1 - latent = torch.zeros([batch_size, 64, 1024], device=self.device) + length = round((seconds * 44100 / 2048) / 2) * 2 + latent = torch.zeros([batch_size, 64, length], device=self.device) return ({"samples":latent, "type": "audio"}, ) class VAEEncodeAudio: From 02cac1d48783725d5994d37ed03e9f3dc32ec504 Mon Sep 17 00:00:00 2001 From: Robin Huang Date: Fri, 28 Jun 2024 13:09:39 -0700 Subject: [PATCH 21/24] Revert "Add macOs integration test for default workflow. (#3898)" (#3904) This reverts commit 97b409cd486b1a3b1114ae204802c81f3727d1a2. --- .../conda-environments/mac-environment.yml | 26 ---------- .github/workflows/mac-integration-test.yml | 50 ------------------- 2 files changed, 76 deletions(-) delete mode 100644 .github/conda-environments/mac-environment.yml delete mode 100644 .github/workflows/mac-integration-test.yml diff --git a/.github/conda-environments/mac-environment.yml b/.github/conda-environments/mac-environment.yml deleted file mode 100644 index 33832b854..000000000 --- a/.github/conda-environments/mac-environment.yml +++ /dev/null @@ -1,26 +0,0 @@ -name: comfyui -channels: - - pytorch-nightly - - defaults -dependencies: - - python>=3.9 - - pytorch-nightly::pytorch - - torchvision - - torchaudio - - pip: - - pip - # comfyui requirements - - einops - - transformers>=4.25.1 - - safetensors>=0.4.2 - - aiohttp - - pyyaml - - Pillow - - scipy - - tqdm - - psutil - # comfy-action requirements - - requests - - google-cloud-storage - - comfy-cli - - charset-normalizer diff --git a/.github/workflows/mac-integration-test.yml b/.github/workflows/mac-integration-test.yml deleted file mode 100644 index 6a0482b1b..000000000 --- a/.github/workflows/mac-integration-test.yml +++ /dev/null @@ -1,50 +0,0 @@ -name: (macOS) ComfyUI Integration Tests -on: - push: - branches: - - master - paths-ignore: - - 'app/**' - - 'input/**' - - 'output/**' - - 'model/**' - - 'notebook/**' - - 'script_example/**' - - 'tests/**' - - 'tests-ui/**' - - '.github/**' - - '.ci/**' - - 'web/**' - workflow_dispatch: - pull_request: - branches: - - master - paths-ignore: - - 'app/**' - - 'input/**' - - 'output/**' - - 'model/**' - - 'notebook/**' - - 'script_example/**' - - 'tests/**' - - 'tests-ui/**' - - '.github/**' - - '.ci/**' - - 'web/**' - -jobs: - test-workflows: - runs-on: [self-hosted, m2] - steps: - - name: Test ComfyUI Workflows - uses: comfy-org/comfy-action@main - with: - os: mac - models-json: '{"v1-5-pruned-emaonly.ckpt": {"url": "https://huggingface.co/runwayml/stable-diffusion-v1-5/resolve/main/v1-5-pruned-emaonly.ckpt", "directory": "checkpoints"}}' - workflow_filenames: "default.json" - gcs_bucket_name: 'comfy-ci-results' - google_credentials: ${{ secrets.GCS_SERVICE_ACCOUNT_JSON }} - output_prefix: 'ComfyUI' - conda_env_file: '.github/conda-environments/mac-environment.yml' - timeout: 50 - From c39cf7fff09c728d573835a48106e2133c446e11 Mon Sep 17 00:00:00 2001 From: Robin Huang Date: Fri, 28 Jun 2024 13:09:55 -0700 Subject: [PATCH 22/24] Revert "Add integration test for Linux with Nvidia GPU. #3884 (#3895)" (#3905) This reverts commit 449bf52923896236ed645da2ff05aadbf2b0d536. --- .../conda-environments/linux-environment.yml | 28 ----------- .github/workflows/linux-integration-test.yml | 50 ------------------- 2 files changed, 78 deletions(-) delete mode 100644 .github/conda-environments/linux-environment.yml delete mode 100644 .github/workflows/linux-integration-test.yml diff --git a/.github/conda-environments/linux-environment.yml b/.github/conda-environments/linux-environment.yml deleted file mode 100644 index c9ebd6406..000000000 --- a/.github/conda-environments/linux-environment.yml +++ /dev/null @@ -1,28 +0,0 @@ -name: comfyui -channels: - - pytorch - - nvidia - - defaults -dependencies: - - python>=3.9 - - pip - - pytorch - - torchvision - - torchaudio - - pytorch-cuda=12.1 - - pip: - # comfyui requirements - - einops - - transformers>=4.25.1 - - safetensors>=0.4.2 - - aiohttp - - pyyaml - - Pillow - - scipy - - tqdm - - psutil - # comfy-action requirements - - requests - - google-cloud-storage - - comfy-cli - - charset-normalizer \ No newline at end of file diff --git a/.github/workflows/linux-integration-test.yml b/.github/workflows/linux-integration-test.yml deleted file mode 100644 index eed84a77c..000000000 --- a/.github/workflows/linux-integration-test.yml +++ /dev/null @@ -1,50 +0,0 @@ -name: (Linux) ComfyUI Integration Tests -on: - push: - branches: - - master - paths-ignore: - - 'app/**' - - 'input/**' - - 'output/**' - - 'model/**' - - 'notebook/**' - - 'script_example/**' - - 'tests/**' - - 'tests-ui/**' - - '.github/**' - - '.ci/**' - - 'web/**' - workflow_dispatch: - pull_request: - branches: - - master - paths-ignore: - - 'app/**' - - 'input/**' - - 'output/**' - - 'model/**' - - 'notebook/**' - - 'script_example/**' - - 'tests/**' - - 'tests-ui/**' - - '.github/**' - - '.ci/**' - - 'web/**' - -jobs: - test-workflows: - runs-on: [self-hosted, Linux, t4] - steps: - - name: Test ComfyUI Workflows - uses: comfy-org/comfy-action@main - with: - os: linux - cuda_version: 12.1 - models-json: '{"v1-5-pruned-emaonly.ckpt": {"url": "https://huggingface.co/runwayml/stable-diffusion-v1-5/resolve/main/v1-5-pruned-emaonly.ckpt", "directory": "checkpoints"}}' - workflow_filenames: "default.json" - gcs_bucket_name: 'comfy-ci-results' - google_credentials: ${{ secrets.GCS_SERVICE_ACCOUNT_JSON }} - output_prefix: 'ComfyUI' - conda_env_file: '.github/conda-environments/linux-environment.yml' - timeout: 50 From fbb7a1f1b663b170d49718590aa1cb4093586f20 Mon Sep 17 00:00:00 2001 From: comfyanonymous Date: Sat, 29 Jun 2024 01:33:22 -0400 Subject: [PATCH 23/24] PreviewAudio node. --- comfy_extras/nodes_audio.py | 16 +++++++++++++++- web/extensions/core/uploadAudio.js | 6 +++--- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/comfy_extras/nodes_audio.py b/comfy_extras/nodes_audio.py index 6481ed3ba..9950d58ed 100644 --- a/comfy_extras/nodes_audio.py +++ b/comfy_extras/nodes_audio.py @@ -6,6 +6,7 @@ import os import io import json import struct +import random from comfy.cli_args import args class EmptyLatentAudio: @@ -118,7 +119,6 @@ class SaveAudio: self.output_dir = folder_paths.get_output_directory() self.type = "output" self.prefix_append = "" - self.compress_level = 4 @classmethod def INPUT_TYPES(s): @@ -168,6 +168,19 @@ class SaveAudio: return { "ui": { "audio": results } } +class PreviewAudio(SaveAudio): + def __init__(self): + self.output_dir = folder_paths.get_temp_directory() + self.type = "temp" + self.prefix_append = "_temp_" + ''.join(random.choice("abcdefghijklmnopqrstupvxyz") for x in range(5)) + + @classmethod + def INPUT_TYPES(s): + return {"required": + {"audio": ("AUDIO", ), }, + "hidden": {"prompt": "PROMPT", "extra_pnginfo": "EXTRA_PNGINFO"}, + } + class LoadAudio: SUPPORTED_FORMATS = ('.wav', '.mp3', '.ogg', '.flac', '.aiff', '.aif') @@ -214,4 +227,5 @@ NODE_CLASS_MAPPINGS = { "VAEDecodeAudio": VAEDecodeAudio, "SaveAudio": SaveAudio, "LoadAudio": LoadAudio, + "PreviewAudio": PreviewAudio, } diff --git a/web/extensions/core/uploadAudio.js b/web/extensions/core/uploadAudio.js index 12a2a196b..0ac9cb807 100644 --- a/web/extensions/core/uploadAudio.js +++ b/web/extensions/core/uploadAudio.js @@ -70,7 +70,7 @@ async function uploadFile( app.registerExtension({ name: "Comfy.AudioWidget", async beforeRegisterNodeDef(nodeType, nodeData) { - if (["LoadAudio", "SaveAudio"].includes(nodeType.comfyClass)) { + if (["LoadAudio", "SaveAudio", "PreviewAudio"].includes(nodeType.comfyClass)) { nodeData.input.required.audioUI = ["AUDIO_UI"] } }, @@ -103,7 +103,7 @@ app.registerExtension({ if (!audios) return const audio = audios[0] audioUIWidget.element.src = api.apiURL( - getResourceURL(audio.subfolder, audio.filename, "output") + getResourceURL(audio.subfolder, audio.filename, audio.type) ) audioUIWidget.element.classList.remove("empty-audio-widget") } @@ -118,7 +118,7 @@ app.registerExtension({ if ("audio" in output) { const audioUIWidget = node.widgets.find((w) => w.name === "audioUI"); const audio = output.audio[0]; - audioUIWidget.element.src = api.apiURL(getResourceURL(audio.subfolder, audio.filename, "output")); + audioUIWidget.element.src = api.apiURL(getResourceURL(audio.subfolder, audio.filename, audio.type)); audioUIWidget.element.classList.remove("empty-audio-widget"); } } From 05e831697a03984032863b2aa5d6443e422713e6 Mon Sep 17 00:00:00 2001 From: comfyanonymous Date: Sat, 29 Jun 2024 11:59:48 -0400 Subject: [PATCH 24/24] Switch to the real cfg++ method in the samplers. The old _pp ones will be updated automatically to the regular ones with 2x the cfg. My fault for not checking what the "_pp" samplers actually did. --- comfy/k_diffusion/sampling.py | 12 ++++++------ comfy/samplers.py | 2 +- web/scripts/app.js | 8 ++++++++ 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/comfy/k_diffusion/sampling.py b/comfy/k_diffusion/sampling.py index f8091bb3f..763d8cc78 100644 --- a/comfy/k_diffusion/sampling.py +++ b/comfy/k_diffusion/sampling.py @@ -998,7 +998,7 @@ def sample_deis(model, x, sigmas, extra_args=None, callback=None, disable=None, return x_next @torch.no_grad() -def sample_euler_pp(model, x, sigmas, extra_args=None, callback=None, disable=None): +def sample_euler_cfg_pp(model, x, sigmas, extra_args=None, callback=None, disable=None): extra_args = {} if extra_args is None else extra_args temp = [0] @@ -1013,16 +1013,16 @@ def sample_euler_pp(model, x, sigmas, extra_args=None, callback=None, disable=No for i in trange(len(sigmas) - 1, disable=disable): sigma_hat = sigmas[i] denoised = model(x, sigma_hat * s_in, **extra_args) - d = to_d(x - denoised + temp[0], sigma_hat, denoised) + d = to_d(x, sigma_hat, temp[0]) if callback is not None: callback({'x': x, 'i': i, 'sigma': sigmas[i], 'sigma_hat': sigma_hat, 'denoised': denoised}) dt = sigmas[i + 1] - sigma_hat # Euler method - x = x + d * dt + x = denoised + d * sigmas[i + 1] return x @torch.no_grad() -def sample_euler_ancestral_pp(model, x, sigmas, extra_args=None, callback=None, disable=None, eta=1., s_noise=1., noise_sampler=None): +def sample_euler_ancestral_cfg_pp(model, x, sigmas, extra_args=None, callback=None, disable=None, eta=1., s_noise=1., noise_sampler=None): """Ancestral sampling with Euler method steps.""" extra_args = {} if extra_args is None else extra_args noise_sampler = default_noise_sampler(x) if noise_sampler is None else noise_sampler @@ -1041,10 +1041,10 @@ def sample_euler_ancestral_pp(model, x, sigmas, extra_args=None, callback=None, sigma_down, sigma_up = get_ancestral_step(sigmas[i], sigmas[i + 1], eta=eta) if callback is not None: callback({'x': x, 'i': i, 'sigma': sigmas[i], 'sigma_hat': sigmas[i], 'denoised': denoised}) - d = to_d(x - denoised + temp[0], sigmas[i], denoised) + d = to_d(x, sigmas[i], temp[0]) # Euler method dt = sigma_down - sigmas[i] - x = x + d * dt + x = denoised + d * sigma_down if sigmas[i + 1] > 0: x = x + noise_sampler(sigmas[i], sigmas[i + 1]) * s_noise * sigma_up return x diff --git a/comfy/samplers.py b/comfy/samplers.py index 7f7114dbb..c0aa12916 100644 --- a/comfy/samplers.py +++ b/comfy/samplers.py @@ -537,7 +537,7 @@ class Sampler: sigma = float(sigmas[0]) return math.isclose(max_sigma, sigma, rel_tol=1e-05) or sigma > max_sigma -KSAMPLER_NAMES = ["euler", "euler_pp", "euler_ancestral", "euler_ancestral_pp", "heun", "heunpp2","dpm_2", "dpm_2_ancestral", +KSAMPLER_NAMES = ["euler", "euler_cfg_pp", "euler_ancestral", "euler_ancestral_cfg_pp", "heun", "heunpp2","dpm_2", "dpm_2_ancestral", "lms", "dpm_fast", "dpm_adaptive", "dpmpp_2s_ancestral", "dpmpp_sde", "dpmpp_sde_gpu", "dpmpp_2m", "dpmpp_2m_sde", "dpmpp_2m_sde_gpu", "dpmpp_3m_sde", "dpmpp_3m_sde_gpu", "ddpm", "lcm", "ipndm", "ipndm_v", "deis"] diff --git a/web/scripts/app.js b/web/scripts/app.js index 43df61065..3a91aa239 100644 --- a/web/scripts/app.js +++ b/web/scripts/app.js @@ -1966,6 +1966,14 @@ export class ComfyApp { if (widget.value.startsWith("sample_")) { widget.value = widget.value.slice(7); } + if (widget.value === "euler_pp" || widget.value === "euler_ancestral_pp") { + widget.value = widget.value.slice(0, -3); + for (let w of node.widgets) { + if (w.name == "cfg") { + w.value *= 2.0; + } + } + } } } if (node.type == "KSampler" || node.type == "KSamplerAdvanced" || node.type == "PrimitiveNode") {