diff --git a/.github/ISSUE_TEMPLATE/bug-report.yml b/.github/ISSUE_TEMPLATE/bug-report.yml new file mode 100644 index 000000000..87bef3681 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug-report.yml @@ -0,0 +1,45 @@ +name: Bug Report +description: "Something is broken inside of ComfyUI. (Do not use this if you're just having issues and need help, or if the issue relates to a custom node)" +labels: [ "Potential Bug" ] +body: + - type: markdown + attributes: + value: | + Before submitting a **Bug Report**, please ensure the following: + + **1:** You are running the latest version of ComfyUI. + **2:** You have looked at the existing bug reports and made sure this isn't already reported. + **3:** This is an actual bug in ComfyUI, not just a support question and not caused by an custom node. A bug is when you can specify exact steps to replicate what went wrong and others will be able to repeat your steps and see the same issue happen. + + If unsure, ask on the [ComfyUI Matrix Space](https://app.element.io/#/room/%23comfyui_space%3Amatrix.org) or the [Comfy Org Discord](https://discord.gg/comfyorg) first. + - type: textarea + attributes: + label: Expected Behavior + description: "What you expected to happen." + validations: + required: true + - type: textarea + attributes: + label: Actual Behavior + description: "What actually happened. Please include a screenshot of the issue if possible." + validations: + required: true + - type: textarea + attributes: + label: Steps to Reproduce + description: "Describe how to reproduce the issue. Please be sure to attach a workflow JSON or PNG, ideally one that doesn't require custom nodes to test. If the bug open happens when certain custom nodes are used, most likely that custom node is what has the bug rather than ComfyUI, in which case it should be reported to the node's author." + validations: + required: true + - type: textarea + attributes: + label: Debug Logs + description: "Please copy the output from your terminal logs here." + render: powershell + validations: + required: true + - type: textarea + attributes: + label: Other + description: "Any other additional information you think might be helpful." + validations: + required: false diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 000000000..2c519ede5 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,8 @@ +blank_issues_enabled: true +contact_links: + - name: ComfyUI Matrix Space + url: https://app.element.io/#/room/%23comfyui_space%3Amatrix.org + about: The ComfyUI Matrix Space is available for support and general discussion related to ComfyUI (Matrix is like Discord but open source). + - name: Comfy Org Discord + url: https://discord.gg/comfyorg + about: The Comfy Org Discord is available for support and general discussion related to ComfyUI. diff --git a/.github/ISSUE_TEMPLATE/feature-request.yml b/.github/ISSUE_TEMPLATE/feature-request.yml new file mode 100644 index 000000000..419721b63 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature-request.yml @@ -0,0 +1,32 @@ +name: Feature Request +description: "You have an idea for something new you would like to see added to ComfyUI's core." +labels: [ "Feature" ] +body: + - type: markdown + attributes: + value: | + Before submitting a **Feature Request**, please ensure the following: + + **1:** You are running the latest version of ComfyUI. + **2:** You have looked to make sure there is not already a feature that does what you need, and there is not already a Feature Request listed for the same idea. + **3:** This is something that makes sense to add to ComfyUI Core, and wouldn't make more sense as a custom node. + + If unsure, ask on the [ComfyUI Matrix Space](https://app.element.io/#/room/%23comfyui_space%3Amatrix.org) or the [Comfy Org Discord](https://discord.gg/comfyorg) first. + - type: textarea + attributes: + label: Feature Idea + description: "Describe the feature you want to see." + validations: + required: true + - type: textarea + attributes: + label: Existing Solutions + description: "Please search through available custom nodes / extensions to see if there are existing custom solutions for this. If so, please link the options you found here as a reference." + validations: + required: false + - type: textarea + attributes: + label: Other + description: "Any other additional information you think might be helpful." + validations: + required: false diff --git a/.github/ISSUE_TEMPLATE/user-support.yml b/.github/ISSUE_TEMPLATE/user-support.yml new file mode 100644 index 000000000..df28804c6 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/user-support.yml @@ -0,0 +1,32 @@ +name: User Support +description: "Use this if you need help with something, or you're experiencing an issue." +labels: [ "User Support" ] +body: + - type: markdown + attributes: + value: | + Before submitting a **User Report** issue, please ensure the following: + + **1:** You are running the latest version of ComfyUI. + **2:** You have made an effort to find public answers to your question before asking here. In other words, you googled it first, and scrolled through recent help topics. + + If unsure, ask on the [ComfyUI Matrix Space](https://app.element.io/#/room/%23comfyui_space%3Amatrix.org) or the [Comfy Org Discord](https://discord.gg/comfyorg) first. + - type: textarea + attributes: + label: Your question + description: "Post your question here. Please be as detailed as possible." + validations: + required: true + - type: textarea + attributes: + label: Logs + description: "If your question relates to an issue you're experiencing, please go to `Server` -> `Logs` -> potentially set `View Type` to `Debug` as well, then copypaste all the text into here." + render: powershell + validations: + required: false + - type: textarea + attributes: + label: Other + description: "Any other additional information you think might be helpful." + validations: + required: false diff --git a/.github/workflows/test-browser.yml b/.github/workflows/test-browser.yml index 049b1cde2..65483b00f 100644 --- a/.github/workflows/test-browser.yml +++ b/.github/workflows/test-browser.yml @@ -7,6 +7,8 @@ name: Playwright Browser Tests CI on: + push: + branches: [ main, master ] pull_request: branches: [ main, master ] @@ -24,7 +26,7 @@ jobs: with: repository: "huchenlei/ComfyUI_frontend" path: "ComfyUI_frontend" - ref: "23a4ee83144d9bf22e7218b9c78c71f8544c8b27" + ref: "fcc54d803e5b6a9b08a462a1d94899318c96dcbb" - uses: actions/setup-node@v3 with: node-version: lts/* diff --git a/.gitignore b/.gitignore index 9f0389241..a9beebe73 100644 --- a/.gitignore +++ b/.gitignore @@ -9,10 +9,12 @@ __pycache__/ !custom_nodes/example_node.py.example extra_model_paths.yaml /.vs +.vscode/ .idea/ venv/ /web/extensions/* !/web/extensions/logging.js.example !/web/extensions/core/ /tests-ui/data/object_info.json -/user/ \ No newline at end of file +/user/ +*.log \ No newline at end of file diff --git a/comfy/cldm/cldm.py b/comfy/cldm/cldm.py index 28076dd92..f8a161594 100644 --- a/comfy/cldm/cldm.py +++ b/comfy/cldm/cldm.py @@ -289,7 +289,8 @@ class ControlNet(nn.Module): guided_hint = self.input_hint_block(hint, emb, context) - outs = [] + out_output = [] + out_middle = [] hs = [] if self.num_classes is not None: @@ -304,10 +305,10 @@ class ControlNet(nn.Module): guided_hint = None else: h = module(h, emb, context) - outs.append(zero_conv(h, emb, context)) + out_output.append(zero_conv(h, emb, context)) h = self.middle_block(h, emb, context) - outs.append(self.middle_block_out(h, emb, context)) + out_middle.append(self.middle_block_out(h, emb, context)) - return outs + return {"middle": out_middle, "output": out_output} diff --git a/comfy/cldm/mmdit.py b/comfy/cldm/mmdit.py new file mode 100644 index 000000000..ee0282bc7 --- /dev/null +++ b/comfy/cldm/mmdit.py @@ -0,0 +1,86 @@ +import torch +from typing import Dict, Optional +import comfy.ldm.modules.diffusionmodules.mmdit + +class ControlNet(comfy.ldm.modules.diffusionmodules.mmdit.MMDiT): + def __init__( + self, + num_blocks = None, + dtype = None, + device = None, + operations = None, + **kwargs, + ): + super().__init__(dtype=dtype, device=device, operations=operations, final_layer=False, num_blocks=num_blocks, **kwargs) + # controlnet_blocks + self.controlnet_blocks = torch.nn.ModuleList([]) + for _ in range(len(self.joint_blocks)): + self.controlnet_blocks.append(operations.Linear(self.hidden_size, self.hidden_size, device=device, dtype=dtype)) + + self.pos_embed_input = comfy.ldm.modules.diffusionmodules.mmdit.PatchEmbed( + None, + self.patch_size, + self.in_channels, + self.hidden_size, + bias=True, + strict_img_size=False, + dtype=dtype, + device=device, + operations=operations + ) + + def forward( + self, + x: torch.Tensor, + timesteps: torch.Tensor, + y: Optional[torch.Tensor] = None, + context: Optional[torch.Tensor] = None, + hint = None, + ) -> torch.Tensor: + + #weird sd3 controlnet specific stuff + y = torch.zeros_like(y) + + if self.context_processor is not None: + context = self.context_processor(context) + + hw = x.shape[-2:] + x = self.x_embedder(x) + self.cropped_pos_embed(hw, device=x.device).to(dtype=x.dtype, device=x.device) + x += self.pos_embed_input(hint) + + c = self.t_embedder(timesteps, dtype=x.dtype) + if y is not None and self.y_embedder is not None: + y = self.y_embedder(y) + c = c + y + + if context is not None: + context = self.context_embedder(context) + + if self.register_length > 0: + context = torch.cat( + ( + repeat(self.register, "1 ... -> b ...", b=x.shape[0]), + default(context, torch.Tensor([]).type_as(x)), + ), + 1, + ) + + output = [] + + blocks = len(self.joint_blocks) + for i in range(blocks): + context, x = self.joint_blocks[i]( + context, + x, + c=c, + use_checkpoint=self.use_checkpoint, + ) + + out = self.controlnet_blocks[i](x) + count = self.depth // blocks + if i == blocks - 1: + count -= 1 + for j in range(count): + output.append(out) + + return {"output": output} diff --git a/comfy/controlnet.py b/comfy/controlnet.py index 8cf4a61a6..d00395139 100644 --- a/comfy/controlnet.py +++ b/comfy/controlnet.py @@ -7,10 +7,12 @@ import comfy.model_management import comfy.model_detection import comfy.model_patcher import comfy.ops +import comfy.latent_formats import comfy.cldm.cldm import comfy.t2i_adapter.adapter import comfy.ldm.cascade.controlnet +import comfy.cldm.mmdit def broadcast_image_to(tensor, target_batch_size, batched_number): @@ -37,6 +39,8 @@ class ControlBase: self.cond_hint = None self.strength = 1.0 self.timestep_percent_range = (0.0, 1.0) + self.latent_format = None + self.vae = None self.global_average_pooling = False self.timestep_range = None self.compression_ratio = 8 @@ -47,10 +51,12 @@ class ControlBase: self.device = device self.previous_controlnet = None - def set_cond_hint(self, cond_hint, strength=1.0, timestep_percent_range=(0.0, 1.0)): + def set_cond_hint(self, cond_hint, strength=1.0, timestep_percent_range=(0.0, 1.0), vae=None): self.cond_hint_original = cond_hint self.strength = strength self.timestep_percent_range = timestep_percent_range + if self.latent_format is not None: + self.vae = vae return self def pre_run(self, model, percent_to_timestep_function): @@ -83,43 +89,35 @@ class ControlBase: c.global_average_pooling = self.global_average_pooling c.compression_ratio = self.compression_ratio c.upscale_algorithm = self.upscale_algorithm + c.latent_format = self.latent_format + c.vae = self.vae def inference_memory_requirements(self, dtype): if self.previous_controlnet is not None: return self.previous_controlnet.inference_memory_requirements(dtype) return 0 - def control_merge(self, control_input, control_output, control_prev, output_dtype): + def control_merge(self, control, control_prev, output_dtype): out = {'input':[], 'middle':[], 'output': []} - if control_input is not None: - for i in range(len(control_input)): - key = 'input' - x = control_input[i] - if x is not None: - x *= self.strength - if x.dtype != output_dtype: - x = x.to(output_dtype) - out[key].insert(0, x) - - if control_output is not None: + for key in control: + control_output = control[key] + applied_to = set() for i in range(len(control_output)): - if i == (len(control_output) - 1): - key = 'middle' - index = 0 - else: - key = 'output' - index = i x = control_output[i] if x is not None: if self.global_average_pooling: x = torch.mean(x, dim=(2, 3), keepdim=True).repeat(1, 1, x.shape[2], x.shape[3]) - x *= self.strength + if x not in applied_to: #memory saving strategy, allow shared tensors and only apply strength to shared tensors once + applied_to.add(x) + x *= self.strength + if x.dtype != output_dtype: x = x.to(output_dtype) out[key].append(x) + if control_prev is not None: for x in ['input', 'middle', 'output']: o = out[x] @@ -134,20 +132,22 @@ class ControlBase: if o[i].shape[0] < prev_val.shape[0]: o[i] = prev_val + o[i] else: - o[i] += prev_val + o[i] = prev_val + o[i] #TODO: change back to inplace add if shared tensors stop being an issue return out class ControlNet(ControlBase): - def __init__(self, control_model=None, global_average_pooling=False, device=None, load_device=None, manual_cast_dtype=None): + def __init__(self, control_model=None, global_average_pooling=False, compression_ratio=8, latent_format=None, device=None, load_device=None, manual_cast_dtype=None): super().__init__(device) self.control_model = control_model self.load_device = load_device if control_model is not None: self.control_model_wrapped = comfy.model_patcher.ModelPatcher(self.control_model, load_device=load_device, offload_device=comfy.model_management.unet_offload_device()) + self.compression_ratio = compression_ratio self.global_average_pooling = global_average_pooling self.model_sampling_current = None self.manual_cast_dtype = manual_cast_dtype + self.latent_format = latent_format def get_control(self, x_noisy, t, cond, batched_number): control_prev = None @@ -170,7 +170,17 @@ class ControlNet(ControlBase): if self.cond_hint is not None: del self.cond_hint self.cond_hint = None - self.cond_hint = comfy.utils.common_upscale(self.cond_hint_original, x_noisy.shape[3] * self.compression_ratio, x_noisy.shape[2] * self.compression_ratio, self.upscale_algorithm, "center").to(dtype).to(self.device) + compression_ratio = self.compression_ratio + if self.vae is not None: + compression_ratio *= self.vae.downscale_ratio + self.cond_hint = comfy.utils.common_upscale(self.cond_hint_original, x_noisy.shape[3] * compression_ratio, x_noisy.shape[2] * compression_ratio, self.upscale_algorithm, "center") + if self.vae is not None: + loaded_models = comfy.model_management.loaded_models(only_currently_used=True) + self.cond_hint = self.vae.encode(self.cond_hint.movedim(1, -1)) + comfy.model_management.load_models_gpu(loaded_models) + if self.latent_format is not None: + self.cond_hint = self.latent_format.process_in(self.cond_hint) + self.cond_hint = self.cond_hint.to(device=self.device, dtype=dtype) if x_noisy.shape[0] != self.cond_hint.shape[0]: self.cond_hint = broadcast_image_to(self.cond_hint, x_noisy.shape[0], batched_number) @@ -182,7 +192,7 @@ class ControlNet(ControlBase): x_noisy = self.model_sampling_current.calculate_input(t, x_noisy) control = self.control_model(x=x_noisy.to(dtype), hint=self.cond_hint, timesteps=timestep.float(), context=context.to(dtype), y=y) - return self.control_merge(None, control, control_prev, output_dtype) + return self.control_merge(control, control_prev, output_dtype) def copy(self): c = ControlNet(None, global_average_pooling=self.global_average_pooling, load_device=self.load_device, manual_cast_dtype=self.manual_cast_dtype) @@ -322,6 +332,39 @@ class ControlLora(ControlNet): def inference_memory_requirements(self, dtype): return comfy.utils.calculate_parameters(self.control_weights) * comfy.model_management.dtype_size(dtype) + ControlBase.inference_memory_requirements(self, dtype) +def load_controlnet_mmdit(sd): + new_sd = comfy.model_detection.convert_diffusers_mmdit(sd, "") + model_config = comfy.model_detection.model_config_from_unet(new_sd, "", True) + num_blocks = comfy.model_detection.count_blocks(new_sd, 'joint_blocks.{}.') + for k in sd: + new_sd[k] = sd[k] + + supported_inference_dtypes = model_config.supported_inference_dtypes + + controlnet_config = model_config.unet_config + unet_dtype = comfy.model_management.unet_dtype(supported_dtypes=supported_inference_dtypes) + load_device = comfy.model_management.get_torch_device() + manual_cast_dtype = comfy.model_management.unet_manual_cast(unet_dtype, load_device) + if manual_cast_dtype is not None: + operations = comfy.ops.manual_cast + else: + operations = comfy.ops.disable_weight_init + + control_model = comfy.cldm.mmdit.ControlNet(num_blocks=num_blocks, operations=operations, device=load_device, dtype=unet_dtype, **controlnet_config) + missing, unexpected = control_model.load_state_dict(new_sd, strict=False) + + if len(missing) > 0: + logging.warning("missing controlnet keys: {}".format(missing)) + + if len(unexpected) > 0: + logging.debug("unexpected controlnet keys: {}".format(unexpected)) + + latent_format = comfy.latent_formats.SD3() + latent_format.shift_factor = 0 #SD3 controlnet weirdness + control = ControlNet(control_model, compression_ratio=1, latent_format=latent_format, load_device=load_device, manual_cast_dtype=manual_cast_dtype) + return control + + def load_controlnet(ckpt_path, model=None): controlnet_data = comfy.utils.load_torch_file(ckpt_path, safe_load=True) if "lora_controlnet" in controlnet_data: @@ -374,6 +417,8 @@ def load_controlnet(ckpt_path, model=None): if len(leftover_keys) > 0: logging.warning("leftover keys: {}".format(leftover_keys)) controlnet_data = new_sd + elif "controlnet_blocks.0.weight" in controlnet_data: #SD3 diffusers format + return load_controlnet_mmdit(controlnet_data) pth_key = 'control_model.zero_convs.0.0.weight' pth = False @@ -490,12 +535,11 @@ class T2IAdapter(ControlBase): self.control_input = self.t2i_model(self.cond_hint.to(x_noisy.dtype)) self.t2i_model.cpu() - control_input = list(map(lambda a: None if a is None else a.clone(), self.control_input)) - mid = None - if self.t2i_model.xl == True: - mid = control_input[-1:] - control_input = control_input[:-1] - return self.control_merge(control_input, mid, control_prev, x_noisy.dtype) + control_input = {} + for k in self.control_input: + control_input[k] = list(map(lambda a: None if a is None else a.clone(), self.control_input[k])) + + return self.control_merge(control_input, control_prev, x_noisy.dtype) def copy(self): c = T2IAdapter(self.t2i_model, self.channels_in, self.compression_ratio, self.upscale_algorithm) diff --git a/comfy/k_diffusion/deis.py b/comfy/k_diffusion/deis.py new file mode 100644 index 000000000..607410656 --- /dev/null +++ b/comfy/k_diffusion/deis.py @@ -0,0 +1,121 @@ +#Taken from: https://github.com/zju-pi/diff-sampler/blob/main/gits-main/solver_utils.py +#under Apache 2 license +import torch +import numpy as np + +# A pytorch reimplementation of DEIS (https://github.com/qsh-zh/deis). +############################# +### Utils for DEIS solver ### +############################# +#---------------------------------------------------------------------------- +# Transfer from the input time (sigma) used in EDM to that (t) used in DEIS. + +def edm2t(edm_steps, epsilon_s=1e-3, sigma_min=0.002, sigma_max=80): + vp_sigma = lambda beta_d, beta_min: lambda t: (np.e ** (0.5 * beta_d * (t ** 2) + beta_min * t) - 1) ** 0.5 + vp_sigma_inv = lambda beta_d, beta_min: lambda sigma: ((beta_min ** 2 + 2 * beta_d * (sigma ** 2 + 1).log()).sqrt() - beta_min) / beta_d + vp_beta_d = 2 * (np.log(torch.tensor(sigma_min).cpu() ** 2 + 1) / epsilon_s - np.log(torch.tensor(sigma_max).cpu() ** 2 + 1)) / (epsilon_s - 1) + vp_beta_min = np.log(torch.tensor(sigma_max).cpu() ** 2 + 1) - 0.5 * vp_beta_d + t_steps = vp_sigma_inv(vp_beta_d.clone().detach().cpu(), vp_beta_min.clone().detach().cpu())(edm_steps.clone().detach().cpu()) + return t_steps, vp_beta_min, vp_beta_d + vp_beta_min + +#---------------------------------------------------------------------------- + +def cal_poly(prev_t, j, taus): + poly = 1 + for k in range(prev_t.shape[0]): + if k == j: + continue + poly *= (taus - prev_t[k]) / (prev_t[j] - prev_t[k]) + return poly + +#---------------------------------------------------------------------------- +# Transfer from t to alpha_t. + +def t2alpha_fn(beta_0, beta_1, t): + return torch.exp(-0.5 * t ** 2 * (beta_1 - beta_0) - t * beta_0) + +#---------------------------------------------------------------------------- + +def cal_intergrand(beta_0, beta_1, taus): + with torch.inference_mode(mode=False): + taus = taus.clone() + beta_0 = beta_0.clone() + beta_1 = beta_1.clone() + with torch.enable_grad(): + taus.requires_grad_(True) + alpha = t2alpha_fn(beta_0, beta_1, taus) + log_alpha = alpha.log() + log_alpha.sum().backward() + d_log_alpha_dtau = taus.grad + integrand = -0.5 * d_log_alpha_dtau / torch.sqrt(alpha * (1 - alpha)) + return integrand + +#---------------------------------------------------------------------------- + +def get_deis_coeff_list(t_steps, max_order, N=10000, deis_mode='tab'): + """ + Get the coefficient list for DEIS sampling. + + Args: + t_steps: A pytorch tensor. The time steps for sampling. + max_order: A `int`. Maximum order of the solver. 1 <= max_order <= 4 + N: A `int`. Use how many points to perform the numerical integration when deis_mode=='tab'. + deis_mode: A `str`. Select between 'tab' and 'rhoab'. Type of DEIS. + Returns: + A pytorch tensor. A batch of generated samples or sampling trajectories if return_inters=True. + """ + if deis_mode == 'tab': + t_steps, beta_0, beta_1 = edm2t(t_steps) + C = [] + for i, (t_cur, t_next) in enumerate(zip(t_steps[:-1], t_steps[1:])): + order = min(i+1, max_order) + if order == 1: + C.append([]) + else: + taus = torch.linspace(t_cur, t_next, N) # split the interval for integral appximation + dtau = (t_next - t_cur) / N + prev_t = t_steps[[i - k for k in range(order)]] + coeff_temp = [] + integrand = cal_intergrand(beta_0, beta_1, taus) + for j in range(order): + poly = cal_poly(prev_t, j, taus) + coeff_temp.append(torch.sum(integrand * poly) * dtau) + C.append(coeff_temp) + + elif deis_mode == 'rhoab': + # Analytical solution, second order + def get_def_intergral_2(a, b, start, end, c): + coeff = (end**3 - start**3) / 3 - (end**2 - start**2) * (a + b) / 2 + (end - start) * a * b + return coeff / ((c - a) * (c - b)) + + # Analytical solution, third order + def get_def_intergral_3(a, b, c, start, end, d): + coeff = (end**4 - start**4) / 4 - (end**3 - start**3) * (a + b + c) / 3 \ + + (end**2 - start**2) * (a*b + a*c + b*c) / 2 - (end - start) * a * b * c + return coeff / ((d - a) * (d - b) * (d - c)) + + C = [] + for i, (t_cur, t_next) in enumerate(zip(t_steps[:-1], t_steps[1:])): + order = min(i, max_order) + if order == 0: + C.append([]) + else: + prev_t = t_steps[[i - k for k in range(order+1)]] + if order == 1: + coeff_cur = ((t_next - prev_t[1])**2 - (t_cur - prev_t[1])**2) / (2 * (t_cur - prev_t[1])) + coeff_prev1 = (t_next - t_cur)**2 / (2 * (prev_t[1] - t_cur)) + coeff_temp = [coeff_cur, coeff_prev1] + elif order == 2: + coeff_cur = get_def_intergral_2(prev_t[1], prev_t[2], t_cur, t_next, t_cur) + coeff_prev1 = get_def_intergral_2(t_cur, prev_t[2], t_cur, t_next, prev_t[1]) + coeff_prev2 = get_def_intergral_2(t_cur, prev_t[1], t_cur, t_next, prev_t[2]) + coeff_temp = [coeff_cur, coeff_prev1, coeff_prev2] + elif order == 3: + coeff_cur = get_def_intergral_3(prev_t[1], prev_t[2], prev_t[3], t_cur, t_next, t_cur) + coeff_prev1 = get_def_intergral_3(t_cur, prev_t[2], prev_t[3], t_cur, t_next, prev_t[1]) + coeff_prev2 = get_def_intergral_3(t_cur, prev_t[1], prev_t[3], t_cur, t_next, prev_t[2]) + coeff_prev3 = get_def_intergral_3(t_cur, prev_t[1], prev_t[2], t_cur, t_next, prev_t[3]) + coeff_temp = [coeff_cur, coeff_prev1, coeff_prev2, coeff_prev3] + C.append(coeff_temp) + return C + diff --git a/comfy/ldm/cascade/controlnet.py b/comfy/ldm/cascade/controlnet.py index 5dac59394..7a52c3c26 100644 --- a/comfy/ldm/cascade/controlnet.py +++ b/comfy/ldm/cascade/controlnet.py @@ -90,4 +90,4 @@ class ControlNet(nn.Module): proj_outputs = [None for _ in range(max(self.proj_blocks) + 1)] for i, idx in enumerate(self.proj_blocks): proj_outputs[idx] = self.projections[i](x) - return proj_outputs + return {"input": proj_outputs[::-1]} diff --git a/comfy/ldm/modules/diffusionmodules/mmdit.py b/comfy/ldm/modules/diffusionmodules/mmdit.py index 20d3a321a..927451534 100644 --- a/comfy/ldm/modules/diffusionmodules/mmdit.py +++ b/comfy/ldm/modules/diffusionmodules/mmdit.py @@ -745,6 +745,8 @@ class MMDiT(nn.Module): qkv_bias: bool = True, context_processor_layers = None, context_size = 4096, + num_blocks = None, + final_layer = True, dtype = None, #TODO device = None, operations = None, @@ -766,7 +768,10 @@ class MMDiT(nn.Module): # apply magic --> this defines a head_size of 64 self.hidden_size = 64 * depth num_heads = depth + if num_blocks is None: + num_blocks = depth + self.depth = depth self.num_heads = num_heads self.x_embedder = PatchEmbed( @@ -821,7 +826,7 @@ class MMDiT(nn.Module): mlp_ratio=mlp_ratio, qkv_bias=qkv_bias, attn_mode=attn_mode, - pre_only=i == depth - 1, + pre_only=(i == num_blocks - 1) and final_layer, rmsnorm=rmsnorm, scale_mod_only=scale_mod_only, swiglu=swiglu, @@ -830,11 +835,12 @@ class MMDiT(nn.Module): device=device, operations=operations ) - for i in range(depth) + for i in range(num_blocks) ] ) - self.final_layer = FinalLayer(self.hidden_size, patch_size, self.out_channels, dtype=dtype, device=device, operations=operations) + if final_layer: + self.final_layer = FinalLayer(self.hidden_size, patch_size, self.out_channels, dtype=dtype, device=device, operations=operations) if compile_core: assert False @@ -893,6 +899,7 @@ class MMDiT(nn.Module): x: torch.Tensor, c_mod: torch.Tensor, context: Optional[torch.Tensor] = None, + control = None, ) -> torch.Tensor: if self.register_length > 0: context = torch.cat( @@ -905,13 +912,20 @@ class MMDiT(nn.Module): # context is B, L', D # x is B, L, D - for block in self.joint_blocks: - context, x = block( + blocks = len(self.joint_blocks) + for i in range(blocks): + context, x = self.joint_blocks[i]( context, x, c=c_mod, use_checkpoint=self.use_checkpoint, ) + if control is not None: + control_o = control.get("output") + if i < len(control_o): + add = control_o[i] + if add is not None: + x += add x = self.final_layer(x, c_mod) # (N, T, patch_size ** 2 * out_channels) return x @@ -922,6 +936,7 @@ class MMDiT(nn.Module): t: torch.Tensor, y: Optional[torch.Tensor] = None, context: Optional[torch.Tensor] = None, + control = None, ) -> torch.Tensor: """ Forward pass of DiT. @@ -943,7 +958,7 @@ class MMDiT(nn.Module): if context is not None: context = self.context_embedder(context) - x = self.forward_core_with_concat(x, c, context) + x = self.forward_core_with_concat(x, c, context, control) x = self.unpatchify(x, hw=hw) # (N, out_channels, H, W) return x[:,:,:hw[-2],:hw[-1]] @@ -956,7 +971,8 @@ class OpenAISignatureMMDITWrapper(MMDiT): timesteps: torch.Tensor, context: Optional[torch.Tensor] = None, y: Optional[torch.Tensor] = None, + control = None, **kwargs, ) -> torch.Tensor: - return super().forward(x, timesteps, context=context, y=y) + return super().forward(x, timesteps, context=context, y=y, control=control) diff --git a/comfy/model_base.py b/comfy/model_base.py index f45b375de..80f6667ec 100644 --- a/comfy/model_base.py +++ b/comfy/model_base.py @@ -627,3 +627,12 @@ class StableAudio1(BaseModel): cross_attn = torch.cat([cross_attn.to(device), seconds_start_embed.repeat((cross_attn.shape[0], 1, 1)), seconds_total_embed.repeat((cross_attn.shape[0], 1, 1))], dim=1) out['c_crossattn'] = comfy.conds.CONDRegular(cross_attn) return out + + def state_dict_for_saving(self, clip_state_dict=None, vae_state_dict=None, clip_vision_state_dict=None): + sd = super().state_dict_for_saving(clip_state_dict=clip_state_dict, vae_state_dict=vae_state_dict, clip_vision_state_dict=clip_vision_state_dict) + d = {"conditioner.conditioners.seconds_start.": self.seconds_start_embedder.state_dict(), "conditioner.conditioners.seconds_total.": self.seconds_total_embedder.state_dict()} + for k in d: + s = d[k] + for l in s: + sd["{}{}".format(k, l)] = s[l] + return sd diff --git a/comfy/model_detection.py b/comfy/model_detection.py index e09dd381a..0b678480f 100644 --- a/comfy/model_detection.py +++ b/comfy/model_detection.py @@ -41,7 +41,9 @@ def detect_unet_config(state_dict, key_prefix): unet_config["in_channels"] = state_dict['{}x_embedder.proj.weight'.format(key_prefix)].shape[1] patch_size = state_dict['{}x_embedder.proj.weight'.format(key_prefix)].shape[2] unet_config["patch_size"] = patch_size - unet_config["out_channels"] = state_dict['{}final_layer.linear.weight'.format(key_prefix)].shape[0] // (patch_size * patch_size) + final_layer = '{}final_layer.linear.weight'.format(key_prefix) + if final_layer in state_dict: + unet_config["out_channels"] = state_dict[final_layer].shape[0] // (patch_size * patch_size) unet_config["depth"] = state_dict['{}x_embedder.proj.weight'.format(key_prefix)].shape[0] // 64 unet_config["input_size"] = None @@ -435,10 +437,11 @@ def model_config_from_diffusers_unet(state_dict): return None def convert_diffusers_mmdit(state_dict, output_prefix=""): - depth = count_blocks(state_dict, 'transformer_blocks.{}.') - if depth > 0: + num_blocks = count_blocks(state_dict, 'transformer_blocks.{}.') + if num_blocks > 0: + depth = state_dict["pos_embed.proj.weight"].shape[0] // 64 out_sd = {} - sd_map = comfy.utils.mmdit_to_diffusers({"depth": depth}, output_prefix=output_prefix) + sd_map = comfy.utils.mmdit_to_diffusers({"depth": depth, "num_blocks": num_blocks}, output_prefix=output_prefix) for k in sd_map: weight = state_dict.get(k, None) if weight is not None: diff --git a/comfy/sd.py b/comfy/sd.py index ea6e9b663..dda0887ba 100644 --- a/comfy/sd.py +++ b/comfy/sd.py @@ -236,7 +236,7 @@ class VAE: self.first_stage_model = AutoencodingEngine(regularizer_config={'target': "comfy.ldm.models.autoencoder.DiagonalGaussianRegularizer"}, encoder_config={'target': "comfy.ldm.modules.diffusionmodules.model.Encoder", 'params': ddconfig}, decoder_config={'target': "comfy.ldm.modules.diffusionmodules.model.Decoder", 'params': ddconfig}) - elif "decoder.layers.0.weight_v" in sd: + elif "decoder.layers.1.layers.0.beta" in sd: self.first_stage_model = AudioOobleckVAE() self.memory_used_encode = lambda shape, dtype: (1000 * shape[2]) * model_management.dtype_size(dtype) self.memory_used_decode = lambda shape, dtype: (1000 * shape[2] * 2048) * model_management.dtype_size(dtype) diff --git a/comfy/supported_models.py b/comfy/supported_models.py index 761498dbc..21fdb7ec7 100644 --- a/comfy/supported_models.py +++ b/comfy/supported_models.py @@ -543,13 +543,16 @@ class StableAudio(supported_models_base.BASE): seconds_total_sd = utils.state_dict_prefix_replace(state_dict, {"conditioner.conditioners.seconds_total.": ""}, filter_keys=True) return model_base.StableAudio1(self, seconds_start_embedder_weights=seconds_start_sd, seconds_total_embedder_weights=seconds_total_sd, device=device) - def process_unet_state_dict(self, state_dict): for k in list(state_dict.keys()): if k.endswith(".cross_attend_norm.beta") or k.endswith(".ff_norm.beta") or k.endswith(".pre_norm.beta"): #These weights are all zero state_dict.pop(k) return state_dict + def process_unet_state_dict_for_saving(self, state_dict): + replace_prefix = {"": "model.model."} + return utils.state_dict_prefix_replace(state_dict, replace_prefix) + def clip_target(self, state_dict={}): return supported_models_base.ClipTarget(sa_t5.SAT5Tokenizer, sa_t5.SAT5Model) diff --git a/comfy/t2i_adapter/adapter.py b/comfy/t2i_adapter/adapter.py index e9a606b1c..10ea18e32 100644 --- a/comfy/t2i_adapter/adapter.py +++ b/comfy/t2i_adapter/adapter.py @@ -153,7 +153,13 @@ class Adapter(nn.Module): features.append(None) features.append(x) - return features + features = features[::-1] + + if self.xl: + return {"input": features[1:], "middle": features[:1]} + else: + return {"input": features} + class LayerNorm(nn.LayerNorm): @@ -290,4 +296,4 @@ class Adapter_light(nn.Module): features.append(None) features.append(x) - return features + return {"input": features[::-1]} diff --git a/comfy/utils.py b/comfy/utils.py index ed6c58a64..48618e076 100644 --- a/comfy/utils.py +++ b/comfy/utils.py @@ -298,7 +298,8 @@ def mmdit_to_diffusers(mmdit_config, output_prefix=""): key_map = {} depth = mmdit_config.get("depth", 0) - for i in range(depth): + num_blocks = mmdit_config.get("num_blocks", depth) + for i in range(num_blocks): block_from = "transformer_blocks.{}".format(i) block_to = "{}joint_blocks.{}".format(output_prefix, i) diff --git a/comfy_extras/nodes_audio.py b/comfy_extras/nodes_audio.py index 57d0a20ab..9950d58ed 100644 --- a/comfy_extras/nodes_audio.py +++ b/comfy_extras/nodes_audio.py @@ -3,6 +3,11 @@ import torch import comfy.model_management import folder_paths import os +import io +import json +import struct +import random +from comfy.cli_args import args class EmptyLatentAudio: def __init__(self): @@ -10,15 +15,16 @@ class EmptyLatentAudio: @classmethod def INPUT_TYPES(s): - return {"required": {}} + return {"required": {"seconds": ("FLOAT", {"default": 47.6, "min": 1.0, "max": 1000.0, "step": 0.1})}} RETURN_TYPES = ("LATENT",) FUNCTION = "generate" CATEGORY = "_for_testing/audio" - def generate(self): + def generate(self, seconds): batch_size = 1 - latent = torch.zeros([batch_size, 64, 1024], device=self.device) + length = round((seconds * 44100 / 2048) / 2) * 2 + latent = torch.zeros([batch_size, 64, length], device=self.device) return ({"samples":latent, "type": "audio"}, ) class VAEEncodeAudio: @@ -53,12 +59,66 @@ class VAEDecodeAudio: audio = vae.decode(samples["samples"]).movedim(-1, 1) return ({"waveform": audio, "sample_rate": 44100}, ) + +def create_vorbis_comment_block(comment_dict, last_block): + vendor_string = b'ComfyUI' + vendor_length = len(vendor_string) + + comments = [] + for key, value in comment_dict.items(): + comment = f"{key}={value}".encode('utf-8') + comments.append(struct.pack('I', len(comment_data))[1:] + comment_data + + return comment_block + +def insert_or_replace_vorbis_comment(flac_io, comment_dict): + if len(comment_dict) == 0: + return flac_io + + flac_io.seek(4) + + blocks = [] + last_block = False + + while not last_block: + header = flac_io.read(4) + last_block = (header[0] & 0x80) != 0 + block_type = header[0] & 0x7F + block_length = struct.unpack('>I', b'\x00' + header[1:])[0] + block_data = flac_io.read(block_length) + + if block_type == 4 or block_type == 1: + pass + else: + header = bytes([(header[0] & (~0x80))]) + header[1:] + blocks.append(header + block_data) + + blocks.append(create_vorbis_comment_block(comment_dict, last_block=True)) + + new_flac_io = io.BytesIO() + new_flac_io.write(b'fLaC') + for block in blocks: + new_flac_io.write(block) + + new_flac_io.write(flac_io.read()) + return new_flac_io + + class SaveAudio: def __init__(self): self.output_dir = folder_paths.get_output_directory() self.type = "output" self.prefix_append = "" - self.compress_level = 4 @classmethod def INPUT_TYPES(s): @@ -78,11 +138,27 @@ class SaveAudio: filename_prefix += self.prefix_append full_output_folder, filename, counter, subfolder, filename_prefix = folder_paths.get_save_image_path(filename_prefix, self.output_dir) results = list() + + metadata = {} + if not args.disable_metadata: + if prompt is not None: + metadata["prompt"] = json.dumps(prompt) + if extra_pnginfo is not None: + for x in extra_pnginfo: + metadata[x] = json.dumps(extra_pnginfo[x]) + for (batch_number, waveform) in enumerate(audio["waveform"]): - #TODO: metadata filename_with_batch_num = filename.replace("%batch_num%", str(batch_number)) file = f"{filename_with_batch_num}_{counter:05}_.flac" - torchaudio.save(os.path.join(full_output_folder, file), waveform, audio["sample_rate"], format="FLAC") + + buff = io.BytesIO() + torchaudio.save(buff, waveform, audio["sample_rate"], format="FLAC") + + buff = insert_or_replace_vorbis_comment(buff, metadata) + + with open(os.path.join(full_output_folder, file), 'wb') as f: + f.write(buff.getbuffer()) + results.append({ "filename": file, "subfolder": subfolder, @@ -92,12 +168,32 @@ class SaveAudio: return { "ui": { "audio": results } } +class PreviewAudio(SaveAudio): + def __init__(self): + self.output_dir = folder_paths.get_temp_directory() + self.type = "temp" + self.prefix_append = "_temp_" + ''.join(random.choice("abcdefghijklmnopqrstupvxyz") for x in range(5)) + + @classmethod + def INPUT_TYPES(s): + return {"required": + {"audio": ("AUDIO", ), }, + "hidden": {"prompt": "PROMPT", "extra_pnginfo": "EXTRA_PNGINFO"}, + } + class LoadAudio: + SUPPORTED_FORMATS = ('.wav', '.mp3', '.ogg', '.flac', '.aiff', '.aif') + @classmethod def INPUT_TYPES(s): input_dir = folder_paths.get_input_directory() - files = [f for f in os.listdir(input_dir) if os.path.isfile(os.path.join(input_dir, f))] - return {"required": {"audio": [sorted(files), ]}, } + files = [ + f for f in os.listdir(input_dir) + if (os.path.isfile(os.path.join(input_dir, f)) + and f.endswith(LoadAudio.SUPPORTED_FORMATS) + ) + ] + return {"required": {"audio": (sorted(files), {"audio_upload": True})}} CATEGORY = "_for_testing/audio" @@ -131,4 +227,5 @@ NODE_CLASS_MAPPINGS = { "VAEDecodeAudio": VAEDecodeAudio, "SaveAudio": SaveAudio, "LoadAudio": LoadAudio, + "PreviewAudio": PreviewAudio, } diff --git a/comfy_extras/nodes_sd3.py b/comfy_extras/nodes_sd3.py index d0303aec5..548b1ad6a 100644 --- a/comfy_extras/nodes_sd3.py +++ b/comfy_extras/nodes_sd3.py @@ -80,8 +80,23 @@ class CLIPTextEncodeSD3: return ([[cond, {"pooled_output": pooled}]], ) +class ControlNetApplySD3(nodes.ControlNetApplyAdvanced): + @classmethod + def INPUT_TYPES(s): + return {"required": {"positive": ("CONDITIONING", ), + "negative": ("CONDITIONING", ), + "control_net": ("CONTROL_NET", ), + "vae": ("VAE", ), + "image": ("IMAGE", ), + "strength": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 10.0, "step": 0.01}), + "start_percent": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 1.0, "step": 0.001}), + "end_percent": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 1.0, "step": 0.001}) + }} + CATEGORY = "_for_testing/sd3" + NODE_CLASS_MAPPINGS = { "TripleCLIPLoader": TripleCLIPLoader, "EmptySD3LatentImage": EmptySD3LatentImage, "CLIPTextEncodeSD3": CLIPTextEncodeSD3, + "ControlNetApplySD3": ControlNetApplySD3, } diff --git a/nodes.py b/nodes.py index 99645b81c..04775d290 100644 --- a/nodes.py +++ b/nodes.py @@ -783,7 +783,7 @@ class ControlNetApplyAdvanced: CATEGORY = "conditioning" - def apply_controlnet(self, positive, negative, control_net, image, strength, start_percent, end_percent): + def apply_controlnet(self, positive, negative, control_net, image, strength, start_percent, end_percent, vae=None): if strength == 0: return (positive, negative) @@ -800,7 +800,7 @@ class ControlNetApplyAdvanced: if prev_cnet in cnets: c_net = cnets[prev_cnet] else: - c_net = control_net.copy().set_cond_hint(control_hint, strength, (start_percent, end_percent)) + c_net = control_net.copy().set_cond_hint(control_hint, strength, (start_percent, end_percent), vae) c_net.set_previous_controlnet(prev_cnet) cnets[prev_cnet] = c_net diff --git a/web/extensions/core/uploadAudio.js b/web/extensions/core/uploadAudio.js new file mode 100644 index 000000000..0ac9cb807 --- /dev/null +++ b/web/extensions/core/uploadAudio.js @@ -0,0 +1,178 @@ +import { app } from "../../scripts/app.js" +import { api } from "../../scripts/api.js" + +function splitFilePath(path) { + const folder_separator = path.lastIndexOf("/") + if (folder_separator === -1) { + return ["", path] + } + return [ + path.substring(0, folder_separator), + path.substring(folder_separator + 1) + ] +} + +function getResourceURL(subfolder, filename, type = "input") { + const params = [ + "filename=" + encodeURIComponent(filename), + "type=" + type, + "subfolder=" + subfolder, + app.getPreviewFormatParam().substring(1), + app.getRandParam().substring(1) + ].join("&") + + return `/view?${params}` +} + +async function uploadFile( + audioWidget, + audioUIWidget, + file, + updateNode, + pasted = false +) { + try { + // Wrap file in formdata so it includes filename + const body = new FormData() + body.append("image", file) + if (pasted) body.append("subfolder", "pasted") + const resp = await api.fetchApi("/upload/image", { + method: "POST", + body + }) + + if (resp.status === 200) { + const data = await resp.json() + // Add the file to the dropdown list and update the widget value + let path = data.name + if (data.subfolder) path = data.subfolder + "/" + path + + if (!audioWidget.options.values.includes(path)) { + audioWidget.options.values.push(path) + } + + if (updateNode) { + audioUIWidget.element.src = api.apiURL( + getResourceURL(...splitFilePath(path)) + ) + audioWidget.value = path + } + } else { + alert(resp.status + " - " + resp.statusText) + } + } catch (error) { + alert(error) + } +} + +// AudioWidget MUST be registered first, as AUDIOUPLOAD depends on AUDIO_UI to be +// present. +app.registerExtension({ + name: "Comfy.AudioWidget", + async beforeRegisterNodeDef(nodeType, nodeData) { + if (["LoadAudio", "SaveAudio", "PreviewAudio"].includes(nodeType.comfyClass)) { + nodeData.input.required.audioUI = ["AUDIO_UI"] + } + }, + getCustomWidgets() { + return { + AUDIO_UI(node, inputName) { + const audio = document.createElement("audio") + audio.controls = true + audio.classList.add("comfy-audio") + audio.setAttribute("name", "media") + + const audioUIWidget = node.addDOMWidget( + inputName, + /* name=*/ "audioUI", + audio + ) + // @ts-ignore + // TODO: Sort out the DOMWidget type. + audioUIWidget.serialize = false + + const isOutputNode = node.constructor.nodeData.output_node + if (isOutputNode) { + // Hide the audio widget when there is no audio initially. + audioUIWidget.element.classList.add("empty-audio-widget") + // Populate the audio widget UI on node execution. + const onExecuted = node.onExecuted + node.onExecuted = function(message) { + onExecuted?.apply(this, arguments) + const audios = message.audio + if (!audios) return + const audio = audios[0] + audioUIWidget.element.src = api.apiURL( + getResourceURL(audio.subfolder, audio.filename, audio.type) + ) + audioUIWidget.element.classList.remove("empty-audio-widget") + } + } + return { widget: audioUIWidget } + } + } + }, + onNodeOutputsUpdated(nodeOutputs) { + for (const [nodeId, output] of Object.entries(nodeOutputs)) { + const node = app.graph.getNodeById(Number.parseInt(nodeId)); + if ("audio" in output) { + const audioUIWidget = node.widgets.find((w) => w.name === "audioUI"); + const audio = output.audio[0]; + audioUIWidget.element.src = api.apiURL(getResourceURL(audio.subfolder, audio.filename, audio.type)); + audioUIWidget.element.classList.remove("empty-audio-widget"); + } + } + }, +}) + +app.registerExtension({ + name: "Comfy.UploadAudio", + async beforeRegisterNodeDef(nodeType, nodeData) { + if (nodeData?.input?.required?.audio?.[1]?.audio_upload === true) { + nodeData.input.required.upload = ["AUDIOUPLOAD"] + } + }, + getCustomWidgets() { + return { + AUDIOUPLOAD(node, inputName) { + // The widget that allows user to select file. + const audioWidget = node.widgets.find(w => w.name === "audio") + const audioUIWidget = node.widgets.find(w => w.name === "audioUI") + + const onAudioWidgetUpdate = () => { + audioUIWidget.element.src = api.apiURL( + getResourceURL(...splitFilePath(audioWidget.value)) + ) + } + // Initially load default audio file to audioUIWidget. + if (audioWidget.value) { + onAudioWidgetUpdate() + } + audioWidget.callback = onAudioWidgetUpdate + + const fileInput = document.createElement("input") + fileInput.type = "file" + fileInput.accept = "audio/*" + fileInput.style.display = "none" + fileInput.onchange = () => { + if (fileInput.files.length) { + uploadFile(audioWidget, audioUIWidget, fileInput.files[0], true) + } + } + // The widget to pop up the upload dialog. + const uploadWidget = node.addWidget( + "button", + inputName, + /* value=*/ "", + () => { + fileInput.click() + } + ) + uploadWidget.label = "choose file to upload" + uploadWidget.serialize = false + + return { widget: uploadWidget } + } + } + } +}) diff --git a/web/scripts/app.js b/web/scripts/app.js index 725538563..3a91aa239 100644 --- a/web/scripts/app.js +++ b/web/scripts/app.js @@ -3,7 +3,7 @@ import { ComfyWidgets, initWidgets } from "./widgets.js"; import { ComfyUI, $el } from "./ui.js"; import { api } from "./api.js"; import { defaultGraph } from "./defaultGraph.js"; -import { getPngMetadata, getWebpMetadata, importA1111, getLatentMetadata } from "./pnginfo.js"; +import { getPngMetadata, getWebpMetadata, getFlacMetadata, importA1111, getLatentMetadata } from "./pnginfo.js"; import { addDomClippingSetting } from "./domWidget.js"; import { createImageHost, calculateImageGrid } from "./ui/imagePreview.js"; import { ComfyAppMenu } from "./ui/menu/index.js"; @@ -71,7 +71,7 @@ export class ComfyApp { * Stores the execution output data for each node * @type {Record} */ - this.nodeOutputs = {}; + this._nodeOutputs = {}; /** * Stores the preview image data for each node @@ -86,6 +86,15 @@ export class ComfyApp { this.shiftDown = false; } + get nodeOutputs() { + return this._nodeOutputs; + } + + set nodeOutputs(value) { + this._nodeOutputs = value; + this.#invokeExtensions("onNodeOutputsUpdated", value); + } + getPreviewFormatParam() { let preview_format = this.ui.settings.getSettingValue("Comfy.PreviewFormat"); if(preview_format) @@ -1957,6 +1966,14 @@ export class ComfyApp { if (widget.value.startsWith("sample_")) { widget.value = widget.value.slice(7); } + if (widget.value === "euler_pp" || widget.value === "euler_ancestral_pp") { + widget.value = widget.value.slice(0, -3); + for (let w of node.widgets) { + if (w.name == "cfg") { + w.value *= 2.0; + } + } + } } } if (node.type == "KSampler" || node.type == "KSamplerAdvanced" || node.type == "PrimitiveNode") { @@ -2268,6 +2285,19 @@ export class ComfyApp { const workflow = pngInfo?.workflow || pngInfo?.Workflow; const prompt = pngInfo?.prompt || pngInfo?.Prompt; + if (workflow) { + this.loadGraphData(JSON.parse(workflow), true, true, fileName); + } else if (prompt) { + this.loadApiJson(JSON.parse(prompt), fileName); + } else { + this.showErrorOnFileLoad(file); + } + } else if (file.type === "audio/flac") { + const pngInfo = await getFlacMetadata(file); + // Support loading workflows from that webp custom node. + const workflow = pngInfo?.workflow; + const prompt = pngInfo?.prompt; + if (workflow) { this.loadGraphData(JSON.parse(workflow), true, true, fileName); } else if (prompt) { @@ -2321,7 +2351,6 @@ export class ComfyApp { node.id = isNaN(+id) ? id : +id; node.title = data._meta?.title ?? node.title app.graph.add(node); - graph.add(node); } this.changeWorkflow(() => { diff --git a/web/scripts/changeTracker.js b/web/scripts/changeTracker.js index 59901d5fc..041c83122 100644 --- a/web/scripts/changeTracker.js +++ b/web/scripts/changeTracker.js @@ -3,7 +3,6 @@ import { api } from "./api.js"; import { clone } from "./utils.js"; - export class ChangeTracker { static MAX_HISTORY = 50; #app; @@ -170,6 +169,17 @@ export class ChangeTracker { return v; }; + // Detects nodes being added via the node search dialog + const onNodeAdded = LiteGraph.LGraph.prototype.onNodeAdded; + LiteGraph.LGraph.prototype.onNodeAdded = function () { + const v = onNodeAdded?.apply(this, arguments); + const ct = changeTracker(); + if (!ct.isOurLoad) { + ct.checkState(); + } + return v; + }; + // Store node outputs api.addEventListener("executed", ({ detail }) => { const prompt = app.workflowManager.queuedPrompts[detail.prompt_id]; diff --git a/web/scripts/pnginfo.js b/web/scripts/pnginfo.js index 7132fb60f..2c03cf74a 100644 --- a/web/scripts/pnginfo.js +++ b/web/scripts/pnginfo.js @@ -163,6 +163,78 @@ export function getLatentMetadata(file) { }); } + +function getString(dataView, offset, length) { + let string = ''; + for (let i = 0; i < length; i++) { + string += String.fromCharCode(dataView.getUint8(offset + i)); + } + return string; +} + +// Function to parse the Vorbis Comment block +function parseVorbisComment(dataView) { + let offset = 0; + const vendorLength = dataView.getUint32(offset, true); + offset += 4; + const vendorString = getString(dataView, offset, vendorLength); + offset += vendorLength; + + const userCommentListLength = dataView.getUint32(offset, true); + offset += 4; + const comments = {}; + for (let i = 0; i < userCommentListLength; i++) { + const commentLength = dataView.getUint32(offset, true); + offset += 4; + const comment = getString(dataView, offset, commentLength); + offset += commentLength; + + const [key, value] = comment.split('='); + + comments[key] = value; + } + + return comments; +} + +// Function to read a FLAC file and parse Vorbis comments +export function getFlacMetadata(file) { + return new Promise((r) => { + const reader = new FileReader(); + reader.onload = function(event) { + const arrayBuffer = event.target.result; + const dataView = new DataView(arrayBuffer); + + // Verify the FLAC signature + const signature = String.fromCharCode(...new Uint8Array(arrayBuffer, 0, 4)); + if (signature !== 'fLaC') { + console.error('Not a valid FLAC file'); + return; + } + + // Parse metadata blocks + let offset = 4; + let vorbisComment = null; + while (offset < dataView.byteLength) { + const isLastBlock = dataView.getUint8(offset) & 0x80; + const blockType = dataView.getUint8(offset) & 0x7F; + const blockSize = dataView.getUint32(offset, false) & 0xFFFFFF; + offset += 4; + + if (blockType === 4) { // Vorbis Comment block type + vorbisComment = parseVorbisComment(new DataView(arrayBuffer, offset, blockSize)); + } + + offset += blockSize; + if (isLastBlock) break; + } + + r(vorbisComment); + }; + reader.readAsArrayBuffer(file); + }); +} + export async function importA1111(graph, parameters) { const p = parameters.lastIndexOf("\nSteps:"); if (p > -1) { diff --git a/web/scripts/ui.js b/web/scripts/ui.js index 87b4887eb..2c47412c9 100644 --- a/web/scripts/ui.js +++ b/web/scripts/ui.js @@ -373,7 +373,7 @@ export class ComfyUI { const fileInput = $el("input", { id: "comfy-file-input", type: "file", - accept: ".json,image/png,.latent,.safetensors,image/webp", + accept: ".json,image/png,.latent,.safetensors,image/webp,audio/flac", style: {display: "none"}, parent: document.body, onchange: () => { @@ -603,7 +603,6 @@ export class ComfyUI { onclick: () => app.refreshComboInNodes() }), $el("button", {id: "comfy-clipspace-button", textContent: "Clipspace", onclick: () => app.openClipspace()}), - $el("button", {id: "comfy-reset-view-button", textContent: "Reset View", onclick: () => app.resetView()}), $el("button", { id: "comfy-clear-button", textContent: "Clear", onclick: () => { if (!confirmClear.value || confirm("Clear workflow?")) { diff --git a/web/scripts/ui/menu/menu.css b/web/scripts/ui/menu/menu.css index 20eeab2cf..afaed3fb0 100644 --- a/web/scripts/ui/menu/menu.css +++ b/web/scripts/ui/menu/menu.css @@ -19,8 +19,12 @@ padding: 4px 8px; box-sizing: border-box; margin: 0; + transition: box-shadow 0.1s; } +.comfyui-button:active { + box-shadow: inset 1px 1px 10px rgba(0, 0, 0, 0.5); +} .comfyui-button:disabled { opacity: 0.5; cursor: not-allowed; diff --git a/web/scripts/ui/menu/queueButton.js b/web/scripts/ui/menu/queueButton.js index 3c29ab090..608f4cc9b 100644 --- a/web/scripts/ui/menu/queueButton.js +++ b/web/scripts/ui/menu/queueButton.js @@ -13,8 +13,8 @@ export class ComfyQueueButton { queuePrompt = async (e) => { this.#internalQueueSize += this.queueOptions.batchCount; - // Hold shift to queue front - await this.app.queuePrompt(-e.shiftKey, this.queueOptions.batchCount); + // Hold shift to queue front, event is undefined when auto-queue is enabled + await this.app.queuePrompt(e?.shiftKey ? -1 : 0, this.queueOptions.batchCount); }; constructor(app) { diff --git a/web/scripts/workflows.js b/web/scripts/workflows.js index 16bbc9976..d38b6f5fc 100644 --- a/web/scripts/workflows.js +++ b/web/scripts/workflows.js @@ -301,11 +301,11 @@ export class ComfyWorkflow { load = async () => { if (this.isOpen) { - await this.manager.app.loadGraphData(this.changeTracker.activeState, true, this); + await this.manager.app.loadGraphData(this.changeTracker.activeState, true, true, this); } else { const data = await this.getWorkflowData(); if (!data) return; - await this.manager.app.loadGraphData(data, true, this); + await this.manager.app.loadGraphData(data, true, true, this); } }; diff --git a/web/style.css b/web/style.css index 8091a489f..e983b652a 100644 --- a/web/style.css +++ b/web/style.css @@ -632,3 +632,7 @@ dialog::backdrop { border-top: none; } } + +audio.comfy-audio.empty-audio-widget { + display: none; +}