From f1ebfe4a95dc74c49f40b4c06928bed7a6c858e1 Mon Sep 17 00:00:00 2001 From: comfyanonymous Date: Sun, 23 Jun 2024 13:21:18 -0400 Subject: [PATCH] workflow --- comfy/model_patcher.py | 10 ++- comfy_extras/nodes_advanced_samplers.py | 74 +++++++++++++++++++ ...ows_sd3_medium_example_workflow_basic.json | 1 + 3 files changed, 82 insertions(+), 3 deletions(-) create mode 100644 my_workflows/comfy_example_workflows_sd3_medium_example_workflow_basic.json diff --git a/comfy/model_patcher.py b/comfy/model_patcher.py index bf8787768..b949031e9 100644 --- a/comfy/model_patcher.py +++ b/comfy/model_patcher.py @@ -51,6 +51,12 @@ def set_model_options_patch_replace(model_options, patch, name, block_name, numb model_options["transformer_options"] = to return model_options +def set_model_options_post_cfg_function(model_options, post_cfg_function, disable_cfg1_optimization=False): + model_options["sampler_post_cfg_function"] = model_options.get("sampler_post_cfg_function", []) + [post_cfg_function] + if disable_cfg1_optimization: + model_options["disable_cfg1_optimization"] = True + return model_options + class ModelPatcher: def __init__(self, model, load_device, offload_device, size=0, current_device=None, weight_inplace_update=False): self.size = size @@ -122,9 +128,7 @@ class ModelPatcher: self.model_options["disable_cfg1_optimization"] = True def set_model_sampler_post_cfg_function(self, post_cfg_function, disable_cfg1_optimization=False): - self.model_options["sampler_post_cfg_function"] = self.model_options.get("sampler_post_cfg_function", []) + [post_cfg_function] - if disable_cfg1_optimization: - self.model_options["disable_cfg1_optimization"] = True + self.model_options = set_model_options_post_cfg_function(self.model_options, post_cfg_function, disable_cfg1_optimization) def set_model_unet_function_wrapper(self, unet_wrapper_function: UnetWrapperFunction): self.model_options["model_function_wrapper"] = unet_wrapper_function diff --git a/comfy_extras/nodes_advanced_samplers.py b/comfy_extras/nodes_advanced_samplers.py index d973def81..7aa89c644 100644 --- a/comfy_extras/nodes_advanced_samplers.py +++ b/comfy_extras/nodes_advanced_samplers.py @@ -56,6 +56,80 @@ class SamplerLCMUpscale: sampler = comfy.samplers.KSAMPLER(sample_lcm_upscale, extra_options={"total_upscale": scale_ratio, "upscale_steps": scale_steps, "upscale_method": upscale_method}) return (sampler, ) +from comfy.k_diffusion.sampling import to_d +import comfy.model_patcher + +@torch.no_grad() +def sample_euler_cfgpp(model, x, sigmas, extra_args=None, callback=None, disable=None): + extra_args = {} if extra_args is None else extra_args + + temp = [0] + def post_cfg_function(args): + temp[0] = args["uncond_denoised"] + return args["denoised"] + + model_options = extra_args.get("model_options", {}).copy() + extra_args["model_options"] = comfy.model_patcher.set_model_options_post_cfg_function(model_options, post_cfg_function, disable_cfg1_optimization=True) + + s_in = x.new_ones([x.shape[0]]) + for i in trange(len(sigmas) - 1, disable=disable): + sigma_hat = sigmas[i] + denoised = model(x, sigma_hat * s_in, **extra_args) + d = to_d(x, sigma_hat, temp[0]) + if callback is not None: + callback({'x': x, 'i': i, 'sigma': sigmas[i], 'sigma_hat': sigma_hat, 'denoised': denoised}) + dt = sigmas[i + 1] - sigma_hat + x = denoised + sigmas[i + 1] * d + return x + +@torch.no_grad() +def sample_euler_cfgpp_alt(model, x, sigmas, extra_args=None, callback=None, disable=None): + extra_args = {} if extra_args is None else extra_args + + temp = [0] + def post_cfg_function(args): + temp[0] = args["uncond_denoised"] + return args["denoised"] + + model_options = extra_args.get("model_options", {}).copy() + extra_args["model_options"] = comfy.model_patcher.set_model_options_post_cfg_function(model_options, post_cfg_function, disable_cfg1_optimization=True) + + s_in = x.new_ones([x.shape[0]]) + for i in trange(len(sigmas) - 1, disable=disable): + sigma_hat = sigmas[i] + denoised = model(x, sigma_hat * s_in, **extra_args) + d = to_d(x - denoised + temp[0], sigma_hat, denoised) + if callback is not None: + callback({'x': x, 'i': i, 'sigma': sigmas[i], 'sigma_hat': sigma_hat, 'denoised': denoised}) + dt = sigmas[i + 1] - sigma_hat + # Euler method + x = x + d * dt + return x + +class SamplerEulerCFGpp: + @classmethod + def INPUT_TYPES(s): + return {"required": + {"version": (["regular", "alternative"],),} + } + RETURN_TYPES = ("SAMPLER",) + # CATEGORY = "sampling/custom_sampling/samplers" + CATEGORY = "_for_testing" + + FUNCTION = "get_sampler" + + def get_sampler(self, version): + if version == "regular": + sampler = comfy.samplers.KSAMPLER(sample_euler_cfgpp) + else: + sampler = comfy.samplers.KSAMPLER(sample_euler_cfgpp_alt) + return (sampler, ) + NODE_CLASS_MAPPINGS = { "SamplerLCMUpscale": SamplerLCMUpscale, + "SamplerEulerCFGpp": SamplerEulerCFGpp, +} + +NODE_DISPLAY_NAME_MAPPINGS = { + "SamplerEulerCFGpp": "SamplerEulerCFG++", } diff --git a/my_workflows/comfy_example_workflows_sd3_medium_example_workflow_basic.json b/my_workflows/comfy_example_workflows_sd3_medium_example_workflow_basic.json new file mode 100644 index 000000000..935e14473 --- /dev/null +++ b/my_workflows/comfy_example_workflows_sd3_medium_example_workflow_basic.json @@ -0,0 +1 @@ +{"last_node_id":272,"last_link_id":599,"nodes":[{"id":11,"type":"TripleCLIPLoader","pos":[-1885,-49],"size":{"0":315,"1":106},"flags":{},"order":0,"mode":0,"outputs":[{"name":"CLIP","type":"CLIP","links":[5,94],"shape":3,"slot_index":0}],"properties":{"Node name for S&R":"TripleCLIPLoader"},"widgets_values":["clip_g.safetensors","clip_l.safetensors","t5xxl_fp8_e4m3fn.safetensors"]},{"id":71,"type":"CLIPTextEncode","pos":[-1869.2871546875003,560.071803930664],"size":{"0":380.4615783691406,"1":102.07693481445312},"flags":{},"order":6,"mode":0,"inputs":[{"name":"clip","type":"CLIP","link":94}],"outputs":[{"name":"CONDITIONING","type":"CONDITIONING","links":[93,580],"shape":3,"slot_index":0}],"title":"CLIP Text Encode (Negative Prompt)","properties":{"Node name for S&R":"CLIPTextEncode"},"widgets_values":["bad quality, poor quality, doll, disfigured, jpg, toy, bad anatomy, missing limbs, missing fingers, 3d, cgi"],"color":"#322","bgcolor":"#533"},{"id":68,"type":"ConditioningSetTimestepRange","pos":[-1010,167],"size":{"0":317.4000244140625,"1":82},"flags":{},"order":10,"mode":0,"inputs":[{"name":"conditioning","type":"CONDITIONING","link":90}],"outputs":[{"name":"CONDITIONING","type":"CONDITIONING","links":[91],"shape":3,"slot_index":0}],"properties":{"Node name for S&R":"ConditioningSetTimestepRange"},"widgets_values":[0.1,1]},{"id":70,"type":"ConditioningSetTimestepRange","pos":[-1006,314],"size":{"0":317.4000244140625,"1":82},"flags":{},"order":8,"mode":0,"inputs":[{"name":"conditioning","type":"CONDITIONING","link":93,"slot_index":0}],"outputs":[{"name":"CONDITIONING","type":"CONDITIONING","links":[92],"shape":3,"slot_index":0}],"properties":{"Node name for S&R":"ConditioningSetTimestepRange"},"widgets_values":[0,0.1]},{"id":67,"type":"ConditioningZeroOut","pos":[-1370,337],"size":{"0":211.60000610351562,"1":26},"flags":{},"order":9,"mode":0,"inputs":[{"name":"conditioning","type":"CONDITIONING","link":580}],"outputs":[{"name":"CONDITIONING","type":"CONDITIONING","links":[90],"shape":3,"slot_index":0}],"properties":{"Node name for S&R":"ConditioningZeroOut"}},{"id":266,"type":"Note","pos":[-2352,576],"size":{"0":308.061279296875,"1":102.86902618408203},"flags":{},"order":1,"mode":0,"properties":{"text":""},"widgets_values":["Resolution should be around 1 megapixel and width/height must be multiple of 64"],"color":"#432","bgcolor":"#653"},{"id":13,"type":"ModelSamplingSD3","pos":[-974,-220],"size":{"0":315,"1":58},"flags":{"collapsed":false},"order":7,"mode":0,"inputs":[{"name":"model","type":"MODEL","link":565}],"outputs":[{"name":"MODEL","type":"MODEL","links":[591],"shape":3,"slot_index":0}],"properties":{"Node name for S&R":"ModelSamplingSD3"},"widgets_values":[3]},{"id":69,"type":"ConditioningCombine","pos":[-662,165],"size":{"0":228.39999389648438,"1":46},"flags":{},"order":11,"mode":0,"inputs":[{"name":"conditioning_1","type":"CONDITIONING","link":91},{"name":"conditioning_2","type":"CONDITIONING","link":92}],"outputs":[{"name":"CONDITIONING","type":"CONDITIONING","links":[592],"shape":3,"slot_index":0}],"properties":{"Node name for S&R":"ConditioningCombine"}},{"id":233,"type":"PreviewImage","pos":[535.1143750722272,-147.92548481673106],"size":{"0":604.7489624023438,"1":592.15576171875},"flags":{},"order":14,"mode":0,"inputs":[{"name":"images","type":"IMAGE","link":599}],"properties":{"Node name for S&R":"PreviewImage"}},{"id":252,"type":"CheckpointLoaderSimple","pos":[-2314,-203],"size":{"0":746.7357788085938,"1":98},"flags":{},"order":2,"mode":0,"outputs":[{"name":"MODEL","type":"MODEL","links":[565],"shape":3,"slot_index":0},{"name":"CLIP","type":"CLIP","links":[],"shape":3,"slot_index":1},{"name":"VAE","type":"VAE","links":[557],"shape":3,"slot_index":2}],"properties":{"Node name for S&R":"CheckpointLoaderSimple"},"widgets_values":["ComfyUI_00001_.safetensors"]},{"id":135,"type":"EmptySD3LatentImage","pos":[-2352,438],"size":{"0":315,"1":106},"flags":{},"order":3,"mode":0,"inputs":[],"outputs":[{"name":"LATENT","type":"LATENT","links":[593],"shape":3,"slot_index":0}],"properties":{"Node name for S&R":"EmptySD3LatentImage"},"widgets_values":[1024,1024,1]},{"id":6,"type":"CLIPTextEncode","pos":[-1876.2871546875003,284.0718039306641],"size":{"0":389.06927490234375,"1":207.84902954101562},"flags":{},"order":5,"mode":0,"inputs":[{"name":"clip","type":"CLIP","link":5}],"outputs":[{"name":"CONDITIONING","type":"CONDITIONING","links":[595],"shape":3,"slot_index":0}],"properties":{"Node name for S&R":"CLIPTextEncode"},"widgets_values":["a female character with long, flowing hair that appears to be made of ethereal, swirling patterns resembling the Northern Lights or Aurora Borealis. The background is dominated by deep blues and purples, creating a mysterious and dramatic atmosphere. The character's face is serene, with pale skin and striking features. She wears a dark-colored outfit with subtle patterns. The overall style of the artwork is reminiscent of fantasy or supernatural genres"],"color":"#232","bgcolor":"#353"},{"id":272,"type":"PrimitiveNode","pos":[-2342,278],"size":{"0":210,"1":82},"flags":{},"order":4,"mode":0,"outputs":[{"name":"INT","type":"INT","links":[597],"slot_index":0,"widget":{"name":"seed"}}],"title":"Seed","properties":{"Run widget replace on values":false},"widgets_values":[945512652412924,"fixed"]},{"id":271,"type":"KSampler","pos":[-269,-179],"size":{"0":315,"1":446},"flags":{},"order":12,"mode":0,"inputs":[{"name":"model","type":"MODEL","link":591},{"name":"positive","type":"CONDITIONING","link":595},{"name":"negative","type":"CONDITIONING","link":592},{"name":"latent_image","type":"LATENT","link":593},{"name":"seed","type":"INT","link":597,"widget":{"name":"seed"},"slot_index":4}],"outputs":[{"name":"LATENT","type":"LATENT","links":[596],"shape":3,"slot_index":0}],"properties":{"Node name for S&R":"KSampler"},"widgets_values":[945512652412924,"fixed",30,4.5,"dpmpp_2m","sgm_uniform",1]},{"id":231,"type":"VAEDecode","pos":[141,-177],"size":{"0":210,"1":46},"flags":{},"order":13,"mode":0,"inputs":[{"name":"samples","type":"LATENT","link":596},{"name":"vae","type":"VAE","link":557}],"outputs":[{"name":"IMAGE","type":"IMAGE","links":[599],"shape":3,"slot_index":0}],"properties":{"Node name for S&R":"VAEDecode"}}],"links":[[5,11,0,6,0,"CLIP"],[90,67,0,68,0,"CONDITIONING"],[91,68,0,69,0,"CONDITIONING"],[92,70,0,69,1,"CONDITIONING"],[93,71,0,70,0,"CONDITIONING"],[94,11,0,71,0,"CLIP"],[557,252,2,231,1,"VAE"],[565,252,0,13,0,"MODEL"],[580,71,0,67,0,"CONDITIONING"],[591,13,0,271,0,"MODEL"],[592,69,0,271,2,"CONDITIONING"],[593,135,0,271,3,"LATENT"],[595,6,0,271,1,"CONDITIONING"],[596,271,0,231,0,"LATENT"],[597,272,0,271,4,"INT"],[599,231,0,233,0,"IMAGE"]],"groups":[{"title":"Load Models","bounding":[-2410,-339,969,488],"color":"#3f789e","font_size":24},{"title":"Input","bounding":[-2409,181,972,523],"color":"#3f789e","font_size":24},{"title":"Output","bounding":[464,-273,741,814],"color":"#3f789e","font_size":24}],"config":{},"extra":{"workspace_info":{"id":"67da901c-1aa7-49bb-b58a-990c2b6e3f1e","saveLock":false,"cloudID":null,"coverMediaPath":null},"ds":{"scale":0.8264462809917354,"offset":[2855.540000000004,776.4699999999997]}},"version":0.4} \ No newline at end of file