diff --git a/comfy/lora.py b/comfy/lora.py index 037431775..f657ec2e7 100644 --- a/comfy/lora.py +++ b/comfy/lora.py @@ -12,15 +12,24 @@ LORA_CLIP_MAP = { def load_lora(lora, to_load): + patch_dict = {} loaded_keys = set() for x in to_load: + #import json + #with open('/apdcephfs_cq8/share_1367250/xuhuaren/kaiyuan/fork/ComfyUI/result_lora.json', 'w') as fp: + # dict_keys_list = list(lora.keys()) + # json.dump(dict_keys_list, fp) + #import pdb + # dpdb.set_trace() + + + alpha_name = "{}.alpha".format(x) alpha = None if alpha_name in lora.keys(): alpha = lora[alpha_name].item() loaded_keys.add(alpha_name) - dora_scale_name = "{}.dora_scale".format(x) dora_scale = None if dora_scale_name in lora.keys(): @@ -31,6 +40,7 @@ def load_lora(lora, to_load): diffusers_lora = "{}_lora.up.weight".format(x) diffusers2_lora = "{}.lora_B.weight".format(x) diffusers3_lora = "{}.lora.up.weight".format(x) + diffusers4_lora = "{}_lora_B_weight".format(x) transformers_lora = "{}.lora_linear_layer.up.weight".format(x) A_name = None @@ -50,6 +60,10 @@ def load_lora(lora, to_load): A_name = diffusers3_lora B_name = "{}.lora.down.weight".format(x) mid_name = None + elif diffusers4_lora in lora.keys(): + A_name = diffusers4_lora + B_name = "{}_lora_A_weight".format(x) + mid_name = None elif transformers_lora in lora.keys(): A_name = transformers_lora B_name ="{}.lora_linear_layer.down.weight".format(x) @@ -63,6 +77,8 @@ def load_lora(lora, to_load): patch_dict[to_load[x]] = ("lora", (lora[A_name], lora[B_name], alpha, mid, dora_scale)) loaded_keys.add(A_name) loaded_keys.add(B_name) + #import pdb + #pdb.set_trace() ######## loha diff --git a/comfy/model_patcher.py b/comfy/model_patcher.py index b949031e9..6c3adc52b 100644 --- a/comfy/model_patcher.py +++ b/comfy/model_patcher.py @@ -349,6 +349,7 @@ class ModelPatcher: return self.model def calculate_weight(self, patches, weight, key): + for p in patches: strength = p[0] v = p[1] @@ -383,6 +384,7 @@ class ModelPatcher: else: weight += function(strength * comfy.model_management.cast_to_device(w1, weight.device, weight.dtype)) elif patch_type == "lora": #lora/locon + mat1 = comfy.model_management.cast_to_device(v[0], weight.device, torch.float32) mat2 = comfy.model_management.cast_to_device(v[1], weight.device, torch.float32) dora_scale = v[4] @@ -404,6 +406,8 @@ class ModelPatcher: weight += function(((strength * alpha) * lora_diff).type(weight.dtype)) except Exception as e: logging.error("ERROR {} {} {}".format(patch_type, key, e)) + + elif patch_type == "lokr": w1 = v[0] w2 = v[1] @@ -450,6 +454,7 @@ class ModelPatcher: weight += function(((strength * alpha) * lora_diff).type(weight.dtype)) except Exception as e: logging.error("ERROR {} {} {}".format(patch_type, key, e)) + elif patch_type == "loha": w1a = v[0] w1b = v[1] diff --git a/custom_nodes/convert_hunyuan_to_comfyui_lora.py b/custom_nodes/convert_hunyuan_to_comfyui_lora.py new file mode 100644 index 000000000..5ed2e015b --- /dev/null +++ b/custom_nodes/convert_hunyuan_to_comfyui_lora.py @@ -0,0 +1,16 @@ + +checkpoint_path2 = "/apdcephfs_cq8/share_1367250/xuhuaren/kaiyuan/fork/ComfyUI/models/loras/adapter_model.safetensors" +save_path = "/apdcephfs_cq8/share_1367250/xuhuaren/kaiyuan/fork/ComfyUI/models/loras/adapter_model_convert_diffusers.safetensors" +from safetensors import safe_open +from safetensors.torch import save_file + + +lora_state_dict = {} +with safe_open(checkpoint_path2, framework="pt", device=0) as f: + for k in f.keys(): + new_key = "lora_unet_" + "_".join(k[17:].split(".")) + lora_state_dict[new_key] = f.get_tensor(k) # remove 'basemodel.model' + +save_file(lora_state_dict, save_path) + +