Upload 2 files

added convert script and test lora

convert_lora.py

@@ -0,0 +1,164 @@
+import argparse
+import os
+from collections import defaultdict
+from pathlib import Path
+
+import torch
+from safetensors.torch import load_file, save_file
+
+
+def convert_diffusers_to_hunyuan_video_lora(diffusers_state_dict):
+    converted_state_dict = {k: diffusers_state_dict.pop(k) for k in list(diffusers_state_dict.keys())}
+
+    TRANSFORMER_KEYS_RENAME_DICT = {
+        "img_in": "x_embedder",
+        "time_in.mlp.0": "time_text_embed.timestep_embedder.linear_1",
+        "time_in.mlp.2": "time_text_embed.timestep_embedder.linear_2",
+        "guidance_in.mlp.0": "time_text_embed.guidance_embedder.linear_1",
+        "guidance_in.mlp.2": "time_text_embed.guidance_embedder.linear_2",
+        "vector_in.in_layer": "time_text_embed.text_embedder.linear_1",
+        "vector_in.out_layer": "time_text_embed.text_embedder.linear_2",
+        ".double_blocks": ".transformer_blocks",
+        ".single_blocks": ".single_transformer_blocks",
+        "img_attn_q_norm": "attn.norm_q",
+        "img_attn_k_norm": "attn.norm_k",
+        "img_attn_proj": "attn.to_out.0",
+        "txt_attn_q_norm": "attn.norm_added_q",
+        "txt_attn_k_norm": "attn.norm_added_k",
+        "txt_attn_proj": "attn.to_add_out",
+        "img_mod.linear": "norm1.linear",
+        "img_norm1": "norm1.norm",
+        "img_norm2": "norm2",
+        "txt_mlp": "ff_context",
+        "img_mlp": "ff",
+        "txt_mod.linear": "norm1_context.linear",
+        "txt_norm1": "norm1.norm",
+        "txt_norm2": "norm2_context",
+        "modulation.linear": "norm.linear",
+        "pre_norm": "norm.norm",
+        "final_layer.norm_final": "norm_out.norm",
+        "final_layer.linear": "proj_out",
+    #   "linear2": "proj_out",
+        "fc1": "net.0.proj",
+        "fc2": "net.2",
+        "input_embedder": "proj_in",
+        # txt_in
+        "individual_token_refiner.blocks": "token_refiner.refiner_blocks",
+        "final_layer.adaLN_modulation.1": "norm_out.linear",
+    #   "t_embedder.mlp.0": "time_text_embed.timestep_embedder.linear_1",
+    #   "t_embedder.mlp.2": "time_text_embed.timestep_embedder.linear_2",
+        "c_embedder": "time_text_embed.text_embedder",
+        "txt_in": "context_embedder",
+    #   "mlp": "ff",
+    }
+
+    TRANSFORMER_KEYS_RENAME_DICT_REVERSE = {v: k for k, v in TRANSFORMER_KEYS_RENAME_DICT.items()}
+
+    for key in list(converted_state_dict.keys()):
+        if "norm_out.linear" in key:
+            weight = converted_state_dict.pop(key)
+            scale, shift = weight.chunk(2, dim=0)
+            new_weight = torch.cat([shift, scale], dim=0)
+            converted_state_dict[key] = new_weight
+
+        if "to_q" in key:
+            if "single_transformer_blocks" in key:
+                to_q = converted_state_dict.pop(key)
+                to_k = converted_state_dict.pop(key.replace("to_q", "to_k"))
+                to_v = converted_state_dict.pop(key.replace("to_q", "to_v"))
+                to_out = converted_state_dict.pop(key.replace("attn.to_q", "proj_mlp"))
+                rename_attn_key = "linear1"
+                if "lora_A" in key:
+                    converted_state_dict[key.replace("attn.to_q", rename_attn_key)] = to_q
+                else:
+                    qkv_mlp = torch.cat([to_q, to_k, to_v, to_out], dim=0)
+                    converted_state_dict[key.replace("attn.to_q", rename_attn_key)] = qkv_mlp
+            else:
+                to_q = converted_state_dict.pop(key)
+                to_k = converted_state_dict.pop(key.replace("to_q", "to_k"))
+                to_v = converted_state_dict.pop(key.replace("to_q", "to_v"))
+                if "token_refiner" in key:
+                    rename_attn_key = "self_attn_qkv"
+                    if "lora_A" in key:
+                        converted_state_dict[key.replace("attn.to_q", rename_attn_key)] = to_q
+                    else:
+                        qkv = torch.cat([to_q, to_k, to_v], dim=0)
+                        converted_state_dict[key.replace("attn.to_q", rename_attn_key)] = qkv
+                else:
+                    rename_attn_key = "img_attn_qkv"
+                    if "lora_A" in key:
+                        converted_state_dict[key.replace("attn.to_q", rename_attn_key)] = to_q
+                    else:
+                        qkv = torch.cat([to_q, to_k, to_v], dim=0)
+                        converted_state_dict[key.replace("attn.to_q", rename_attn_key)] = qkv
+
+        if "add_q_proj" in key:
+            to_q = converted_state_dict.pop(key)
+            to_k = converted_state_dict.pop(key.replace("add_q_proj", "add_k_proj"))
+            to_v = converted_state_dict.pop(key.replace("add_q_proj", "add_v_proj"))
+            rename_attn_key = "txt_attn_qkv"
+            if "lora_A" in key:
+                converted_state_dict[key.replace("attn.add_q_proj", rename_attn_key)] = to_q
+            else:
+                qkv = torch.cat([to_q, to_k, to_v], dim=0)
+                converted_state_dict[key.replace("attn.add_q_proj", rename_attn_key)] = qkv
+
+    for key in list(converted_state_dict.keys()):
+        new_key = key[:]
+        if "token_refiner" in key and "attn.to_out.0" in new_key:
+            new_key = new_key.replace("attn.to_out.0", "self_attn_proj")
+        if "token_refiner" in key and "ff" in new_key:
+            new_key = new_key.replace("ff", "mlp")
+        if "token_refiner" in key and "norm_out.linear" in new_key:
+            new_key = new_key.replace("norm_out.linear", "adaLN_modulation.1")
+        if "context_embedder" in key and "time_text_embed.text_embedder.linear_1" in new_key:
+            new_key = new_key.replace("time_text_embed.text_embedder.linear_1", "c_embedder.linear_1")
+        if "context_embedder" in key and "time_text_embed.text_embedder.linear_2" in new_key:
+            new_key = new_key.replace("time_text_embed.text_embedder.linear_2", "c_embedder.linear_2")
+        if "context_embedder" in key and "time_text_embed.timestep_embedder.linear_1" in new_key:
+            new_key = new_key.replace("time_text_embed.timestep_embedder.linear_1", "t_embedder.mlp.0")
+        if "context_embedder" in key and "time_text_embed.timestep_embedder.linear_2" in new_key:
+            new_key = new_key.replace("time_text_embed.timestep_embedder.linear_2", "t_embedder.mlp.2")
+        if "single_transformer_blocks" in key and "proj_out" in new_key:
+            new_key = new_key.replace("proj_out", "linear2")
+        for replace_key, rename_key in TRANSFORMER_KEYS_RENAME_DICT_REVERSE.items():
+            new_key = new_key.replace(replace_key, rename_key)
+        converted_state_dict[new_key] = converted_state_dict.pop(key)
+
+    # Remove "transformer." prefix
+    for key in list(converted_state_dict.keys()):
+        if key.startswith("transformer."):
+            converted_state_dict[key[len("transformer."):]] = converted_state_dict.pop(key)
+
+    # Add back "diffusion_model." prefix
+    for key in list(converted_state_dict.keys()):
+        converted_state_dict[f"diffusion_model.{key}"] = converted_state_dict.pop(key)
+
+    return converted_state_dict
+
+
+def get_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--ckpt_path", type=str, required=True)
+    parser.add_argument("--output_path_or_name", type=str, required=True)
+    return parser.parse_args()
+
+
+if __name__ == "__main__":
+    args = get_args()
+
+    if args.ckpt_path.endswith(".pt"):
+        diffusers_state_dict = torch.load(args.ckpt_path, map_location="cpu", weights_only=True)
+    elif args.ckpt_path.endswith(".safetensors"):
+        diffusers_state_dict = load_file(args.ckpt_path)
+
+    original_format_state_dict = convert_diffusers_to_hunyuan_video_lora(diffusers_state_dict)
+
+    output_path_or_name = Path(args.output_path_or_name)
+    if output_path_or_name.as_posix().endswith(".safetensors"):
+        os.makedirs(output_path_or_name.parent, exist_ok=True)
+        save_file(original_format_state_dict, output_path_or_name)
+    else:
+        os.makedirs(output_path_or_name, exist_ok=True)
+        output_path_or_name = output_path_or_name / "pytorch_lora_weights.safetensors"
+        save_file(original_format_state_dict, output_path_or_name)

wf_0400.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:474982b3da41876ed4a28e252b4b58ea69e558f4019accef5c962e9876926a3b
+size 374925440