docs: add conversion script

convert_qwen2_to_llama.py

@@ -0,0 +1,182 @@
+# coding=utf-8
+# Converts the 2nd version of the Qwen models in the same format as LLaMA2.
+# Usage: python convert_qwen2_to_llama.py --input_dir magnum-72b-v1 --output_dir magnum-72b-v1-llamaify --save_safetensors --continue_conversion
+# Original script: https://github.com/Minami-su/character_AI_open/blob/main/llamafy_qwen_v2.py
+
+import json
+import os
+from collections import OrderedDict
+from typing import Any, Dict, Optional
+
+import fire
+import torch
+from safetensors import safe_open
+from safetensors.torch import save_file
+from tqdm import tqdm
+from transformers.modeling_utils import (
+    SAFE_WEIGHTS_INDEX_NAME,
+    SAFE_WEIGHTS_NAME,
+    WEIGHTS_INDEX_NAME,
+    WEIGHTS_NAME,
+    shard_checkpoint,
+)
+from transformers.utils import check_min_version
+
+try:
+    check_min_version("4.34.0")
+except Exception:
+    raise ValueError("Please upgrade `transformers` to 4.34.0")
+
+CONFIG_NAME = "config.json"
+
+
+def load_existing_shards(
+    output_dir: str, save_safetensors: bool
+) -> Dict[str, torch.Tensor]:
+    existing_state_dict = OrderedDict()
+    weights_name = SAFE_WEIGHTS_NAME if save_safetensors else WEIGHTS_NAME
+    index_name = SAFE_WEIGHTS_INDEX_NAME if save_safetensors else WEIGHTS_INDEX_NAME
+
+    if os.path.exists(os.path.join(output_dir, index_name)):
+        with open(os.path.join(output_dir, index_name), "r", encoding="utf-8") as f:
+            index = json.load(f)
+
+        for shard_file in tqdm(
+            index["weight_map"].values(), desc="Loading existing shards"
+        ):
+            if os.path.exists(os.path.join(output_dir, shard_file)):
+                if save_safetensors:
+                    with safe_open(
+                        os.path.join(output_dir, shard_file),
+                        framework="pt",
+                        device="cpu",
+                    ) as f:
+                        for key in f.keys():
+                            existing_state_dict[key] = f.get_tensor(key)
+                else:
+                    shard = torch.load(
+                        os.path.join(output_dir, shard_file), map_location="cpu"
+                    )
+                    existing_state_dict.update(shard)
+
+    return existing_state_dict
+
+
+def save_weight(
+    input_dir: str,
+    output_dir: str,
+    shard_size: str,
+    save_safetensors: bool,
+    continue_conversion: bool,
+) -> str:
+    qwen_state_dict: Dict[str, torch.Tensor] = OrderedDict()
+    for filepath in tqdm(os.listdir(input_dir), desc="Load weights"):
+        if os.path.isfile(os.path.join(input_dir, filepath)) and filepath.endswith(
+            ".safetensors"
+        ):
+            with safe_open(
+                os.path.join(input_dir, filepath), framework="pt", device="cpu"
+            ) as f:
+                for key in f.keys():
+                    qwen_state_dict[key] = f.get_tensor(key)
+
+    llama2_state_dict: Dict[str, torch.Tensor] = OrderedDict()
+    if continue_conversion:
+        llama2_state_dict = load_existing_shards(output_dir, save_safetensors)
+
+    torch_dtype = None
+    for key, value in tqdm(qwen_state_dict.items(), desc="Convert format"):
+        if torch_dtype is None:
+            torch_dtype = value.dtype
+        if "self_attn.o_proj" in key:
+            llama2_state_dict[key] = value
+            bias_key = key.replace(".weight", ".bias")
+            if bias_key not in llama2_state_dict:
+                llama2_state_dict[bias_key] = torch.zeros_like(value[:, 0]).squeeze()
+        else:
+            llama2_state_dict[key] = value
+
+    weights_name = SAFE_WEIGHTS_NAME if save_safetensors else WEIGHTS_NAME
+    shards, index = shard_checkpoint(
+        llama2_state_dict, max_shard_size=shard_size, weights_name=weights_name
+    )
+
+    for shard_file, shard in tqdm(shards.items(), desc="Save weights"):
+        if save_safetensors:
+            save_file(
+                shard, os.path.join(output_dir, shard_file), metadata={"format": "pt"}
+            )
+        else:
+            torch.save(shard, os.path.join(output_dir, shard_file))
+
+    if index is None:
+        print(f"Model weights saved in {os.path.join(output_dir, weights_name)}")
+    else:
+        index_name = SAFE_WEIGHTS_INDEX_NAME if save_safetensors else WEIGHTS_INDEX_NAME
+        with open(os.path.join(output_dir, index_name), "w", encoding="utf-8") as f:
+            json.dump(index, f, indent=2, sort_keys=True)
+        print(f"Model weights saved in {output_dir}")
+
+    return str(torch_dtype).replace("torch.", "")
+
+
+def save_config(input_dir: str, output_dir: str, torch_dtype: str):
+    with open(os.path.join(input_dir, CONFIG_NAME), "r", encoding="utf-8") as f:
+        qwen_config_dict: Dict[str, Any] = json.load(f)
+
+    llama2_config_dict: Dict[str, Any] = OrderedDict()
+    llama2_config_dict["architectures"] = ["LlamaForCausalLM"]
+    llama2_config_dict["attention_bias"] = True
+    llama2_config_dict["attention_dropout"] = qwen_config_dict["attention_dropout"]
+    llama2_config_dict["hidden_act"] = "silu"
+    llama2_config_dict["hidden_size"] = qwen_config_dict["hidden_size"]
+    llama2_config_dict["initializer_range"] = qwen_config_dict["initializer_range"]
+    llama2_config_dict["intermediate_size"] = qwen_config_dict["intermediate_size"]
+    llama2_config_dict["max_position_embeddings"] = 32767  # Qwen2-72B-Instruct
+    llama2_config_dict["max_window_layers"] = qwen_config_dict["max_window_layers"]
+    llama2_config_dict["model_type"] = "llama"
+    llama2_config_dict["num_attention_heads"] = qwen_config_dict["num_attention_heads"]
+    llama2_config_dict["num_hidden_layers"] = qwen_config_dict["num_hidden_layers"]
+    llama2_config_dict["num_key_value_heads"] = qwen_config_dict["num_key_value_heads"]
+    llama2_config_dict["pretraining_tp"] = 1
+    llama2_config_dict["rms_norm_eps"] = qwen_config_dict["rms_norm_eps"]
+    llama2_config_dict["rope_theta"] = qwen_config_dict["rope_theta"]
+    llama2_config_dict["rope_scaling"] = None
+    llama2_config_dict["sliding_window"] = qwen_config_dict["sliding_window"]
+    llama2_config_dict["tie_word_embeddings"] = qwen_config_dict["tie_word_embeddings"]
+    llama2_config_dict["torch_dtype"] = torch_dtype
+    llama2_config_dict["transformers_version"] = "4.37.0"
+    llama2_config_dict["use_cache"] = True
+    llama2_config_dict["use_sliding_window"] = qwen_config_dict["use_sliding_window"]
+    llama2_config_dict["vocab_size"] = qwen_config_dict["vocab_size"]
+
+    with open(os.path.join(output_dir, CONFIG_NAME), "w", encoding="utf-8") as f:
+        json.dump(llama2_config_dict, f, indent=2)
+    print(f"Model config saved in {os.path.join(output_dir, CONFIG_NAME)}")
+
+
+def llamafy_qwen_v2(
+    input_dir: str,
+    output_dir: str,
+    shard_size: Optional[str] = "4GB",
+    save_safetensors: Optional[bool] = False,
+    continue_conversion: Optional[bool] = False,
+):
+    if not continue_conversion:
+        try:
+            os.makedirs(output_dir, exist_ok=False)
+        except Exception as e:
+            raise ValueError(
+                "Output dir already exists. Use --continue_conversion to resume."
+            ) from e
+    else:
+        os.makedirs(output_dir, exist_ok=True)
+
+    torch_dtype = save_weight(
+        input_dir, output_dir, shard_size, save_safetensors, continue_conversion
+    )
+    save_config(input_dir, output_dir, torch_dtype)
+
+
+if __name__ == "__main__":
+    fire.Fire(llamafy_qwen_v2)