manifest.json

{
  "name": "cosyvoice3-coreml",
  "version": "1.0.0",
  "language": "zh",
  "library": "fluidaudio",
  "description": "CoreML conversions of CosyVoice3 Mandarin TTS (Qwen2-0.5B LLM + Flow mel generator + HiFT vocoder).",
  "pipeline_tag": "text-to-speech",
  "sample_rate_hz": 24000,
  "compute": {
    "target_platform": "Apple Silicon (M-series)",
    "min_os": "macOS 14 / iOS 17",
    "neural_engine": ["LLM-Prefill", "LLM-Decode", "HiFT"],
    "cpu_and_gpu": ["Flow"]
  },
  "model_graph": {
    "llm_hidden_dim": 896,
    "llm_layers": 24,
    "llm_query_heads": 14,
    "llm_kv_heads": 2,
    "llm_head_dim": 64,
    "llm_text_vocab": 151936,
    "speech_vocab": 6761,
    "speech_sos": 6561,
    "speech_eos": 6562,
    "speech_task_id": 6563,
    "mel_bins": 80,
    "mel_hop": 480,
    "mel_nfft": 1920
  },
  "models": [
    {
      "name": "LLM-Prefill-T256-M768-fp16",
      "paths": {
        "mlpackage": "LLM-Prefill-T256-M768-fp16.mlpackage",
        "mlmodelc":  "LLM-Prefill-T256-M768-fp16.mlmodelc"
      },
      "dtype": "fp16",
      "compute_units": "cpuAndNeuralEngine",
      "purpose": "Qwen2 prefill over 256-token context, initializes 768-slot KV cache.",
      "size_bytes": 729042944,
      "inputs": {
        "inputs_embeds": "[1, 256, 896] fp16",
        "attention_mask": "[1, 256] int32",
        "position_ids": "[1, 256] int32"
      },
      "outputs": {
        "logits": "[1, 256, 6761] fp16 (speech vocab)",
        "kv_k_out": "[24, 1, 2, 768, 64] fp16",
        "kv_v_out": "[24, 1, 2, 768, 64] fp16"
      }
    },
    {
      "name": "LLM-Decode-M768-fp16",
      "paths": {
        "mlpackage": "LLM-Decode-M768-fp16.mlpackage",
        "mlmodelc":  "LLM-Decode-M768-fp16.mlmodelc"
      },
      "dtype": "fp16",
      "compute_units": "cpuAndNeuralEngine",
      "purpose": "Single-step AR decode against a 768-slot KV cache.",
      "size_bytes": 728567808,
      "inputs": {
        "inputs_embeds": "[1, 1, 896] fp16",
        "cur_len": "[1] int32",
        "kv_k_in": "[24, 1, 2, 768, 64] fp16",
        "kv_v_in": "[24, 1, 2, 768, 64] fp16"
      },
      "outputs": {
        "logits": "[1, 1, 6761] fp16",
        "kv_k_out": "[24, 1, 2, 768, 64] fp16",
        "kv_v_out": "[24, 1, 2, 768, 64] fp16"
      }
    },
    {
      "name": "Flow-N250-fp16",
      "paths": {
        "mlpackage": "Flow-N250-fp16.mlpackage",
        "mlmodelc":  "Flow-N250-fp16.mlmodelc"
      },
      "dtype": "fp16",
      "compute_units": "cpuAndGPU",
      "purpose": "Speech tokens -> 80-bin log-mel @ 24 kHz. Must run with cpuAndGPU: pure CPU overflows the fused LayerNorm and produces NaNs; ANE refuses to compile this graph (ANECCompile fails). GPU path uses fp32 accumulators internally and is stable + ~3x faster than the previous fp32/cpuOnly shipping config.",
      "size_bytes": 669208054,
      "inputs": {
        "token_total": "[1, 250] int32 (prompt_ids || new_ids, right-padded)",
        "num_prompt_tokens": "[1] int32",
        "prompt_feat": "[1, 500, 80] fp32 (right-padded)",
        "embedding": "[1, 192] fp32 (CAMPPlus speaker embedding)"
      },
      "outputs": {
        "mel": "[1, 80, 500] fp32 (full buffer; slice to num_prompt_mel..num_prompt_mel+2*N_new)",
        "num_prompt_mel": "[1] int32"
      }
    },
    {
      "name": "HiFT-T500-fp16",
      "paths": {
        "mlpackage": "HiFT-T500-fp16.mlpackage",
        "mlmodelc":  "HiFT-T500-fp16.mlmodelc"
      },
      "dtype": "fp16",
      "compute_units": "cpuAndNeuralEngine",
      "purpose": "Mel -> 24 kHz PCM via iSTFT-based vocoder.",
      "size_bytes": 46448640,
      "inputs": {
        "mel": "[1, 80, 500] fp16 (right-padded)",
        "num_valid_frames": "[1] int32"
      },
      "outputs": {
        "audio": "[1, 240000] fp16 (clip to 480 * num_valid_frames samples)"
      }
    }
  ],
  "embeddings": [
    {
      "name": "embeddings-runtime-fp32",
      "path": "embeddings/embeddings-runtime-fp32.safetensors",
      "shape": [151936, 896],
      "dtype": "fp32",
      "size_bytes": 568770400,
      "purpose": "Qwen2 model.embed_tokens.weight at post-.float() runtime dtype. Required for bit-exact parity with Python reference. Swift mmaps this file."
    },
    {
      "name": "speech_embedding-fp16",
      "path": "embeddings/speech_embedding-fp16.safetensors",
      "shape": [6761, 896],
      "dtype": "fp16",
      "size_bytes": 12115808,
      "purpose": "CosyVoice3 speech_embedding table. Row-lookup per decoded speech token in the decode loop."
    }
  ],
  "tokenizer": {
    "kind": "qwen2-bpe",
    "vocab_file": "tokenizer/vocab.json",
    "merges_file": "tokenizer/merges.txt",
    "config_file": "tokenizer/tokenizer_config.json",
    "special_tokens_file": "tokenizer/special_tokens.json",
    "base_vocab_size": 151936,
    "special_token_count": 281,
    "special_token_id_range": [151643, 151923],
    "required_tokens": {
      "endofprompt": 151646,
      "endoftext": 151643,
      "im_start": 151644,
      "im_end": 151645
    }
  },
  "voices": [
    {
      "voice_id": "cosyvoice3-default-zh",
      "files": {
        "tensors":  "voices/cosyvoice3-default-zh.safetensors",
        "metadata": "voices/cosyvoice3-default-zh.json"
      },
      "reference_wav": "CosyVoice upstream zero_shot_prompt.wav",
      "prompt_utterance": "希望你以后能够做的比我还好呦。",
      "n_speech": 87,
      "mel_frames": 174,
      "size_bytes": 57244
    }
  ],
  "additional_voices_repo": "FluidInference/cosyvoice3-voices-zh",
  "swift": {
    "library": "FluidAudio",
    "manager": "CosyVoice3TtsManager",
    "public_api": "synthesize(text: String, promptAssets: CosyVoice3PromptAssets) async throws -> SynthesisResult",
    "default_voice": "voices/cosyvoice3-default-zh.safetensors"
  },
  "license": "Apache-2.0",
  "upstream": "FunAudioLLM/CosyVoice3"
}