Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+samples/input_zh.wav filter=lfs diff=lfs merge=lfs -text

README.md

@@ -1,13 +1,29 @@
 ---
-title: Index Echo S2st 2b
-emoji: 🦀
-colorFrom: purple
-colorTo: green
+title: Index-Echo-S2ST-2B
+emoji: 🎙️
+colorFrom: pink
+colorTo: purple
 sdk: gradio
 sdk_version: 6.29.0
-python_version: '3.12'
 app_file: app.py
-pinned: false
+short_description: Voice-preserving speech dubbing across zh/en/es/ja
+python_version: "3.12"
+startup_duration_timeout: 1h
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# Index-Echo-S2ST-2B — voice-preserving speech dubbing
+
+Speak in **Chinese or English** and get the same sentence back in **English,
+Spanish, Japanese or Chinese** — in your own voice. The model ([IndexTeam /
+Index-Echo-S2ST-2B](https://huggingface.co/IndexTeam/Index-Echo-S2ST-2B),
+Apache-2.0) chains a 2B speech-translation LM → a ~25M lang-conditioned
+Hidden2CVMapper → CosyVoice3 0.5B, with the source audio used as the voice
+prompt so the speaker's timbre is preserved.
+
+Six dubbing directions are supported: zh→en/es/ja and en→zh/es/ja. The source
+language is detected automatically from the ST transcript (CJK characters ⇒
+Chinese source, otherwise English). Dubbing into the same language as the
+source is not supported. Input audio must be ≤ 30 s.
+
+The sample inputs under `samples/` are the model authors' own verification
+samples (Apache-2.0).

app.py

@@ -0,0 +1,334 @@
+"""Gradio demo for IndexTeam/Index-Echo-S2ST-2B: voice-preserving speech-to-speech dubbing.
+
+Pipeline: 2B speech-translation LM (ST LM, frozen) -> ~25M Hidden2CVMapper
+(lang-conditioned) -> CosyVoice3 0.5B (frozen, source audio as voice prompt).
+6 directions: zh->en/es/ja + en->zh/es/ja. Source language is auto-detected
+from the ST transcript (CJK chars => zh source, else en source).
+
+Disk strategy: the raw snapshot is ~17 GB and the ZeroGPU weight pack adds
+~14 GB on top; each weight blob is unlinked from disk as soon as its consumer
+has read it into RAM (safe on Linux: open mmaps and even fully-read files
+keep working after unlink), so the pack never overlaps with the raw snapshot.
+"""
+import spaces  # MUST come before any torch / CUDA-touching import
+
+import os
+import sys
+import time
+
+import gradio as gr
+import torch
+from huggingface_hub import snapshot_download
+
+REPO_ID = "IndexTeam/Index-Echo-S2ST-2B"
+HERE = os.path.dirname(os.path.abspath(__file__))
+MODEL_DIR = os.path.join(HERE, "dubbing_2b_fulldir_cv3")
+
+# ---- interposition: unlink weight blobs once loaded into RAM --------------
+_CONSUMED = set()
+
+
+def _unlink(path):
+    """Delete a fully-read weight blob inside MODEL_DIR; mappings stay valid."""
+    if not os.path.realpath(path).startswith(os.path.realpath(MODEL_DIR) + os.sep):
+        return
+    try:
+        size_gb = os.path.getsize(path) / 1e9
+        os.unlink(path)
+        _CONSUMED.add(path)
+        print(f"[app] freed {os.path.basename(path)} ({size_gb:.2f} GB)", flush=True)
+    except OSError:
+        pass
+
+
+def _install_progressive_cleanup():
+    """Patch the loaders the vendored pipeline uses so each safetensors/torch
+    blob is unlinked right after its contents reach RAM."""
+    import safetensors.torch as st
+
+    _orig_load_file = st.load_file
+
+    def load_file_and_unlink(filename, **kw):
+        out = _orig_load_file(filename, **kw)
+        if isinstance(filename, str) and filename.endswith(".safetensors"):
+            _unlink(filename)
+        return out
+
+    st.load_file = load_file_and_unlink
+    # code/stlm/stlm_api.py, code/stlm/audio_model.py and the CV3 model's
+    # _load_sd do `from safetensors.torch import load_file`, i.e. bind a fresh
+    # reference at import time; patching the module attr before those imports
+    # run is enough because they happen later, inside DubPipeline.__init__.
+
+    _orig_torch_load = torch.load
+
+    def torch_load_and_unlink(f, *a, **kw):
+        out = _orig_torch_load(f, *a, **kw)
+        if isinstance(f, str) and f.endswith(".pt"):
+            _unlink(f)
+        return out
+
+    torch.load = torch_load_and_unlink
+
+    # transformers keeps the mmap open on from_pretrained safetensors blobs
+    # (stlm_llm 4.55 GB via AutoModelForCausalLM, sharded in this package;
+    # CosyVoice-BlankEN 1 GB via Qwen2ForCausalLM). Unlink every safetensors
+    # blob in the loaded dir right after the call returns — the open mappings
+    # stay valid on Linux.
+    from transformers import PreTrainedModel
+
+    _orig_from_pretrained = PreTrainedModel.from_pretrained.__func__
+
+    def from_pretrained_and_unlink(cls, path, *a, **kw):
+        out = _orig_from_pretrained(cls, path, *a, **kw)
+        try:
+            if os.path.isdir(str(path)):
+                for name in sorted(os.listdir(path)):
+                    if name.endswith(".safetensors"):
+                        _unlink(os.path.join(path, name))
+        except OSError:
+            pass
+        return out
+
+    PreTrainedModel.from_pretrained = classmethod(from_pretrained_and_unlink)
+
+
+# Set to True as the very first statement of the @spaces.GPU function body,
+# i.e. only inside the forked GPU worker where real CUDA exists.
+_REAL_CUDA = [False]
+
+
+def _install_torchaudio_save_fallback():
+    """torchaudio 2.11 delegates .save() to torchcodec (an FFmpeg-bound package
+    that is not installed here), so the vendored pipeline's
+    `torchaudio.save(out_wav, w, sr)` in code/tts/pipeline.py dies with
+    "ImportError: TorchCodec is required for save_with_torchcodec".
+    soundfile is installed (and is what the pipeline uses to *read* audio), so
+    redirect .save() to it. Called from inside @spaces.GPU only, to keep the
+    parent process free of avoidable library mutation."""
+    import numpy as np
+    import soundfile as sf
+    import torchaudio
+
+    def save(uri, src, sample_rate, **kw):
+        if hasattr(src, "detach"):
+            src = src.detach().cpu()
+        arr = np.asarray(src.numpy() if hasattr(src, "numpy") else src)
+        if arr.ndim == 2:
+            arr = arr.T  # (channels, T) -> (T, channels) for soundfile
+        sf.write(uri, arr, int(sample_rate))
+
+    torchaudio.save = save
+    print("[app] torchaudio.save -> soundfile fallback installed", flush=True)
+
+
+def _install_zerogpu_stream_shim():
+    """CosyVoice3Model.__init__ builds a side stream:
+    `torch.cuda.stream(torch.cuda.Stream(self.device)) if torch.cuda.is_available()`
+    ZeroGPU fakes is_available() -> True on the *boot* CPU, where there is no
+    GPU, so constructing a real stream there is a trap twice over:
+
+      1. it reaches the real driver and raises "no CUDA-capable device"; and
+      2. worse, the real path calls C++ getStreamFromPool ->
+         c10::cuda::device_count(), which caches 0 devices in a process-global
+         static. ZeroGPU forks its GPU worker from this process, so the child
+         inherits the cached 0 and torch.init() dies with
+         "No CUDA GPUs are available" before the user function ever runs.
+
+    So the parent must NEVER construct a real stream. Here torch.cuda.Stream
+    stays a real CLASS (transformers annotates `torch.cuda.Stream | None`, so a
+    function would break PEP 604 unions) but instantiation yields an inert
+    CPU-side stand-in until _REAL_CUDA is set inside the @spaces.GPU worker,
+    where the genuine constructor is used (spaces' unpatch() does not restore
+    these two attrs, so we dispatch on the flag instead)."""
+    from contextlib import nullcontext
+
+    real_Stream = torch.cuda.Stream
+    real_stream = torch.cuda.stream
+
+    class _CPUStreamFallback:
+        """Inert stand-in used only in the parent, before a GPU is attached."""
+
+        cuda_stream = 0
+        device = torch.device("cuda")
+
+        def synchronize(self):
+            return None
+
+        def wait_stream(self, *a, **kw):
+            return None
+
+        def query(self):
+            return True
+
+        def __enter__(self):
+            return self
+
+        def __exit__(self, *a):
+            return False
+
+    class Stream(real_Stream):
+        def __new__(cls, *a, **kw):
+            if _REAL_CUDA[0]:
+                return real_Stream.__new__(cls, *a, **kw)
+            fb = object.__new__(_CPUStreamFallback)
+            fb._zerogpu_fake = True
+            return fb
+
+        # torch's own Stream defines only __new__ (all the work happens there)
+        # and inherits object.__init__, relying on CPython's rule that a
+        # subclass overriding __new__ is allowed to ignore extra __init__ args.
+        # Calling real_Stream.__init__ explicitly would break that rule with
+        # "object.__init__() takes exactly one argument". Stay a no-op.
+        def __init__(self, *a, **kw):
+            pass
+
+    def stream(s):
+        if getattr(s, "_zerogpu_fake", False):
+            return nullcontext()
+        return real_stream(s)
+
+    torch.cuda.Stream = Stream
+    torch.cuda.stream = stream
+
+
+_install_zerogpu_stream_shim()
+
+_install_progressive_cleanup()
+
+print(f"[app] downloading {REPO_ID} snapshot (~17 GB, first boot only)...", flush=True)
+_t0 = time.time()
+snapshot_download(
+    repo_id=REPO_ID,
+    local_dir=MODEL_DIR,
+    max_workers=16,
+    ignore_patterns=["bridge/mapper_best.pt", "__pycache__/*", "*.pyc",
+                     ".ms_upload_cache"],
+)
+print(f"[app] snapshot done in {time.time() - _t0:.0f}s", flush=True)
+
+# The vendored inference stack (modeling_dubbing.py + code/) lives inside the
+# model snapshot; DubbingBridgeModel puts code/ and code/tts on sys.path itself.
+sys.path.insert(0, MODEL_DIR)
+from modeling_dubbing import DubbingBridgeModel  # noqa: E402
+
+print("[app] loading pipeline (ST LM 2B + mapper + CosyVoice3)...", flush=True)
+_t0 = time.time()
+model = DubbingBridgeModel.from_pretrained(MODEL_DIR, device="cuda")
+print(f"[app] pipeline ready in {time.time() - _t0:.0f}s", flush=True)
+print(f"[app] unlinked {len(_CONSUMED)} raw blobs", flush=True)
+
+SAMPLE_RATE = model.sample_rate  # 24000
+LANG_NAMES = {"en": "English", "es": "Spanish", "ja": "Japanese", "zh": "Chinese"}
+LANG_CODES = {v: k for k, v in LANG_NAMES.items()}
+
+
+def _has_cjk(text: str) -> bool:
+    return any("一" <= ch <= "鿿" for ch in text)
+
+
+@spaces.GPU(duration=120)
+def dub(audio_path: str, target_lang: str):
+    """Dub source speech into a target language, preserving the original voice.
+
+    Args:
+        audio_path: path to source speech (Chinese or English, <= 30 s).
+        target_lang: target language — "English", "Spanish", "Japanese" or
+            "Chinese" (a code en/es/ja/zh also works).
+
+    Returns:
+        (audio (sample_rate, waveform), source transcript markdown,
+        translation markdown) for the dubbed speech.
+    """
+    if not audio_path:
+        raise gr.Error("Please upload or record some source speech first.")
+    _REAL_CUDA[0] = True  # we are inside the forked GPU worker now
+    _install_torchaudio_save_fallback()
+    lang = LANG_CODES.get(target_lang, str(target_lang).lower())
+    if lang not in LANG_NAMES:
+        raise gr.Error(f"Unknown target language: {target_lang!r}")
+    t0 = time.time()
+    try:
+        wav, sr, info = model.dub(audio_path, lang=lang, return_info=True)
+    except AssertionError as e:
+        msg = str(e) or "unsupported dubbing direction"
+        if "30" in msg or "extract speech token" in msg:
+            raise gr.Error("Input audio is too long — please keep it under 30 seconds.")
+        raise gr.Error(
+            "This direction is not supported (the source language was detected "
+            "as the same as the target). Supported: zh→en/es/ja, en→zh/es/ja. "
+            f"({msg})"
+        )
+    elapsed = time.time() - t0
+    src_text, tgt_raw, tgt_cv = info["zh"], info["tgt_raw"], info["tgt_cv"]
+    src_lang = info.get("src_lang", "zh" if _has_cjk(src_text) else "en")
+    print(f"[dub] {src_lang}->{lang} in {elapsed:.1f}s: {src_text!r} -> {tgt_raw!r}",
+          flush=True)
+    audio = (sr, wav.detach().cpu().numpy().flatten())
+    transcript_md = (f"**Detected source ({LANG_NAMES.get(src_lang, src_lang)}):** "
+                     f"{src_text}")
+    translation_md = (f"**TTS input ({LANG_NAMES[lang]}):** "
+                      f"{tgt_cv if lang == 'ja' else tgt_raw}")
+    return audio, transcript_md, translation_md
+
+
+with gr.Blocks(theme=gr.themes.Citrus(), title="Index-Echo-S2ST-2B Dubbing") as demo:
+    gr.Markdown(
+        "# Index-Echo-S2ST-2B — voice-preserving speech dubbing\n"
+        "Speak in **Chinese or English**; get the same sentence back in "
+        "**English, Spanish, Japanese or Chinese** — in your own voice.\n\n"
+        "2B speech-translation LM → ~25M Hidden2CVMapper → CosyVoice3 0.5B. "
+        "Runs on ZeroGPU."
+    )
+    with gr.Row():
+        with gr.Column():
+            audio_in = gr.Audio(
+                sources=["upload", "microphone"],
+                type="filepath",
+                label="Source speech (Chinese or English, ≤ 30 s)",
+            )
+            lang_dd = gr.Dropdown(
+                choices=["English", "Spanish", "Japanese", "Chinese"],
+                value="English",
+                label="Target language",
+                info="Source language is detected automatically; dubbing into "
+                     "the same language as the source is not supported.",
+            )
+            btn = gr.Button("Dub speech", variant="primary")
+        with gr.Column():
+            audio_out = gr.Audio(label="Dubbed speech")
+            src_md = gr.Markdown()
+            tgt_md = gr.Markdown()
+    btn.click(
+        fn=dub,
+        inputs=[audio_in, lang_dd],
+        outputs=[audio_out, src_md, tgt_md],
+        api_name="dub",
+        show_progress="full",
+    )
+
+    with gr.Accordion("Supported directions", open=False):
+        gr.Markdown(
+            "| Source | Targets |\n|---|---|\n"
+            "| Chinese | English · Spanish · Japanese |\n"
+            "| English | Chinese · Spanish · Japanese |"
+        )
+    gr.Examples(
+        examples=[
+            ["samples/input_zh.wav", "English"],
+            ["samples/input_zh.wav", "Spanish"],
+            ["samples/input_zh.wav", "Japanese"],
+            ["samples/input_en.wav", "Chinese"],
+            ["samples/input_en.wav", "Spanish"],
+            ["samples/input_en.wav", "Japanese"],
+        ],
+        inputs=[audio_in, lang_dd],
+        fn=dub,
+        outputs=[audio_out, src_md, tgt_md],
+        cache_examples=True,
+        cache_mode="lazy",
+        label="Try the authors' own sample inputs (one per direction)",
+    )
+
+demo.queue(default_concurrency_limit=1)
+demo.launch(mcp_server=True, ssr_mode=False)

requirements.txt

@@ -0,0 +1,46 @@
+# Index-Echo-S2ST-2B vendored-inference-stack requirements (authors' venv_unify pins, see model card)
+# NOTE: torch/torchaudio pinned to the ZeroGPU-supported set (2.8.0/2.9.1/2.10.0/2.11.0);
+# plain PyPI builds (no +cu129 suffix) so the ZeroGPU version check passes.
+torch==2.11.0
+torchaudio==2.11.0
+transformers==5.6.0
+librosa==1.0.0
+onnxruntime==1.30.0
+wetext==0.0.4
+kaldifst==1.8.0
+conformer==0.3.2
+HyperPyYAML==1.2.3
+lightning==2.6.6
+x-transformers==2.11.24
+einx==0.4.3
+einops>=0.8.0
+frozendict==2.4.7
+pyworld==0.3.5
+soundfile==0.13.1
+modelscope==1.37.1
+safetensors==0.7.0
+numpy==2.4.6
+openai-whisper==20250625
+# Matcha-TTS (vendored in the snapshot, used by CosyVoice3's flow/HiFT) imports
+# GEGLU/GELU/ApproximateGELU and AdaLayerNorm/AdaLayerNormZero from
+# diffusers.models.attention — re-exports that only exist up to 0.34.x
+# (also the authors' own pin, cosyvoice_repo/requirements.txt).
+diffusers==0.29.0
+# matcha/utils/__init__.py is executed on any matcha.utils.* import (parent pkg init):
+# it pulls rich_utils (rich, hydra) and utils (gdown, wget, matplotlib) -> required
+rich==13.7.1
+hydra-core==1.3.7
+gdown==5.1.0
+wget==3.2
+matplotlib==3.10.7
+# wetext/kaldifst deps (pypi metadata), whisper deps, HyperPyYAML deps
+ruamel.yaml>=0.17.28,<0.19.0
+inflect
+tiktoken
+pyarrow
+regex
+scipy
+tqdm
+# NOTE: pyopenjtalk / pykakasi / jaconv / deprecated / wrapt are vendored in the
+# model repo at code/ja_ext/ and put on sys.path by the pipeline itself.
+# Do NOT pip-install them here — installed copies would shadow the vendored ones.

samples/input_zh.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d01464c28a57ffb40b4d0126f8faf5081b73f6dbb5592fc41108009b14f064e8
+size 198478