README.md

@@ -23,5 +23,10 @@ Chain the three demos together on `media-files/` into an end-to-end pipeline:
 3. `tts` — clone the original speaker's voice (from the isolated vocal stem) and synthesize
    the translated text in that voice, then remix with the original instrumental stem
 
-Stem separation and transcription are already working end to end (see `demucs/README.md` and
+All three stages now have a working, if narrow, proof of concept:
-`whisperx/README.md`); translation and voice-cloning TTS are not yet implemented.
+
+- Stem separation (`demucs/README.md`) and transcription (`whisperx/README.md`) run end to end
+  on the full sample clip.
+- Translation + voice-cloning TTS (`tts/README.md`) works for a single short sentence
+  (English → Spanish, cloned onto the isolated narrator voice via MOSS-TTS). Not yet wired up
+  to run over the full transcript or to remix with the instrumental stem.

tts/README.md

@@ -1,3 +1,55 @@
 # tts
 
 Text-to-speech service demos and proofs-of-concept.
+
+## Voice-clone + translate demo (MOSS-TTS)
+
+`clone_and_translate_demo.py` translates a short English sentence spoken by the narrator in
+our sample clip to Spanish (via `Helsinki-NLP/opus-mt-en-es`), then uses
+[MOSS-TTS](https://github.com/OpenMOSS/MOSS-TTS) to synthesize the Spanish text in the
+narrator's cloned voice. The reference audio (`reference/narrator_ref_en.wav`) is trimmed
+from demucs's isolated vocals stem for the exact same sentence, so the English original and
+the Spanish clone can be compared directly — this is the first working slice of the full
+audio-translation pipeline sketched in the top-level README.
+
+### Setup
+
+MOSS-TTS ships as a full repo you install editable (not a plain `pip install`), and it isn't
+part of this git repo — it's vendored outside it since it's a large third-party project:
+
+```bash
+git clone --depth 1 https://github.com/OpenMOSS/MOSS-TTS.git /path/to/vendor/MOSS-TTS
+python3 -m venv .venv && . .venv/bin/activate
+pip install --extra-index-url https://download.pytorch.org/whl/cu128 \
+  -e "/path/to/vendor/MOSS-TTS[torch-runtime]"
+pip install sentencepiece  # for the MarianMT translation step
+```
+
+### Running
+
+```bash
+SITE=$(python3 -c "import site; print(site.getsitepackages()[0])")
+export LD_LIBRARY_PATH=$(find "$SITE/nvidia" -maxdepth 2 -type d -name lib | paste -sd: -)
+python3 clone_and_translate_demo.py
+```
+
+Output goes to `output/narrator_cloned_es.wav` (48kHz stereo, ~11s).
+
+### Gotchas found running this on an RTX 3090
+
+- **The flagship 8B `MOSS-TTS-v1.5` checkpoint OOMs on a 24GB 3090** — loading the model plus
+  its audio tokenizer already uses ~22.5GB, leaving no room for generation. Use the 4B
+  `MOSS-TTS-Local-Transformer-v1.5` checkpoint instead (also natively 48kHz stereo); it's
+  the option the project itself recommends for smaller GPUs.
+- **`torchaudio.load()`/`.save()` fail with `Could not load libtorchcodec`** on this setup.
+  The real cause (visible via `ctypes.CDLL` directly, not the wrapped exception) is a missing
+  `libnppicc.so.12` — torchcodec links against NVIDIA's NPP library for its video-decode path,
+  but `moss-tts`'s dependencies don't pull it in since MOSS-TTS only needs audio. Fix:
+  `pip install nvidia-npp-cu12`, then add all `nvidia/*/lib` wheel directories to
+  `LD_LIBRARY_PATH` before running (see the `SITE=...` line above) — the pip-installed CUDA
+  libraries aren't on the default linker search path. This also affects code paths *inside*
+  MOSS-TTS's own processor (it uses `torchaudio.load()` to read reference audio), so the fix
+  has to be in place before the script imports it, not just around your own save calls.
+- Saving the generated audio with plain `soundfile.write()` instead of `torchaudio.save()`
+  avoids depending on torchcodec at all for the output side, but doesn't help with the
+  reference-audio loading path above.

tts/clone_and_translate_demo.py

@@ -0,0 +1,120 @@
+"""
+Voice-clone + translate demo.
+
+Takes a short English sentence spoken by the narrator in our sample clip
+(the same sentence used as the reference audio, for an apples-to-apples
+comparison), machine-translates it to Spanish, then uses MOSS-TTS to
+synthesize the Spanish text in the narrator's cloned voice.
+
+Reference audio (tts/reference/narrator_ref_en.wav) was trimmed from
+demucs's isolated vocals stem for this same clip.
+"""
+
+import importlib.util
+from pathlib import Path
+
+import soundfile as sf
+import torch
+from transformers import AutoModel, AutoProcessor, MarianMTModel, MarianTokenizer
+
+HERE = Path(__file__).parent
+REFERENCE_AUDIO = HERE / "reference" / "narrator_ref_en.wav"
+OUTPUT_DIR = HERE / "output"
+
+TEXT_EN = (
+    "We shall show you in this film typical examples of children who have "
+    "been well brought up, the kind who grow up to be strong, healthy, and "
+    "happy like these young folks."
+)
+
+device = "cuda" if torch.cuda.is_available() else "cpu"
+dtype = torch.bfloat16 if device == "cuda" else torch.float32
+
+
+def translate_to_spanish(text: str) -> str:
+    model_name = "Helsinki-NLP/opus-mt-en-es"
+    tokenizer = MarianTokenizer.from_pretrained(model_name)
+    model = MarianMTModel.from_pretrained(model_name).to(device)
+    batch = tokenizer([text], return_tensors="pt", padding=True).to(device)
+    generated = model.generate(**batch, max_new_tokens=256)
+    return tokenizer.decode(generated[0], skip_special_tokens=True)
+
+
+def resolve_attn_implementation() -> str:
+    if (
+        device == "cuda"
+        and importlib.util.find_spec("flash_attn") is not None
+        and dtype in {torch.float16, torch.bfloat16}
+    ):
+        major, _ = torch.cuda.get_device_capability()
+        if major >= 8:
+            return "flash_attention_2"
+    return "sdpa" if device == "cuda" else "eager"
+
+
+def main() -> None:
+    text_es = translate_to_spanish(TEXT_EN)
+    print(f"[EN] {TEXT_EN}")
+    print(f"[ES] {text_es}")
+
+    # Disable the broken cuDNN SDPA backend (per MOSS-TTS README).
+    torch.backends.cuda.enable_cudnn_sdp(False)
+    torch.backends.cuda.enable_flash_sdp(True)
+    torch.backends.cuda.enable_mem_efficient_sdp(True)
+    torch.backends.cuda.enable_math_sdp(True)
+
+    # The flagship MOSS-TTS-v1.5 (8B) doesn't fit alongside its audio
+    # tokenizer + generation overhead on a 24GB 3090 (OOMs at ~23.3GB used).
+    # The 4B Local-Transformer variant is the recommended smaller-GPU option
+    # and also natively outputs 48kHz stereo.
+    pretrained = "OpenMOSS-Team/MOSS-TTS-Local-Transformer-v1.5"
+    attn_implementation = resolve_attn_implementation()
+    print(f"[INFO] attn_implementation={attn_implementation}")
+
+    processor = AutoProcessor.from_pretrained(pretrained, trust_remote_code=True)
+    processor.audio_tokenizer = processor.audio_tokenizer.to(device)
+
+    model = AutoModel.from_pretrained(
+        pretrained,
+        trust_remote_code=True,
+        attn_implementation=attn_implementation,
+        dtype=dtype,
+    ).to(device)
+    model.eval()
+
+    conversation = [
+        processor.build_user_message(
+            text=text_es,
+            language="Spanish",
+            reference=[str(REFERENCE_AUDIO)],
+        )
+    ]
+
+    OUTPUT_DIR.mkdir(exist_ok=True, parents=True)
+    with torch.no_grad():
+        batch = processor([conversation], mode="generation")
+        outputs = model.generate(
+            input_ids=batch["input_ids"].to(device),
+            attention_mask=batch["attention_mask"].to(device),
+            max_new_tokens=4096,
+            do_sample=True,
+            audio_temperature=1.7,
+            audio_top_p=0.8,
+            audio_top_k=25,
+            audio_repetition_penalty=1.0,
+        )
+        message = processor.decode(outputs)[0]
+        audio = message.audio_codes_list[0]
+        # torchaudio.save() depends on torchcodec, whose bundled shared libs
+        # failed to load in this environment; soundfile (already a MOSS-TTS
+        # dependency) sidesteps that entirely.
+        data = audio.detach().cpu().to(torch.float32).numpy()
+        if data.ndim == 2:
+            data = data.T  # soundfile wants (frames, channels), not (channels, frames)
+        out_path = OUTPUT_DIR / "narrator_cloned_es.wav"
+        sf.write(str(out_path), data, processor.model_config.sampling_rate)
+        print(f"[INFO] Saved {out_path}")
+
+
+if __name__ == "__main__":
+    main()