diff --git a/README.md b/README.md index f2637a1..1b1a829 100644 --- a/README.md +++ b/README.md @@ -23,5 +23,10 @@ Chain the three demos together on `media-files/` into an end-to-end pipeline: 3. `tts` — clone the original speaker's voice (from the isolated vocal stem) and synthesize the translated text in that voice, then remix with the original instrumental stem -Stem separation and transcription are already working end to end (see `demucs/README.md` and -`whisperx/README.md`); translation and voice-cloning TTS are not yet implemented. +All three stages now have a working, if narrow, proof of concept: + +- Stem separation (`demucs/README.md`) and transcription (`whisperx/README.md`) run end to end + on the full sample clip. +- Translation + voice-cloning TTS (`tts/README.md`) works for a single short sentence + (English → Spanish, cloned onto the isolated narrator voice via MOSS-TTS). Not yet wired up + to run over the full transcript or to remix with the instrumental stem. diff --git a/tts/README.md b/tts/README.md index 0451ccb..687f804 100644 --- a/tts/README.md +++ b/tts/README.md @@ -1,3 +1,55 @@ # tts Text-to-speech service demos and proofs-of-concept. + +## Voice-clone + translate demo (MOSS-TTS) + +`clone_and_translate_demo.py` translates a short English sentence spoken by the narrator in +our sample clip to Spanish (via `Helsinki-NLP/opus-mt-en-es`), then uses +[MOSS-TTS](https://github.com/OpenMOSS/MOSS-TTS) to synthesize the Spanish text in the +narrator's cloned voice. The reference audio (`reference/narrator_ref_en.wav`) is trimmed +from demucs's isolated vocals stem for the exact same sentence, so the English original and +the Spanish clone can be compared directly — this is the first working slice of the full +audio-translation pipeline sketched in the top-level README. + +### Setup + +MOSS-TTS ships as a full repo you install editable (not a plain `pip install`), and it isn't +part of this git repo — it's vendored outside it since it's a large third-party project: + +```bash +git clone --depth 1 https://github.com/OpenMOSS/MOSS-TTS.git /path/to/vendor/MOSS-TTS +python3 -m venv .venv && . .venv/bin/activate +pip install --extra-index-url https://download.pytorch.org/whl/cu128 \ + -e "/path/to/vendor/MOSS-TTS[torch-runtime]" +pip install sentencepiece # for the MarianMT translation step +``` + +### Running + +```bash +SITE=$(python3 -c "import site; print(site.getsitepackages()[0])") +export LD_LIBRARY_PATH=$(find "$SITE/nvidia" -maxdepth 2 -type d -name lib | paste -sd: -) +python3 clone_and_translate_demo.py +``` + +Output goes to `output/narrator_cloned_es.wav` (48kHz stereo, ~11s). + +### Gotchas found running this on an RTX 3090 + +- **The flagship 8B `MOSS-TTS-v1.5` checkpoint OOMs on a 24GB 3090** — loading the model plus + its audio tokenizer already uses ~22.5GB, leaving no room for generation. Use the 4B + `MOSS-TTS-Local-Transformer-v1.5` checkpoint instead (also natively 48kHz stereo); it's + the option the project itself recommends for smaller GPUs. +- **`torchaudio.load()`/`.save()` fail with `Could not load libtorchcodec`** on this setup. + The real cause (visible via `ctypes.CDLL` directly, not the wrapped exception) is a missing + `libnppicc.so.12` — torchcodec links against NVIDIA's NPP library for its video-decode path, + but `moss-tts`'s dependencies don't pull it in since MOSS-TTS only needs audio. Fix: + `pip install nvidia-npp-cu12`, then add all `nvidia/*/lib` wheel directories to + `LD_LIBRARY_PATH` before running (see the `SITE=...` line above) — the pip-installed CUDA + libraries aren't on the default linker search path. This also affects code paths *inside* + MOSS-TTS's own processor (it uses `torchaudio.load()` to read reference audio), so the fix + has to be in place before the script imports it, not just around your own save calls. +- Saving the generated audio with plain `soundfile.write()` instead of `torchaudio.save()` + avoids depending on torchcodec at all for the output side, but doesn't help with the + reference-audio loading path above. diff --git a/tts/clone_and_translate_demo.py b/tts/clone_and_translate_demo.py new file mode 100644 index 0000000..fe4e5aa --- /dev/null +++ b/tts/clone_and_translate_demo.py @@ -0,0 +1,120 @@ +""" +Voice-clone + translate demo. + +Takes a short English sentence spoken by the narrator in our sample clip +(the same sentence used as the reference audio, for an apples-to-apples +comparison), machine-translates it to Spanish, then uses MOSS-TTS to +synthesize the Spanish text in the narrator's cloned voice. + +Reference audio (tts/reference/narrator_ref_en.wav) was trimmed from +demucs's isolated vocals stem for this same clip. +""" + +import importlib.util +from pathlib import Path + +import soundfile as sf +import torch +from transformers import AutoModel, AutoProcessor, MarianMTModel, MarianTokenizer + +HERE = Path(__file__).parent +REFERENCE_AUDIO = HERE / "reference" / "narrator_ref_en.wav" +OUTPUT_DIR = HERE / "output" + +TEXT_EN = ( + "We shall show you in this film typical examples of children who have " + "been well brought up, the kind who grow up to be strong, healthy, and " + "happy like these young folks." +) + +device = "cuda" if torch.cuda.is_available() else "cpu" +dtype = torch.bfloat16 if device == "cuda" else torch.float32 + + +def translate_to_spanish(text: str) -> str: + model_name = "Helsinki-NLP/opus-mt-en-es" + tokenizer = MarianTokenizer.from_pretrained(model_name) + model = MarianMTModel.from_pretrained(model_name).to(device) + batch = tokenizer([text], return_tensors="pt", padding=True).to(device) + generated = model.generate(**batch, max_new_tokens=256) + return tokenizer.decode(generated[0], skip_special_tokens=True) + + +def resolve_attn_implementation() -> str: + if ( + device == "cuda" + and importlib.util.find_spec("flash_attn") is not None + and dtype in {torch.float16, torch.bfloat16} + ): + major, _ = torch.cuda.get_device_capability() + if major >= 8: + return "flash_attention_2" + return "sdpa" if device == "cuda" else "eager" + + +def main() -> None: + text_es = translate_to_spanish(TEXT_EN) + print(f"[EN] {TEXT_EN}") + print(f"[ES] {text_es}") + + # Disable the broken cuDNN SDPA backend (per MOSS-TTS README). + torch.backends.cuda.enable_cudnn_sdp(False) + torch.backends.cuda.enable_flash_sdp(True) + torch.backends.cuda.enable_mem_efficient_sdp(True) + torch.backends.cuda.enable_math_sdp(True) + + # The flagship MOSS-TTS-v1.5 (8B) doesn't fit alongside its audio + # tokenizer + generation overhead on a 24GB 3090 (OOMs at ~23.3GB used). + # The 4B Local-Transformer variant is the recommended smaller-GPU option + # and also natively outputs 48kHz stereo. + pretrained = "OpenMOSS-Team/MOSS-TTS-Local-Transformer-v1.5" + attn_implementation = resolve_attn_implementation() + print(f"[INFO] attn_implementation={attn_implementation}") + + processor = AutoProcessor.from_pretrained(pretrained, trust_remote_code=True) + processor.audio_tokenizer = processor.audio_tokenizer.to(device) + + model = AutoModel.from_pretrained( + pretrained, + trust_remote_code=True, + attn_implementation=attn_implementation, + dtype=dtype, + ).to(device) + model.eval() + + conversation = [ + processor.build_user_message( + text=text_es, + language="Spanish", + reference=[str(REFERENCE_AUDIO)], + ) + ] + + OUTPUT_DIR.mkdir(exist_ok=True, parents=True) + with torch.no_grad(): + batch = processor([conversation], mode="generation") + outputs = model.generate( + input_ids=batch["input_ids"].to(device), + attention_mask=batch["attention_mask"].to(device), + max_new_tokens=4096, + do_sample=True, + audio_temperature=1.7, + audio_top_p=0.8, + audio_top_k=25, + audio_repetition_penalty=1.0, + ) + message = processor.decode(outputs)[0] + audio = message.audio_codes_list[0] + # torchaudio.save() depends on torchcodec, whose bundled shared libs + # failed to load in this environment; soundfile (already a MOSS-TTS + # dependency) sidesteps that entirely. + data = audio.detach().cpu().to(torch.float32).numpy() + if data.ndim == 2: + data = data.T # soundfile wants (frames, channels), not (channels, frames) + out_path = OUTPUT_DIR / "narrator_cloned_es.wav" + sf.write(str(out_path), data, processor.model_config.sampling_rate) + print(f"[INFO] Saved {out_path}") + + +if __name__ == "__main__": + main()