audio-demo/tts/clone_and_translate_demo.py

"""
Voice-clone + translate demo.

Takes a short English sentence spoken by the narrator in our sample clip
(the same sentence used as the reference audio, for an apples-to-apples
comparison), machine-translates it to Spanish, then uses MOSS-TTS to
synthesize the Spanish text in the narrator's cloned voice.

Reference audio (tts/reference/narrator_ref_en.wav) was trimmed from
demucs's isolated vocals stem for this same clip.
"""

import importlib.util
from pathlib import Path

import soundfile as sf
import torch
from transformers import AutoModel, AutoProcessor, MarianMTModel, MarianTokenizer

HERE = Path(__file__).parent
REFERENCE_AUDIO = HERE / "reference" / "narrator_ref_en.wav"
OUTPUT_DIR = HERE / "output"

TEXT_EN = (
    "We shall show you in this film typical examples of children who have "
    "been well brought up, the kind who grow up to be strong, healthy, and "
    "happy like these young folks."
)

device = "cuda" if torch.cuda.is_available() else "cpu"
dtype = torch.bfloat16 if device == "cuda" else torch.float32


def translate_to_spanish(text: str) -> str:
    model_name = "Helsinki-NLP/opus-mt-en-es"
    tokenizer = MarianTokenizer.from_pretrained(model_name)
    model = MarianMTModel.from_pretrained(model_name).to(device)
    batch = tokenizer([text], return_tensors="pt", padding=True).to(device)
    generated = model.generate(**batch, max_new_tokens=256)
    return tokenizer.decode(generated[0], skip_special_tokens=True)


def resolve_attn_implementation() -> str:
    if (
        device == "cuda"
        and importlib.util.find_spec("flash_attn") is not None
        and dtype in {torch.float16, torch.bfloat16}
    ):
        major, _ = torch.cuda.get_device_capability()
        if major >= 8:
            return "flash_attention_2"
    return "sdpa" if device == "cuda" else "eager"


def main() -> None:
    text_es = translate_to_spanish(TEXT_EN)
    print(f"[EN] {TEXT_EN}")
    print(f"[ES] {text_es}")

    # Disable the broken cuDNN SDPA backend (per MOSS-TTS README).
    torch.backends.cuda.enable_cudnn_sdp(False)
    torch.backends.cuda.enable_flash_sdp(True)
    torch.backends.cuda.enable_mem_efficient_sdp(True)
    torch.backends.cuda.enable_math_sdp(True)

    # The flagship MOSS-TTS-v1.5 (8B) doesn't fit alongside its audio
    # tokenizer + generation overhead on a 24GB 3090 (OOMs at ~23.3GB used).
    # The 4B Local-Transformer variant is the recommended smaller-GPU option
    # and also natively outputs 48kHz stereo.
    pretrained = "OpenMOSS-Team/MOSS-TTS-Local-Transformer-v1.5"
    attn_implementation = resolve_attn_implementation()
    print(f"[INFO] attn_implementation={attn_implementation}")

    processor = AutoProcessor.from_pretrained(pretrained, trust_remote_code=True)
    processor.audio_tokenizer = processor.audio_tokenizer.to(device)

    model = AutoModel.from_pretrained(
        pretrained,
        trust_remote_code=True,
        attn_implementation=attn_implementation,
        dtype=dtype,
    ).to(device)
    model.eval()

    conversation = [
        processor.build_user_message(
            text=text_es,
            language="Spanish",
            reference=[str(REFERENCE_AUDIO)],
        )
    ]

    OUTPUT_DIR.mkdir(exist_ok=True, parents=True)
    with torch.no_grad():
        batch = processor([conversation], mode="generation")
        outputs = model.generate(
            input_ids=batch["input_ids"].to(device),
            attention_mask=batch["attention_mask"].to(device),
            max_new_tokens=4096,
            do_sample=True,
            audio_temperature=1.7,
            audio_top_p=0.8,
            audio_top_k=25,
            audio_repetition_penalty=1.0,
        )
        message = processor.decode(outputs)[0]
        audio = message.audio_codes_list[0]
        # torchaudio.save() depends on torchcodec, whose bundled shared libs
        # failed to load in this environment; soundfile (already a MOSS-TTS
        # dependency) sidesteps that entirely.
        data = audio.detach().cpu().to(torch.float32).numpy()
        if data.ndim == 2:
            data = data.T  # soundfile wants (frames, channels), not (channels, frames)
        out_path = OUTPUT_DIR / "narrator_cloned_es.wav"
        sf.write(str(out_path), data, processor.model_config.sampling_rate)
        print(f"[INFO] Saved {out_path}")


if __name__ == "__main__":
    main()