diff --git a/.gitignore b/.gitignore index df109b8..1a5cae0 100644 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,7 @@ venv/ *.safetensors models/ weights/ +ckpts/ # Audio I/O scratch *.wav @@ -33,3 +34,6 @@ media-files/video/*.mp4 # Compressed exports of demucs stems — keep committed for review without # local playback (the raw separated/*.wav stems stay local, per *.wav above) !demucs/*.mp3 + +# Small public-domain demo evidence clip for the auk voice-editing demo +!auk/*.wav diff --git a/README.md b/README.md index f2637a1..db266ab 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,9 @@ Audio processing demos: WhisperX, Demucs, and TTS technology proofs-of-concept. - `whisperx/` — speech-to-text with word-level alignment and diarization - `demucs/` — source separation (vocals/instrumentals stem splitting) -- `tts/` — text-to-speech service demos +- `tts/` — text-to-speech service demos (voice cloning via MOSS-TTS) +- `auk/` — voice *editing* demos (emotion/pitch/speed/content editing on existing recordings, + as opposed to `tts/`'s clone-and-generate approach) Each subdirectory has its own README with setup and run instructions. diff --git a/auk/README.md b/auk/README.md new file mode 100644 index 0000000..e9d27ff --- /dev/null +++ b/auk/README.md @@ -0,0 +1,67 @@ +# auk + +Voice-*editing* demos using [AuK](https://github.com/Tencent-Hunyuan/AuK) (Tencent-Hunyuan) — a +1.5B foundation model for speech generation **and editing**. Unlike MOSS-TTS's `instruction` +field (steers style at generation time, from a reference clip), AuK edits an *existing* +recording directly: same speaker, same words, same duration, different delivery. It also does +content editing (rewrite/insert/remove words), pitch/speed/volume editing, de-accenting, and +speech enhancement/separation — all through one natural-language instruction interface. + +## Demo + +`emotion_edit_demo.py` takes the same public-domain narrator clip used by the whisperx/tts demos +(`tts/reference/narrator_ref_en.wav`, 11.7s, isolated via demucs) and edits its emotion to +"excited" while preserving the voice, words, and exact duration. + +### Setup + +AuK is vendored outside this repo (large third-party project, editable install), same pattern as +MOSS-TTS: + +```bash +git clone https://github.com/Tencent-Hunyuan/AuK.git /path/to/vendor/AuK +python3 -m venv .venv && . .venv/bin/activate +pip install -e /path/to/vendor/AuK + +# Two runtime deps AuK needs but doesn't declare (see Gotchas): +pip install audioread "huggingface_hub<1.0,>=0.34.0" + +# Model weights (~17GB total) +pip install -U "huggingface_hub[cli]" +hf download tencent/AuK --local-dir ./ckpts/AuK +hf download Qwen/Qwen2.5-Omni-3B --local-dir ./ckpts/Qwen2.5-Omni-3B +``` + +### Running + +```bash +python3 emotion_edit_demo.py +``` + +Output goes to `narrator_excited.wav` (24kHz mono, ~11.7s — exact source duration preserved). + +## Gotchas found running this on an RTX 3090 + +- **`qwen_omni_utils` needs `audioread`, but nothing declares it as a dependency** — same class + of packaging gap as demucs's missing `numpy` earlier in this repo. Install it explicitly or + the first real inference call fails with `ModuleNotFoundError: No module named 'audioread'` + (model loading itself succeeds first, so this only surfaces once generation actually starts). +- **`pip install "huggingface_hub[cli]"` breaks `transformers`** — it pulls `huggingface_hub` + 2.x-style 1.31.0, but `transformers` (installed as an AuK dependency) requires `<1.0`. Only + needed transiently to fetch the weights; downgrade back to `<1.0,>=0.34.0` before running + inference, or install the CLI in a separate throwaway environment. +- **`--cpu_offload` is not optional on a 24GB card** — AuK's own documented VRAM figures + (24.78–25.00GiB *without* offload) exceed the RTX 3090's 24GB entirely, even dedicating the + whole card. Always pass `--cpu_offload` (or `cpu_offload=True` in the Python API). +- **Real practical length limit is much shorter than it looks: ~25–30s, not 5 minutes.** AuK's + ComfyUI docs mention a "30-second source/reference-plus-target budget" as if it were a + ComfyUI-specific constraint — it isn't. A 25s clip worked fine even with `--cpu_offload`; an + 80s clip OOM'd (tried to allocate another 8.9GB mid-attention on top of the ~17GB already in + use) because attention memory scales fast with sequence length. Editing a full multi-minute + podcast take means chunking it into ≤~25s segments and concatenating, not a single pass — + worth planning for prosody continuity across chunk boundaries. +- **Default checkpoint path** baked into `auk-infer`/`AukInfer` points at wherever you cloned + the vendor repo, not this directory — always pass `--ckpt`/`--qwen_path` (CLI) or + `config_path`/`ckpt_path`/`qwen_path` (Python API) explicitly. +- **Output is 24kHz mono**, notably lower fidelity than MOSS-TTS's 48kHz stereo output — worth + factoring in if AuK-edited audio needs to sit alongside MOSS-TTS output in the same pipeline. diff --git a/auk/emotion_edit_demo.py b/auk/emotion_edit_demo.py new file mode 100644 index 0000000..14c3eeb --- /dev/null +++ b/auk/emotion_edit_demo.py @@ -0,0 +1,47 @@ +""" +Voice-editing demo using Tencent's AuK (https://github.com/Tencent-Hunyuan/AuK). + +Unlike MOSS-TTS's `instruction` field (steers style at generation time), +AuK edits emotion on an *existing* recording directly - same speaker, same +words, same duration, different delivery. Demonstrates on the public-domain +narrator clip already used by the whisperx/tts demos. + +Run from the auk/ directory with its .venv activated. +""" + +from pathlib import Path + +from auk.infer.infer_auk import AukInfer, save_audio + +HERE = Path(__file__).parent +CONFIG = HERE / "ckpts" / "AuK" / "config.yaml" +CKPT = HERE / "ckpts" / "AuK" / "auk_base.safetensors" +QWEN_PATH = HERE / "ckpts" / "Qwen2.5-Omni-3B" +INPUT_AUDIO = HERE.parent / "tts" / "reference" / "narrator_ref_en.wav" +OUTPUT_PATH = HERE / "narrator_excited.wav" + +INSTRUCTION = "Change the emotion to excited." + + +def main() -> None: + engine = AukInfer( + str(CONFIG), str(CKPT), qwen_path=str(QWEN_PATH), cpu_offload=True + ) + + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": INSTRUCTION}, + {"type": "audio", "audio": str(INPUT_AUDIO)}, + ], + } + ] + + audio, sr = engine.generate(messages) + save_audio(audio, sr, str(OUTPUT_PATH)) + print(f"[INFO] Saved {OUTPUT_PATH}") + + +if __name__ == "__main__": + main() diff --git a/auk/narrator_excited.wav b/auk/narrator_excited.wav new file mode 100644 index 0000000..8d15c43 Binary files /dev/null and b/auk/narrator_excited.wav differ