From b4fef19993cd5ecb57ce656ae3c6221dcfc24ab1 Mon Sep 17 00:00:00 2001 From: Steve Calcott Date: Sun, 11 Oct 2026 18:09:59 +0000 Subject: [PATCH] Add Stable Audio 3 Small-SFX to sound effects tab, with model toggle and save-to-library Runs entirely on CPU (no VRAM contention with ComfyUI), and sounds noticeably better than MOSS-SoundEffect per listening test, so it is now the default model. Toggle keeps MOSS-SoundEffect available as a fallback. Saved clips land in the-service-station/sfx_library/ on disk, no db/ metadata system per explicit scope. Co-Authored-By: Claude Sonnet 5 --- sfx-stable-audio-3/generate.py | 32 +++++++++++ webui/app.py | 98 ++++++++++++++++++++++++++++------ 2 files changed, 113 insertions(+), 17 deletions(-) create mode 100644 sfx-stable-audio-3/generate.py diff --git a/sfx-stable-audio-3/generate.py b/sfx-stable-audio-3/generate.py new file mode 100644 index 0000000..c133d1f --- /dev/null +++ b/sfx-stable-audio-3/generate.py @@ -0,0 +1,32 @@ +""" +Stable Audio 3 Small-SFX CLI: generate a sound effect/ambience clip from a +text description. Runs on CPU by design - this checkpoint is the +"lightweight, no GPU required" model in Stable Audio 3's lineup, so it +never contends with ComfyUI/other models for VRAM (unlike generate_sfx.py's +MOSS-SoundEffect, which needs the 3090 and ComfyUI's memory freed first). +""" + +import argparse + +import torchaudio +from stable_audio_3 import StableAudioModel + + +def main() -> None: + parser = argparse.ArgumentParser(description="Stable Audio 3 Small-SFX text-to-sound-effect") + parser.add_argument("--text", required=True, help="Description of the sound effect/ambience") + parser.add_argument("--output", required=True, help="Output wav path") + parser.add_argument("--duration", type=float, default=10.0, help="Clip duration in seconds (max 120)") + parser.add_argument("--seed", type=int, default=-1, help="Random seed, or -1 for random") + args = parser.parse_args() + + model = StableAudioModel.from_pretrained("small-sfx", device="cpu") + sample_rate = model.model.sample_rate + + audio = model.generate(prompt=args.text, duration=args.duration, seed=args.seed) + torchaudio.save(args.output, audio[0].cpu(), sample_rate) + print(f"[INFO] Saved {args.output}") + + +if __name__ == "__main__": + main() diff --git a/webui/app.py b/webui/app.py index 827218a..1f4b43e 100644 --- a/webui/app.py +++ b/webui/app.py @@ -28,6 +28,7 @@ import glob import json import os import re +import shutil import subprocess import tempfile import time @@ -51,6 +52,16 @@ TTS_PYTHON = TTS_DIR / ".venv" / "bin" / "python3" TTS_GENERATE = TTS_DIR / "generate.py" TTS_GENERATE_SFX = TTS_DIR / "generate_sfx.py" +# Stable Audio 3 Small-SFX - a separate isolated venv/repo, not the tts/ one. +# Runs on CPU by design (the model's own "lightweight, no GPU" tier), so it +# never needs ComfyUI's VRAM freed first, unlike MOSS-SoundEffect below. +SFX_SA3_DIR = REPO_ROOT / "sfx-stable-audio-3" +SFX_SA3_PYTHON = SFX_SA3_DIR / ".venv" / "bin" / "python3" +SFX_SA3_GENERATE = SFX_SA3_DIR / "generate.py" + +SFX_MODEL_SA3 = "Stable Audio 3 (Small-SFX)" +SFX_MODEL_MOSS = "MOSS-SoundEffect" + MODEL_BASE = "Base (full quality)" MODEL_FLASH = "Flash (fast draft, 4-step)" @@ -59,6 +70,10 @@ MODEL_FLASH = "Flash (fast draft, 4-step)" SERVICE_STATION_DIR = Path(os.environ.get("SERVICE_STATION_REPO", "/root/projects/the-service-station")) SS_VOICES_DIR = SERVICE_STATION_DIR / "voices" +# Saved sound effects just live on disk here - no metadata/db, per Steve's +# "just on disk FOR NOW is fine". +SFX_LIBRARY_DIR = SERVICE_STATION_DIR / "sfx_library" + WORKDIR = Path(tempfile.mkdtemp(prefix="audio-demo-webui-")) # AuK's Emotion task has exactly this vocabulary (vendor/AuK/docs/COOKBOOK.md @@ -419,30 +434,58 @@ def generate_speech(reference: str, text: str, language: str): return str(out_path), transcript -def generate_sound_effect(description: str): - """MOSS-SoundEffect - text-only ambience/SFX generation, no reference - audio or character voice involved (different generation primitive from - the AuK/MOSS-TTS voice-cloning flow above, so it's a standalone section - rather than wired into that pipeline's history/reference state). +def generate_sound_effect(description: str, model_choice: str, duration: float): + """Text-only ambience/SFX generation, no reference audio or character + voice involved (different generation primitive from the AuK/MOSS-TTS + voice-cloning flow above, so it's a standalone section rather than + wired into that pipeline's history/reference state). + Two model backends: Stable Audio 3 Small-SFX (own venv, CPU-only, no + ComfyUI VRAM contention, `duration` matters) and MOSS-SoundEffect (tts/ + venv, needs the GPU + ComfyUI's memory freed first, length is implicit + in its own generation rather than a parameter). No whisper verification here - there's no speech content to check.""" if not description or not description.strip(): raise gr.Error("Enter a description of the sound effect/ambience.") - _free_comfyui_memory() out_path = WORKDIR / f"sfx_{int(time.time())}.wav" - env = None - if TTS_LD_LIBRARY_PATH: - env = os.environ.copy() - env["LD_LIBRARY_PATH"] = TTS_LD_LIBRARY_PATH - _run( - [str(TTS_PYTHON), str(TTS_GENERATE_SFX), "--text", description.strip(), "--output", str(out_path)], - cwd=TTS_DIR, - env=env, - ) + if model_choice == SFX_MODEL_SA3: + _run( + [ + str(SFX_SA3_PYTHON), str(SFX_SA3_GENERATE), + "--text", description.strip(), + "--output", str(out_path), + "--duration", str(duration), + ], + cwd=SFX_SA3_DIR, + ) + else: + _free_comfyui_memory() + env = None + if TTS_LD_LIBRARY_PATH: + env = os.environ.copy() + env["LD_LIBRARY_PATH"] = TTS_LD_LIBRARY_PATH + _run( + [str(TTS_PYTHON), str(TTS_GENERATE_SFX), "--text", description.strip(), "--output", str(out_path)], + cwd=TTS_DIR, + env=env, + ) return str(out_path) +def save_sound_effect_to_library(audio_path: str, description: str) -> str: + """Copy the current clip into SFX_LIBRARY_DIR so it survives past + WORKDIR (a tempdir, wiped on restart). On-disk only, no metadata/db - + per Steve's "just on disk FOR NOW is fine".""" + if not audio_path: + raise gr.Error("Generate a sound effect first.") + SFX_LIBRARY_DIR.mkdir(parents=True, exist_ok=True) + slug = _slugify(description or "") or "sfx" + dest = SFX_LIBRARY_DIR / f"{slug}_{int(time.time())}.wav" + shutil.copy2(audio_path, dest) + return f"Saved to {dest}" + + # --------------------------------------------------------------------------- # Characters (the-service-station cast) # --------------------------------------------------------------------------- @@ -846,11 +889,16 @@ with gr.Blocks(title="Audio-demo pipeline studio") as demo: ) with gr.Tab("Sound Effects"): gr.Markdown( - "# Sound effects (MOSS-SoundEffect)\n" + "# Sound effects\n" "Text-only ambience/SFX generation - a separate model from the " "voice pipeline in Pipeline Studio, no reference audio or " "character voice involved." ) + sfx_model = gr.Radio( + choices=[SFX_MODEL_SA3, SFX_MODEL_MOSS], + value=SFX_MODEL_SA3, + label="Model", + ) sfx_description = gr.Textbox( label="Description", placeholder=( @@ -859,9 +907,25 @@ with gr.Blocks(title="Audio-demo pipeline studio") as demo: ), lines=2, ) + sfx_duration = gr.Slider( + minimum=1, maximum=60, value=10, step=1, + label="Duration (seconds) - Stable Audio 3 only", + ) sfx_btn = gr.Button("Generate sound effect", variant="primary") sfx_audio = gr.Audio(label="Generated sound effect", type="filepath") - sfx_btn.click(generate_sound_effect, inputs=[sfx_description], outputs=[sfx_audio]) + with gr.Row(): + sfx_save_btn = gr.Button("Save to library") + sfx_save_status = gr.Markdown() + sfx_btn.click( + generate_sound_effect, + inputs=[sfx_description, sfx_model, sfx_duration], + outputs=[sfx_audio], + ) + sfx_save_btn.click( + save_sound_effect_to_library, + inputs=[sfx_audio, sfx_description], + outputs=[sfx_save_status], + ) with gr.Tab("Characters"): gr.Markdown(