Add Stable Audio 3 Small-SFX to sound effects tab, with model toggle and save-to-library
Runs entirely on CPU (no VRAM contention with ComfyUI), and sounds noticeably better than MOSS-SoundEffect per listening test, so it is now the default model. Toggle keeps MOSS-SoundEffect available as a fallback. Saved clips land in the-service-station/sfx_library/ on disk, no db/ metadata system per explicit scope. Co-Authored-By: Claude Sonnet 5 <[email protected]>
This commit is contained in:
steve-adminandClaude Sonnet 5 committed
1 parent
042c438b8d
commit
b4fef19993
2 files changed
+113
-17
No files matched your search
+81
-17
@@ -28,6 +28,7 @@ import glob
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import tempfile
|
||||
import time
|
||||
@@ -51,6 +52,16 @@ TTS_PYTHON = TTS_DIR / ".venv" / "bin" / "python3"
|
||||
TTS_GENERATE = TTS_DIR / "generate.py"
|
||||
TTS_GENERATE_SFX = TTS_DIR / "generate_sfx.py"
|
||||
|
||||
# Stable Audio 3 Small-SFX - a separate isolated venv/repo, not the tts/ one.
|
||||
# Runs on CPU by design (the model's own "lightweight, no GPU" tier), so it
|
||||
# never needs ComfyUI's VRAM freed first, unlike MOSS-SoundEffect below.
|
||||
SFX_SA3_DIR = REPO_ROOT / "sfx-stable-audio-3"
|
||||
SFX_SA3_PYTHON = SFX_SA3_DIR / ".venv" / "bin" / "python3"
|
||||
SFX_SA3_GENERATE = SFX_SA3_DIR / "generate.py"
|
||||
|
||||
SFX_MODEL_SA3 = "Stable Audio 3 (Small-SFX)"
|
||||
SFX_MODEL_MOSS = "MOSS-SoundEffect"
|
||||
|
||||
MODEL_BASE = "Base (full quality)"
|
||||
MODEL_FLASH = "Flash (fast draft, 4-step)"
|
||||
|
||||
@@ -59,6 +70,10 @@ MODEL_FLASH = "Flash (fast draft, 4-step)"
|
||||
SERVICE_STATION_DIR = Path(os.environ.get("SERVICE_STATION_REPO", "/root/projects/the-service-station"))
|
||||
SS_VOICES_DIR = SERVICE_STATION_DIR / "voices"
|
||||
|
||||
# Saved sound effects just live on disk here - no metadata/db, per Steve's
|
||||
# "just on disk FOR NOW is fine".
|
||||
SFX_LIBRARY_DIR = SERVICE_STATION_DIR / "sfx_library"
|
||||
|
||||
WORKDIR = Path(tempfile.mkdtemp(prefix="audio-demo-webui-"))
|
||||
|
||||
# AuK's Emotion task has exactly this vocabulary (vendor/AuK/docs/COOKBOOK.md
|
||||
@@ -419,30 +434,58 @@ def generate_speech(reference: str, text: str, language: str):
|
||||
return str(out_path), transcript
|
||||
|
||||
|
||||
def generate_sound_effect(description: str):
|
||||
"""MOSS-SoundEffect - text-only ambience/SFX generation, no reference
|
||||
audio or character voice involved (different generation primitive from
|
||||
the AuK/MOSS-TTS voice-cloning flow above, so it's a standalone section
|
||||
rather than wired into that pipeline's history/reference state).
|
||||
def generate_sound_effect(description: str, model_choice: str, duration: float):
|
||||
"""Text-only ambience/SFX generation, no reference audio or character
|
||||
voice involved (different generation primitive from the AuK/MOSS-TTS
|
||||
voice-cloning flow above, so it's a standalone section rather than
|
||||
wired into that pipeline's history/reference state).
|
||||
Two model backends: Stable Audio 3 Small-SFX (own venv, CPU-only, no
|
||||
ComfyUI VRAM contention, `duration` matters) and MOSS-SoundEffect (tts/
|
||||
venv, needs the GPU + ComfyUI's memory freed first, length is implicit
|
||||
in its own generation rather than a parameter).
|
||||
No whisper verification here - there's no speech content to check."""
|
||||
if not description or not description.strip():
|
||||
raise gr.Error("Enter a description of the sound effect/ambience.")
|
||||
|
||||
_free_comfyui_memory()
|
||||
out_path = WORKDIR / f"sfx_{int(time.time())}.wav"
|
||||
env = None
|
||||
if TTS_LD_LIBRARY_PATH:
|
||||
env = os.environ.copy()
|
||||
env["LD_LIBRARY_PATH"] = TTS_LD_LIBRARY_PATH
|
||||
|
||||
_run(
|
||||
[str(TTS_PYTHON), str(TTS_GENERATE_SFX), "--text", description.strip(), "--output", str(out_path)],
|
||||
cwd=TTS_DIR,
|
||||
env=env,
|
||||
)
|
||||
if model_choice == SFX_MODEL_SA3:
|
||||
_run(
|
||||
[
|
||||
str(SFX_SA3_PYTHON), str(SFX_SA3_GENERATE),
|
||||
"--text", description.strip(),
|
||||
"--output", str(out_path),
|
||||
"--duration", str(duration),
|
||||
],
|
||||
cwd=SFX_SA3_DIR,
|
||||
)
|
||||
else:
|
||||
_free_comfyui_memory()
|
||||
env = None
|
||||
if TTS_LD_LIBRARY_PATH:
|
||||
env = os.environ.copy()
|
||||
env["LD_LIBRARY_PATH"] = TTS_LD_LIBRARY_PATH
|
||||
_run(
|
||||
[str(TTS_PYTHON), str(TTS_GENERATE_SFX), "--text", description.strip(), "--output", str(out_path)],
|
||||
cwd=TTS_DIR,
|
||||
env=env,
|
||||
)
|
||||
return str(out_path)
|
||||
|
||||
|
||||
def save_sound_effect_to_library(audio_path: str, description: str) -> str:
|
||||
"""Copy the current clip into SFX_LIBRARY_DIR so it survives past
|
||||
WORKDIR (a tempdir, wiped on restart). On-disk only, no metadata/db -
|
||||
per Steve's "just on disk FOR NOW is fine"."""
|
||||
if not audio_path:
|
||||
raise gr.Error("Generate a sound effect first.")
|
||||
SFX_LIBRARY_DIR.mkdir(parents=True, exist_ok=True)
|
||||
slug = _slugify(description or "") or "sfx"
|
||||
dest = SFX_LIBRARY_DIR / f"{slug}_{int(time.time())}.wav"
|
||||
shutil.copy2(audio_path, dest)
|
||||
return f"Saved to {dest}"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Characters (the-service-station cast)
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -846,11 +889,16 @@ with gr.Blocks(title="Audio-demo pipeline studio") as demo:
|
||||
)
|
||||
with gr.Tab("Sound Effects"):
|
||||
gr.Markdown(
|
||||
"# Sound effects (MOSS-SoundEffect)\n"
|
||||
"# Sound effects\n"
|
||||
"Text-only ambience/SFX generation - a separate model from the "
|
||||
"voice pipeline in Pipeline Studio, no reference audio or "
|
||||
"character voice involved."
|
||||
)
|
||||
sfx_model = gr.Radio(
|
||||
choices=[SFX_MODEL_SA3, SFX_MODEL_MOSS],
|
||||
value=SFX_MODEL_SA3,
|
||||
label="Model",
|
||||
)
|
||||
sfx_description = gr.Textbox(
|
||||
label="Description",
|
||||
placeholder=(
|
||||
@@ -859,9 +907,25 @@ with gr.Blocks(title="Audio-demo pipeline studio") as demo:
|
||||
),
|
||||
lines=2,
|
||||
)
|
||||
sfx_duration = gr.Slider(
|
||||
minimum=1, maximum=60, value=10, step=1,
|
||||
label="Duration (seconds) - Stable Audio 3 only",
|
||||
)
|
||||
sfx_btn = gr.Button("Generate sound effect", variant="primary")
|
||||
sfx_audio = gr.Audio(label="Generated sound effect", type="filepath")
|
||||
sfx_btn.click(generate_sound_effect, inputs=[sfx_description], outputs=[sfx_audio])
|
||||
with gr.Row():
|
||||
sfx_save_btn = gr.Button("Save to library")
|
||||
sfx_save_status = gr.Markdown()
|
||||
sfx_btn.click(
|
||||
generate_sound_effect,
|
||||
inputs=[sfx_description, sfx_model, sfx_duration],
|
||||
outputs=[sfx_audio],
|
||||
)
|
||||
sfx_save_btn.click(
|
||||
save_sound_effect_to_library,
|
||||
inputs=[sfx_audio, sfx_description],
|
||||
outputs=[sfx_save_status],
|
||||
)
|
||||
|
||||
with gr.Tab("Characters"):
|
||||
gr.Markdown(
|
||||
|
||||
Reference in new issue
Block a user