Add MOSS-SoundEffect generation (CLI + webui section)
New tts/generate_sfx.py mirrors generate.py (MOSS-TTS) but for the MOSS-SoundEffect checkpoint: text-only ambience/SFX generation, no reference audio or voice cloning. Needed PYTORCH_ALLOC_CONF= expandable_segments:True baked in - this checkpoints weights alone use ~22.3GB of this boxs ~23.56GB usable VRAM, leaving generate() itself no room without it (confirmed via OOM otherwise). webui/app.py gets a new standalone "5. Sound effects" section in the Pipeline Studio tab (separate from the AuK/MOSS-TTS voice flow - this is a different generation primitive with no reference/history state). Verified end-to-end: three real test clips (door creak, market ambience, car pass) generated cleanly, approved by Steve as "definitely passable," then wired into the GUI and confirmed working through app.py itself. Co-Authored-By: Claude Sonnet 5 <[email protected]> Claude-Session: https://claude.ai/code/session_01SN15Vxn4svKuhYxFsVtMJf
This commit is contained in:
1 parent
f038ba609d
commit
f4e5b9e20a
2 files changed
+143
No files matched your search
@@ -49,6 +49,7 @@ AUK_FLASH_CKPT = AUK_DIR / "ckpts" / "AuK-Flash" / "auk_flash.safetensors"
|
||||
AUK_QWEN_PATH = AUK_DIR / "ckpts" / "Qwen2.5-Omni-3B"
|
||||
TTS_PYTHON = TTS_DIR / ".venv" / "bin" / "python3"
|
||||
TTS_GENERATE = TTS_DIR / "generate.py"
|
||||
TTS_GENERATE_SFX = TTS_DIR / "generate_sfx.py"
|
||||
|
||||
MODEL_BASE = "Base (full quality)"
|
||||
MODEL_FLASH = "Flash (fast draft, 4-step)"
|
||||
@@ -418,6 +419,30 @@ def generate_speech(reference: str, text: str, language: str):
|
||||
return str(out_path), transcript
|
||||
|
||||
|
||||
def generate_sound_effect(description: str):
|
||||
"""MOSS-SoundEffect - text-only ambience/SFX generation, no reference
|
||||
audio or character voice involved (different generation primitive from
|
||||
the AuK/MOSS-TTS voice-cloning flow above, so it's a standalone section
|
||||
rather than wired into that pipeline's history/reference state).
|
||||
No whisper verification here - there's no speech content to check."""
|
||||
if not description or not description.strip():
|
||||
raise gr.Error("Enter a description of the sound effect/ambience.")
|
||||
|
||||
_free_comfyui_memory()
|
||||
out_path = WORKDIR / f"sfx_{int(time.time())}.wav"
|
||||
env = None
|
||||
if TTS_LD_LIBRARY_PATH:
|
||||
env = os.environ.copy()
|
||||
env["LD_LIBRARY_PATH"] = TTS_LD_LIBRARY_PATH
|
||||
|
||||
_run(
|
||||
[str(TTS_PYTHON), str(TTS_GENERATE_SFX), "--text", description.strip(), "--output", str(out_path)],
|
||||
cwd=TTS_DIR,
|
||||
env=env,
|
||||
)
|
||||
return str(out_path)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Characters (the-service-station cast)
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -781,6 +806,24 @@ with gr.Blocks(title="Audio-demo pipeline studio") as demo:
|
||||
save_char_btn = gr.Button("Save as new character (status: draft)")
|
||||
save_char_status = gr.Markdown()
|
||||
|
||||
with gr.Row():
|
||||
with gr.Column():
|
||||
gr.Markdown(
|
||||
"## 5. Sound effects (MOSS-SoundEffect)\n"
|
||||
"Text-only ambience/SFX generation - no reference audio or character "
|
||||
"voice involved, a separate model from the voice pipeline above."
|
||||
)
|
||||
sfx_description = gr.Textbox(
|
||||
label="Description",
|
||||
placeholder=(
|
||||
'e.g. "A door creaking open slowly, then closing with a soft thud." '
|
||||
'or "A busy outdoor market with distant chatter and footsteps."'
|
||||
),
|
||||
lines=2,
|
||||
)
|
||||
sfx_btn = gr.Button("Generate sound effect", variant="primary")
|
||||
sfx_audio = gr.Audio(label="Generated sound effect", type="filepath")
|
||||
|
||||
strip_btn.click(strip_vocals, inputs=[input_audio], outputs=[vocals_out, no_vocals_out])
|
||||
use_vocals_btn.click(
|
||||
use_as_reference, inputs=[vocals_out, history_state],
|
||||
@@ -819,6 +862,7 @@ with gr.Blocks(title="Audio-demo pipeline studio") as demo:
|
||||
inputs=[new_char_name, new_char_role, new_char_prose, gen_language, current_ref_audio, history_state],
|
||||
outputs=[save_char_status],
|
||||
)
|
||||
sfx_btn.click(generate_sound_effect, inputs=[sfx_description], outputs=[sfx_audio])
|
||||
|
||||
with gr.Tab("Characters"):
|
||||
gr.Markdown(
|
||||
|
||||
Reference in new issue
Block a user