From f780fbda0164e3506150a05c00cec4e6a8ae443e Mon Sep 17 00:00:00 2001 From: Steve Calcott Date: Tue, 15 Sep 2026 23:30:34 +0000 Subject: [PATCH] Refresh webui: fix Speed no-op, Whisper verification, Characters tab Speed editing was a silent no-op - AuK needs --gen_seconds or it just preserves the input duration (vendor/AuK/docs/COOKBOOK.md 3.2). Fixed by computing the target from the current reference duration / speed factor, with a UI warning that longer references risk a memory-swap stall on this box (reproduced building a character voice: a 27s target hung 20+ minutes swapping with 0%% GPU use, a 12s source completed in seconds). Added Whisper content verification (faster-whisper, small/CPU/int8) after every AuK edit and MOSS generation. Motivated by a real incident building the-service-station cast: a chained AuK edit leaked fragments of its own instruction text into a character reference, undetected through a full listen-and-approve round because nothing had transcribed the reference itself, only generated lines. History is now per-instruction rather than per-click, so Undo is granular and a save action can derive a sensibly-scoped generation_recipe. New Characters tab: browses the-service-station repo's CHARACTER.md cards (name/role/status/reference audio/emotion_library), lets you create a new character (writes a fresh CHARACTER.md, always status: draft) or generate an emotion candidate for an existing one (writes to voices//emotion-cache/.wav, same path the automated pipeline's own fallback uses). Approving anything stays a manual step - this tab creates candidates for a human to listen to and promote, it never self-approves. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01Y1oQ4h4WE4R9KRvMQLr5Ey --- webui/app.py | 536 +++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 438 insertions(+), 98 deletions(-) diff --git a/webui/app.py b/webui/app.py index c95345e..5634ee3 100644 --- a/webui/app.py +++ b/webui/app.py @@ -1,24 +1,41 @@ """ Pipeline studio: Demucs (strip) -> AuK (edit voice, with preview/undo) -> -MOSS-TTS (generate speech in the edited voice). +MOSS-TTS (generate speech in the edited voice). Plus a Characters tab for +browsing the-service-station's cast and creating new characters/emotion +candidates without SSH + manual YAML editing. Orchestrates across the three existing isolated venvs (demucs/, auk/, tts/) via subprocess, rather than merging their conflicting dependencies (MOSS-TTS needs transformers 5.0.0, AuK needs 4.57.6) into one environment. Run with: webui/.venv/bin/python webui/app.py + +2026-09-15 refresh, after building the-service-station's cast surfaced real +gaps in this UI: +- Speed editing was a silent no-op (AuK needs --gen_seconds or it just + preserves the input's duration - see vendor/AuK/docs/COOKBOOK.md 3.2). +- AuK's Emotion task has a fixed 8-word vocabulary; anything else needs the + Timbre task's free-form template instead. +- A chained AuK edit can leak fragments of its own instruction text into + the spoken content. This went undetected through a full listen-and- + approve cycle on a real character voice because nothing had ever + transcribed the *reference* audio, only generated lines. Every AuK edit + and MOSS generation here now gets transcribed automatically so that's + visible immediately instead of discovered later by ear. """ +import glob import json import os +import re import subprocess import tempfile import time -import urllib.error -import urllib.request +from datetime import date from pathlib import Path import gradio as gr +import yaml REPO_ROOT = Path(__file__).resolve().parent.parent DEMUCS_DIR = REPO_ROOT / "demucs" @@ -32,8 +49,17 @@ AUK_QWEN_PATH = AUK_DIR / "ckpts" / "Qwen2.5-Omni-3B" TTS_PYTHON = TTS_DIR / ".venv" / "bin" / "python3" TTS_GENERATE = TTS_DIR / "generate.py" +# The-service-station checkout on this same box - where character cards +# live. Override with the env var if running this UI somewhere else. +SERVICE_STATION_DIR = Path(os.environ.get("SERVICE_STATION_REPO", "/root/projects/the-service-station")) +SS_VOICES_DIR = SERVICE_STATION_DIR / "voices" + WORKDIR = Path(tempfile.mkdtemp(prefix="audio-demo-webui-")) +# AuK's Emotion task has exactly this vocabulary (vendor/AuK/docs/COOKBOOK.md +# 4.1) - anything else needs the Timbre task's free-form template instead +# (see auk_emotion_instruction below). Kept as the dropdown's options since +# these are the only words guaranteed to hit the dedicated task. EMOTIONS = ["happy", "angry", "sad", "fearful", "surprised", "disgusted", "calm", "excited"] PITCH_OPTIONS = ["-3", "-2", "-1", "+1", "+2", "+3"] SPEED_OPTIONS = ["0.5", "0.75", "1.25", "1.5", "2.0"] @@ -45,16 +71,32 @@ LANGUAGES = [ COMFYUI_BASE = "http://localhost:8188" +def auk_emotion_instruction(word: str) -> str: + """Same logic as the-service-station/workflows/generate_episode_lines.py's + auk_emotion_instruction() - duplicated rather than shared because this + repo and the-service-station use separate venvs/dependency sets, but + kept in sync deliberately. AuK's Emotion task only understands the 8 + words in EMOTIONS; anything else goes through Timbre's free-form + description instead, which has no fixed vocabulary.""" + if word in EMOTIONS: + return f"Change the emotion to {word}." + return f'Keep the spoken content unchanged and change the timbre to: "the same voice, sounding {word}".' + + def _comfyui_queue_empty() -> bool: try: + import urllib.error + import urllib.request with urllib.request.urlopen(f"{COMFYUI_BASE}/queue", timeout=3) as resp: data = json.loads(resp.read()) return not data.get("queue_running") and not data.get("queue_pending") - except (urllib.error.URLError, OSError): + except Exception: return True # ComfyUI not reachable -> nothing to worry about def _free_comfyui_memory() -> None: + import urllib.error + import urllib.request if not _comfyui_queue_empty(): return try: @@ -95,8 +137,20 @@ def _ensure_wav(path: str) -> str: return str(out_path) +def _duration(path: str) -> float: + r = subprocess.run( + ["ffprobe", "-v", "error", "-show_entries", "format=duration", + "-of", "default=noprint_wrappers=1:nokey=1", path], + capture_output=True, text=True, + ) + return float(r.stdout.strip()) + + def _ld_library_path_for_venv(venv_dir: Path) -> str: - site_packages = next(venv_dir.glob(".venv/lib/python*/site-packages")) + matches = list(venv_dir.glob(".venv/lib/python*/site-packages")) + if not matches: + return "" + site_packages = matches[0] nvidia_dir = site_packages / "nvidia" if not nvidia_dir.is_dir(): return "" @@ -107,6 +161,28 @@ def _ld_library_path_for_venv(venv_dir: Path) -> str: TTS_LD_LIBRARY_PATH = _ld_library_path_for_venv(TTS_DIR) +# --------------------------------------------------------------------------- +# Whisper content verification - added 2026-09-15 after a chained AuK edit +# leaked fragments of its own instruction text into a character's spoken +# reference, undetected through a full listen-and-approve round because +# nobody had transcribed the reference itself. Loaded once, lazily, since +# most sessions of this UI won't need it immediately. +# --------------------------------------------------------------------------- +_whisper_model = None + + +def _whisper_transcribe(path: str) -> str: + global _whisper_model + try: + if _whisper_model is None: + from faster_whisper import WhisperModel + _whisper_model = WhisperModel("small", device="cpu", compute_type="int8") + segments, _ = _whisper_model.transcribe(path, language="en", beam_size=5) + return " ".join(s.text.strip() for s in segments) + except Exception as e: + return f"(transcription failed: {e})" + + # --------------------------------------------------------------------------- # Stage 1: input / demucs stripping # --------------------------------------------------------------------------- @@ -134,6 +210,12 @@ def strip_vocals(input_audio: str): # --------------------------------------------------------------------------- # Stage 2: AuK voice editing, with history for undo/reset +# +# history_state is a list of {"path": str, "instruction": str|None} dicts, +# one entry per actual AuK call (not per button click - a click can queue +# several edits, e.g. enhance+emotion, and each becomes its own history +# entry so "Save as character" can derive a sensibly-scoped +# generation_recipe and Undo is granular). # --------------------------------------------------------------------------- @@ -141,14 +223,17 @@ def use_as_reference(audio_path: str, history: list): if not audio_path: raise gr.Error("No audio to use yet.") audio_path = _ensure_wav(audio_path) - history = [audio_path] - return audio_path, history, _history_label(history) + history = [{"path": audio_path, "instruction": None}] + transcript = _whisper_transcribe(audio_path) + return audio_path, history, _history_label(history), transcript def _history_label(history: list) -> str: - if not history: - return "No reference set yet." - return f"Edit history: {len(history) - 1} edit(s) applied on top of the original reference." + n = len(history) - 1 + if n <= 0: + return "No edits applied yet." if history else "No reference set yet." + steps = "\n".join(f"{i}. {h['instruction']}" for i, h in enumerate(history[1:], 1)) + return f"{n} edit(s) applied on top of the original reference:\n{steps}" ENHANCE_INSTRUCTIONS = { @@ -168,6 +253,7 @@ def apply_edit( history: list, enhance: str, emotion: str, + other_emotion: str, pitch: str, speed: str, custom_instruction: str, @@ -181,12 +267,18 @@ def apply_edit( if enhance and ENHANCE_INSTRUCTIONS.get(enhance): instructions.append(ENHANCE_INSTRUCTIONS[enhance]) if emotion and emotion != "none": - instructions.append(f"Change the emotion to {emotion}.") + instructions.append(auk_emotion_instruction(emotion)) + if other_emotion and other_emotion.strip(): + # Free-text emotion word, not one of AuK's 8 - always goes through + # the Timbre task's free-form template (see auk_emotion_instruction). + instructions.append(auk_emotion_instruction(other_emotion.strip())) if pitch and pitch != "none": direction = "Raise" if pitch.startswith("+") else "Lower" instructions.append(f"{direction} the pitch by {pitch.lstrip('+-')} semitone(s).") + speed_gen_seconds = None if speed and speed != "none": - instructions.append(f"Adjust the speech speed to {speed}x.") + speed_gen_seconds = _duration(current_ref) / float(speed) + instructions.append((f"Adjust the speech speed to {speed}x.", speed_gen_seconds)) if custom_instruction and custom_instruction.strip(): instructions.append(custom_instruction.strip()) @@ -197,7 +289,11 @@ def apply_edit( ref = current_ref out_dir = WORKDIR / f"auk_{int(time.time())}" out_dir.mkdir(parents=True, exist_ok=True) + last_transcript = "" for i, instruction in enumerate(instructions): + gen_seconds = None + if isinstance(instruction, tuple): + instruction, gen_seconds = instruction out_path = out_dir / f"edit_{i}.wav" cmd = [ str(AUK_INFER), @@ -208,25 +304,30 @@ def apply_edit( "--output", str(out_path), "--cpu_offload", ] + if gen_seconds is not None: + cmd += ["--gen_seconds", str(round(gen_seconds, 2))] _run(cmd, cwd=AUK_DIR) ref = str(out_path) + last_transcript = _whisper_transcribe(ref) + history = history + [{"path": ref, "instruction": instruction}] - history = history + [ref] - return ref, history, _history_label(history) + return ref, history, _history_label(history), last_transcript def undo_edit(history: list): if len(history) <= 1: raise gr.Error("Nothing to undo.") history = history[:-1] - return history[-1], history, _history_label(history) + transcript = _whisper_transcribe(history[-1]["path"]) + return history[-1]["path"], history, _history_label(history), transcript def reset_to_original(history: list): if not history: raise gr.Error("No reference set yet.") history = [history[0]] - return history[0], history, _history_label(history) + transcript = _whisper_transcribe(history[0]["path"]) + return history[0]["path"], history, _history_label(history), transcript # --------------------------------------------------------------------------- @@ -258,7 +359,182 @@ def generate_speech(reference: str, text: str, language: str): cwd=TTS_DIR, env=env, ) - return str(out_path) + transcript = _whisper_transcribe(str(out_path)) + return str(out_path), transcript + + +# --------------------------------------------------------------------------- +# Characters (the-service-station cast) +# --------------------------------------------------------------------------- + + +def _slugify(text: str) -> str: + return re.sub(r"[^a-z0-9]+", "-", text.lower()).strip("-") + + +def read_character_cards() -> dict: + """name -> parsed card dict (frontmatter + prose), read-only. Never + round-trips these through yaml.dump - only used for display and for + resolving a base reference_audio, never for rewriting an existing file + (that would silently strip hand-authored inline comments).""" + cards = {} + if not SS_VOICES_DIR.is_dir(): + return cards + for card_path in sorted(SS_VOICES_DIR.glob("*/CHARACTER.md")): + try: + text = card_path.read_text(encoding="utf-8") + parts = text.split("---", 2) + if len(parts) < 3: + continue + frontmatter = yaml.safe_load(parts[1]) or {} + frontmatter["_prose"] = parts[2].strip() + frontmatter["_dir"] = card_path.parent + cards[frontmatter.get("name", card_path.parent.name)] = frontmatter + except Exception as e: + cards[card_path.parent.name] = {"_error": str(e), "_dir": card_path.parent} + return cards + + +def character_names() -> list[str]: + return sorted(read_character_cards().keys()) + + +def load_character(name: str): + if not name: + return "Pick a character.", None + cards = read_character_cards() + card = cards.get(name) + if not card: + return f"No card found for '{name}'.", None + if "_error" in card: + return f"Could not parse {name}'s card: {card['_error']}", None + + ref_audio = card.get("reference_audio") + ref_path = str(SERVICE_STATION_DIR / ref_audio) if ref_audio else None + + lib = card.get("emotion_library") or {} + lib_lines = [f"- **{emo}**: {entry.get('status', '?')}" for emo, entry in lib.items()] or ["(none approved yet)"] + + cache_dir = card["_dir"] / "emotion-cache" + cached = sorted(p.stem for p in cache_dir.glob("*.wav")) if cache_dir.is_dir() else [] + cache_lines = [f"- {e} (cached, not yet approved)" for e in cached] or ["(none pending)"] + + info = ( + f"**{name}** — {card.get('role', '?')}\n\n" + f"Status: `{card.get('status', '?')}` | Engine: `{card.get('engine', '?')}` | " + f"Default emotion (documentation only): `{card.get('default_emotion', '?')}`\n\n" + f"### Approved emotion_library\n" + "\n".join(lib_lines) + "\n\n" + f"### Pending (emotion-cache, not approved)\n" + "\n".join(cache_lines) + "\n\n" + f"---\n{card.get('_prose', '')}" + ) + return info, ref_path + + +def create_character( + name: str, role: str, prose: str, language: str, + current_ref: str, history: list, +): + if not name or not name.strip(): + raise gr.Error("Give the character a name.") + if not current_ref or not history: + raise gr.Error("Set up a reference and edit it above first - there's nothing to save yet.") + + slug = _slugify(name) + char_dir = SS_VOICES_DIR / slug / "webui-created" + if (SS_VOICES_DIR / slug / "CHARACTER.md").exists(): + raise gr.Error(f"voices/{slug}/CHARACTER.md already exists - this UI won't overwrite an existing character.") + char_dir.mkdir(parents=True, exist_ok=True) + + source_dest = char_dir / f"{slug}_source.wav" + ref_dest = char_dir / f"{slug}_ref_v1.wav" + subprocess.run(["cp", history[0]["path"], str(source_dest)], check=True) + subprocess.run(["cp", current_ref, str(ref_dest)], check=True) + + recipe = [ + {"tool": "auk", "instruction": h["instruction"]} + for h in history[1:] if h["instruction"] + ] + + frontmatter = { + "name": name.strip(), + "role": role.strip() or "Standing guest", + "engine": "moss-tts", + "model": "OpenMOSS-Team/MOSS-TTS-Local-Transformer-v1.5", + "reference_audio": str(ref_dest.relative_to(SERVICE_STATION_DIR)), + "source_clip": str(source_dest.relative_to(SERVICE_STATION_DIR)), + "source_clip_note": "Created via the webui pipeline studio - see generation_recipe for the exact edit chain.", + "generation_recipe": recipe, + "seed": None, + "emotion_library": {}, + "language": language, + "status": "draft", + "last_updated": date.today().isoformat(), + "provenance": "webui", + "appearances": [], + } + frontmatter_yaml = yaml.dump(frontmatter, sort_keys=False, allow_unicode=True) + body = f"---\n{frontmatter_yaml}---\n\n# {name.strip()}\n\n{prose.strip()}\n" + (SS_VOICES_DIR / slug / "CHARACTER.md").write_text(body, encoding="utf-8") + + return ( + f"Created voices/{slug}/CHARACTER.md with status: draft. " + f"Not committed - review, listen, and commit it yourself (or ask an agent to) " + f"once you're happy, same as every other voice in this project." + ) + + +def generate_emotion_candidate(character_name: str, emotion_word: str): + if not character_name: + raise gr.Error("Pick a character first.") + if not emotion_word or not emotion_word.strip(): + raise gr.Error("Enter an emotion word.") + emotion_word = emotion_word.strip() + + cards = read_character_cards() + card = cards.get(character_name) + if not card or "_error" in card: + raise gr.Error(f"Could not load {character_name}'s card.") + + base_reference = str(SERVICE_STATION_DIR / card["reference_audio"]) + cache_dir = card["_dir"] / "emotion-cache" + cache_dir.mkdir(parents=True, exist_ok=True) + cache_path = cache_dir / f"{_slugify(emotion_word)}.wav" + + _free_comfyui_memory() + instruction = auk_emotion_instruction(emotion_word) + cmd = [ + str(AUK_INFER), "--ckpt", str(AUK_CKPT), "--qwen_path", str(AUK_QWEN_PATH), + "--audio", base_reference, "--instruction", instruction, + "--output", str(cache_path), "--cpu_offload", + ] + _run(cmd, cwd=AUK_DIR) + + ref_transcript = _whisper_transcribe(str(cache_path)) + + # Preview clone so the candidate can actually be judged, not just the + # edited reference in isolation. + env = None + if TTS_LD_LIBRARY_PATH: + env = os.environ.copy() + env["LD_LIBRARY_PATH"] = TTS_LD_LIBRARY_PATH + preview_text = f"Testing the {emotion_word} emotion for {character_name}." + preview_path = WORKDIR / f"emotion_preview_{int(time.time())}.wav" + _run( + [str(TTS_PYTHON), str(TTS_GENERATE), "--text", preview_text, + "--reference", str(cache_path), "--language", "English", "--output", str(preview_path)], + cwd=TTS_DIR, env=env, + ) + preview_transcript = _whisper_transcribe(str(preview_path)) + + status = ( + f"Saved to `{cache_path.relative_to(SERVICE_STATION_DIR)}` (cached, **not approved**).\n\n" + f"Reference transcript: {ref_transcript}\n\n" + f"Preview clone transcript: {preview_transcript}\n\n" + f"If this sounds right, promote it manually per `.claude/skills/produce-episode/" + f"SKILL.md`'s \"Adding an emotion to a character's library\" - this UI won't do " + f"that step for you." + ) + return str(cache_path), str(preview_path), status # --------------------------------------------------------------------------- @@ -266,92 +542,156 @@ def generate_speech(reference: str, text: str, language: str): # --------------------------------------------------------------------------- with gr.Blocks(title="Audio-demo pipeline studio") as demo: - gr.Markdown( - "# Pipeline studio\n" - "Strip a voice with **Demucs**, edit it with **AuK**, then generate new " - "speech in that voice with **MOSS-TTS**. Each stage shells out to its own " - "isolated venv, so this box's GPU is the real constraint — heavy edits " - "(long clips) can OOM AuK; keep reference clips short (under ~25s)." - ) - - reference_state = gr.State(None) - history_state = gr.State([]) - - with gr.Row(): - with gr.Column(): - gr.Markdown("## 1. Input") - input_audio = gr.Audio(label="Upload or record", type="filepath") - strip_btn = gr.Button("Strip vocals with Demucs") - with gr.Row(): - vocals_out = gr.Audio(label="Vocals (isolated)", type="filepath") - no_vocals_out = gr.Audio(label="Background (no vocals)", type="filepath") - with gr.Row(): - use_vocals_btn = gr.Button("Use vocals as reference") - use_no_vocals_btn = gr.Button("Use background as reference") - use_original_btn = gr.Button("Use uploaded audio as-is") - - with gr.Column(): - gr.Markdown("## 2. Voice editor (AuK)") - current_ref_audio = gr.Audio(label="Current reference", type="filepath", interactive=False) - history_label = gr.Markdown("No reference set yet.") - enhance_dd = gr.Dropdown( - list(ENHANCE_INSTRUCTIONS.keys()), value="none", - label="Enhance (denoise/dereverb/restore) - runs first, before character edits", - ) - emotion_dd = gr.Dropdown(["none"] + EMOTIONS, value="none", label="Emotion") - pitch_dd = gr.Dropdown(["none"] + PITCH_OPTIONS, value="none", label="Pitch (semitones)") - speed_dd = gr.Dropdown(["none"] + SPEED_OPTIONS, value="none", label="Speed") - custom_instruction = gr.Textbox( - label="Custom instruction (advanced, optional)", - placeholder='e.g. "Lower the pitch very slightly."', - ) + with gr.Tabs(): + with gr.Tab("Pipeline Studio"): gr.Markdown( - "*Note: pitch editing has been unreliable in testing — even " - "\"1 semitone\" has overshot to ~4+ semitones on some voices. " - "Preview before committing.*" + "# Pipeline studio\n" + "Strip a voice with **Demucs**, edit it with **AuK**, then generate new " + "speech in that voice with **MOSS-TTS**. Each stage shells out to its own " + "isolated venv, so this box's GPU is the real constraint — heavy edits " + "(long clips) can OOM AuK; keep reference clips short (under ~25s)." ) - apply_btn = gr.Button("Apply edit(s)", variant="primary") + + reference_state = gr.State(None) + history_state = gr.State([]) + with gr.Row(): - undo_btn = gr.Button("Undo last edit") - reset_btn = gr.Button("Reset to original") + with gr.Column(): + gr.Markdown("## 1. Input") + input_audio = gr.Audio(label="Upload or record", type="filepath") + strip_btn = gr.Button("Strip vocals with Demucs") + with gr.Row(): + vocals_out = gr.Audio(label="Vocals (isolated)", type="filepath") + no_vocals_out = gr.Audio(label="Background (no vocals)", type="filepath") + with gr.Row(): + use_vocals_btn = gr.Button("Use vocals as reference") + use_no_vocals_btn = gr.Button("Use background as reference") + use_original_btn = gr.Button("Use uploaded audio as-is") - with gr.Column(): - gr.Markdown("## 3. Generate (MOSS-TTS)") - gen_text = gr.Textbox(label="Text to speak", lines=6) - gen_language = gr.Dropdown(LANGUAGES, value="English", label="Language") - generate_btn = gr.Button("Generate speech", variant="primary") - generated_audio = gr.Audio(label="Generated speech", type="filepath") + with gr.Column(): + gr.Markdown("## 2. Voice editor (AuK)") + current_ref_audio = gr.Audio(label="Current reference", type="filepath", interactive=False) + ref_transcript_box = gr.Textbox(label="Whisper transcript of current reference", interactive=False) + history_label = gr.Markdown("No reference set yet.") + enhance_dd = gr.Dropdown( + list(ENHANCE_INSTRUCTIONS.keys()), value="none", + label="Enhance (denoise/dereverb/restore) - runs first, before character edits", + ) + emotion_dd = gr.Dropdown( + ["none"] + EMOTIONS, value="none", + label="Emotion (AuK's fixed vocabulary - anything else, use the field below)", + ) + other_emotion_tb = gr.Textbox( + label="Other emotion word (not in AuK's list)", + placeholder='e.g. "bemused" - automatically wrapped as a Timbre instruction instead', + ) + pitch_dd = gr.Dropdown(["none"] + PITCH_OPTIONS, value="none", label="Pitch (semitones)") + speed_dd = gr.Dropdown(["none"] + SPEED_OPTIONS, value="none", label="Speed") + custom_instruction = gr.Textbox( + label="Custom instruction (advanced, optional)", + placeholder='e.g. "Lower the pitch very slightly."', + ) + gr.Markdown( + "*Pitch editing has been unreliable in testing — even \"1 semitone\" " + "has overshot to ~4+ semitones on some voices. Prefer a free-form " + "custom instruction like \"a little lower in register\" over the Pitch " + "dropdown for character work.*\n\n" + "*Speed now sets AuK's required `--gen_seconds` automatically. On " + "references longer than ~15s this can stall (a 27s target hung this " + "box for 20+ minutes swapping to disk with 0% GPU use, while a 12s " + "source completed in seconds) - trim your reference shorter first if " + "using Speed.*" + ) + apply_btn = gr.Button("Apply edit(s)", variant="primary") + with gr.Row(): + undo_btn = gr.Button("Undo last edit") + reset_btn = gr.Button("Reset to original") - strip_btn.click(strip_vocals, inputs=[input_audio], outputs=[vocals_out, no_vocals_out]) - use_vocals_btn.click( - use_as_reference, inputs=[vocals_out, history_state], - outputs=[current_ref_audio, history_state, history_label], - ) - use_no_vocals_btn.click( - use_as_reference, inputs=[no_vocals_out, history_state], - outputs=[current_ref_audio, history_state, history_label], - ) - use_original_btn.click( - use_as_reference, inputs=[input_audio, history_state], - outputs=[current_ref_audio, history_state, history_label], - ) - apply_btn.click( - apply_edit, - inputs=[current_ref_audio, history_state, enhance_dd, emotion_dd, pitch_dd, speed_dd, custom_instruction], - outputs=[current_ref_audio, history_state, history_label], - ) - undo_btn.click( - undo_edit, inputs=[history_state], - outputs=[current_ref_audio, history_state, history_label], - ) - reset_btn.click( - reset_to_original, inputs=[history_state], - outputs=[current_ref_audio, history_state, history_label], - ) - generate_btn.click( - generate_speech, inputs=[current_ref_audio, gen_text, gen_language], - outputs=[generated_audio], - ) + with gr.Column(): + gr.Markdown("## 3. Generate (MOSS-TTS)") + gen_text = gr.Textbox(label="Text to speak", lines=6) + gen_language = gr.Dropdown(LANGUAGES, value="English", label="Language") + generate_btn = gr.Button("Generate speech", variant="primary") + generated_audio = gr.Audio(label="Generated speech", type="filepath") + gen_transcript_box = gr.Textbox(label="Whisper transcript", interactive=False) + + gr.Markdown("## 4. Save as a the-service-station character") + new_char_name = gr.Textbox(label="Name") + new_char_role = gr.Textbox(label="Role", placeholder="e.g. Standing guest - weather") + new_char_prose = gr.Textbox(label="Character brief (prose)", lines=4) + save_char_btn = gr.Button("Save as new character (status: draft)") + save_char_status = gr.Markdown() + + strip_btn.click(strip_vocals, inputs=[input_audio], outputs=[vocals_out, no_vocals_out]) + use_vocals_btn.click( + use_as_reference, inputs=[vocals_out, history_state], + outputs=[current_ref_audio, history_state, history_label, ref_transcript_box], + ) + use_no_vocals_btn.click( + use_as_reference, inputs=[no_vocals_out, history_state], + outputs=[current_ref_audio, history_state, history_label, ref_transcript_box], + ) + use_original_btn.click( + use_as_reference, inputs=[input_audio, history_state], + outputs=[current_ref_audio, history_state, history_label, ref_transcript_box], + ) + apply_btn.click( + apply_edit, + inputs=[current_ref_audio, history_state, enhance_dd, emotion_dd, other_emotion_tb, + pitch_dd, speed_dd, custom_instruction], + outputs=[current_ref_audio, history_state, history_label, ref_transcript_box], + ) + undo_btn.click( + undo_edit, inputs=[history_state], + outputs=[current_ref_audio, history_state, history_label, ref_transcript_box], + ) + reset_btn.click( + reset_to_original, inputs=[history_state], + outputs=[current_ref_audio, history_state, history_label, ref_transcript_box], + ) + generate_btn.click( + generate_speech, inputs=[current_ref_audio, gen_text, gen_language], + outputs=[generated_audio, gen_transcript_box], + ) + save_char_btn.click( + create_character, + inputs=[new_char_name, new_char_role, new_char_prose, gen_language, current_ref_audio, history_state], + outputs=[save_char_status], + ) + + with gr.Tab("Characters"): + gr.Markdown( + f"# The Service Station cast\n" + f"Reading from `{SERVICE_STATION_DIR}`. Approving a character or an emotion " + f"is a manual step (editing the card) - this tab creates drafts and " + f"candidates for you to listen to and promote yourself, it never marks " + f"anything `approved` on its own." + ) + with gr.Row(): + char_dd = gr.Dropdown(label="Character", choices=character_names()) + refresh_btn = gr.Button("Refresh") + char_info = gr.Markdown() + char_audio = gr.Audio(label="Reference audio", type="filepath", interactive=False) + + gr.Markdown("## Generate a new emotion candidate") + with gr.Row(): + emotion_char_dd = gr.Dropdown(label="Character", choices=character_names()) + emotion_word_tb = gr.Textbox(label="Emotion word", placeholder="e.g. surprised, bemused, wistful") + gen_emotion_btn = gr.Button("Generate candidate", variant="primary") + emotion_ref_audio = gr.Audio(label="Edited reference (candidate)", type="filepath", interactive=False) + emotion_preview_audio = gr.Audio(label="Preview clone", type="filepath", interactive=False) + emotion_status = gr.Markdown() + + refresh_btn.click( + lambda: (gr.update(choices=character_names()), gr.update(choices=character_names())), + outputs=[char_dd, emotion_char_dd], + ) + char_dd.change(load_character, inputs=[char_dd], outputs=[char_info, char_audio]) + gen_emotion_btn.click( + generate_emotion_candidate, + inputs=[emotion_char_dd, emotion_word_tb], + outputs=[emotion_ref_audio, emotion_preview_audio, emotion_status], + ) if __name__ == "__main__": demo.launch(server_name="0.0.0.0", server_port=7860)