Refresh webui: fix Speed no-op, Whisper verification, Characters tab
Speed editing was a silent no-op - AuK needs --gen_seconds or it just preserves the input duration (vendor/AuK/docs/COOKBOOK.md 3.2). Fixed by computing the target from the current reference duration / speed factor, with a UI warning that longer references risk a memory-swap stall on this box (reproduced building a character voice: a 27s target hung 20+ minutes swapping with 0%% GPU use, a 12s source completed in seconds). Added Whisper content verification (faster-whisper, small/CPU/int8) after every AuK edit and MOSS generation. Motivated by a real incident building the-service-station cast: a chained AuK edit leaked fragments of its own instruction text into a character reference, undetected through a full listen-and-approve round because nothing had transcribed the reference itself, only generated lines. History is now per-instruction rather than per-click, so Undo is granular and a save action can derive a sensibly-scoped generation_recipe. New Characters tab: browses the-service-station repo's CHARACTER.md cards (name/role/status/reference audio/emotion_library), lets you create a new character (writes a fresh CHARACTER.md, always status: draft) or generate an emotion candidate for an existing one (writes to voices/<name>/emotion-cache/<emotion>.wav, same path the automated pipeline's own fallback uses). Approving anything stays a manual step - this tab creates candidates for a human to listen to and promote, it never self-approves. Co-Authored-By: Claude Sonnet 5 <[email protected]> Claude-Session: https://claude.ai/code/session_01Y1oQ4h4WE4R9KRvMQLr5Ey
This commit is contained in:
1 parent
a65532f3b2
commit
f780fbda01
1 file changed
+438
-98
+438
-98
@@ -1,24 +1,41 @@
|
|||||||
"""
|
"""
|
||||||
Pipeline studio: Demucs (strip) -> AuK (edit voice, with preview/undo) ->
|
Pipeline studio: Demucs (strip) -> AuK (edit voice, with preview/undo) ->
|
||||||
MOSS-TTS (generate speech in the edited voice).
|
MOSS-TTS (generate speech in the edited voice). Plus a Characters tab for
|
||||||
|
browsing the-service-station's cast and creating new characters/emotion
|
||||||
|
candidates without SSH + manual YAML editing.
|
||||||
|
|
||||||
Orchestrates across the three existing isolated venvs (demucs/, auk/, tts/)
|
Orchestrates across the three existing isolated venvs (demucs/, auk/, tts/)
|
||||||
via subprocess, rather than merging their conflicting dependencies
|
via subprocess, rather than merging their conflicting dependencies
|
||||||
(MOSS-TTS needs transformers 5.0.0, AuK needs 4.57.6) into one environment.
|
(MOSS-TTS needs transformers 5.0.0, AuK needs 4.57.6) into one environment.
|
||||||
|
|
||||||
Run with: webui/.venv/bin/python webui/app.py
|
Run with: webui/.venv/bin/python webui/app.py
|
||||||
|
|
||||||
|
2026-09-15 refresh, after building the-service-station's cast surfaced real
|
||||||
|
gaps in this UI:
|
||||||
|
- Speed editing was a silent no-op (AuK needs --gen_seconds or it just
|
||||||
|
preserves the input's duration - see vendor/AuK/docs/COOKBOOK.md 3.2).
|
||||||
|
- AuK's Emotion task has a fixed 8-word vocabulary; anything else needs the
|
||||||
|
Timbre task's free-form template instead.
|
||||||
|
- A chained AuK edit can leak fragments of its own instruction text into
|
||||||
|
the spoken content. This went undetected through a full listen-and-
|
||||||
|
approve cycle on a real character voice because nothing had ever
|
||||||
|
transcribed the *reference* audio, only generated lines. Every AuK edit
|
||||||
|
and MOSS generation here now gets transcribed automatically so that's
|
||||||
|
visible immediately instead of discovered later by ear.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
import glob
|
||||||
import json
|
import json
|
||||||
import os
|
import os
|
||||||
|
import re
|
||||||
import subprocess
|
import subprocess
|
||||||
import tempfile
|
import tempfile
|
||||||
import time
|
import time
|
||||||
import urllib.error
|
from datetime import date
|
||||||
import urllib.request
|
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
import gradio as gr
|
import gradio as gr
|
||||||
|
import yaml
|
||||||
|
|
||||||
REPO_ROOT = Path(__file__).resolve().parent.parent
|
REPO_ROOT = Path(__file__).resolve().parent.parent
|
||||||
DEMUCS_DIR = REPO_ROOT / "demucs"
|
DEMUCS_DIR = REPO_ROOT / "demucs"
|
||||||
@@ -32,8 +49,17 @@ AUK_QWEN_PATH = AUK_DIR / "ckpts" / "Qwen2.5-Omni-3B"
|
|||||||
TTS_PYTHON = TTS_DIR / ".venv" / "bin" / "python3"
|
TTS_PYTHON = TTS_DIR / ".venv" / "bin" / "python3"
|
||||||
TTS_GENERATE = TTS_DIR / "generate.py"
|
TTS_GENERATE = TTS_DIR / "generate.py"
|
||||||
|
|
||||||
|
# The-service-station checkout on this same box - where character cards
|
||||||
|
# live. Override with the env var if running this UI somewhere else.
|
||||||
|
SERVICE_STATION_DIR = Path(os.environ.get("SERVICE_STATION_REPO", "/root/projects/the-service-station"))
|
||||||
|
SS_VOICES_DIR = SERVICE_STATION_DIR / "voices"
|
||||||
|
|
||||||
WORKDIR = Path(tempfile.mkdtemp(prefix="audio-demo-webui-"))
|
WORKDIR = Path(tempfile.mkdtemp(prefix="audio-demo-webui-"))
|
||||||
|
|
||||||
|
# AuK's Emotion task has exactly this vocabulary (vendor/AuK/docs/COOKBOOK.md
|
||||||
|
# 4.1) - anything else needs the Timbre task's free-form template instead
|
||||||
|
# (see auk_emotion_instruction below). Kept as the dropdown's options since
|
||||||
|
# these are the only words guaranteed to hit the dedicated task.
|
||||||
EMOTIONS = ["happy", "angry", "sad", "fearful", "surprised", "disgusted", "calm", "excited"]
|
EMOTIONS = ["happy", "angry", "sad", "fearful", "surprised", "disgusted", "calm", "excited"]
|
||||||
PITCH_OPTIONS = ["-3", "-2", "-1", "+1", "+2", "+3"]
|
PITCH_OPTIONS = ["-3", "-2", "-1", "+1", "+2", "+3"]
|
||||||
SPEED_OPTIONS = ["0.5", "0.75", "1.25", "1.5", "2.0"]
|
SPEED_OPTIONS = ["0.5", "0.75", "1.25", "1.5", "2.0"]
|
||||||
@@ -45,16 +71,32 @@ LANGUAGES = [
|
|||||||
COMFYUI_BASE = "http://localhost:8188"
|
COMFYUI_BASE = "http://localhost:8188"
|
||||||
|
|
||||||
|
|
||||||
|
def auk_emotion_instruction(word: str) -> str:
|
||||||
|
"""Same logic as the-service-station/workflows/generate_episode_lines.py's
|
||||||
|
auk_emotion_instruction() - duplicated rather than shared because this
|
||||||
|
repo and the-service-station use separate venvs/dependency sets, but
|
||||||
|
kept in sync deliberately. AuK's Emotion task only understands the 8
|
||||||
|
words in EMOTIONS; anything else goes through Timbre's free-form
|
||||||
|
description instead, which has no fixed vocabulary."""
|
||||||
|
if word in EMOTIONS:
|
||||||
|
return f"Change the emotion to {word}."
|
||||||
|
return f'Keep the spoken content unchanged and change the timbre to: "the same voice, sounding {word}".'
|
||||||
|
|
||||||
|
|
||||||
def _comfyui_queue_empty() -> bool:
|
def _comfyui_queue_empty() -> bool:
|
||||||
try:
|
try:
|
||||||
|
import urllib.error
|
||||||
|
import urllib.request
|
||||||
with urllib.request.urlopen(f"{COMFYUI_BASE}/queue", timeout=3) as resp:
|
with urllib.request.urlopen(f"{COMFYUI_BASE}/queue", timeout=3) as resp:
|
||||||
data = json.loads(resp.read())
|
data = json.loads(resp.read())
|
||||||
return not data.get("queue_running") and not data.get("queue_pending")
|
return not data.get("queue_running") and not data.get("queue_pending")
|
||||||
except (urllib.error.URLError, OSError):
|
except Exception:
|
||||||
return True # ComfyUI not reachable -> nothing to worry about
|
return True # ComfyUI not reachable -> nothing to worry about
|
||||||
|
|
||||||
|
|
||||||
def _free_comfyui_memory() -> None:
|
def _free_comfyui_memory() -> None:
|
||||||
|
import urllib.error
|
||||||
|
import urllib.request
|
||||||
if not _comfyui_queue_empty():
|
if not _comfyui_queue_empty():
|
||||||
return
|
return
|
||||||
try:
|
try:
|
||||||
@@ -95,8 +137,20 @@ def _ensure_wav(path: str) -> str:
|
|||||||
return str(out_path)
|
return str(out_path)
|
||||||
|
|
||||||
|
|
||||||
|
def _duration(path: str) -> float:
|
||||||
|
r = subprocess.run(
|
||||||
|
["ffprobe", "-v", "error", "-show_entries", "format=duration",
|
||||||
|
"-of", "default=noprint_wrappers=1:nokey=1", path],
|
||||||
|
capture_output=True, text=True,
|
||||||
|
)
|
||||||
|
return float(r.stdout.strip())
|
||||||
|
|
||||||
|
|
||||||
def _ld_library_path_for_venv(venv_dir: Path) -> str:
|
def _ld_library_path_for_venv(venv_dir: Path) -> str:
|
||||||
site_packages = next(venv_dir.glob(".venv/lib/python*/site-packages"))
|
matches = list(venv_dir.glob(".venv/lib/python*/site-packages"))
|
||||||
|
if not matches:
|
||||||
|
return ""
|
||||||
|
site_packages = matches[0]
|
||||||
nvidia_dir = site_packages / "nvidia"
|
nvidia_dir = site_packages / "nvidia"
|
||||||
if not nvidia_dir.is_dir():
|
if not nvidia_dir.is_dir():
|
||||||
return ""
|
return ""
|
||||||
@@ -107,6 +161,28 @@ def _ld_library_path_for_venv(venv_dir: Path) -> str:
|
|||||||
TTS_LD_LIBRARY_PATH = _ld_library_path_for_venv(TTS_DIR)
|
TTS_LD_LIBRARY_PATH = _ld_library_path_for_venv(TTS_DIR)
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Whisper content verification - added 2026-09-15 after a chained AuK edit
|
||||||
|
# leaked fragments of its own instruction text into a character's spoken
|
||||||
|
# reference, undetected through a full listen-and-approve round because
|
||||||
|
# nobody had transcribed the reference itself. Loaded once, lazily, since
|
||||||
|
# most sessions of this UI won't need it immediately.
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
_whisper_model = None
|
||||||
|
|
||||||
|
|
||||||
|
def _whisper_transcribe(path: str) -> str:
|
||||||
|
global _whisper_model
|
||||||
|
try:
|
||||||
|
if _whisper_model is None:
|
||||||
|
from faster_whisper import WhisperModel
|
||||||
|
_whisper_model = WhisperModel("small", device="cpu", compute_type="int8")
|
||||||
|
segments, _ = _whisper_model.transcribe(path, language="en", beam_size=5)
|
||||||
|
return " ".join(s.text.strip() for s in segments)
|
||||||
|
except Exception as e:
|
||||||
|
return f"(transcription failed: {e})"
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# Stage 1: input / demucs stripping
|
# Stage 1: input / demucs stripping
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
@@ -134,6 +210,12 @@ def strip_vocals(input_audio: str):
|
|||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# Stage 2: AuK voice editing, with history for undo/reset
|
# Stage 2: AuK voice editing, with history for undo/reset
|
||||||
|
#
|
||||||
|
# history_state is a list of {"path": str, "instruction": str|None} dicts,
|
||||||
|
# one entry per actual AuK call (not per button click - a click can queue
|
||||||
|
# several edits, e.g. enhance+emotion, and each becomes its own history
|
||||||
|
# entry so "Save as character" can derive a sensibly-scoped
|
||||||
|
# generation_recipe and Undo is granular).
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
@@ -141,14 +223,17 @@ def use_as_reference(audio_path: str, history: list):
|
|||||||
if not audio_path:
|
if not audio_path:
|
||||||
raise gr.Error("No audio to use yet.")
|
raise gr.Error("No audio to use yet.")
|
||||||
audio_path = _ensure_wav(audio_path)
|
audio_path = _ensure_wav(audio_path)
|
||||||
history = [audio_path]
|
history = [{"path": audio_path, "instruction": None}]
|
||||||
return audio_path, history, _history_label(history)
|
transcript = _whisper_transcribe(audio_path)
|
||||||
|
return audio_path, history, _history_label(history), transcript
|
||||||
|
|
||||||
|
|
||||||
def _history_label(history: list) -> str:
|
def _history_label(history: list) -> str:
|
||||||
if not history:
|
n = len(history) - 1
|
||||||
return "No reference set yet."
|
if n <= 0:
|
||||||
return f"Edit history: {len(history) - 1} edit(s) applied on top of the original reference."
|
return "No edits applied yet." if history else "No reference set yet."
|
||||||
|
steps = "\n".join(f"{i}. {h['instruction']}" for i, h in enumerate(history[1:], 1))
|
||||||
|
return f"{n} edit(s) applied on top of the original reference:\n{steps}"
|
||||||
|
|
||||||
|
|
||||||
ENHANCE_INSTRUCTIONS = {
|
ENHANCE_INSTRUCTIONS = {
|
||||||
@@ -168,6 +253,7 @@ def apply_edit(
|
|||||||
history: list,
|
history: list,
|
||||||
enhance: str,
|
enhance: str,
|
||||||
emotion: str,
|
emotion: str,
|
||||||
|
other_emotion: str,
|
||||||
pitch: str,
|
pitch: str,
|
||||||
speed: str,
|
speed: str,
|
||||||
custom_instruction: str,
|
custom_instruction: str,
|
||||||
@@ -181,12 +267,18 @@ def apply_edit(
|
|||||||
if enhance and ENHANCE_INSTRUCTIONS.get(enhance):
|
if enhance and ENHANCE_INSTRUCTIONS.get(enhance):
|
||||||
instructions.append(ENHANCE_INSTRUCTIONS[enhance])
|
instructions.append(ENHANCE_INSTRUCTIONS[enhance])
|
||||||
if emotion and emotion != "none":
|
if emotion and emotion != "none":
|
||||||
instructions.append(f"Change the emotion to {emotion}.")
|
instructions.append(auk_emotion_instruction(emotion))
|
||||||
|
if other_emotion and other_emotion.strip():
|
||||||
|
# Free-text emotion word, not one of AuK's 8 - always goes through
|
||||||
|
# the Timbre task's free-form template (see auk_emotion_instruction).
|
||||||
|
instructions.append(auk_emotion_instruction(other_emotion.strip()))
|
||||||
if pitch and pitch != "none":
|
if pitch and pitch != "none":
|
||||||
direction = "Raise" if pitch.startswith("+") else "Lower"
|
direction = "Raise" if pitch.startswith("+") else "Lower"
|
||||||
instructions.append(f"{direction} the pitch by {pitch.lstrip('+-')} semitone(s).")
|
instructions.append(f"{direction} the pitch by {pitch.lstrip('+-')} semitone(s).")
|
||||||
|
speed_gen_seconds = None
|
||||||
if speed and speed != "none":
|
if speed and speed != "none":
|
||||||
instructions.append(f"Adjust the speech speed to {speed}x.")
|
speed_gen_seconds = _duration(current_ref) / float(speed)
|
||||||
|
instructions.append((f"Adjust the speech speed to {speed}x.", speed_gen_seconds))
|
||||||
if custom_instruction and custom_instruction.strip():
|
if custom_instruction and custom_instruction.strip():
|
||||||
instructions.append(custom_instruction.strip())
|
instructions.append(custom_instruction.strip())
|
||||||
|
|
||||||
@@ -197,7 +289,11 @@ def apply_edit(
|
|||||||
ref = current_ref
|
ref = current_ref
|
||||||
out_dir = WORKDIR / f"auk_{int(time.time())}"
|
out_dir = WORKDIR / f"auk_{int(time.time())}"
|
||||||
out_dir.mkdir(parents=True, exist_ok=True)
|
out_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
last_transcript = ""
|
||||||
for i, instruction in enumerate(instructions):
|
for i, instruction in enumerate(instructions):
|
||||||
|
gen_seconds = None
|
||||||
|
if isinstance(instruction, tuple):
|
||||||
|
instruction, gen_seconds = instruction
|
||||||
out_path = out_dir / f"edit_{i}.wav"
|
out_path = out_dir / f"edit_{i}.wav"
|
||||||
cmd = [
|
cmd = [
|
||||||
str(AUK_INFER),
|
str(AUK_INFER),
|
||||||
@@ -208,25 +304,30 @@ def apply_edit(
|
|||||||
"--output", str(out_path),
|
"--output", str(out_path),
|
||||||
"--cpu_offload",
|
"--cpu_offload",
|
||||||
]
|
]
|
||||||
|
if gen_seconds is not None:
|
||||||
|
cmd += ["--gen_seconds", str(round(gen_seconds, 2))]
|
||||||
_run(cmd, cwd=AUK_DIR)
|
_run(cmd, cwd=AUK_DIR)
|
||||||
ref = str(out_path)
|
ref = str(out_path)
|
||||||
|
last_transcript = _whisper_transcribe(ref)
|
||||||
|
history = history + [{"path": ref, "instruction": instruction}]
|
||||||
|
|
||||||
history = history + [ref]
|
return ref, history, _history_label(history), last_transcript
|
||||||
return ref, history, _history_label(history)
|
|
||||||
|
|
||||||
|
|
||||||
def undo_edit(history: list):
|
def undo_edit(history: list):
|
||||||
if len(history) <= 1:
|
if len(history) <= 1:
|
||||||
raise gr.Error("Nothing to undo.")
|
raise gr.Error("Nothing to undo.")
|
||||||
history = history[:-1]
|
history = history[:-1]
|
||||||
return history[-1], history, _history_label(history)
|
transcript = _whisper_transcribe(history[-1]["path"])
|
||||||
|
return history[-1]["path"], history, _history_label(history), transcript
|
||||||
|
|
||||||
|
|
||||||
def reset_to_original(history: list):
|
def reset_to_original(history: list):
|
||||||
if not history:
|
if not history:
|
||||||
raise gr.Error("No reference set yet.")
|
raise gr.Error("No reference set yet.")
|
||||||
history = [history[0]]
|
history = [history[0]]
|
||||||
return history[0], history, _history_label(history)
|
transcript = _whisper_transcribe(history[0]["path"])
|
||||||
|
return history[0]["path"], history, _history_label(history), transcript
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
@@ -258,7 +359,182 @@ def generate_speech(reference: str, text: str, language: str):
|
|||||||
cwd=TTS_DIR,
|
cwd=TTS_DIR,
|
||||||
env=env,
|
env=env,
|
||||||
)
|
)
|
||||||
return str(out_path)
|
transcript = _whisper_transcribe(str(out_path))
|
||||||
|
return str(out_path), transcript
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Characters (the-service-station cast)
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def _slugify(text: str) -> str:
|
||||||
|
return re.sub(r"[^a-z0-9]+", "-", text.lower()).strip("-")
|
||||||
|
|
||||||
|
|
||||||
|
def read_character_cards() -> dict:
|
||||||
|
"""name -> parsed card dict (frontmatter + prose), read-only. Never
|
||||||
|
round-trips these through yaml.dump - only used for display and for
|
||||||
|
resolving a base reference_audio, never for rewriting an existing file
|
||||||
|
(that would silently strip hand-authored inline comments)."""
|
||||||
|
cards = {}
|
||||||
|
if not SS_VOICES_DIR.is_dir():
|
||||||
|
return cards
|
||||||
|
for card_path in sorted(SS_VOICES_DIR.glob("*/CHARACTER.md")):
|
||||||
|
try:
|
||||||
|
text = card_path.read_text(encoding="utf-8")
|
||||||
|
parts = text.split("---", 2)
|
||||||
|
if len(parts) < 3:
|
||||||
|
continue
|
||||||
|
frontmatter = yaml.safe_load(parts[1]) or {}
|
||||||
|
frontmatter["_prose"] = parts[2].strip()
|
||||||
|
frontmatter["_dir"] = card_path.parent
|
||||||
|
cards[frontmatter.get("name", card_path.parent.name)] = frontmatter
|
||||||
|
except Exception as e:
|
||||||
|
cards[card_path.parent.name] = {"_error": str(e), "_dir": card_path.parent}
|
||||||
|
return cards
|
||||||
|
|
||||||
|
|
||||||
|
def character_names() -> list[str]:
|
||||||
|
return sorted(read_character_cards().keys())
|
||||||
|
|
||||||
|
|
||||||
|
def load_character(name: str):
|
||||||
|
if not name:
|
||||||
|
return "Pick a character.", None
|
||||||
|
cards = read_character_cards()
|
||||||
|
card = cards.get(name)
|
||||||
|
if not card:
|
||||||
|
return f"No card found for '{name}'.", None
|
||||||
|
if "_error" in card:
|
||||||
|
return f"Could not parse {name}'s card: {card['_error']}", None
|
||||||
|
|
||||||
|
ref_audio = card.get("reference_audio")
|
||||||
|
ref_path = str(SERVICE_STATION_DIR / ref_audio) if ref_audio else None
|
||||||
|
|
||||||
|
lib = card.get("emotion_library") or {}
|
||||||
|
lib_lines = [f"- **{emo}**: {entry.get('status', '?')}" for emo, entry in lib.items()] or ["(none approved yet)"]
|
||||||
|
|
||||||
|
cache_dir = card["_dir"] / "emotion-cache"
|
||||||
|
cached = sorted(p.stem for p in cache_dir.glob("*.wav")) if cache_dir.is_dir() else []
|
||||||
|
cache_lines = [f"- {e} (cached, not yet approved)" for e in cached] or ["(none pending)"]
|
||||||
|
|
||||||
|
info = (
|
||||||
|
f"**{name}** — {card.get('role', '?')}\n\n"
|
||||||
|
f"Status: `{card.get('status', '?')}` | Engine: `{card.get('engine', '?')}` | "
|
||||||
|
f"Default emotion (documentation only): `{card.get('default_emotion', '?')}`\n\n"
|
||||||
|
f"### Approved emotion_library\n" + "\n".join(lib_lines) + "\n\n"
|
||||||
|
f"### Pending (emotion-cache, not approved)\n" + "\n".join(cache_lines) + "\n\n"
|
||||||
|
f"---\n{card.get('_prose', '')}"
|
||||||
|
)
|
||||||
|
return info, ref_path
|
||||||
|
|
||||||
|
|
||||||
|
def create_character(
|
||||||
|
name: str, role: str, prose: str, language: str,
|
||||||
|
current_ref: str, history: list,
|
||||||
|
):
|
||||||
|
if not name or not name.strip():
|
||||||
|
raise gr.Error("Give the character a name.")
|
||||||
|
if not current_ref or not history:
|
||||||
|
raise gr.Error("Set up a reference and edit it above first - there's nothing to save yet.")
|
||||||
|
|
||||||
|
slug = _slugify(name)
|
||||||
|
char_dir = SS_VOICES_DIR / slug / "webui-created"
|
||||||
|
if (SS_VOICES_DIR / slug / "CHARACTER.md").exists():
|
||||||
|
raise gr.Error(f"voices/{slug}/CHARACTER.md already exists - this UI won't overwrite an existing character.")
|
||||||
|
char_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
|
source_dest = char_dir / f"{slug}_source.wav"
|
||||||
|
ref_dest = char_dir / f"{slug}_ref_v1.wav"
|
||||||
|
subprocess.run(["cp", history[0]["path"], str(source_dest)], check=True)
|
||||||
|
subprocess.run(["cp", current_ref, str(ref_dest)], check=True)
|
||||||
|
|
||||||
|
recipe = [
|
||||||
|
{"tool": "auk", "instruction": h["instruction"]}
|
||||||
|
for h in history[1:] if h["instruction"]
|
||||||
|
]
|
||||||
|
|
||||||
|
frontmatter = {
|
||||||
|
"name": name.strip(),
|
||||||
|
"role": role.strip() or "Standing guest",
|
||||||
|
"engine": "moss-tts",
|
||||||
|
"model": "OpenMOSS-Team/MOSS-TTS-Local-Transformer-v1.5",
|
||||||
|
"reference_audio": str(ref_dest.relative_to(SERVICE_STATION_DIR)),
|
||||||
|
"source_clip": str(source_dest.relative_to(SERVICE_STATION_DIR)),
|
||||||
|
"source_clip_note": "Created via the webui pipeline studio - see generation_recipe for the exact edit chain.",
|
||||||
|
"generation_recipe": recipe,
|
||||||
|
"seed": None,
|
||||||
|
"emotion_library": {},
|
||||||
|
"language": language,
|
||||||
|
"status": "draft",
|
||||||
|
"last_updated": date.today().isoformat(),
|
||||||
|
"provenance": "webui",
|
||||||
|
"appearances": [],
|
||||||
|
}
|
||||||
|
frontmatter_yaml = yaml.dump(frontmatter, sort_keys=False, allow_unicode=True)
|
||||||
|
body = f"---\n{frontmatter_yaml}---\n\n# {name.strip()}\n\n{prose.strip()}\n"
|
||||||
|
(SS_VOICES_DIR / slug / "CHARACTER.md").write_text(body, encoding="utf-8")
|
||||||
|
|
||||||
|
return (
|
||||||
|
f"Created voices/{slug}/CHARACTER.md with status: draft. "
|
||||||
|
f"Not committed - review, listen, and commit it yourself (or ask an agent to) "
|
||||||
|
f"once you're happy, same as every other voice in this project."
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def generate_emotion_candidate(character_name: str, emotion_word: str):
|
||||||
|
if not character_name:
|
||||||
|
raise gr.Error("Pick a character first.")
|
||||||
|
if not emotion_word or not emotion_word.strip():
|
||||||
|
raise gr.Error("Enter an emotion word.")
|
||||||
|
emotion_word = emotion_word.strip()
|
||||||
|
|
||||||
|
cards = read_character_cards()
|
||||||
|
card = cards.get(character_name)
|
||||||
|
if not card or "_error" in card:
|
||||||
|
raise gr.Error(f"Could not load {character_name}'s card.")
|
||||||
|
|
||||||
|
base_reference = str(SERVICE_STATION_DIR / card["reference_audio"])
|
||||||
|
cache_dir = card["_dir"] / "emotion-cache"
|
||||||
|
cache_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
cache_path = cache_dir / f"{_slugify(emotion_word)}.wav"
|
||||||
|
|
||||||
|
_free_comfyui_memory()
|
||||||
|
instruction = auk_emotion_instruction(emotion_word)
|
||||||
|
cmd = [
|
||||||
|
str(AUK_INFER), "--ckpt", str(AUK_CKPT), "--qwen_path", str(AUK_QWEN_PATH),
|
||||||
|
"--audio", base_reference, "--instruction", instruction,
|
||||||
|
"--output", str(cache_path), "--cpu_offload",
|
||||||
|
]
|
||||||
|
_run(cmd, cwd=AUK_DIR)
|
||||||
|
|
||||||
|
ref_transcript = _whisper_transcribe(str(cache_path))
|
||||||
|
|
||||||
|
# Preview clone so the candidate can actually be judged, not just the
|
||||||
|
# edited reference in isolation.
|
||||||
|
env = None
|
||||||
|
if TTS_LD_LIBRARY_PATH:
|
||||||
|
env = os.environ.copy()
|
||||||
|
env["LD_LIBRARY_PATH"] = TTS_LD_LIBRARY_PATH
|
||||||
|
preview_text = f"Testing the {emotion_word} emotion for {character_name}."
|
||||||
|
preview_path = WORKDIR / f"emotion_preview_{int(time.time())}.wav"
|
||||||
|
_run(
|
||||||
|
[str(TTS_PYTHON), str(TTS_GENERATE), "--text", preview_text,
|
||||||
|
"--reference", str(cache_path), "--language", "English", "--output", str(preview_path)],
|
||||||
|
cwd=TTS_DIR, env=env,
|
||||||
|
)
|
||||||
|
preview_transcript = _whisper_transcribe(str(preview_path))
|
||||||
|
|
||||||
|
status = (
|
||||||
|
f"Saved to `{cache_path.relative_to(SERVICE_STATION_DIR)}` (cached, **not approved**).\n\n"
|
||||||
|
f"Reference transcript: {ref_transcript}\n\n"
|
||||||
|
f"Preview clone transcript: {preview_transcript}\n\n"
|
||||||
|
f"If this sounds right, promote it manually per `.claude/skills/produce-episode/"
|
||||||
|
f"SKILL.md`'s \"Adding an emotion to a character's library\" - this UI won't do "
|
||||||
|
f"that step for you."
|
||||||
|
)
|
||||||
|
return str(cache_path), str(preview_path), status
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
@@ -266,92 +542,156 @@ def generate_speech(reference: str, text: str, language: str):
|
|||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
with gr.Blocks(title="Audio-demo pipeline studio") as demo:
|
with gr.Blocks(title="Audio-demo pipeline studio") as demo:
|
||||||
gr.Markdown(
|
with gr.Tabs():
|
||||||
"# Pipeline studio\n"
|
with gr.Tab("Pipeline Studio"):
|
||||||
"Strip a voice with **Demucs**, edit it with **AuK**, then generate new "
|
|
||||||
"speech in that voice with **MOSS-TTS**. Each stage shells out to its own "
|
|
||||||
"isolated venv, so this box's GPU is the real constraint — heavy edits "
|
|
||||||
"(long clips) can OOM AuK; keep reference clips short (under ~25s)."
|
|
||||||
)
|
|
||||||
|
|
||||||
reference_state = gr.State(None)
|
|
||||||
history_state = gr.State([])
|
|
||||||
|
|
||||||
with gr.Row():
|
|
||||||
with gr.Column():
|
|
||||||
gr.Markdown("## 1. Input")
|
|
||||||
input_audio = gr.Audio(label="Upload or record", type="filepath")
|
|
||||||
strip_btn = gr.Button("Strip vocals with Demucs")
|
|
||||||
with gr.Row():
|
|
||||||
vocals_out = gr.Audio(label="Vocals (isolated)", type="filepath")
|
|
||||||
no_vocals_out = gr.Audio(label="Background (no vocals)", type="filepath")
|
|
||||||
with gr.Row():
|
|
||||||
use_vocals_btn = gr.Button("Use vocals as reference")
|
|
||||||
use_no_vocals_btn = gr.Button("Use background as reference")
|
|
||||||
use_original_btn = gr.Button("Use uploaded audio as-is")
|
|
||||||
|
|
||||||
with gr.Column():
|
|
||||||
gr.Markdown("## 2. Voice editor (AuK)")
|
|
||||||
current_ref_audio = gr.Audio(label="Current reference", type="filepath", interactive=False)
|
|
||||||
history_label = gr.Markdown("No reference set yet.")
|
|
||||||
enhance_dd = gr.Dropdown(
|
|
||||||
list(ENHANCE_INSTRUCTIONS.keys()), value="none",
|
|
||||||
label="Enhance (denoise/dereverb/restore) - runs first, before character edits",
|
|
||||||
)
|
|
||||||
emotion_dd = gr.Dropdown(["none"] + EMOTIONS, value="none", label="Emotion")
|
|
||||||
pitch_dd = gr.Dropdown(["none"] + PITCH_OPTIONS, value="none", label="Pitch (semitones)")
|
|
||||||
speed_dd = gr.Dropdown(["none"] + SPEED_OPTIONS, value="none", label="Speed")
|
|
||||||
custom_instruction = gr.Textbox(
|
|
||||||
label="Custom instruction (advanced, optional)",
|
|
||||||
placeholder='e.g. "Lower the pitch very slightly."',
|
|
||||||
)
|
|
||||||
gr.Markdown(
|
gr.Markdown(
|
||||||
"*Note: pitch editing has been unreliable in testing — even "
|
"# Pipeline studio\n"
|
||||||
"\"1 semitone\" has overshot to ~4+ semitones on some voices. "
|
"Strip a voice with **Demucs**, edit it with **AuK**, then generate new "
|
||||||
"Preview before committing.*"
|
"speech in that voice with **MOSS-TTS**. Each stage shells out to its own "
|
||||||
|
"isolated venv, so this box's GPU is the real constraint — heavy edits "
|
||||||
|
"(long clips) can OOM AuK; keep reference clips short (under ~25s)."
|
||||||
)
|
)
|
||||||
apply_btn = gr.Button("Apply edit(s)", variant="primary")
|
|
||||||
|
reference_state = gr.State(None)
|
||||||
|
history_state = gr.State([])
|
||||||
|
|
||||||
with gr.Row():
|
with gr.Row():
|
||||||
undo_btn = gr.Button("Undo last edit")
|
with gr.Column():
|
||||||
reset_btn = gr.Button("Reset to original")
|
gr.Markdown("## 1. Input")
|
||||||
|
input_audio = gr.Audio(label="Upload or record", type="filepath")
|
||||||
|
strip_btn = gr.Button("Strip vocals with Demucs")
|
||||||
|
with gr.Row():
|
||||||
|
vocals_out = gr.Audio(label="Vocals (isolated)", type="filepath")
|
||||||
|
no_vocals_out = gr.Audio(label="Background (no vocals)", type="filepath")
|
||||||
|
with gr.Row():
|
||||||
|
use_vocals_btn = gr.Button("Use vocals as reference")
|
||||||
|
use_no_vocals_btn = gr.Button("Use background as reference")
|
||||||
|
use_original_btn = gr.Button("Use uploaded audio as-is")
|
||||||
|
|
||||||
with gr.Column():
|
with gr.Column():
|
||||||
gr.Markdown("## 3. Generate (MOSS-TTS)")
|
gr.Markdown("## 2. Voice editor (AuK)")
|
||||||
gen_text = gr.Textbox(label="Text to speak", lines=6)
|
current_ref_audio = gr.Audio(label="Current reference", type="filepath", interactive=False)
|
||||||
gen_language = gr.Dropdown(LANGUAGES, value="English", label="Language")
|
ref_transcript_box = gr.Textbox(label="Whisper transcript of current reference", interactive=False)
|
||||||
generate_btn = gr.Button("Generate speech", variant="primary")
|
history_label = gr.Markdown("No reference set yet.")
|
||||||
generated_audio = gr.Audio(label="Generated speech", type="filepath")
|
enhance_dd = gr.Dropdown(
|
||||||
|
list(ENHANCE_INSTRUCTIONS.keys()), value="none",
|
||||||
|
label="Enhance (denoise/dereverb/restore) - runs first, before character edits",
|
||||||
|
)
|
||||||
|
emotion_dd = gr.Dropdown(
|
||||||
|
["none"] + EMOTIONS, value="none",
|
||||||
|
label="Emotion (AuK's fixed vocabulary - anything else, use the field below)",
|
||||||
|
)
|
||||||
|
other_emotion_tb = gr.Textbox(
|
||||||
|
label="Other emotion word (not in AuK's list)",
|
||||||
|
placeholder='e.g. "bemused" - automatically wrapped as a Timbre instruction instead',
|
||||||
|
)
|
||||||
|
pitch_dd = gr.Dropdown(["none"] + PITCH_OPTIONS, value="none", label="Pitch (semitones)")
|
||||||
|
speed_dd = gr.Dropdown(["none"] + SPEED_OPTIONS, value="none", label="Speed")
|
||||||
|
custom_instruction = gr.Textbox(
|
||||||
|
label="Custom instruction (advanced, optional)",
|
||||||
|
placeholder='e.g. "Lower the pitch very slightly."',
|
||||||
|
)
|
||||||
|
gr.Markdown(
|
||||||
|
"*Pitch editing has been unreliable in testing — even \"1 semitone\" "
|
||||||
|
"has overshot to ~4+ semitones on some voices. Prefer a free-form "
|
||||||
|
"custom instruction like \"a little lower in register\" over the Pitch "
|
||||||
|
"dropdown for character work.*\n\n"
|
||||||
|
"*Speed now sets AuK's required `--gen_seconds` automatically. On "
|
||||||
|
"references longer than ~15s this can stall (a 27s target hung this "
|
||||||
|
"box for 20+ minutes swapping to disk with 0% GPU use, while a 12s "
|
||||||
|
"source completed in seconds) - trim your reference shorter first if "
|
||||||
|
"using Speed.*"
|
||||||
|
)
|
||||||
|
apply_btn = gr.Button("Apply edit(s)", variant="primary")
|
||||||
|
with gr.Row():
|
||||||
|
undo_btn = gr.Button("Undo last edit")
|
||||||
|
reset_btn = gr.Button("Reset to original")
|
||||||
|
|
||||||
strip_btn.click(strip_vocals, inputs=[input_audio], outputs=[vocals_out, no_vocals_out])
|
with gr.Column():
|
||||||
use_vocals_btn.click(
|
gr.Markdown("## 3. Generate (MOSS-TTS)")
|
||||||
use_as_reference, inputs=[vocals_out, history_state],
|
gen_text = gr.Textbox(label="Text to speak", lines=6)
|
||||||
outputs=[current_ref_audio, history_state, history_label],
|
gen_language = gr.Dropdown(LANGUAGES, value="English", label="Language")
|
||||||
)
|
generate_btn = gr.Button("Generate speech", variant="primary")
|
||||||
use_no_vocals_btn.click(
|
generated_audio = gr.Audio(label="Generated speech", type="filepath")
|
||||||
use_as_reference, inputs=[no_vocals_out, history_state],
|
gen_transcript_box = gr.Textbox(label="Whisper transcript", interactive=False)
|
||||||
outputs=[current_ref_audio, history_state, history_label],
|
|
||||||
)
|
gr.Markdown("## 4. Save as a the-service-station character")
|
||||||
use_original_btn.click(
|
new_char_name = gr.Textbox(label="Name")
|
||||||
use_as_reference, inputs=[input_audio, history_state],
|
new_char_role = gr.Textbox(label="Role", placeholder="e.g. Standing guest - weather")
|
||||||
outputs=[current_ref_audio, history_state, history_label],
|
new_char_prose = gr.Textbox(label="Character brief (prose)", lines=4)
|
||||||
)
|
save_char_btn = gr.Button("Save as new character (status: draft)")
|
||||||
apply_btn.click(
|
save_char_status = gr.Markdown()
|
||||||
apply_edit,
|
|
||||||
inputs=[current_ref_audio, history_state, enhance_dd, emotion_dd, pitch_dd, speed_dd, custom_instruction],
|
strip_btn.click(strip_vocals, inputs=[input_audio], outputs=[vocals_out, no_vocals_out])
|
||||||
outputs=[current_ref_audio, history_state, history_label],
|
use_vocals_btn.click(
|
||||||
)
|
use_as_reference, inputs=[vocals_out, history_state],
|
||||||
undo_btn.click(
|
outputs=[current_ref_audio, history_state, history_label, ref_transcript_box],
|
||||||
undo_edit, inputs=[history_state],
|
)
|
||||||
outputs=[current_ref_audio, history_state, history_label],
|
use_no_vocals_btn.click(
|
||||||
)
|
use_as_reference, inputs=[no_vocals_out, history_state],
|
||||||
reset_btn.click(
|
outputs=[current_ref_audio, history_state, history_label, ref_transcript_box],
|
||||||
reset_to_original, inputs=[history_state],
|
)
|
||||||
outputs=[current_ref_audio, history_state, history_label],
|
use_original_btn.click(
|
||||||
)
|
use_as_reference, inputs=[input_audio, history_state],
|
||||||
generate_btn.click(
|
outputs=[current_ref_audio, history_state, history_label, ref_transcript_box],
|
||||||
generate_speech, inputs=[current_ref_audio, gen_text, gen_language],
|
)
|
||||||
outputs=[generated_audio],
|
apply_btn.click(
|
||||||
)
|
apply_edit,
|
||||||
|
inputs=[current_ref_audio, history_state, enhance_dd, emotion_dd, other_emotion_tb,
|
||||||
|
pitch_dd, speed_dd, custom_instruction],
|
||||||
|
outputs=[current_ref_audio, history_state, history_label, ref_transcript_box],
|
||||||
|
)
|
||||||
|
undo_btn.click(
|
||||||
|
undo_edit, inputs=[history_state],
|
||||||
|
outputs=[current_ref_audio, history_state, history_label, ref_transcript_box],
|
||||||
|
)
|
||||||
|
reset_btn.click(
|
||||||
|
reset_to_original, inputs=[history_state],
|
||||||
|
outputs=[current_ref_audio, history_state, history_label, ref_transcript_box],
|
||||||
|
)
|
||||||
|
generate_btn.click(
|
||||||
|
generate_speech, inputs=[current_ref_audio, gen_text, gen_language],
|
||||||
|
outputs=[generated_audio, gen_transcript_box],
|
||||||
|
)
|
||||||
|
save_char_btn.click(
|
||||||
|
create_character,
|
||||||
|
inputs=[new_char_name, new_char_role, new_char_prose, gen_language, current_ref_audio, history_state],
|
||||||
|
outputs=[save_char_status],
|
||||||
|
)
|
||||||
|
|
||||||
|
with gr.Tab("Characters"):
|
||||||
|
gr.Markdown(
|
||||||
|
f"# The Service Station cast\n"
|
||||||
|
f"Reading from `{SERVICE_STATION_DIR}`. Approving a character or an emotion "
|
||||||
|
f"is a manual step (editing the card) - this tab creates drafts and "
|
||||||
|
f"candidates for you to listen to and promote yourself, it never marks "
|
||||||
|
f"anything `approved` on its own."
|
||||||
|
)
|
||||||
|
with gr.Row():
|
||||||
|
char_dd = gr.Dropdown(label="Character", choices=character_names())
|
||||||
|
refresh_btn = gr.Button("Refresh")
|
||||||
|
char_info = gr.Markdown()
|
||||||
|
char_audio = gr.Audio(label="Reference audio", type="filepath", interactive=False)
|
||||||
|
|
||||||
|
gr.Markdown("## Generate a new emotion candidate")
|
||||||
|
with gr.Row():
|
||||||
|
emotion_char_dd = gr.Dropdown(label="Character", choices=character_names())
|
||||||
|
emotion_word_tb = gr.Textbox(label="Emotion word", placeholder="e.g. surprised, bemused, wistful")
|
||||||
|
gen_emotion_btn = gr.Button("Generate candidate", variant="primary")
|
||||||
|
emotion_ref_audio = gr.Audio(label="Edited reference (candidate)", type="filepath", interactive=False)
|
||||||
|
emotion_preview_audio = gr.Audio(label="Preview clone", type="filepath", interactive=False)
|
||||||
|
emotion_status = gr.Markdown()
|
||||||
|
|
||||||
|
refresh_btn.click(
|
||||||
|
lambda: (gr.update(choices=character_names()), gr.update(choices=character_names())),
|
||||||
|
outputs=[char_dd, emotion_char_dd],
|
||||||
|
)
|
||||||
|
char_dd.change(load_character, inputs=[char_dd], outputs=[char_info, char_audio])
|
||||||
|
gen_emotion_btn.click(
|
||||||
|
generate_emotion_candidate,
|
||||||
|
inputs=[emotion_char_dd, emotion_word_tb],
|
||||||
|
outputs=[emotion_ref_audio, emotion_preview_audio, emotion_status],
|
||||||
|
)
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
demo.launch(server_name="0.0.0.0", server_port=7860)
|
demo.launch(server_name="0.0.0.0", server_port=7860)
|
||||||
Reference in new issue
Block a user