diff --git a/.gitignore b/.gitignore index be0e442..68aa1fd 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ .env surreal_data/ notebook_data/ +services/logs/ diff --git a/CLAUDE.md b/CLAUDE.md index 381cb66..7f76da7 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -37,6 +37,27 @@ Neither `surrealdb` nor `open_notebook` do local model inference themselves, so A separate local `llama.cpp` server exists on this machine (managed via `~/llamacppctl-server_neu/llamacppctl/`, container name `llama_cpp_server`) but is **not** wired into this stack. It can be added later as an additional OpenAI-compatible provider through the Open Notebook UI (Settings → AI Providers) if needed — note it competes for the same GPU 1/2 VRAM as Ollama, so don't run large models on both simultaneously. +### Text-to-speech / speech-to-text + +Open Notebook's `openai_compatible` provider type talks to any endpoint implementing OpenAI's audio API shape (`POST /audio/speech`, `POST /audio/transcriptions` — see `esperanto.providers.tts.openai_compatible` / `.stt.openai_compatible` inside the app container for the exact contract). Since neither Ollama nor OpenRouter support TTS/STT, two thin wrapper servers in `services/` expose the locally installed tools this way: + +- `services/tts_server.py` — wraps `chatterbox-tts` (via `~/chatterbox-tts-cli/chatterbox_cli_v4.py`), run with the `chatterbox` conda env (`~/miniforge3/envs/chatterbox/bin/python`). +- `services/stt_server.py` — wraps `faster-whisper` (`large-v3` model), run with the base `python3`. + +Both are plain background processes (`nohup ... &`), **not** systemd units — they don't survive a reboot; restart manually if needed: + +```bash +cd ~/open-notebook/services +CUDA_VISIBLE_DEVICES=2 TTS_PORT=8901 nohup ~/miniforge3/envs/chatterbox/bin/python tts_server.py > logs/tts_server.log 2>&1 & +CUDA_VISIBLE_DEVICES=2 STT_PORT=8902 nohup python3 stt_server.py > logs/stt_server.log 2>&1 & +``` + +They're pinned to `CUDA_VISIBLE_DEVICES=2` deliberately — GPU 1 tends to fill up with Ollama's resident models (`OLLAMA_KEEP_ALIVE=-1` keeps them loaded), so TTS/STT get GPU 2 to themselves rather than risking an OOM race with Ollama. + +Reachability from the `open_notebook` container requires **two things**, both already done: the `extra_hosts: host.docker.internal:host-gateway` entry in `docker-compose.yml`, and UFW rules allowing inbound `8901/tcp` and `8902/tcp` (same pattern as the existing `11434/tcp` rule for Ollama — UFW defaults to deny-incoming, so any new host-side port a container needs to reach must be explicitly opened). + +Registered in Open Notebook as `openai_compatible` credentials with `base_url=http://host.docker.internal:8901` (TTS) / `:8902` (STT), and set as `default_text_to_speech_model` / `default_speech_to_text_model`. + ## Secrets `.env` (gitignored) holds `OPEN_NOTEBOOK_ENCRYPTION_KEY` (encrypts provider credentials stored in SurrealDB), `SURREAL_PASSWORD`, and `OPENROUTER_API_KEY`. `.env.example` is the committed template. Provider API keys can also be entered directly in the Open Notebook UI after first startup — the project's own docs recommend this over env vars for better security since they get encrypted at rest. diff --git a/services/stt_server.py b/services/stt_server.py new file mode 100644 index 0000000..01047ca --- /dev/null +++ b/services/stt_server.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python3 +"""OpenAI-compatible STT wrapper around faster-whisper, for Open Notebook. + +Implements exactly the endpoint esperanto's OpenAICompatibleSpeechToTextModel calls: + POST /audio/transcriptions (multipart: file, model, language?, prompt?) -> {"text": ...} + +Start: + python3 stt_server.py + +Env vars: + STT_HOST (default 0.0.0.0), STT_PORT (default 8902) + WHISPER_MODEL (default large-v3), WHISPER_COMPUTE_TYPE (default float16) + CUDA_VISIBLE_DEVICES should be set by the caller (e.g. "1,2") to keep GPU 0 free. +""" +from __future__ import annotations + +import os +import tempfile +from pathlib import Path + +from faster_whisper import WhisperModel +from fastapi import FastAPI, File, Form, UploadFile + +app = FastAPI(title="faster-whisper STT (OpenAI-compatible)", version="1.0") + +_MODEL_SIZE = os.environ.get("WHISPER_MODEL", "large-v3") +_COMPUTE_TYPE = os.environ.get("WHISPER_COMPUTE_TYPE", "float16") +_model = WhisperModel(_MODEL_SIZE, device="cuda", compute_type=_COMPUTE_TYPE) + + +@app.get("/health") +def health(): + return {"status": "ok", "model": _MODEL_SIZE} + + +@app.post("/audio/transcriptions") +async def transcriptions( + file: UploadFile = File(...), + model: str = Form(default="whisper-1"), + language: str | None = Form(default=None), + prompt: str | None = Form(default=None), +): + suffix = Path(file.filename or "audio").suffix or ".wav" + with tempfile.NamedTemporaryFile(suffix=suffix, delete=False) as tmp: + tmp.write(await file.read()) + tmp_path = tmp.name + + try: + segments, _info = _model.transcribe( + tmp_path, + language=language, + initial_prompt=prompt, + ) + text = "".join(segment.text for segment in segments).strip() + finally: + os.unlink(tmp_path) + + return {"text": text} + + +if __name__ == "__main__": + import uvicorn + uvicorn.run(app, host=os.environ.get("STT_HOST", "0.0.0.0"), + port=int(os.environ.get("STT_PORT", "8902"))) diff --git a/services/tts_server.py b/services/tts_server.py new file mode 100644 index 0000000..6c9b6ab --- /dev/null +++ b/services/tts_server.py @@ -0,0 +1,112 @@ +#!/usr/bin/env python3 +"""OpenAI-compatible TTS wrapper around chatterbox-tts, for Open Notebook. + +Implements exactly the endpoint esperanto's OpenAICompatibleTextToSpeechModel calls: + POST /audio/speech {model, voice, input, response_format} -> raw audio bytes + +Start: + ~/miniforge3/envs/chatterbox/bin/python tts_server.py + +Env vars: + TTS_HOST (default 0.0.0.0), TTS_PORT (default 8901) + CUDA_VISIBLE_DEVICES should be set by the caller (e.g. "1,2") to keep GPU 0 free. +""" +from __future__ import annotations + +import os +import subprocess +import sys +import tempfile +from pathlib import Path + +sys.path.insert(0, str(Path.home() / "chatterbox-tts-cli")) +import chatterbox_cli_v4 as tts # noqa: E402 + +import torch +import torchaudio as ta +from fastapi import FastAPI, HTTPException, Response +from pydantic import BaseModel + +app = FastAPI(title="Chatterbox TTS (OpenAI-compatible)", version="1.0") + +_DEVICE = tts.get_device(None) +_model_cache: dict[str, tuple] = {} + + +def _get_model(lang: str): + key = "en" if lang == "en" else "multi" + if key not in _model_cache: + _model_cache[key] = tts.load_model(lang, _DEVICE, t3_model="v3") + return _model_cache[key] + + +class SpeechRequest(BaseModel): + model: str | None = None + voice: str = "de" + input: str + response_format: str = "mp3" + + +_FORMAT_CONTENT_TYPE = { + "mp3": "audio/mpeg", + "wav": "audio/wav", + "opus": "audio/opus", + "flac": "audio/flac", + "aac": "audio/aac", +} + + +@app.get("/health") +def health(): + return {"status": "ok", "device": _DEVICE} + + +@app.get("/audio/voices") +def voices(): + return {"voices": [{"id": lang, "name": lang} for lang in sorted(tts.SUPPORTED_LANGS)]} + + +@app.post("/audio/speech") +def speech(req: SpeechRequest): + lang = req.voice if req.voice in tts.SUPPORTED_LANGS else "de" + + raw = tts.clean_raw_text(req.input) + raw_chunks = tts.split_into_sentences(raw, max_len=400) + chunks = [tts.preprocess_tts_text(c, lang=lang, pronunciation_dict=None) for c in raw_chunks] + chunks = [c for c in chunks if c.strip()] + if not chunks: + raise HTTPException(status_code=422, detail="Kein synthetisierbarer Text übrig.") + + model, model_kind, sr = _get_model(lang) + + wavs = [] + for chunk in chunks: + wavs.append(tts.generate_chunk(model, model_kind, chunk, lang, None)) + final = wavs[0] if len(wavs) == 1 else torch.cat(wavs, dim=-1) + + with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as wav_tmp: + wav_path = wav_tmp.name + ta.save(wav_path, final, sr) + + fmt = req.response_format if req.response_format in _FORMAT_CONTENT_TYPE else "wav" + try: + if fmt == "wav": + audio_bytes = Path(wav_path).read_bytes() + else: + out_path = wav_path.replace(".wav", f".{fmt}") + subprocess.run( + ["ffmpeg", "-y", "-i", wav_path, out_path], + check=True, capture_output=True, + ) + audio_bytes = Path(out_path).read_bytes() + os.unlink(out_path) + finally: + os.unlink(wav_path) + + return Response(content=audio_bytes, media_type=_FORMAT_CONTENT_TYPE[fmt]) + + +if __name__ == "__main__": + import uvicorn + uvicorn.run(app, host=os.environ.get("TTS_HOST", "0.0.0.0"), + port=int(os.environ.get("TTS_PORT", "8901")))