diff --git a/.gitignore b/.gitignore index e00fe4c..479d860 100644 --- a/.gitignore +++ b/.gitignore @@ -32,3 +32,7 @@ venv/ 3 TODO.md + +# Local voice models +config/voices/*.onnx +config/voices/*.onnx.json diff --git a/BEDIENUNGSANLEITUNG.md b/BEDIENUNGSANLEITUNG.md index 1c5be23..c396874 100644 --- a/BEDIENUNGSANLEITUNG.md +++ b/BEDIENUNGSANLEITUNG.md @@ -135,7 +135,7 @@ pip install -e .[local] # installiert faster-whisper + piper-tts ### 2.2 Installation ```bash -cd voice-assistant-scaffold +cd my_voice_assistant_v3 python3 -m venv .venv source .venv/bin/activate pip install -U pip @@ -647,7 +647,7 @@ VA_PROFILE=hybrid make run # oder: systemctl --user restart voice-assis **Prüfen** (egal welche Richtung): ```bash -curl http://127.0.0.1:11434/v1/models # Ollama (bzw. :8001 für llama.cpp) +curl -s http://127.0.0.1:11434/v1/models | jq # Ollama (bzw. :8001 für llama.cpp) # danach im Admin → Status den LLM-Provider/das Modell kontrollieren oder kurz testen ``` @@ -776,16 +776,16 @@ https://va.beispiel.de/ ← remote über Reverse-Proxy (alles, inkl. Mikr ┌─────────────────────────────────────────────────────────┐ │ 👄 Voice Assistant ............... [🇩🇪 ▾] [ ⋮ ] │ ├─────────────────────────────────────────────────────────┤ ┌─ Menü (⋮) ──────────────┐ -│ │ │ VORLESEN │ -│ Nachrichtenverlauf │ │ ▣ 📱 Im Gerät │ -│ (eigene Nachrichten: blaue Blase rechts) │ │ ▢ ☁ Server │ -│ (Assistent: graue Blase links) │ │ ▢ ✨ Beste Qualität │ +│ Angemeldet als ....................... Abmelden │ │ VORLESEN │ +├─────────────────────────────────────────────────────────┤ │ ▣ 📱 Im Gerät │ +│ Nachrichtenverlauf │ │ ▢ ⚡ Schnell │ +│ (eigene Nachrichten: blaue Blase rechts) │ │ ▢ ✨ Hohe Qualität │ +│ (Assistent: graue Blase links) │ │ ▢ ☁ Cloud │ │ │ │ ────────────────────── │ ├─────────────────────────────┬───────────────────────────┤ │ ✎ Neues Gespräch │ │ Texteingabe … [Senden] │ [🎤] │ │ 🌙 Tag-/Nachtmodus │ -└─────────────────────────────┴───────────────────────────┘ │ ────────────────────── │ - Statuszeile: „denkt …" / „verarbeite Sprache …" / leer │ Angemeldet · Admin · Ab │ - └──────────────────────────┘ +└─────────────────────────────┴───────────────────────────┘ │ ⚙ Admin-Bereich │ + Statuszeile: „denkt …" / „verarbeite Sprache …" / leer └──────────────────────────┘ ``` **Kopfzeile (immer schlank):** @@ -796,14 +796,17 @@ https://va.beispiel.de/ ← remote über Reverse-Proxy (alles, inkl. Mikr | **Sprache ▾** | Antwortsprache (→ § 6.6): `🔄 Flex` = folgt der gesprochenen Sprache · feste Sprache (🇩🇪/🇬🇧/…) = Antwort + Stimme in dieser Sprache | | **⋮ Menü** | Öffnet die Einstellungen (unten) | +**Identitätsleiste (direkt unter der Kopfzeile):** links „Angemeldet als …" (SSO-Identität; +„Gast" ohne SSO), rechts **Abmelden** (Link zum SSO-Logout). + **Im ⋮-Menü:** | Element | Funktion | |---------|----------| -| **Vorlesen** | Wie wird die Antwort vorgelesen? `📱 Im Gerät` = das Handy liest selbst vor (kein Server-Audio → spart Daten) · `☁ Server` = piper (lokal, zuverlässig) · `✨ Beste Qualität` = chatterbox (natürliche Stimme). „Im Gerät" erscheint nur, wo der Browser es unterstützt. | +| **Vorlesen** | Wie wird die Antwort vorgelesen? `📱 Im Gerät` = das Handy liest selbst vor (kein Server-Audio → spart Daten; fällt auf Server zurück, wenn der Browser keine Stimmen hat) · `⚡ Schnell` = piper (lokal) · `✨ Hohe Qualität` = chatterbox (natürliche Stimme) · `☁ Cloud` = OpenRouter (Internet nötig). „Im Gerät" erscheint nur, wo der Browser es unterstützt. | | **✎ Neues Gespräch** | Frische Sitzung (Verlauf zurücksetzen) | | **🌙 Tag-/Nachtmodus** | Heller/dunkler Modus; folgt sonst dem Betriebssystem | -| **Konto** | „Angemeldet als …", **⚙ Admin** (nur Admins → § 7.5), **Abmelden** | +| **⚙ Admin-Bereich** | Öffnet das Admin-Panel — nur für Admin-Nutzer sichtbar (→ § 7.5) | **Unten:** @@ -1321,18 +1324,18 @@ Der Cache wird automatisch geleert. **Weg 2 — REST-API** (alle Sprachen, sofort wirksam): ```bash # Französischen Eintrag hinzufügen (kein Neustart nötig): -curl -X POST http://localhost:8080/api/admin/pronunciation/fr \ - -H "Authorization: Bearer " \ +curl -s -X POST $URL/api/admin/pronunciation/fr \ + -H "X-Admin-Key: $ADMIN_API_KEY" \ -H "Content-Type: application/json" \ - -d '{"section":"terms","key":"Schlüter","value":"Chluteur"}' + -d '{"section":"terms","key":"Schlüter","value":"Chluteur"}' | jq # Eintrag löschen: -curl -X DELETE http://localhost:8080/api/admin/pronunciation/fr/terms/Schlüter \ - -H "Authorization: Bearer " +curl -s -X DELETE $URL/api/admin/pronunciation/fr/terms/Schlüter \ + -H "X-Admin-Key: $ADMIN_API_KEY" | jq # Alle Einträge einer Sprache anzeigen: -curl http://localhost:8080/api/admin/pronunciation/ru \ - -H "Authorization: Bearer " +curl -s $URL/api/admin/pronunciation/ru \ + -H "X-Admin-Key: $ADMIN_API_KEY" | jq ``` **Weg 3 — YAML-Datei direkt editieren** (alle Sprachen): @@ -1353,7 +1356,7 @@ Spricht den Testsatz mit der aktuell eingestellten Stimme und Sprache. **Oder via curl:** ```bash -curl -s -X POST http://localhost:8080/api/speak \ +curl -s -X POST $URL/api/speak \ -H "Authorization: Bearer $TOKEN" \ -H "Content-Type: application/json" \ -d '{"text":"Hallo, ich bin Dieter Schlüter.","tts_provider":"piper","language":"fr"}' \ @@ -1408,12 +1411,12 @@ DEFAULT_LANGUAGE_MODE=fix # global: fix | flex (Admin: Tab „⚙ Einstellungen Pro Nutzer (dauerhaft): ```bash # Feste Sprache (Fix): -curl -X PUT $URL/api/me/prefs -H "Authorization: Bearer $TOKEN" \ - -H 'Content-Type: application/json' -d '{"language":"en","language_mode":"fix"}' +curl -s -X PUT $URL/api/me/prefs -H "Authorization: Bearer $TOKEN" \ + -H 'Content-Type: application/json' -d '{"language":"en","language_mode":"fix"}' | jq # Flex (Sprache folgt automatisch): -curl -X PUT $URL/api/me/prefs -H "Authorization: Bearer $TOKEN" \ - -H 'Content-Type: application/json' -d '{"language_mode":"flex"}' +curl -s -X PUT $URL/api/me/prefs -H "Authorization: Bearer $TOKEN" \ + -H 'Content-Type: application/json' -d '{"language_mode":"flex"}' | jq ``` Pro Aufruf: `{"text":"…","language":"en","language_mode":"fix"}` im Body. @@ -1894,7 +1897,7 @@ curl -s -X POST $URL/api/me/memories \ curl -s $URL/api/me/memories -H "Authorization: Bearer $TOKEN" | jq # Erinnerung löschen: -curl -s -X DELETE $URL/api/me/memories/ -H "Authorization: Bearer $TOKEN" +curl -s -X DELETE $URL/api/me/memories/ -H "Authorization: Bearer $TOKEN" | jq ``` In der Datenbank direkt ansehen/löschen (nötig z. B. wenn eine falsch extrahierte diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..4eb982d --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,69 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +> Sprache: Code-Kommentare, Doku und Commits sind auf Deutsch. Antworte und kommentiere auf Deutsch. + +## Befehle + +```bash +make install # venv anlegen + pip install -e .[test] +make run # Gateway (uvicorn --reload, Port aus .env, Standard 8080) +make test # gesamte Pytest-Suite (offline, ohne Netz/Kosten) +make smoke # echter End-to-End-Test gegen OpenRouter (kostet wenig) +make stop / restart # Gateway + llama.cpp stoppen / neu starten + +# Einzelner Test / einzelner Fall: +. .venv/bin/activate && pytest tests/test_routing.py +. .venv/bin/activate && pytest tests/test_routing.py::test_name -q +``` + +Profil beim Start wählen (überschreibt `.env`): `VA_PROFILE=cloud|hybrid|local-dev make run`. +Lokales LLM (llama.cpp-Docker) für `hybrid`/`local-dev`: `make llm-up` → `make llm-status` → `make llm-down`. +Backend wechseln (llama.cpp ↔ Ollama, teilen sich die GPU, nie gleichzeitig): `make llm-llamacpp` / `make llm-ollama`. + +Vor `make run`: wenn `.env` fehlt, wird sie aus `.env.example` erzeugt. Für Cloud/Hybrid `OPENROUTER_API_KEY` setzen. + +## Architektur (großes Bild) + +Modulares FastAPI-Gateway für einen seniorengerechten Sprachassistenten. Leitprinzip: **jede Achse (STT/LLM/TTS, Audio-I/O, lokal vs. remote) austauschbar, ohne Kern-Code zu ändern.** Ausführliche Referenz: `Docs/voice-assistant-architecture.md` und `BEDIENUNGSANLEITUNG.md`. + +### Zwei zentrale Mechanismen, die man verstehen muss + +1. **Geschichtete Konfiguration mit Profilen** (`app/config.py`). Präzedenz, höhere Ebene gewinnt: + `Defaults < config/voice-assistant.toml ([defaults]+[profiles.]) < ENV/.env < Nutzer-Prefs < Session-Route < Request`. + `VA_PROFILE` aktiviert ein Profil; `TomlProfileSource` hängt die TOML-Werte als pydantic-settings-Quelle **unter** ENV ein. **Secrets nie in die TOML** — nur in die Umgebung. Profile: `cloud` (alles OpenRouter), `hybrid` (LLM lokal), `local-dev` (alles lokal). + +2. **Route-Auflösung + Registries** (`app/dependencies.py`). `resolve_route(user, session_id, overrides)` liefert pro Aufruf eine `ResolvedRoute` (Endpunkte, STT/LLM/TTS-Provider, Sprache) nach obiger Präzedenz. Provider kommen aus `STT_REGISTRY`/`LLM_REGISTRY`/`TTS_REGISTRY` (`name -> factory(settings)`). **Neuen Provider = ein Registry-Eintrag + ABC-Implementierung** in `app/providers/{stt,llm,tts}/`. Unbekannter Name → `UnknownComponentError` → HTTP 422. + +### Speech-Pipeline (`app/pipeline/`, orchestriert in `app/core/orchestrator.py`) + +Kernidee: **semantische** Antwort ≠ **gesprochene** Antwort. Stufen: +`raw_transcript → cleaned_transcript (input_cleaner) → semantic_response (LLM) → spoken_response (spoken_response_adapter) → tts_ready_text (tts_normalizer) → TTS`. + +- `spoken_response_adapter`: macht LLM-Text sprechbar (Markdown raus, Listen → Ordinalwörter). +- `tts_normalizer`: füllt nur Phonemizer-Lücken (Ordinalia, Einheiten, Abkürzungen, YAML-Lexikon `config/pronunciation..yaml`). Stufe provider-abhängig (`TTS_NORMALIZE_LEVEL`): piper→`full`, Cloud-TTS→`light`. Deutsche Ordinalzahlen in `german_numbers.py`. +- `sentence_chunker`: inkrementelle Satzsegmentierung fürs satzweise Streaming-TTS; trennt bewusst **nicht** bei „1. Mai", „z. B." usw. + +Der Orchestrator schreibt synthetisiertes Audio **zusätzlich** in den Output-Endpunkt (`open→write_chunk→flush→close`) **und** gibt es als HTTP-Stream zurück. Bei `/api/chat` mit `session_id` lädt die API-Schicht den Verlauf aus dem Store, gibt ihn als `history` ans LLM und speichert beide Turns; ohne `session_id` zustandslos. + +### Querschnitt + +- **Persistenz** (`app/store.py`): `Store`-Interface + `SQLiteStore` (Nutzer, Sessions, Verlauf, Erinnerungen) — DB in `data/` (gitignored). +- **Auth** (`app/auth.py`): Bearer-Token (`require_user`) + Admin-Schutz (`require_admin`). `main.py` hat zwei Middlewares: Metriken + Web-UI-Gate (statische Seiten hinter Auth/SSO; `/api`, `/ws`, `/health` ausgenommen). `AUTH_ENABLED=false` → anonymer Nutzer (LAN-Dev). +- **Resilienz** (`app/providers/fallback.py`): Fallback-Ketten je Modul (`*_FALLBACK`). Metriken in `app/metrics.py` (`/api/metrics`, JSON + Prometheus). Tageskontingent `app/quota.py` (429). +- **Sicherheit** (`app/safety/`): zweistufige Notfall-Eskalation — schnelle Stichwort-Heuristik im Hot-Path (`emergency.py`) + LLM-Klassifikation als nicht-blockierender Hintergrund-Task (`llm_classifier.py`). +- **Auto-Erinnerungen** (`app/core/memory_extractor.py`): nach N Turns destilliert ein LLM dauerhafte Fakten als Hintergrund-Task. +- **Echtzeit** (`app/api/ws.py`): `/ws/chat` (Token-/Audio-Streaming) und `/ws/voice` (Audio→STT→Pipeline, VAD, Barge-in via `interrupt`). +- **Admin** (`app/api/admin.py`, `app/admin_llm.py`, `app/audit.py`, `app/runtime_config.py`): Live-Config, Backend-Wechsel, Gateway-Neustart, Lexika, Live-Log — schreibende Aktionen werden ins Audit-Log geschrieben. +- **Frontend** (`app/web/`): Tailwind ohne Build-Schritt, mobiltauglich. Geräte-TTS (Browser-SpeechSynthesis) mit Fallback auf Server-Audio. + +### Bewusste Platzhalter (nicht „kaputt") + +Audio-Endpunkte (`app/audio/endpoints/`) liefern leere Chunks — nur Auswahl/Lifecycle sind verdrahtet, kein echtes Hardware-I/O. `app/audio/transport_router.py` (Ebene 4 lokal/remote) existiert, ist aber noch nicht aktiv; lokal/remote trägt vorerst der Provider-Name. + +## Provider & lokale Module + +- Cloud: OpenRouter (STT/LLM/TTS). Lokales LLM: OpenAI-kompatibel (`local_openai_compatible.py`) gegen llama.cpp oder Ollama. +- Lokale KI ist **optional**: `pip install -e .[local]` (faster-whisper, piper). Beim Serverstart werden lokale Modelle vorgeladen (`app/core/warmup.py`). +- `chatterbox`-TTS (eigener GPU-HTTP-Dienst, Voice-Cloning) ist als wählbarer Provider angebunden (job-basiert `/speak`→`/status`→`/audio`). diff --git a/Docs/voice-assistant-architecture.md b/Docs/voice-assistant-architecture.md index 66b5315..15b3e49 100644 --- a/Docs/voice-assistant-architecture.md +++ b/Docs/voice-assistant-architecture.md @@ -170,7 +170,6 @@ ein No-op; `LoopbackOutput` sammelt die Chunks (testbar ohne Hardware). | `GET /api/devices` | verfügbare Audio-Endpunkte + Capabilities | | `POST /api/sessions/{id}/route` | bevorzugte Geräte/Provider/Sprache je Session | | `GET /api/config` | aktives Profil + aufgelöste Route (ohne Secrets) | -| `POST /api/admin/users` | Nutzer anlegen (Admin-Key) → Token einmalig | | `GET /api/me` · `PUT /api/me/prefs` | aktueller Nutzer + dauerhafte Präferenzen | | `GET/POST/DELETE /api/me/memories` | Langzeit-Erinnerungen des Nutzers | | `GET /api/metrics` | Metriken (JSON / Prometheus) | @@ -181,6 +180,32 @@ Endpunkt-/Provider-Auswahl ist über **Request-Body** (pro Aufruf), **Session** (`?session_id=…`) und **Defaults/Profil** steuerbar. Verwendete Route erscheint als `X-*`-Header bzw. im `?debug`-JSON. +### 7.1 Admin-API (alle hinter `require_admin`, Audit-geloggt) + +Trägt das Admin-Web-Panel (5 Bereiche + Übersicht-Dashboard). Schreibende Aktionen +werden ins Audit-Log geschrieben. + +| Methode & Pfad | Zweck | +|--------------------------------------------------|-------| +| `POST /api/admin/users` | Nutzer anlegen → Token einmalig | +| `GET /api/admin/users` | Nutzerliste | +| `PUT/DELETE /api/admin/users/{id}` | Nutzer ändern/löschen | +| `POST /api/admin/users/{id}/token` | Token neu ausstellen | +| `GET/POST/DELETE /api/admin/users/{id}/memories` | Erinnerungen je Nutzer | +| `GET /api/admin/users/{id}/sessions`·`/usage` | Sessions / Kontingent-Nutzung | +| `GET /api/admin/sessions/{id}/messages` | Gesprächsverlauf einsehen | +| `GET/PUT/DELETE /api/admin/config[/{key}]` | Live-Config lesen/setzen (z. B. `top_p`) | +| `GET /api/admin/llm/status` | LLM-/GPU-Status (read-only) | +| `POST /api/admin/llm/backend` | Backend wechseln (Ollama ↔ llama.cpp) | +| `POST /api/admin/gateway/restart` | Gateway aus dem Panel neu starten | +| `GET/POST/DELETE /api/admin/pronunciation/{lang}`| Aussprache-Lexika pflegen | +| `GET /api/admin/usage`·`/emergency-events` | Gesamt-Nutzung / Notfall-Ereignisse | +| `GET /api/admin/db-export` | SQLite-Export | +| `WS /api/admin/log` | Live-Log-Stream | + +Config-Änderungen sind als **Live** (sofort wirksam) oder **Restart** (Neustart nötig) +gekennzeichnet; der Backend-Wechsel und Live-Parameter wie `top_p` laufen ohne Neustart. + ## 8. Stand der Implementierung **Umgesetzt:** FastAPI-Gateway, alle o. g. REST-Endpunkte; OpenRouter-Adapter für @@ -195,7 +220,15 @@ Store, fließt ins LLM)**; **Langzeit-Erinnerungen pro Nutzer (als LLM-Kontext)* über WebSocket (`/ws/voice`: Audio rein → STT → Antwort-Pipeline) mit VAD-Aeusserungs- erkennung und Barge-in (`interrupt`)**; **Resilienz (Fallback-Ketten je Modul, In-Memory-Metriken `/api/metrics`), Tageskontingent pro Nutzer und heuristische -Notfall-Eskalation**; automatisierte Tests. +Notfall-Eskalation**; **Admin-Web-Panel (5 Bereiche + Übersicht-Dashboard) über die +Admin-API (§7.1) inkl. Backend-Wechsel, Gateway-Neustart, Live-Config-Parameter, +LLM-/GPU-Status, Aussprache-Lexika, Live-Log und Audit-Logging schreibender +Aktionen**; automatisierte Tests. + +**Web-/Mobil-Frontend:** schlankes Web-Interface unter `/` (Tailwind, kein Build), +mit **Geräte-TTS** (Browser-SpeechSynthesis auf Mobilgeräten; fällt auf Server-Audio +zurück, wenn keine lokalen Stimmen vorhanden) und **Ton-Presets** (Schnell → piper, +Hohe Qualität → chatterbox, Cloud → openrouter). Auth-Gate (Bearer-Token). **Echtes lokales STT & TTS:** `faster-whisper` (optionale Dependency `.[local]`, CTranslate2) transkribiert real; `piper` (in-process via piper-Python-API, Stimmmodell @@ -228,23 +261,28 @@ Verschlüsselung at-rest/in-transit, Löschkonzept, Einwilligung — „privacy ## 10. Verzeichnisstruktur (Ist) ```text -voice-assistant-scaffold/ +my_voice_assistant_v3/ ├── app/ │ ├── main.py # FastAPI-App + Router-Registrierung │ ├── config.py # Settings, TOML-Profile, Präzedenz │ ├── dependencies.py # Registries, ResolvedRoute, resolve_route, Store-/Router-Singleton │ ├── store.py # Persistenz: Store-Interface + SQLiteStore (Nutzer/Sessions/Verlauf) │ ├── auth.py # Bearer-Token-Auth (require_user) + Admin-Schutz +│ ├── audit.py # Audit-Log schreibender Admin-Aktionen +│ ├── admin_llm.py # LLM-/GPU-Status + Backend-Wechsel (Ollama ↔ llama.cpp) +│ ├── runtime_config.py # Live-Config (zur Laufzeit setzbare Parameter) │ ├── metrics.py # In-Memory-Metriken (Counter/Timer, JSON + Prometheus) │ ├── quota.py # Tageskontingent pro Nutzer (Kostenkontrolle) -│ ├── safety/ # emergency.py: heuristische Notfall-Erkennung/-Eskalation +│ ├── safety/ # emergency.py (Heuristik) + llm_classifier.py (Notfall-Klassifikation) │ ├── errors.py # RoutingError -> HTTP 422 │ ├── schemas.py # Pydantic-Modelle │ ├── api/ # health, chat, speak, transcribe, devices, sessions, config, admin, me, ws -│ ├── core/ # orchestrator +│ ├── core/ # orchestrator, memory_extractor (Auto-Erinnerungen), warmup (Modell-Vorladen) │ ├── audio/ # router, transport_router, vad, endpoints/input|output/* │ ├── pipeline/ # input_cleaner, spoken_response_adapter, tts_normalizer, sentence_chunker, german_numbers -│ └── providers/ # stt/ llm/ tts/ (openrouter + lokale Stubs) + fallback.py +│ ├── providers/ # stt/ llm/ tts/ (openrouter + lokale + chatterbox) + fallback.py +│ ├── utils/ # Hilfsfunktionen +│ └── web/ # Web-/Admin-Frontend (Tailwind, kein Build) ├── config/ # voice-assistant.example.toml (+ lokale .toml, gitignored) ├── data/ # SQLite-DB (gitignored) ├── deploy/ # systemd unit + env-Beispiel diff --git a/README.md b/README.md index fc1fc9f..d28ed79 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,8 @@ Jede Achse — Hardware, Betrieb, Software — ist frei konfigurierbar, ohne Cod - **Gesprächsgedächtnis** pro Session (Verlauf) + Langzeit-Erinnerungen pro Nutzer (manuell + automatisch) - **WebSocket-Streaming:** Token-Streaming (LLM), Audio-Streaming (satzweises TTS), Sprach-Eingang, VAD, Barge-in - **Notfall-Eskalation:** zweistufig (Stichwörter + LLM-Klassifikation im Hintergrund) -- **Web-Interface** unter `/` (Tailwind, kein Build, mobiltauglich) +- **Web-Interface** unter `/` (Tailwind, kein Build, mobiltauglich) mit Geräte-TTS (Browser-Sprachausgabe) und Ton-Presets (Schnell/Hohe Qualität/Cloud) +- **Admin-Panel** (5 Bereiche + Übersicht-Dashboard): Nutzer/Sessions, LLM-/GPU-Status, Backend-Wechsel + Gateway-Neustart, Live-Config, Aussprache-Lexika, Live-Log, Audit-Logging - **Keine Secrets im Code** — API-Keys nur über die Umgebung --- @@ -193,7 +194,7 @@ make llm-status # Container- und HTTP-Status prüfen ## Projektstruktur (Kurzform) ```text -app/ Gateway: config, api/, core/, audio/, pipeline/, providers/ +app/ Gateway: config, api/, core/, audio/, pipeline/, providers/, web/ (Frontend + Admin) config/ voice-assistant.example.toml (lokale .toml ist gitignored) deploy/ systemd-Unit, nginx-Vorlage, env-Beispiel scripts/ voice_loop.py, chat_client.py, smoke_e2e.py, add_pronunciation.py diff --git a/app/api/chat.py b/app/api/chat.py index f56a0db..b0a3fa6 100644 --- a/app/api/chat.py +++ b/app/api/chat.py @@ -13,6 +13,7 @@ from app.dependencies import ( resolve_output_endpoint, get_store, piper_voice_for_language, + voice_for_route, ) from app.core.memory_extractor import maybe_schedule_extraction from app.quota import enforce_quota, record_usage, QuotaExceededError @@ -96,7 +97,7 @@ async def chat( raise HTTPException(status_code=429, detail=str(exc)) # Explizit angeforderte Stimme gewinnt; sonst folgt sie der Routensprache. - voice = payload.voice or piper_voice_for_language(route.tts_provider, route.language) + voice = payload.voice or voice_for_route(route.tts_provider, route.language, route.voice_gender) try: trace, audio = await orchestrator.chat_text( diff --git a/app/api/config.py b/app/api/config.py index 6b1f1e5..c939955 100644 --- a/app/api/config.py +++ b/app/api/config.py @@ -1,3 +1,4 @@ +import httpx from fastapi import APIRouter from app.config import settings, active_profile @@ -12,14 +13,60 @@ from app.dependencies import ( router = APIRouter() +async def _chatterbox_reachable() -> bool: + try: + async with httpx.AsyncClient(timeout=1.5) as client: + r = await client.get(f"{settings.chatterbox_base_url.rstrip('/')}/status") + return r.status_code < 500 + except Exception: + return False + + +async def _cartesia_available() -> bool: + key = settings.cartesia_api_key.strip() + voice_id = settings.cartesia_voice_id.strip() + if not key or not voice_id: + return False + try: + async with httpx.AsyncClient(timeout=5.0) as client: + r = await client.get( + "https://api.cartesia.ai/voices", + headers={"X-API-Key": key, "Cartesia-Version": "2024-06-10"}, + ) + return r.status_code < 400 + except Exception: + return False + + +async def _openrouter_tts_available() -> bool: + """Prüft ob das konfigurierte TTS-Modell auf OpenRouter via /audio/speech erreichbar ist.""" + key = settings.openrouter_api_key.strip() + model = settings.openrouter_tts_model.strip() + if not key or not model: + return False + try: + async with httpx.AsyncClient(timeout=5.0) as client: + r = await client.post( + "https://openrouter.ai/api/v1/audio/speech", + headers={"Authorization": f"Bearer {key}", "Content-Type": "application/json"}, + json={"model": model, "input": "Test", "voice": settings.openrouter_tts_voice or "alloy", + "response_format": "mp3"}, + ) + # 200 = Audio OK, 4xx mit "does not exist" = Modell fehlt + if r.status_code == 200: + return True + body = r.text + return "does not exist" not in body and r.status_code < 500 + except Exception: + return False + + @router.get("/config") async def get_config(): - """Zeigt aktives Profil, aufgeloeste Default-Route und verfuegbare Bausteine. - - Bewusst OHNE Secrets - API-Keys werden nur als 'gesetzt/nicht gesetzt' gemeldet. - """ + """Zeigt aktives Profil, aufgeloeste Default-Route und verfuegbare Bausteine.""" route = resolve_route() audio_router = get_audio_router() + chatterbox_ok, openrouter_tts_ok, cartesia_ok = await _chatterbox_reachable(), await _openrouter_tts_available(), await _cartesia_available() return { "profile": active_profile(), "app_env": settings.app_env, @@ -30,6 +77,9 @@ async def get_config(): "tts_providers": sorted(TTS_REGISTRY), "input_endpoints": [c.model_dump() for c in await audio_router.list_inputs()], "output_endpoints": [c.model_dump() for c in await audio_router.list_outputs()], + "chatterbox_reachable": chatterbox_ok, + "openrouter_tts_available": openrouter_tts_ok, + "cartesia_available": cartesia_ok, }, "secrets": { "openrouter_api_key_set": bool(settings.openrouter_api_key.strip()), diff --git a/app/api/speak.py b/app/api/speak.py index a39e8d1..e93428f 100644 --- a/app/api/speak.py +++ b/app/api/speak.py @@ -12,6 +12,7 @@ from app.dependencies import ( resolve_output_endpoint, get_store, piper_voice_for_language, + voice_for_route, ) from app.quota import enforce_quota, record_usage, QuotaExceededError from app.schemas import SpeakRequest @@ -36,7 +37,7 @@ async def speak( try: route = resolve_route(user, session_id, overrides) # Explizit angefragte Stimme gewinnt; sonst folgt sie der Sprache (Piper-Parität zu /chat). - voice = payload.voice or piper_voice_for_language(route.tts_provider, route.language) + voice = payload.voice or voice_for_route(route.tts_provider, route.language, route.voice_gender) orchestrator = build_orchestrator(route) output = await resolve_output_endpoint(route) except SessionOwnershipError as exc: diff --git a/app/api/ws.py b/app/api/ws.py index 18d03ad..4199f5c 100644 --- a/app/api/ws.py +++ b/app/api/ws.py @@ -27,6 +27,7 @@ from app.dependencies import ( build_orchestrator, resolve_output_endpoint, piper_voice_for_language, + voice_for_route, ) from app.store import ANONYMOUS_USER_ID, SessionOwnershipError from app.audio.vad import EnergyVAD @@ -122,7 +123,7 @@ async def _run_turn( elif effective_voice is not None: voice = effective_voice else: - voice = piper_voice_for_language(route.tts_provider, route.language) + voice = voice_for_route(route.tts_provider, route.language, route.voice_gender) stream = bool(options.get("stream")) # text_only: Geräte-TTS (Web Speech API) spricht selbst -> kein Server-Audio erzeugen/senden. text_only = bool(options.get("text_only")) @@ -233,15 +234,15 @@ async def _voice_turn(websocket, store, user, session_id, audio, fmt, options): audio, fmt=fmt, language=None ) # Stimme folgt der erkannten Sprache (Fallback: Routensprache). - effective_voice = piper_voice_for_language( - route.tts_provider, detected_language or route.language + effective_voice = voice_for_route( + route.tts_provider, detected_language or route.language, route.voice_gender ) else: # Fix: konfigurierte Sprache an Whisper → Whisper übersetzt automatisch. transcript = await orchestrator.stt.transcribe(audio, fmt=fmt, language=route.language) detected_language = None # Stimme folgt der konfigurierten Sprache. - effective_voice = piper_voice_for_language(route.tts_provider, route.language) + effective_voice = voice_for_route(route.tts_provider, route.language, route.voice_gender) except Exception as exc: await websocket.send_json({"type": "error", "status": 502, "detail": str(exc)}) return diff --git a/app/config.py b/app/config.py index 14ce23f..bb2109a 100644 --- a/app/config.py +++ b/app/config.py @@ -141,6 +141,11 @@ class Settings(BaseSettings): piper_voice: str = "de_DE-thorsten-high" # Stimmmodell-Name (ohne .onnx) oder voller Pfad tts_sample_rate: int = 24000 # Ziel-Sample-Rate (das Gateway erwartet 24000 Hz) # --- Chatterbox-TTS (hohe Qualitaet + Voice-Cloning, eigener HTTP-Dienst) - + # --- Cartesia TTS (Cloud, sehr niedrige Latenz) ------------------------- + cartesia_api_key: str = "" + cartesia_voice_id: str = "" # UUID aus cartesia.ai/voices (Fallback) + cartesia_tts_model: str = "sonic-turbo" + cartesia_voices: str = "" # "lang:gender:uuid:name" je Zeile, # = Kommentar # sonic-2 | sonic-multilingual chatterbox_base_url: str = "http://127.0.0.1:9999" chatterbox_voice: str = "" # Pfad zu Referenz-WAV (Voice-Cloning) oder leer chatterbox_lang: str = "de" diff --git a/app/dependencies.py b/app/dependencies.py index 9179ae5..b58407d 100644 --- a/app/dependencies.py +++ b/app/dependencies.py @@ -18,6 +18,7 @@ from app.providers.stt.faster_whisper import FasterWhisperProvider from app.providers.llm.local_openai_compatible import LocalOpenAICompatibleLLM from app.providers.llm.openrouter import OpenRouterLLMProvider from app.providers.tts.openrouter import OpenRouterTTSProvider +from app.providers.tts.cartesia import CartesiaTTSProvider from app.providers.tts.chatterbox import ChatterboxTTSProvider from app.providers.tts.piper import PiperTTSProvider from app.providers.fallback import ( @@ -80,6 +81,10 @@ TTS_REGISTRY = { s.chatterbox_timeout, voices_dir=s.chatterbox_voices_dir, ), + "cartesia": lambda s: CartesiaTTSProvider( + s.cartesia_api_key, s.cartesia_voice_id, s.cartesia_tts_model, s.tts_sample_rate, + voices_str=s.cartesia_voices, + ), "piper": lambda s: PiperTTSProvider( s.piper_bin, s.piper_voices_dir, s.piper_voice, s.tts_sample_rate ), @@ -151,32 +156,74 @@ ROUTE_KEYS = ( "tts_provider", "language", "language_mode", + "voice_gender", ) -# Stimm-Auswahl nach Sprache: (effektive) Sprache → beste Piper-Stimme. -# Gilt in BEIDEN Modi — die gesprochene Stimme folgt immer der Antwortsprache. +# Stimm-Auswahl nach Sprache: (effektive) Sprache → beste Piper-Stimme (gender-agnostisch). LANG_TO_PIPER_VOICE: dict[str, str] = { "de": "de_DE-thorsten-high", "en": "en_US-lessac-high", "fr": "fr_FR-siwis-medium", "es": "es_ES-sharvard-medium", "it": "it_IT-paola-medium", - "nl": "nl_NL-mls-medium", + "pt": "pt_BR-faber-medium", + "pl": "pl_PL-darkman-medium", + "ar": "ar_JO-kareem-medium", "ru": "ru_RU-irina-medium", "zh": "zh_CN-huayan-medium", "cmn": "zh_CN-huayan-medium", } +# Geschlechtsspezifische Piper-Stimmen für Sprachen mit m+f Varianten. +# Präzedenz: PIPER_VOICE_GENDERED > LANG_TO_PIPER_VOICE. +# ES sharvard ist ein Multi-Speaker-Modell: #0=M, #1=F (speaker_id_map: {'M':0,'F':1}) +PIPER_VOICE_GENDERED: dict[tuple[str, str], str] = { + ("de", "f"): "de_DE-kerstin-low", + ("de", "m"): "de_DE-thorsten-high", + ("en", "f"): "en_US-amy-medium", + ("en", "m"): "en_US-lessac-high", + ("fr", "f"): "fr_FR-siwis-medium", + ("fr", "m"): "fr_FR-tom-medium", + ("es", "f"): "es_ES-sharvard-medium#1", + ("es", "m"): "es_ES-sharvard-medium#0", + ("it", "f"): "it_IT-paola-medium", + ("it", "m"): "it_IT-riccardo-x_low", + ("ru", "f"): "ru_RU-irina-medium", + ("ru", "m"): "ru_RU-ruslan-medium", + ("pl", "f"): "pl_PL-gosia-medium", + ("pl", "m"): "pl_PL-darkman-medium", +} + def piper_voice_for_language(tts_provider: str, language: str | None) -> str | None: - """Liefert die zur Sprache passende Piper-Stimme (sonst None → Provider-Default). + """Kompatibilitäts-Alias für voice_for_route ohne gender.""" + return voice_for_route(tts_provider, language, "any") - Nur für Piper sinnvoll: Cloud-TTS (OpenRouter) und Chatterbox haben eigene - Stimmnamen (z. B. "Zephyr") und steuern die Sprache anders. Bei None nimmt der - jeweilige Provider seine konfigurierte Default-Stimme. + +def voice_for_route( + tts_provider: str, language: str | None, voice_gender: str = "any" +) -> str | None: + """Liefert den voice-Parameter für synthesize() passend zum Provider. + + - piper: Piper-Stimmmodell-Name; geschlechtsspezifisch wenn Varianten vorhanden + - cartesia: Geschlechts-Code ("m"/"f"/"any") → CartesiaTTSProvider löst UUID auf + - andere: None (Provider nutzt seinen eigenen Default) """ if tts_provider == "piper" and language: - return LANG_TO_PIPER_VOICE.get(language) + lang = language.lower() + gender = (voice_gender or "any").lower() + # Geschlechtsspezifische Stimme: bevorzugte Richtung, dann Gegenteil + if gender in ("m", "f"): + candidates = [gender, "f" if gender == "m" else "m"] + else: # "any" → weiblich bevorzugt (konsistent mit Cartesia) + candidates = ["f", "m"] + for g in candidates: + voice = PIPER_VOICE_GENDERED.get((lang, g)) + if voice: + return voice + return LANG_TO_PIPER_VOICE.get(lang) + if tts_provider == "cartesia": + return voice_gender or "any" return None @@ -189,6 +236,7 @@ class ResolvedRoute: tts_provider: str language: str language_mode: str = "fix" + voice_gender: str = "any" def as_dict(self) -> dict: return { @@ -199,6 +247,7 @@ class ResolvedRoute: "tts_provider": self.tts_provider, "language": self.language, "language_mode": self.language_mode, + "voice_gender": self.voice_gender, } diff --git a/app/providers/llm/openrouter.py b/app/providers/llm/openrouter.py index 408afa7..f29864a 100644 --- a/app/providers/llm/openrouter.py +++ b/app/providers/llm/openrouter.py @@ -1,3 +1,5 @@ +import asyncio +import time from collections.abc import AsyncIterator import httpx @@ -9,6 +11,37 @@ from app.providers.llm.base import ( sse_delta, ) +_RETRY_STATUS = {404, 429, 500, 502, 503} + + +def _retry_wait(status: int, headers: httpx.Headers, attempt: int, body: str = "") -> float: + """Wartezeit: JSON retry_after_seconds > Retry-After > X-RateLimit-Reset > exponentiell.""" + if status == 429: + if body: + try: + import json as _json + d = _json.loads(body) + secs = d.get("error", {}).get("metadata", {}).get("retry_after_seconds") + if secs: + return max(1.0, min(float(secs) + 0.5, 120.0)) + except Exception: + pass + retry_after = headers.get("Retry-After") + if retry_after: + try: + return max(1.0, min(float(retry_after) + 0.5, 120.0)) + except ValueError: + pass + reset_ms = headers.get("X-RateLimit-Reset") + if reset_ms: + try: + wait = (int(reset_ms) / 1000.0) - time.time() + 0.5 + return max(1.0, min(wait, 120.0)) + except ValueError: + pass + return 30.0 + return 2.0 * attempt + SYSTEM_PROMPT = """ You are a voice assistant for spoken conversations with older adults. @@ -52,9 +85,10 @@ Always optimize your answer for listening, not for reading. class OpenRouterLLMProvider(LLMProvider): - def __init__(self, api_key: str, model: str): + def __init__(self, api_key: str, model: str, max_retries: int = 5): self.api_key = (api_key or "").strip() self.model = (model or "").strip() + self.max_retries = max(1, max_retries) def _build_messages( self, text: str, history: list[dict] | None, language: str | None = None @@ -74,7 +108,6 @@ class OpenRouterLLMProvider(LLMProvider): messages = [{"role": "system", "content": system_content}] if history: messages.extend(history) - # Sprach-Erinnerung direkt an der letzten Nutzer-Nachricht (schlägt History-Trägheit). messages.append({"role": "user", "content": with_lang_reminder(text.strip(), language)}) return messages @@ -89,40 +122,52 @@ class OpenRouterLLMProvider(LLMProvider): "model": self.model, "messages": self._build_messages(text, history, language=language), } - timeout = httpx.Timeout(connect=10.0, read=120.0, write=30.0, pool=10.0) + last_error: Exception | None = None - async with httpx.AsyncClient(timeout=timeout) as client: + for attempt in range(1, self.max_retries + 1): + async with httpx.AsyncClient(timeout=timeout) as client: + try: + response = await client.post( + "https://openrouter.ai/api/v1/chat/completions", + headers={ + "Authorization": f"Bearer {self.api_key}", + "Content-Type": "application/json", + }, + json=payload, + ) + if response.status_code in _RETRY_STATUS and attempt < self.max_retries: + last_error = RuntimeError( + f"OpenRouter LLM error {response.status_code}: {response.text}" + ) + await asyncio.sleep(_retry_wait(response.status_code, response.headers, attempt, response.text)) + continue + response.raise_for_status() + except httpx.HTTPStatusError as exc: + last_error = RuntimeError( + f"OpenRouter LLM error {exc.response.status_code}: {exc.response.text}" + ) + if exc.response.status_code in _RETRY_STATUS and attempt < self.max_retries: + await asyncio.sleep(_retry_wait(exc.response.status_code, exc.response.headers, attempt, exc.response.text)) + continue + raise last_error from exc + except httpx.TimeoutException as exc: + raise RuntimeError("OpenRouter LLM timeout") from exc + except httpx.HTTPError as exc: + raise RuntimeError(f"OpenRouter LLM transport error: {exc}") from exc + + data = response.json() try: - response = await client.post( - "https://openrouter.ai/api/v1/chat/completions", - headers={ - "Authorization": f"Bearer {self.api_key}", - "Content-Type": "application/json", - }, - json=payload, - ) - response.raise_for_status() - except httpx.HTTPStatusError as exc: - raise RuntimeError( - f"OpenRouter LLM error {exc.response.status_code}: {exc.response.text}" - ) from exc - except httpx.TimeoutException as exc: - raise RuntimeError("OpenRouter LLM timeout") from exc - except httpx.HTTPError as exc: - raise RuntimeError(f"OpenRouter LLM transport error: {exc}") from exc + content = data["choices"][0]["message"]["content"] + except (KeyError, IndexError, TypeError) as exc: + raise RuntimeError(f"Unexpected OpenRouter LLM response: {data}") from exc - data = response.json() + if not content or not str(content).strip(): + raise RuntimeError("OpenRouter LLM returned empty content") - try: - content = data["choices"][0]["message"]["content"] - except (KeyError, IndexError, TypeError) as exc: - raise RuntimeError(f"Unexpected OpenRouter LLM response: {data}") from exc + return str(content).strip() - if not content or not str(content).strip(): - raise RuntimeError("OpenRouter LLM returned empty content") - - return str(content).strip() + raise last_error or RuntimeError("OpenRouter LLM: all attempts failed") async def stream( self, @@ -137,30 +182,42 @@ class OpenRouterLLMProvider(LLMProvider): "stream": True, } timeout = httpx.Timeout(connect=10.0, read=120.0, write=30.0, pool=10.0) + last_error: Exception | None = None - async with httpx.AsyncClient(timeout=timeout) as client: - try: - async with client.stream( - "POST", - "https://openrouter.ai/api/v1/chat/completions", - headers={ - "Authorization": f"Bearer {self.api_key}", - "Content-Type": "application/json", - }, - json=payload, - ) as response: - if response.status_code >= 400: - body = await response.aread() - raise RuntimeError( - f"OpenRouter LLM error {response.status_code}: " - f"{body.decode(errors='replace')}" - ) - async for line in response.aiter_lines(): - delta = sse_delta(line) - if delta: - yield delta - except httpx.TimeoutException as exc: - raise RuntimeError("OpenRouter LLM timeout") from exc - except httpx.HTTPError as exc: - raise RuntimeError(f"OpenRouter LLM transport error: {exc}") from exc + for attempt in range(1, self.max_retries + 1): + async with httpx.AsyncClient(timeout=timeout) as client: + try: + async with client.stream( + "POST", + "https://openrouter.ai/api/v1/chat/completions", + headers={ + "Authorization": f"Bearer {self.api_key}", + "Content-Type": "application/json", + }, + json=payload, + ) as response: + if response.status_code in _RETRY_STATUS and attempt < self.max_retries: + body = await response.aread() + last_error = RuntimeError( + f"OpenRouter LLM error {response.status_code}: " + f"{body.decode(errors='replace')}" + ) + await asyncio.sleep(_retry_wait(response.status_code, response.headers, attempt, body.decode(errors='replace'))) + continue + if response.status_code >= 400: + body = await response.aread() + raise RuntimeError( + f"OpenRouter LLM error {response.status_code}: " + f"{body.decode(errors='replace')}" + ) + async for line in response.aiter_lines(): + delta = sse_delta(line) + if delta: + yield delta + return + except httpx.TimeoutException as exc: + raise RuntimeError("OpenRouter LLM timeout") from exc + except httpx.HTTPError as exc: + raise RuntimeError(f"OpenRouter LLM transport error: {exc}") from exc + raise last_error or RuntimeError("OpenRouter LLM: all attempts failed") diff --git a/app/providers/tts/cartesia.py b/app/providers/tts/cartesia.py new file mode 100644 index 0000000..fbcbe21 --- /dev/null +++ b/app/providers/tts/cartesia.py @@ -0,0 +1,142 @@ +import httpx + +from app.providers.tts.base import TTSProvider + +_API_URL = "https://api.cartesia.ai/tts/bytes" +_API_VERSION = "2024-06-10" + +# Sprachen, die sonic-turbo unterstützt (Stand 2026-06). +_LANG_MAP = { + "de": "de", "en": "en", "fr": "fr", "es": "es", + "zh": "zh", "cmn": "zh", "pt": "pt", "ja": "ja", "ko": "ko", "hi": "hi", +} + +# sonic-turbo unterstützt diese Sprachen nicht — Fallback auf Piper empfohlen. +_UNSUPPORTED_LANGS = {"ar", "it", "ru", "nl", "pl", "sv", "tr"} + +_LANG_NAMES = { + "ar": "Arabisch", "it": "Italienisch", "ru": "Russisch", + "nl": "Niederländisch", "pl": "Polnisch", "sv": "Schwedisch", "tr": "Türkisch", +} + +_GENDER_CODES = {"m", "f", "any"} + + +def _parse_voice_map(voices_str: str) -> dict[tuple[str, str], dict]: + """Parst 'lang:gender:uuid:name'-Zeilen in ein (lang,gender)→{uuid,name}-Dict.""" + result = {} + for line in (voices_str or "").splitlines(): + line = line.strip() + if not line or line.startswith("#"): + continue + parts = line.split(":", 3) + if len(parts) != 4: + continue + lang, gender, uuid, name = [p.strip() for p in parts] + if lang and gender in ("m", "f") and uuid: + result[(lang, gender)] = {"uuid": uuid, "name": name} + return result + + +class CartesiaTTSProvider(TTSProvider): + def __init__( + self, + api_key: str, + voice_id: str, + model: str = "sonic-2", + sample_rate: int = 24000, + voices_str: str = "", + ): + self.api_key = (api_key or "").strip() + self.default_voice_id = (voice_id or "").strip() + self.model = (model or "sonic-2").strip() + self.sample_rate = int(sample_rate) + self.voice_map = _parse_voice_map(voices_str) + + def _resolve_voice_id(self, language: str | None, gender_hint: str | None) -> str: + """Wählt UUID aus Voice-Map. Fallback-Kette: f→m→any→default_voice_id.""" + lang = (language or "de").lower() + gender = (gender_hint or "any").strip().lower() + + # Präferenz-Reihenfolge: "any" → weiblich bevorzugt + if gender == "any" or gender not in ("m", "f"): + candidates = ["f", "m"] + elif gender == "f": + candidates = ["f", "m"] + else: + candidates = ["m", "f"] + + for g in candidates: + entry = self.voice_map.get((lang, g)) + if entry: + return entry["uuid"] + + return self.default_voice_id + + async def synthesize( + self, + text: str, + voice: str | None = None, + audio_format: str = "pcm", + language: str | None = None, + ) -> bytes: + if not self.api_key: + raise ValueError("CARTESIA_API_KEY is empty") + if not text or not text.strip(): + raise ValueError("TTS input text is empty") + + # voice ist entweder ein Geschlechts-Code ("m"/"f"/"any") oder eine direkte UUID + if voice and voice not in _GENDER_CODES: + voice_id = voice # direkte UUID-Übergabe + else: + voice_id = self._resolve_voice_id(language, voice) + + if not voice_id: + raise ValueError("CARTESIA_VOICE_ID ist nicht konfiguriert — bitte UUID in voice-assistant.toml eintragen") + + lang_key = (language or "de").lower() + if lang_key in _UNSUPPORTED_LANGS: + lang_label = _LANG_NAMES.get(lang_key, lang_key) + raise RuntimeError( + f"Cartesia unterstützt {lang_label} ({lang_key.upper()}) noch nicht als TTS-Sprache" + " — bitte Piper oder einen anderen Provider wählen" + ) + lang_code = _LANG_MAP.get(lang_key, "de") + + payload = { + "model_id": self.model, + "transcript": text.strip(), + "voice": {"mode": "id", "id": voice_id}, + "output_format": { + "container": "raw", + "encoding": "pcm_s16le", + "sample_rate": self.sample_rate, + }, + "language": lang_code, + } + + timeout = httpx.Timeout(connect=10.0, read=60.0, write=10.0, pool=10.0) + async with httpx.AsyncClient(timeout=timeout) as client: + try: + response = await client.post( + _API_URL, + headers={ + "X-API-Key": self.api_key, + "Cartesia-Version": _API_VERSION, + "Content-Type": "application/json", + }, + json=payload, + ) + response.raise_for_status() + except httpx.HTTPStatusError as exc: + raise RuntimeError( + f"Cartesia TTS error {exc.response.status_code}: {exc.response.text}" + ) from exc + except httpx.TimeoutException as exc: + raise RuntimeError("Cartesia TTS timeout") from exc + except httpx.HTTPError as exc: + raise RuntimeError(f"Cartesia TTS transport error: {exc}") from exc + + if not response.content: + raise RuntimeError("Cartesia TTS returned empty audio") + return response.content diff --git a/app/providers/tts/piper.py b/app/providers/tts/piper.py index e1d102c..f19cd04 100644 --- a/app/providers/tts/piper.py +++ b/app/providers/tts/piper.py @@ -104,6 +104,17 @@ class PiperTTSProvider(TTSProvider): except (OSError, KeyError, ValueError, TypeError): return self.target_rate # Konfig unlesbar -> Resampling überspringen + @staticmethod + def _parse_speaker(voice: str | None) -> tuple[str | None, int | None]: + """Trennt 'stimme#speaker_id' auf. Liefert (stimme, int_id) oder (stimme, None).""" + if voice and "#" in voice: + name, sid = voice.rsplit("#", 1) + try: + return name, int(sid) + except ValueError: + pass + return voice, None + async def synthesize( self, text: str, @@ -114,9 +125,10 @@ class PiperTTSProvider(TTSProvider): if not text or not text.strip(): raise ValueError("TTS input text is empty") - model, config = self._model_paths(voice) + voice_name, speaker_id = self._parse_speaker(voice) + model, config = self._model_paths(voice_name) pcm, native_rate = await asyncio.to_thread( - self._synthesize_sync, str(model), str(config), text.strip() + self._synthesize_sync, str(model), str(config), text.strip(), speaker_id ) if native_rate != self.target_rate: pcm = await asyncio.to_thread(_resample, pcm, native_rate, self.target_rate) @@ -125,26 +137,30 @@ class PiperTTSProvider(TTSProvider): return _wrap_wav(pcm, self.target_rate) return pcm - def _synthesize_sync(self, model: str, config: str, text: str) -> tuple[bytes, int]: + def _synthesize_sync(self, model: str, config: str, text: str, speaker_id: int | None = None) -> tuple[bytes, int]: if _PIPER_LIB: - voice = _load_voice(model) + from piper.config import SynthesisConfig + piper_voice = _load_voice(model) + syn_cfg = SynthesisConfig(speaker_id=speaker_id) if speaker_id is not None else None pcm = bytearray() rate = self.target_rate - for chunk in voice.synthesize(text): + for chunk in piper_voice.synthesize(text, syn_config=syn_cfg): pcm += chunk.audio_int16_bytes rate = chunk.sample_rate if not pcm: raise RuntimeError("Piper lieferte kein Audio") return bytes(pcm), rate # Fallback: piper-Binary (Subprozess pro Aufruf, langsamer). - return self._run_piper_binary(model, config, text) + return self._run_piper_binary(model, config, text, speaker_id) - def _run_piper_binary(self, model: str, config: str, text: str) -> tuple[bytes, int]: + def _run_piper_binary(self, model: str, config: str, text: str, speaker_id: int | None = None) -> tuple[bytes, int]: import subprocess if not (shutil.which(self.bin_path) or Path(self.bin_path).exists()): raise RuntimeError(f"Piper-Binary nicht gefunden: {self.bin_path}") cmd = [self.bin_path, "-m", model, "-c", config, "--output-raw"] + if speaker_id is not None: + cmd += ["--speaker", str(speaker_id)] proc = subprocess.run(cmd, input=text.encode("utf-8"), capture_output=True) if proc.returncode != 0: detail = proc.stderr.decode("utf-8", "replace").strip()[:300] diff --git a/app/web/app.js b/app/web/app.js index 95c7cd3..234caa9 100644 --- a/app/web/app.js +++ b/app/web/app.js @@ -20,6 +20,29 @@ function langOverrides() { } // Nutzer-Präferenzen als Request-Overrides für jeden Turn. +// Geschlechtspräferenz: 'm', 'f', oder 'any' (= weiblich bevorzugt) +let _voiceGender = 'f'; + +function applyGenderChoice(gender) { + // 'any' aus alten Prefs → weiblich als Standardwert + _voiceGender = (gender === 'm') ? 'm' : 'f'; + const btnM = document.getElementById('gender-m'); + const btnF = document.getElementById('gender-f'); + if (!btnM || !btnF) return; + const mActive = (_voiceGender === 'm'); + const fActive = (_voiceGender === 'f'); + // Aktiv: blauer Hintergrund + weiße Schrift + // Inaktiv: kein Hintergrund + dunkle Schrift (explizit, damit kein Vererbungsproblem) + btnM.classList.toggle('bg-blue-600', mActive); + btnM.classList.toggle('text-white', mActive); + btnM.classList.toggle('border-blue-600', mActive); + btnM.classList.toggle('text-slate-700', !mActive); + btnF.classList.toggle('bg-blue-600', fActive); + btnF.classList.toggle('text-white', fActive); + btnF.classList.toggle('border-blue-600', fActive); + btnF.classList.toggle('text-slate-700', !fActive); +} + function overrides() { const out = { ...langOverrides() }; if (isDeviceMode()) { @@ -27,6 +50,7 @@ function overrides() { } else if (ttsSel && ttsSel.value) { out.tts_provider = ttsSel.value; } + out.voice_gender = _voiceGender; return out; } @@ -45,7 +69,7 @@ async function savePrefs(obj) { const TTS_SUPPORTED = typeof window !== "undefined" && "speechSynthesis" in window; const LANG_BCP47 = { de: "de-DE", en: "en-US", fr: "fr-FR", es: "es-ES", - it: "it-IT", nl: "nl-NL", ru: "ru-RU", zh: "zh-CN", + it: "it-IT", pt: "pt-BR", pl: "pl-PL", ar: "ar-SA", ru: "ru-RU", zh: "zh-CN", }; function isMobile() { @@ -60,7 +84,9 @@ let _voices = []; function loadVoices() { _voices = TTS_SUPPORTED ? speechSynthesis.getVoices() : []; } if (TTS_SUPPORTED) { loadVoices(); - speechSynthesis.addEventListener("voiceschanged", loadVoices); + // Stimmen laden asynchron -> bei Eintreffen Preset-Zustand neu bewerten (entgrauen). + speechSynthesis.addEventListener("voiceschanged", () => { loadVoices(); refreshDevicePreset(); }); + setTimeout(() => refreshDevicePreset(), 1500); // Backstop, falls kein voiceschanged kommt } function pickVoice(bcp) { @@ -139,13 +165,6 @@ const PLAYBACK_KEY = "va-playback"; // geräte-lokal: "device" wenn Geräte-TTS // Vorrang: 1. lokale Wahl (localStorage), 2. Server-Pref (tts_provider), 3. Mobil-Default = Gerät. function applyPlaybackChoice(serverProvider) { if (!ttsSel) return; - // "Im Gerät"-Preset ausblenden, wenn der Browser keine Web Speech API hat. - if (!TTS_SUPPORTED) { - const opt = ttsSel.querySelector('option[value="device"]'); - if (opt) opt.remove(); - const btn = document.querySelector('.tts-preset[data-tts="device"]'); - if (btn) btn.classList.add("hidden"); - } const local = localStorage.getItem(PLAYBACK_KEY); if (local === "device" && TTS_SUPPORTED) { ttsSel.value = "device"; @@ -157,15 +176,101 @@ function applyPlaybackChoice(serverProvider) { } else if (!ttsSel.value) { ttsSel.value = "piper"; // Fallback: Server } + refreshDevicePreset(); // graut „Im Gerät" aus, wenn keine Stimmen + syncPresetUI +} + +// Graut „Im Gerät" aus, wenn keine Sprachstimmen vorhanden sind; entgraut automatisch, +// sobald welche da sind. Kein Aufzwingen: device wird nur (wieder) gewählt, wenn es die +// gespeicherte Präferenz ist. Ohne Stimmen NICHT auf der deaktivierten Option sitzen bleiben. +function refreshDevicePreset() { + const btn = document.querySelector('.tts-preset[data-tts="device"]'); + if (!btn || !ttsSel) return; + const ready = deviceVoicesReady(); + btn.disabled = !ready; + btn.classList.toggle("opacity-50", !ready); + btn.classList.toggle("cursor-not-allowed", !ready); + const sub = btn.querySelector(".tts-sub"); + if (sub) sub.textContent = ready + ? "Das Gerät liest vor · spart Daten" + : "keine Sprachstimmen im Browser"; + if (ready) { + if (localStorage.getItem(PLAYBACK_KEY) === "device" && ttsSel.value !== "device") { + ttsSel.value = "device"; // gespeicherte Präferenz wieder aktivieren + } + } else if (ttsSel.value === "device") { + ttsSel.value = "piper"; // Server statt deaktivierter Option (Präferenz bleibt) + } syncPresetUI(); } +// Graut "Hohe Qualität" (Chatterbox) aus, wenn kein GPU-Backend erreichbar ist. +async function refreshChatterboxPreset() { + const btn = document.querySelector('.tts-preset[data-tts="chatterbox"]'); + if (!btn || !ttsSel) return; + try { + const cfg = await fetch("/api/config", { cache: "no-store" }).then(r => r.json()); + const available = cfg.available?.chatterbox_reachable === true; + btn.disabled = !available; + btn.classList.toggle("opacity-50", !available); + btn.classList.toggle("cursor-not-allowed", !available); + const sub = btn.querySelector(".tts-sub"); + if (sub) sub.textContent = available + ? "Chatterbox · natürliche Stimme" + : "Chatterbox · kein GPU-Backend erreichbar"; + if (!available && ttsSel.value === "chatterbox") { + ttsSel.value = "piper"; + syncPresetUI(); + } + } catch (e) { /* ignorieren */ } +} + +// Graut "Cartesia" aus, wenn API-Key oder Voice-ID fehlt. +async function refreshCartesiaPreset() { + const btn = document.querySelector('.tts-preset[data-tts="cartesia"]'); + if (!btn || !ttsSel) return; + try { + const cfg = await fetch("/api/config", { cache: "no-store" }).then(r => r.json()); + const available = cfg.available?.cartesia_available === true; + btn.disabled = !available; + btn.classList.toggle("opacity-50", !available); + btn.classList.toggle("cursor-not-allowed", !available); + const sub = btn.querySelector(".tts-sub"); + if (sub) sub.textContent = available + ? "Cartesia Sonic · natürlich & schnell" + : "Cartesia · kein API-Key konfiguriert"; + if (!available && ttsSel.value === "cartesia") { + ttsSel.value = "piper"; + syncPresetUI(); + } + } catch (e) { /* ignorieren */ } +} + +// Graut "Cloud" (OpenRouter TTS) aus, wenn das TTS-Modell nicht verfügbar ist. +async function refreshOpenrouterTtsPreset() { + const btn = document.querySelector('.tts-preset[data-tts="openrouter"]'); + if (!btn || !ttsSel) return; + try { + const cfg = await fetch("/api/config", { cache: "no-store" }).then(r => r.json()); + const available = cfg.available?.openrouter_tts_available === true; + btn.disabled = !available; + btn.classList.toggle("opacity-50", !available); + btn.classList.toggle("cursor-not-allowed", !available); + const sub = btn.querySelector(".tts-sub"); + if (sub) sub.textContent = available + ? "OpenRouter · benötigt Internet" + : "OpenRouter · kein TTS-Modell verfügbar"; + if (!available && ttsSel.value === "openrouter") { + ttsSel.value = "piper"; + syncPresetUI(); + } + } catch (e) { /* ignorieren */ } +} + // Markiert das aktive Ton-Preset (Häkchen + Rahmen) passend zu #tts. function syncPresetUI() { const val = ttsSel ? ttsSel.value : ""; document.querySelectorAll(".tts-preset").forEach((btn) => { - const active = btn.dataset.tts === val || - (val === "openrouter" && btn.dataset.tts === "chatterbox"); // Cloud -> "Beste Qualität" + const active = btn.dataset.tts === val; btn.classList.toggle("ring-2", active); btn.classList.toggle("ring-blue-500", active); btn.classList.toggle("border-blue-500", active); @@ -223,9 +328,10 @@ function loadMe() { // Flex-Modus -> "flex", sonst die feste Sprache (Fallback: Flex). langSel.value = prefs.language_mode === "flex" ? "flex" - : (prefs.language || "flex"); + : (prefs.language || "de"); } applyPlaybackChoice(prefs.tts_provider); + applyGenderChoice(prefs.voice_gender || 'f'); return me; } catch (e) { $("#identity").textContent = "Verbindung fehlgeschlagen"; @@ -273,6 +379,7 @@ function addMessage(role, text, meta = {}) { div.className = BUBBLE[role] || BUBBLE.assistant; const span = document.createElement("span"); span.className = "bubble-text"; + span.dir = "auto"; span.textContent = text; div.appendChild(span); div._textEl = span; @@ -666,7 +773,25 @@ if (ttsSel) { }); } + +// Gender-Buttons +document.addEventListener("DOMContentLoaded", () => { + const btnM = document.getElementById("gender-m"); + const btnF = document.getElementById("gender-f"); + if (btnM) btnM.addEventListener("click", () => { + applyGenderChoice("m"); + savePrefs({ voice_gender: "m" }); + }); + if (btnF) btnF.addEventListener("click", () => { + applyGenderChoice("f"); + savePrefs({ voice_gender: "f" }); + }); +}); + loadMe(); +refreshChatterboxPreset(); +refreshCartesiaPreset(); +refreshOpenrouterTtsPreset(); // ═══════════════════════════════════════════════════════════════ // ADMIN PANEL @@ -1552,12 +1677,12 @@ const FIELD_META = { default_stt_provider: { ui: "combo", opts: ["openrouter","faster-whisper"], test: "stt" }, default_llm_provider: { ui: "combo", opts: ["openrouter","local-openai-compatible"], test: "llm" }, default_tts_provider: { ui: "combo", opts: ["openrouter","piper","chatterbox"], test: "tts" }, - default_language: { ui: "combo", opts: ["de","en","fr","es","it","nl"], - labels: { de:"Deutsch",en:"Englisch",fr:"Französisch",es:"Spanisch",it:"Italienisch",nl:"Niederländisch" } }, + default_language: { ui: "combo", opts: ["de","en","fr","es","it","pt","pl","ar"], + labels: { de:"Deutsch",en:"Englisch",fr:"Französisch",es:"Spanisch",it:"Italienisch",pt:"Portugiesisch",pl:"Polnisch",ar:"Arabisch" } }, openrouter_llm_model: { ui: "combo", opts: ["google/gemini-3.1-flash-lite","openai/gpt-4.1-mini","google/gemini-2.5-flash","anthropic/claude-haiku-4-5-20251001","meta-llama/llama-3.3-70b-instruct"], test: "llm" }, openrouter_tts_model: { ui: "combo", opts: ["google/gemini-3.1-flash-tts-preview","openai/gpt-4o-mini-tts"], test: "tts", testProvider: "openrouter" }, openrouter_tts_voice: { ui: "combo", opts: ["Zephyr","Puck","Charon","Kore","Fenrir","Leda","Orus","Aoede","Callirrhoe","Enceladus","Algieba","Despina","Achernar"], test: "tts", testProvider: "openrouter" }, - piper_voice: { ui: "combo", opts: ["de_DE-thorsten-high","en_US-ryan-high","en_US-lessac-high","en_GB-cori-high","es_ES-sharvard-medium","fr_FR-siwis-medium","it_IT-paola-medium","nl_NL-mls-medium","ru_RU-irina-medium","zh_CN-huayan-medium"], test: "tts", testProvider: "piper" }, + piper_voice: { ui: "combo", opts: ["de_DE-thorsten-high","de_DE-kerstin-low","en_US-lessac-high","en_US-amy-medium","en_US-ryan-high","en_GB-cori-high","fr_FR-siwis-medium","fr_FR-tom-medium","es_ES-sharvard-medium","it_IT-paola-medium","it_IT-riccardo-x_low","pt_BR-faber-medium","pl_PL-gosia-medium","pl_PL-darkman-medium","ar_JO-kareem-medium","ru_RU-irina-medium","ru_RU-ruslan-medium","zh_CN-huayan-medium"], test: "tts", testProvider: "piper" }, local_llm_system_prompt: { ui: "textarea", test: "llm" }, local_llm_temperature: { ui: "range", min: 0, max: 2, step: 0.1, test: "llm" }, local_llm_top_p: { ui: "range", min: 0, max: 1, step: 0.05, test: "llm" }, diff --git a/app/web/index.html b/app/web/index.html index 002d30f..ea91fc4 100644 --- a/app/web/index.html +++ b/app/web/index.html @@ -163,7 +163,9 @@ - + + + @@ -225,7 +227,9 @@ - + + + @@ -233,11 +237,18 @@ class="h-9 w-9 grid place-items-center rounded-lg border border-slate-300 dark:border-slate-600 hover:bg-slate-100 dark:hover:bg-slate-700 text-xl leading-none transition-colors">⋮ + +
+ lade … + +
+ @@ -255,23 +266,53 @@ class="tts-preset w-full flex items-center gap-3 rounded-lg border border-slate-200 dark:border-slate-700 px-3 py-2 text-left hover:bg-slate-50 dark:hover:bg-slate-700/50 transition-colors"> 📱 Im Gerät - Das Gerät liest vor · spart Daten + Das Gerät liest vor · spart Daten ✓ + + + + + + + +
+

Stimme

+
+ +
@@ -287,16 +328,11 @@ 🌙 Tag-/Nachtmodus - - - -
-

lade …

-
- - -
+
@@ -306,7 +342,7 @@
-