my_voice_assistant_v3_jamulix/app/api/ws.py

"""WebSocket-Echtzeit-Chat und -Sprache.

- /ws/chat : Text rein (JSON pro Turn), Antwort als Event-Folge zurueck.
- /ws/voice: Audio rein (binaere Chunks + Control), Transkription -> selbe Pipeline.

Antwort-Events: ack -> [token*] -> [audio*] -> semantic -> done.
Mit {"stream":true} kommen LLM-Token live, mit {"audio_stream":true} das Audio
satzweise (chunked TTS). /ws/voice sendet zuvor ein transcript-Event.

Barge-in: Ein {"type":"interrupt"}-Frame oder eine neue Eingabe bricht eine laufende
Antwort ab (-> interrupted-Event). Der Antwort-Turn laeuft als abbrechbarer Task.

Spaeter (eigene Increments): echte partielle Live-Transkripte (Streaming-STT-Dienst),
WebRTC.
"""

import asyncio
import json

from fastapi import APIRouter, WebSocket, WebSocketDisconnect

from app.config import settings
from app.errors import RoutingError
from app.dependencies import (
    get_store,
    resolve_route,
    build_orchestrator,
    resolve_output_endpoint,
    piper_voice_for_language,
    voice_for_route,
)
from app.store import ANONYMOUS_USER_ID, SessionOwnershipError
from app.audio.vad import EnergyVAD
from app.core.memory_extractor import maybe_schedule_extraction
from app.quota import enforce_quota, record_usage, QuotaExceededError
from app.auth import authenticate, _cookie_value, CAPABILITY_COOKIE

router = APIRouter()

_OVERRIDE_KEYS = (
    "input_endpoint",
    "output_endpoint",
    "language",
    "stt_provider",
    "llm_provider",
    "tts_provider",
)


def _capability_token(websocket: WebSocket, token: str | None) -> str | None:
    """Token aus ?token=, ?k= oder dem va_token-Cookie (Senioren-Login)."""
    return (
        token
        or websocket.query_params.get("k")
        or _cookie_value(websocket.headers.get("cookie"), CAPABILITY_COOKIE)
    )


def _authenticate(websocket: WebSocket, token: str | None):
    # Forward-Auth (SSO) greift auch beim WS-Handshake: SSOwat injiziert den
    # Identitaets-Header in den Upgrade-Request -> aus websocket.headers lesbar.
    client_host = websocket.client.host if websocket.client else ""
    return authenticate(websocket.headers, client_host, _capability_token(websocket, token))


async def _resolve(user, session_id, options):
    """Loest Route + Orchestrator + Output-Endpunkt auf (kann RoutingError/Ownership werfen)."""
    overrides = {key: options.get(key) for key in _OVERRIDE_KEYS}
    route = resolve_route(user, session_id, overrides)
    orchestrator = build_orchestrator(route)
    output = await resolve_output_endpoint(route)
    return route, orchestrator, output


async def _run_turn(
    websocket, store, user, session_id, route, orchestrator, output, text, options,
    effective_voice: str | None = None,
):
    """Faehrt einen Antwort-Turn und streamt die Events an den Client."""
    conversation = (
        store.get_recent_messages(session_id, settings.history_max_messages)
        if session_id
        else []
    )
    memories = store.get_memories(user.id)
    llm_context = list(conversation)
    user_context_parts = []
    if user.id != ANONYMOUS_USER_ID:
        user_context_parts.append(f"Du sprichst mit {user.display_name}.")
    if memories:
        user_context_parts.append(
            "Was du ueber den Nutzer weisst:\n" + "\n".join(f"- {m.content}" for m in memories)
        )
    if user_context_parts:
        llm_context = [{"role": "system", "content": "\n".join(user_context_parts)}] + llm_context

    try:
        enforce_quota(user, store)
    except QuotaExceededError as exc:
        await websocket.send_json({"type": "error", "status": 429, "detail": str(exc)})
        return

    await websocket.send_json({"type": "ack", "route": route.as_dict()})

    # Stimme: explizit angeforderte gewinnt, sonst die vom Caller (Sprach-Turn)
    # vorberechnete sprachpassende Stimme, sonst folgt sie der Routensprache
    # (greift v. a. beim Text-Chat ohne Spracherkennung).
    explicit_voice = options.get("voice")
    if explicit_voice:
        voice = explicit_voice
    elif effective_voice is not None:
        voice = effective_voice
    else:
        voice = voice_for_route(route.tts_provider, route.language, route.voice_gender)
    stream = bool(options.get("stream"))
    # text_only: Geräte-TTS (Web Speech API) spricht selbst -> kein Server-Audio erzeugen/senden.
    text_only = bool(options.get("text_only"))
    # audio_stream: explizite Anfrage gewinnt, sonst der serverseitige Default (Admin).
    audio_stream = False if text_only else (
        bool(options["audio_stream"]) if "audio_stream" in options
        else settings.audio_stream_default
    )

    on_token = None
    if stream:
        async def on_token(delta):
            await websocket.send_json({"type": "token", "text": delta})

    on_audio = None
    if audio_stream:
        audio_seq = 0

        async def on_audio(chunk):
            nonlocal audio_seq
            await websocket.send_json({"type": "audio", "seq": audio_seq})
            audio_seq += 1
            await websocket.send_bytes(chunk)

    # Filler (Beruhigungs-/Geduldssätze) als eigener Event: Status-Zeile + Geräte-TTS.
    on_filler = None
    if stream or audio_stream:
        async def on_filler(filler_text):
            await websocket.send_json({"type": "filler", "text": filler_text})

    try:
        if stream or audio_stream:
            trace, audio = await orchestrator.chat_stream(
                text,
                language=route.language,
                voice=voice,
                output=output,
                history=llm_context,
                on_token=on_token,
                on_audio=on_audio,
                on_filler=on_filler,
                text_only=text_only,
            )
        else:
            trace, audio = await orchestrator.chat_text(
                text,
                language=route.language,
                voice=voice,
                output=output,
                history=llm_context,
                text_only=text_only,
            )
    except Exception as exc:
        await websocket.send_json({"type": "error", "status": 502, "detail": str(exc)})
        return

    record_usage(user, store, len(text) + len(trace.semantic_response or ""))

    if session_id:
        store.append_message(session_id, user.id, "user", text)
        store.append_message(session_id, user.id, "assistant", trace.semantic_response)
        maybe_schedule_extraction(store, user.id, session_id)

    await websocket.send_json(
        {"type": "semantic", "text": trace.semantic_response, "spoken": trace.spoken_response}
    )
    # Im text_only-Modus kommt kein Audio (Gerät spricht selbst).
    if not audio_stream and not text_only:
        await websocket.send_bytes(audio)
    await websocket.send_json({"type": "done", "audio_format": "pcm", "sample_rate": 24000})


async def _cancel_active(task, websocket) -> None:
    """Bricht einen laufenden Antwort-Turn ab (Barge-in) und meldet 'interrupted'."""
    if task is None or task.done():
        return
    task.cancel()
    try:
        await task
    except asyncio.CancelledError:
        pass
    await websocket.send_json({"type": "interrupted"})


async def _chat_turn(websocket, store, user, session_id, text, options):
    try:
        route, orchestrator, output = await _resolve(user, session_id, options)
    except SessionOwnershipError as exc:
        await websocket.send_json({"type": "error", "status": 403, "detail": str(exc)})
        return
    except RoutingError as exc:
        await websocket.send_json({"type": "error", "status": 422, "detail": str(exc)})
        return
    await _run_turn(websocket, store, user, session_id, route, orchestrator, output, text, options)


async def _voice_turn(websocket, store, user, session_id, audio, fmt, options):
    try:
        route, orchestrator, output = await _resolve(user, session_id, options)
    except SessionOwnershipError as exc:
        await websocket.send_json({"type": "error", "status": 403, "detail": str(exc)})
        return
    except RoutingError as exc:
        await websocket.send_json({"type": "error", "status": 422, "detail": str(exc)})
        return

    try:
        if settings.stt_force_language:
            # Feste Zielsprache statt Auto-Detect -> robuster bei kurzen/verrauschten
            # (Handy-)Clips; verhindert "falsche Sprache erkannt = Blödsinn".
            transcript, _ = await orchestrator.stt.transcribe_detect(
                audio, fmt=fmt, language=route.language
            )
        else:
            # Gesprochene Sprache automatisch erkennen; weicht sie von der Zielsprache
            # ab, wird in die Zielsprache übersetzt (Fremdsprachen-Diktat).
            transcript, detected = await orchestrator.stt.transcribe_detect(
                audio, fmt=fmt, language=None
            )
            if (transcript and transcript.strip() and detected
                    and detected.lower() != (route.language or "").lower()):
                transcript = await orchestrator.translate(transcript, route.language)
        effective_voice = voice_for_route(route.tts_provider, route.language, route.voice_gender)
    except Exception as exc:
        await websocket.send_json({"type": "error", "status": 502, "detail": str(exc)})
        return

    await websocket.send_json({
        "type": "transcript",
        "text": transcript,
    })
    if not transcript or not transcript.strip():
        await websocket.send_json({
            "type": "error",
            "detail": "Keine Sprache erkannt — bitte erneut sprechen.",
        })
        return
    await _run_turn(
        websocket, store, user, session_id, route, orchestrator, output, transcript, options,
        effective_voice=effective_voice,
    )


@router.websocket("/ws/chat")
async def ws_chat(websocket: WebSocket, session_id: str | None = None, token: str | None = None):
    user = _authenticate(websocket, token)
    if user is None:
        await websocket.close(code=1008)
        return
    await websocket.accept()
    store = get_store()
    active = None

    try:
        while True:
            msg = await websocket.receive_json()
            if msg.get("type") == "interrupt":
                await _cancel_active(active, websocket)
                active = None
                continue
            text = (msg.get("text") or "").strip()
            if not text:
                await websocket.send_json({"type": "error", "detail": "empty text"})
                continue
            await _cancel_active(active, websocket)  # Barge-in bei neuer Eingabe
            active = asyncio.create_task(
                _chat_turn(websocket, store, user, session_id, text, msg)
            )
    except WebSocketDisconnect:
        if active and not active.done():
            active.cancel()
        return


@router.websocket("/ws/voice")
async def ws_voice(websocket: WebSocket, session_id: str | None = None, token: str | None = None):
    user = _authenticate(websocket, token)
    if user is None:
        await websocket.close(code=1008)
        return
    await websocket.accept()
    store = get_store()

    audio_buffer = bytearray()
    fmt = "wav"
    active = None
    vad = None
    start_options: dict = {}  # Konfig aus dem start-Frame (Provider, audio_stream, language)

    async def _start_voice(audio: bytes, options: dict):
        nonlocal active
        await _cancel_active(active, websocket)  # Barge-in bei neuer Aeusserung
        active = asyncio.create_task(
            _voice_turn(websocket, store, user, session_id, audio, fmt, options)
        )

    try:
        while True:
            message = await websocket.receive()
            if message["type"] == "websocket.disconnect":
                if active and not active.done():
                    active.cancel()
                return

            if message.get("bytes") is not None:
                audio_buffer.extend(message["bytes"])
                # VAD: Aeusserungsende automatisch erkennen (opt-in via start-Frame).
                if vad is not None and vad.feed(message["bytes"]):
                    audio = bytes(audio_buffer)
                    audio_buffer.clear()
                    vad.reset()
                    await _start_voice(audio, start_options)
                continue

            raw = message.get("text")
            if raw is None:
                continue
            try:
                control = json.loads(raw)
            except ValueError:
                await websocket.send_json({"type": "error", "detail": "invalid control frame"})
                continue

            ctype = control.get("type")
            if ctype == "interrupt":
                await _cancel_active(active, websocket)
                active = None
                continue
            if ctype == "start":
                audio_buffer.clear()
                start_options = control  # Provider/audio_stream/language fuer den Turn merken
                fmt = control.get("format", "wav")
                if control.get("vad"):
                    vad = EnergyVAD(
                        sample_rate=control.get("sample_rate", 16000),
                        threshold=control.get("vad_threshold", 500.0),
                        silence_ms=control.get("vad_silence_ms", 700.0),
                    )
                else:
                    vad = None
                continue
            if ctype != "end":
                continue

            if not audio_buffer:
                await websocket.send_json({"type": "error", "detail": "no audio received"})
                continue

            audio = bytes(audio_buffer)
            audio_buffer.clear()
            if vad is not None:
                vad.reset()
            # start-Frame-Konfig + end-Frame zusammenfuehren (end kann ueberschreiben)
            await _start_voice(audio, {**start_options, **control})
    except WebSocketDisconnect:
        if active and not active.done():
            active.cancel()
        return