feat: Token-Level-LLM-Streaming über WebSocket (#4 Ausbau)

- LLMProvider.stream: Basis-Default (Fallback über complete) + SSE-Streaming
  fuer OpenRouter und lokalen OpenAI-kompatiblen Provider; gemeinsamer Parser sse_delta
- Orchestrator.chat_stream: LLM-Token live via on_token-Callback, danach
  Spoken-Adapter/Normalizer/TTS/Output; Fallback fuer Provider ohne stream
- WS /ws/chat: opt-in {"stream":true} -> ack -> token* -> semantic -> audio -> done
- Tests: 43 gruen (+5: SSE-Parsing, Default-Fallback, WS-Token-Flow)
- Doku aktualisiert; .gitignore: *.wav (generierte Audio-Ausgaben)

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Dieter Schlüter 2026-06-17 04:37:37 +02:00
commit 379e002460
10 changed files with 302 additions and 24 deletions

View file

@ -1,6 +1,8 @@
from collections.abc import AsyncIterator
import httpx
from app.providers.llm.base import LLMProvider
from app.providers.llm.base import LLMProvider, sse_delta
SYSTEM_PROMPT = """
@ -49,12 +51,7 @@ class OpenRouterLLMProvider(LLMProvider):
self.api_key = (api_key or "").strip()
self.model = (model or "").strip()
async def complete(
self,
text: str,
history: list[dict] | None = None,
session_id: str | None = None,
) -> str:
def _build_messages(self, text: str, history: list[dict] | None) -> list[dict]:
if not self.api_key:
raise ValueError("OPENROUTER_API_KEY is empty")
if not self.model:
@ -66,10 +63,17 @@ class OpenRouterLLMProvider(LLMProvider):
if history:
messages.extend(history)
messages.append({"role": "user", "content": text.strip()})
return messages
async def complete(
self,
text: str,
history: list[dict] | None = None,
session_id: str | None = None,
) -> str:
payload = {
"model": self.model,
"messages": messages,
"messages": self._build_messages(text, history),
}
timeout = httpx.Timeout(connect=10.0, read=120.0, write=30.0, pool=10.0)
@ -106,3 +110,42 @@ class OpenRouterLLMProvider(LLMProvider):
return str(content).strip()
async def stream(
self,
text: str,
history: list[dict] | None = None,
session_id: str | None = None,
) -> AsyncIterator[str]:
payload = {
"model": self.model,
"messages": self._build_messages(text, history),
"stream": True,
}
timeout = httpx.Timeout(connect=10.0, read=120.0, write=30.0, pool=10.0)
async with httpx.AsyncClient(timeout=timeout) as client:
try:
async with client.stream(
"POST",
"https://openrouter.ai/api/v1/chat/completions",
headers={
"Authorization": f"Bearer {self.api_key}",
"Content-Type": "application/json",
},
json=payload,
) as response:
if response.status_code >= 400:
body = await response.aread()
raise RuntimeError(
f"OpenRouter LLM error {response.status_code}: "
f"{body.decode(errors='replace')}"
)
async for line in response.aiter_lines():
delta = sse_delta(line)
if delta:
yield delta
except httpx.TimeoutException as exc:
raise RuntimeError("OpenRouter LLM timeout") from exc
except httpx.HTTPError as exc:
raise RuntimeError(f"OpenRouter LLM transport error: {exc}") from exc