feat: Resilienz (Fallback-Ketten) und Metriken (#5)
- Fallback-Provider (app/providers/fallback.py) fuer STT/LLM/TTS: Provider-Kette der Reihe nach; Config *_FALLBACK; build_orchestrator baut Ketten (dedupliziert) - LLM-Stream-Fallback nur solange kein Token gesendet wurde - Metriken (app/metrics.py): In-Memory Counter/Timer, keine externe Dependency - HTTP-Middleware (Requests/Latenz/Status je Pfad); Pipeline-Stufen-Timing stt/llm/tts; Fallback-/Fehlerzaehler; GET /api/metrics (JSON + Prometheus) - Tests: 58 gruen (+6); Doku aktualisiert (README, BEDIENUNGSANLEITUNG, Architektur, .env.example) Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
parent
093da817d8
commit
6422444017
13 changed files with 452 additions and 23 deletions
|
|
@ -3,16 +3,18 @@ import pytest
|
|||
import app.dependencies as deps
|
||||
from app.config import settings
|
||||
from app.store import SQLiteStore
|
||||
from app.metrics import metrics
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def reset_state(tmp_path, monkeypatch):
|
||||
"""Pro Test: frische SQLite-DB, frischer Singleton-Audio-Router, Auth aus.
|
||||
"""Pro Test: frische SQLite-DB, frischer Singleton-Audio-Router, Auth aus, Metriken leer.
|
||||
|
||||
Auth-Tests schalten `settings.auth_enabled` selbst wieder ein.
|
||||
"""
|
||||
deps._store = SQLiteStore(str(tmp_path / "test.db"))
|
||||
deps._audio_router = None
|
||||
metrics.reset()
|
||||
monkeypatch.setattr(settings, "auth_enabled", False)
|
||||
monkeypatch.setattr(settings, "admin_api_key", "")
|
||||
yield
|
||||
|
|
|
|||
129
tests/test_resilience.py
Normal file
129
tests/test_resilience.py
Normal file
|
|
@ -0,0 +1,129 @@
|
|||
import asyncio
|
||||
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
import app.dependencies as deps
|
||||
from app.main import app
|
||||
from app.config import settings
|
||||
from app.metrics import metrics
|
||||
from app.providers.fallback import FallbackLLMProvider
|
||||
|
||||
client = TestClient(app)
|
||||
|
||||
|
||||
def _run(coro):
|
||||
return asyncio.run(coro)
|
||||
|
||||
|
||||
# --- Fallback-Einheiten ----------------------------------------------------
|
||||
|
||||
def test_llm_fallback_uses_second_on_error():
|
||||
class BadLLM:
|
||||
async def complete(self, text, history=None, session_id=None):
|
||||
raise RuntimeError("down")
|
||||
|
||||
class GoodLLM:
|
||||
async def complete(self, text, history=None, session_id=None):
|
||||
return "ok"
|
||||
|
||||
chain = FallbackLLMProvider("llm", [("bad", BadLLM()), ("good", GoodLLM())])
|
||||
assert _run(chain.complete("x")) == "ok"
|
||||
|
||||
counters = metrics.snapshot()["counters"]
|
||||
assert any("provider_fallback_total" in key for key in counters)
|
||||
assert any('provider_error_total{module="llm",provider="bad"}' in key for key in counters)
|
||||
|
||||
|
||||
def test_llm_fallback_all_fail_raises():
|
||||
class BadLLM:
|
||||
async def complete(self, text, history=None, session_id=None):
|
||||
raise RuntimeError("x")
|
||||
|
||||
chain = FallbackLLMProvider("llm", [("a", BadLLM()), ("b", BadLLM())])
|
||||
with pytest.raises(RuntimeError):
|
||||
_run(chain.complete("x"))
|
||||
|
||||
|
||||
def test_llm_stream_fallback_before_first_token():
|
||||
class BadStream:
|
||||
async def complete(self, text, history=None, session_id=None):
|
||||
return "x"
|
||||
|
||||
async def stream(self, text, history=None, session_id=None):
|
||||
raise RuntimeError("boom")
|
||||
yield # macht die Funktion zum Generator
|
||||
|
||||
class GoodStream:
|
||||
async def complete(self, text, history=None, session_id=None):
|
||||
return "ok"
|
||||
|
||||
async def stream(self, text, history=None, session_id=None):
|
||||
yield "he"
|
||||
yield "llo"
|
||||
|
||||
chain = FallbackLLMProvider("llm", [("bad", BadStream()), ("good", GoodStream())])
|
||||
|
||||
async def collect():
|
||||
return [delta async for delta in chain.stream("x")]
|
||||
|
||||
assert _run(collect()) == ["he", "llo"]
|
||||
|
||||
|
||||
# --- Fallback ueber Config + Endpunkt --------------------------------------
|
||||
|
||||
def test_config_llm_fallback_applied(monkeypatch):
|
||||
class BadLLM:
|
||||
async def complete(self, text, history=None, session_id=None):
|
||||
raise RuntimeError("primary down")
|
||||
|
||||
class GoodLLM:
|
||||
async def complete(self, text, history=None, session_id=None):
|
||||
return "rescued"
|
||||
|
||||
class StubTTS:
|
||||
async def synthesize(self, text, voice=None, audio_format="pcm"):
|
||||
return b"A"
|
||||
|
||||
monkeypatch.setitem(deps.LLM_REGISTRY, "bad", lambda s: BadLLM())
|
||||
monkeypatch.setitem(deps.LLM_REGISTRY, "good", lambda s: GoodLLM())
|
||||
monkeypatch.setitem(deps.TTS_REGISTRY, "t", lambda s: StubTTS())
|
||||
monkeypatch.setattr(settings, "llm_fallback", "good")
|
||||
|
||||
resp = client.post(
|
||||
"/api/chat?debug=true",
|
||||
json={"text": "x", "llm_provider": "bad", "tts_provider": "t"},
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
assert resp.json()["trace"]["semantic_response"] == "rescued"
|
||||
|
||||
|
||||
# --- Metriken --------------------------------------------------------------
|
||||
|
||||
def test_metrics_endpoint_records_requests_and_stages(monkeypatch):
|
||||
class StubLLM:
|
||||
async def complete(self, text, history=None, session_id=None):
|
||||
return "hi"
|
||||
|
||||
class StubTTS:
|
||||
async def synthesize(self, text, voice=None, audio_format="pcm"):
|
||||
return b"A"
|
||||
|
||||
monkeypatch.setitem(deps.LLM_REGISTRY, "l", lambda s: StubLLM())
|
||||
monkeypatch.setitem(deps.TTS_REGISTRY, "t", lambda s: StubTTS())
|
||||
|
||||
resp = client.post(
|
||||
"/api/chat?debug=true", json={"text": "x", "llm_provider": "l", "tts_provider": "t"}
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
|
||||
snap = client.get("/api/metrics").json()
|
||||
assert any("http_requests_total" in k and "chat" in k for k in snap["counters"])
|
||||
assert any('stage_duration_seconds{stage="llm"}' in k for k in snap["timers"])
|
||||
assert any('stage_duration_seconds{stage="tts"}' in k for k in snap["timers"])
|
||||
|
||||
|
||||
def test_metrics_prometheus_format(monkeypatch):
|
||||
client.get("/health")
|
||||
text = client.get("/api/metrics?format=prometheus").text
|
||||
assert "http_requests_total" in text
|
||||
Loading…
Add table
Add a link
Reference in a new issue