feat: adaptiver Modell-Router (Circuit-Breaker) + parallele KI-Analyse

Der reale Erst-Scan dauerte ~10 min, weil pro Kandidat sequenziell erst die
rate-limited Free-Modelle (429) probiert wurden. Zwei Hebel beheben das:

1. ModelRouter: programmatische Telemetrie statt LLM-Orchestrator. Pro Modell
   Erfolg/Latenz; ausgefallene/rate-limited Modelle bekommen per Circuit-Breaker
   einen Cooldown und werden übersprungen, das schnellste gesunde Modell zuerst.
   Failsafe-Pruning toter Slugs über OpenRouter /models (24h-Cache). State
   persistent in data/ai_router_state.json.

2. Parallelisierung: ThreadPoolExecutor mit konfigurierbarer max_concurrency
   (Default 8, I/O-gebunden → an API-Rate-Limits gebunden, nicht an CPU-Kerne).
   Klassifikationen laufen nebenläufig, Merge im Hauptthread (keine Locks),
   findings deterministisch sortiert. Fingerprint-Gruppierung: identischer
   Inhalt wird nur einmal klassifiziert, Funde aber für alle URLs emittiert.

Realtest bredelar.info: Frisch-Scan von ~10 min auf 1:01 min; Breaker öffnete
14× (Free-Modelle übersprungen), alle Verdikte von gemini-2.5-flash-lite.

Config: max_concurrency, breaker_failure_threshold, breaker_cooldown_seconds,
refresh_models. Tests: 251 grün (+12: Router-Reihung/Breaker/Pruning-failsafe,
parallel==sequenziell, Dedup, deterministische Reihenfolge).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Dieter Schlüter 2026-06-13 02:51:30 +02:00
commit 125d80e7f9
7 changed files with 415 additions and 65 deletions

View file

@ -12,6 +12,7 @@ from scanner.ai_analyzer import (
_text_fingerprint,
_models_for,
_openrouter_chat,
ModelRouter,
)
from scanner.baseline import BaselineManager
from scanner.config import DEFAULT_CONFIG
@ -26,6 +27,7 @@ def _cfg(**ai_overrides) -> dict:
cfg["ai_analysis"]["enabled"] = True
cfg["ai_analysis"]["site_context"] = "Heimat- und Vereinswebsite über Bergbau"
cfg["ai_analysis"]["image"]["enabled"] = False # Text-Tests: Bild aus
cfg["ai_analysis"]["refresh_models"] = False # Tests: kein echter /models-Netzcall
cfg["ai_analysis"].update(ai_overrides)
return cfg
@ -307,6 +309,109 @@ class TestRedGate:
assert out["level"] == "yellow"
# ---------------------------------------------------------------------------
# Adaptive model router: circuit-breaker + latency ordering + /models pruning
# ---------------------------------------------------------------------------
def _router(catalog_slugs=None):
cfg = {"breaker_failure_threshold": 2, "breaker_cooldown_seconds": 120, "refresh_models": False}
state = {"models": {}, "catalog": {"slugs": catalog_slugs or [], "fetched_at": None}}
return ModelRouter(cfg, state)
class TestModelRouter:
def test_failure_below_threshold_keeps_model_healthy(self):
r = _router()
r.record_failure("m")
assert r.order(["m", "n"])[0] in ("m", "n") # nicht hart aussortiert
def test_breaker_opens_after_threshold_and_sorts_last(self):
r = _router()
r.record_failure("bad"); r.record_failure("bad") # erreicht threshold 2
assert r.order(["bad", "good"]) == ["good", "bad"]
def test_success_records_latency_faster_first(self):
r = _router()
r.record_success("slow", 2.0)
r.record_success("fast", 0.3)
assert r.order(["slow", "fast"]) == ["fast", "slow"]
def test_success_resets_open_breaker(self):
r = _router()
r.record_failure("m"); r.record_failure("m")
r.record_success("m", 0.5)
assert r.order(["m", "n"])[0] == "m" # wieder gesund (Latenz 0.5 < unbekannt? egal: nicht open)
def test_cooldown_expiry_makes_model_healthy_again(self):
r = _router()
r.record_failure("m"); r.record_failure("m")
r._models["m"]["open_until"] = time.time() - 1 # Cooldown abgelaufen
assert r.order(["m", "n"])[-1] != "m" or r.order(["m"]) == ["m"]
def test_prune_drops_dead_slug(self):
r = _router(catalog_slugs=["alive"])
assert r.order(["alive", "dead-404"]) == ["alive"]
def test_prune_failsafe_without_catalog(self):
r = _router(catalog_slugs=[]) # kein Katalog → nicht filtern
assert set(r.order(["a", "b"])) == {"a", "b"}
def test_order_never_empty_even_if_all_pruned(self):
r = _router(catalog_slugs=["other"])
assert r.order(["x", "y"]) # keiner im Katalog → trotzdem nicht leer
def test_refresh_catalog_failsafe_on_network_error(self):
r = _router()
with patch("scanner.ai_analyzer.requests.get", side_effect=Exception("net down")):
r.refresh_catalog() # darf nicht werfen
assert r.order(["a"]) == ["a"]
# ---------------------------------------------------------------------------
# Parallel analysis correctness
# ---------------------------------------------------------------------------
class TestParallel:
def test_all_pages_classified_in_parallel(self, tmp_path, monkeypatch):
monkeypatch.setenv("OPENROUTER_API_KEY", "test")
bm = BaselineManager(tmp_path)
pages = {f"https://x.de/p{i}": {"url": f"https://x.de/p{i}", "status": 200,
"text": _LONG + str(i), "links": {"img": []}}
for i in range(6)}
with patch("scanner.ai_analyzer._openrouter_chat", return_value=_verdict()):
res = run_ai_analysis(_cfg(), bm, {"pages": pages}, {})
assert res["api_calls"] == 6
assert len(bm.load_ai_ledger()["entries"]) == 6
assert res["skipped"] is None
def test_duplicate_text_classified_once_findings_for_each_url(self, tmp_path, monkeypatch):
monkeypatch.setenv("OPENROUTER_API_KEY", "test")
bm = BaselineManager(tmp_path)
same = _LONG + " identisch"
pages = {
"https://x.de/a": {"url": "https://x.de/a", "status": 200, "text": same, "links": {"img": []}},
"https://x.de/b": {"url": "https://x.de/b", "status": 200, "text": same, "links": {"img": []}},
}
v = _verdict("hidden_spam", "high", 0.9)
with patch("scanner.ai_analyzer._openrouter_chat", return_value=v):
res = run_ai_analysis(_cfg(), bm, {"pages": pages}, {})
assert res["api_calls"] == 1 # identischer Text → nur 1 Call
assert len(res["findings"]) == 2 # aber Fund für beide URLs
assert {f["url"] for f in res["findings"]} == {"https://x.de/a", "https://x.de/b"}
def test_findings_sorted_deterministically(self, tmp_path, monkeypatch):
monkeypatch.setenv("OPENROUTER_API_KEY", "test")
bm = BaselineManager(tmp_path)
pages = {f"https://x.de/{c}": {"url": f"https://x.de/{c}", "status": 200,
"text": _LONG + c, "links": {"img": []}}
for c in "cab"}
v = _verdict("off_topic_commercial", "high", 0.9)
with patch("scanner.ai_analyzer._openrouter_chat", return_value=v):
res = run_ai_analysis(_cfg(), bm, {"pages": pages}, {})
urls = [f["url"] for f in res["findings"]]
assert urls == sorted(urls) # deterministisch nach URL sortiert
# ---------------------------------------------------------------------------
# Robustness: hard wall-clock deadline + resilient ledger save
# ---------------------------------------------------------------------------