feat: adaptiver Modell-Router (Circuit-Breaker) + parallele KI-Analyse
Der reale Erst-Scan dauerte ~10 min, weil pro Kandidat sequenziell erst die rate-limited Free-Modelle (429) probiert wurden. Zwei Hebel beheben das: 1. ModelRouter: programmatische Telemetrie statt LLM-Orchestrator. Pro Modell Erfolg/Latenz; ausgefallene/rate-limited Modelle bekommen per Circuit-Breaker einen Cooldown und werden übersprungen, das schnellste gesunde Modell zuerst. Failsafe-Pruning toter Slugs über OpenRouter /models (24h-Cache). State persistent in data/ai_router_state.json. 2. Parallelisierung: ThreadPoolExecutor mit konfigurierbarer max_concurrency (Default 8, I/O-gebunden → an API-Rate-Limits gebunden, nicht an CPU-Kerne). Klassifikationen laufen nebenläufig, Merge im Hauptthread (keine Locks), findings deterministisch sortiert. Fingerprint-Gruppierung: identischer Inhalt wird nur einmal klassifiziert, Funde aber für alle URLs emittiert. Realtest bredelar.info: Frisch-Scan von ~10 min auf 1:01 min; Breaker öffnete 14× (Free-Modelle übersprungen), alle Verdikte von gemini-2.5-flash-lite. Config: max_concurrency, breaker_failure_threshold, breaker_cooldown_seconds, refresh_models. Tests: 251 grün (+12: Router-Reihung/Breaker/Pruning-failsafe, parallel==sequenziell, Dedup, deterministische Reihenfolge). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
parent
31730403b3
commit
125d80e7f9
7 changed files with 415 additions and 65 deletions
|
|
@ -12,6 +12,7 @@ from scanner.ai_analyzer import (
|
|||
_text_fingerprint,
|
||||
_models_for,
|
||||
_openrouter_chat,
|
||||
ModelRouter,
|
||||
)
|
||||
from scanner.baseline import BaselineManager
|
||||
from scanner.config import DEFAULT_CONFIG
|
||||
|
|
@ -26,6 +27,7 @@ def _cfg(**ai_overrides) -> dict:
|
|||
cfg["ai_analysis"]["enabled"] = True
|
||||
cfg["ai_analysis"]["site_context"] = "Heimat- und Vereinswebsite über Bergbau"
|
||||
cfg["ai_analysis"]["image"]["enabled"] = False # Text-Tests: Bild aus
|
||||
cfg["ai_analysis"]["refresh_models"] = False # Tests: kein echter /models-Netzcall
|
||||
cfg["ai_analysis"].update(ai_overrides)
|
||||
return cfg
|
||||
|
||||
|
|
@ -307,6 +309,109 @@ class TestRedGate:
|
|||
assert out["level"] == "yellow"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Adaptive model router: circuit-breaker + latency ordering + /models pruning
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _router(catalog_slugs=None):
|
||||
cfg = {"breaker_failure_threshold": 2, "breaker_cooldown_seconds": 120, "refresh_models": False}
|
||||
state = {"models": {}, "catalog": {"slugs": catalog_slugs or [], "fetched_at": None}}
|
||||
return ModelRouter(cfg, state)
|
||||
|
||||
|
||||
class TestModelRouter:
|
||||
def test_failure_below_threshold_keeps_model_healthy(self):
|
||||
r = _router()
|
||||
r.record_failure("m")
|
||||
assert r.order(["m", "n"])[0] in ("m", "n") # nicht hart aussortiert
|
||||
|
||||
def test_breaker_opens_after_threshold_and_sorts_last(self):
|
||||
r = _router()
|
||||
r.record_failure("bad"); r.record_failure("bad") # erreicht threshold 2
|
||||
assert r.order(["bad", "good"]) == ["good", "bad"]
|
||||
|
||||
def test_success_records_latency_faster_first(self):
|
||||
r = _router()
|
||||
r.record_success("slow", 2.0)
|
||||
r.record_success("fast", 0.3)
|
||||
assert r.order(["slow", "fast"]) == ["fast", "slow"]
|
||||
|
||||
def test_success_resets_open_breaker(self):
|
||||
r = _router()
|
||||
r.record_failure("m"); r.record_failure("m")
|
||||
r.record_success("m", 0.5)
|
||||
assert r.order(["m", "n"])[0] == "m" # wieder gesund (Latenz 0.5 < unbekannt? egal: nicht open)
|
||||
|
||||
def test_cooldown_expiry_makes_model_healthy_again(self):
|
||||
r = _router()
|
||||
r.record_failure("m"); r.record_failure("m")
|
||||
r._models["m"]["open_until"] = time.time() - 1 # Cooldown abgelaufen
|
||||
assert r.order(["m", "n"])[-1] != "m" or r.order(["m"]) == ["m"]
|
||||
|
||||
def test_prune_drops_dead_slug(self):
|
||||
r = _router(catalog_slugs=["alive"])
|
||||
assert r.order(["alive", "dead-404"]) == ["alive"]
|
||||
|
||||
def test_prune_failsafe_without_catalog(self):
|
||||
r = _router(catalog_slugs=[]) # kein Katalog → nicht filtern
|
||||
assert set(r.order(["a", "b"])) == {"a", "b"}
|
||||
|
||||
def test_order_never_empty_even_if_all_pruned(self):
|
||||
r = _router(catalog_slugs=["other"])
|
||||
assert r.order(["x", "y"]) # keiner im Katalog → trotzdem nicht leer
|
||||
|
||||
def test_refresh_catalog_failsafe_on_network_error(self):
|
||||
r = _router()
|
||||
with patch("scanner.ai_analyzer.requests.get", side_effect=Exception("net down")):
|
||||
r.refresh_catalog() # darf nicht werfen
|
||||
assert r.order(["a"]) == ["a"]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Parallel analysis correctness
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestParallel:
|
||||
def test_all_pages_classified_in_parallel(self, tmp_path, monkeypatch):
|
||||
monkeypatch.setenv("OPENROUTER_API_KEY", "test")
|
||||
bm = BaselineManager(tmp_path)
|
||||
pages = {f"https://x.de/p{i}": {"url": f"https://x.de/p{i}", "status": 200,
|
||||
"text": _LONG + str(i), "links": {"img": []}}
|
||||
for i in range(6)}
|
||||
with patch("scanner.ai_analyzer._openrouter_chat", return_value=_verdict()):
|
||||
res = run_ai_analysis(_cfg(), bm, {"pages": pages}, {})
|
||||
assert res["api_calls"] == 6
|
||||
assert len(bm.load_ai_ledger()["entries"]) == 6
|
||||
assert res["skipped"] is None
|
||||
|
||||
def test_duplicate_text_classified_once_findings_for_each_url(self, tmp_path, monkeypatch):
|
||||
monkeypatch.setenv("OPENROUTER_API_KEY", "test")
|
||||
bm = BaselineManager(tmp_path)
|
||||
same = _LONG + " identisch"
|
||||
pages = {
|
||||
"https://x.de/a": {"url": "https://x.de/a", "status": 200, "text": same, "links": {"img": []}},
|
||||
"https://x.de/b": {"url": "https://x.de/b", "status": 200, "text": same, "links": {"img": []}},
|
||||
}
|
||||
v = _verdict("hidden_spam", "high", 0.9)
|
||||
with patch("scanner.ai_analyzer._openrouter_chat", return_value=v):
|
||||
res = run_ai_analysis(_cfg(), bm, {"pages": pages}, {})
|
||||
assert res["api_calls"] == 1 # identischer Text → nur 1 Call
|
||||
assert len(res["findings"]) == 2 # aber Fund für beide URLs
|
||||
assert {f["url"] for f in res["findings"]} == {"https://x.de/a", "https://x.de/b"}
|
||||
|
||||
def test_findings_sorted_deterministically(self, tmp_path, monkeypatch):
|
||||
monkeypatch.setenv("OPENROUTER_API_KEY", "test")
|
||||
bm = BaselineManager(tmp_path)
|
||||
pages = {f"https://x.de/{c}": {"url": f"https://x.de/{c}", "status": 200,
|
||||
"text": _LONG + c, "links": {"img": []}}
|
||||
for c in "cab"}
|
||||
v = _verdict("off_topic_commercial", "high", 0.9)
|
||||
with patch("scanner.ai_analyzer._openrouter_chat", return_value=v):
|
||||
res = run_ai_analysis(_cfg(), bm, {"pages": pages}, {})
|
||||
urls = [f["url"] for f in res["findings"]]
|
||||
assert urls == sorted(urls) # deterministisch nach URL sortiert
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Robustness: hard wall-clock deadline + resilient ledger save
|
||||
# ---------------------------------------------------------------------------
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue