feat: KI-Absicherung gegen Falsch-Negative bei Dienstausfall

Bisher: Konnte die KI kein Verdikt liefern (alle Modellstufen scheitern),
ging der Inhalt still als grün durch — für ein Sicherheitswerkzeug ein
Falsch-Negativ-Risiko ("konnte nicht prüfen" ≠ "sauber").

Zwei Maßnahmen:
1. Retry: _classify wiederholt die ganze Modell-Kette bei Komplettausfall
   bis max_retries (Default 1) mit Backoff — fängt transiente Aussetzer ab.
2. Tracking: nicht prüfbare Inhalte landen in ai_result["unchecked"] (statt
   stillem skip) und werden sichtbar in Terminal, Report (Markdown), diff-only
   und E-Mail — jeweils mit URL. score_ai_findings nimmt sie als Grund auf;
   mit unchecked_level="yellow" heben sie das Level auf mindestens Gelb an
   (Default "warn" = nur Hinweis, Ampel unberührt).

Config: max_retries, retry_backoff_seconds, unchecked_level (ai_analysis).
Tests: 230 grün (+ Retry-Recovery, unchecked-Tracking, unchecked_level).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Dieter Schlüter 2026-06-13 00:50:36 +02:00
commit d06e3d3dfd
9 changed files with 138 additions and 18 deletions

View file

@ -70,14 +70,17 @@ class TestGating:
assert "kein API-Key" in res["skipped"]
m.assert_not_called()
def test_api_failure_is_graceful(self, tmp_path, monkeypatch):
def test_api_failure_tracks_unchecked_not_silent_clean(self, tmp_path, monkeypatch):
monkeypatch.setenv("OPENROUTER_API_KEY", "test")
monkeypatch.setattr("scanner.ai_analyzer.time.sleep", lambda *_: None)
bm = BaselineManager(tmp_path)
with patch("scanner.ai_analyzer._openrouter_chat", return_value=None):
res = run_ai_analysis(_cfg(), bm, _snap(_LONG), {})
assert res["findings"] == []
assert res["api_calls"] == 0
assert res["skipped"] # ein Hinweis wurde gesetzt, aber keine Exception
# Inhalt bleibt UNGEPRÜFT — darf nicht still als clean durchgehen
assert len(res["unchecked"]) == 1
assert res["unchecked"][0]["kind"] == "text"
# ---------------------------------------------------------------------------
@ -248,6 +251,21 @@ class TestScoring:
out = score_ai_findings({"findings": findings}, _cfg())
assert out["score"] == 40
def test_unchecked_level_yellow_raises_from_green(self):
cfg = _cfg()
cfg["ai_analysis"]["unchecked_level"] = "yellow"
out = score_ai_findings(
{"findings": [], "unchecked": [{"kind": "text", "url": "u"}]}, cfg)
assert out["level"] == "yellow"
assert out["exit_code"] == 1
def test_unchecked_level_warn_stays_green_but_notes_reason(self):
cfg = _cfg() # default "warn"
out = score_ai_findings(
{"findings": [], "unchecked": [{"kind": "text", "url": "u"}]}, cfg)
assert out["level"] == "green"
assert any("nicht geprüft" in r for r in out["reasons"])
# ---------------------------------------------------------------------------
# Model escalation chain
@ -291,18 +309,35 @@ class TestEscalation:
entry = next(iter(bm.load_ai_ledger()["entries"].values()))
assert entry["model"] == "paid-3"
def test_all_stages_fail_is_graceful(self, tmp_path, monkeypatch):
def test_all_stages_fail_tracks_unchecked(self, tmp_path, monkeypatch):
monkeypatch.setenv("OPENROUTER_API_KEY", "test")
monkeypatch.setattr("scanner.ai_analyzer.time.sleep", lambda *_: None)
bm = BaselineManager(tmp_path)
cfg = _cfg()
cfg["ai_analysis"]["text"]["models"] = ["free-1", "free-2", "paid-3"]
with patch("scanner.ai_analyzer._openrouter_chat", side_effect=[None, None, None]):
with patch("scanner.ai_analyzer._openrouter_chat", return_value=None):
res = run_ai_analysis(cfg, bm, _snap(_LONG), {})
assert res["findings"] == []
assert res["api_calls"] == 0
assert res["skipped"] # Hinweis gesetzt
assert len(res["unchecked"]) == 1 # ungeprüft erfasst
assert bm.load_ai_ledger()["entries"] == {} # nichts gespeichert
def test_retry_recovers_after_full_chain_failure(self, tmp_path, monkeypatch):
"""Erster Durchlauf der Kette scheitert komplett → Wiederholung rettet das Verdikt."""
monkeypatch.setenv("OPENROUTER_API_KEY", "test")
monkeypatch.setattr("scanner.ai_analyzer.time.sleep", lambda *_: None)
bm = BaselineManager(tmp_path)
cfg = _cfg()
cfg["ai_analysis"]["text"]["models"] = ["m1", "m2"]
cfg["ai_analysis"]["max_retries"] = 1
v = _verdict("clean", "none", 0.9)
# Durchlauf 1: m1,m2 → None,None. Wiederholung: m1 → verdict.
with patch("scanner.ai_analyzer._openrouter_chat", side_effect=[None, None, v]) as m:
res = run_ai_analysis(cfg, bm, _snap(_LONG), {})
assert m.call_count == 3
assert res["unchecked"] == []
assert res["api_calls"] == 1
# ---------------------------------------------------------------------------
# Fingerprint stability