feat: KI-Absicherung gegen Falsch-Negative bei Dienstausfall
Bisher: Konnte die KI kein Verdikt liefern (alle Modellstufen scheitern),
ging der Inhalt still als grün durch — für ein Sicherheitswerkzeug ein
Falsch-Negativ-Risiko ("konnte nicht prüfen" ≠ "sauber").
Zwei Maßnahmen:
1. Retry: _classify wiederholt die ganze Modell-Kette bei Komplettausfall
bis max_retries (Default 1) mit Backoff — fängt transiente Aussetzer ab.
2. Tracking: nicht prüfbare Inhalte landen in ai_result["unchecked"] (statt
stillem skip) und werden sichtbar in Terminal, Report (Markdown), diff-only
und E-Mail — jeweils mit URL. score_ai_findings nimmt sie als Grund auf;
mit unchecked_level="yellow" heben sie das Level auf mindestens Gelb an
(Default "warn" = nur Hinweis, Ampel unberührt).
Config: max_retries, retry_backoff_seconds, unchecked_level (ai_analysis).
Tests: 230 grün (+ Retry-Recovery, unchecked-Tracking, unchecked_level).
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
parent
822f1ca41d
commit
d06e3d3dfd
9 changed files with 138 additions and 18 deletions
|
|
@ -70,14 +70,17 @@ class TestGating:
|
|||
assert "kein API-Key" in res["skipped"]
|
||||
m.assert_not_called()
|
||||
|
||||
def test_api_failure_is_graceful(self, tmp_path, monkeypatch):
|
||||
def test_api_failure_tracks_unchecked_not_silent_clean(self, tmp_path, monkeypatch):
|
||||
monkeypatch.setenv("OPENROUTER_API_KEY", "test")
|
||||
monkeypatch.setattr("scanner.ai_analyzer.time.sleep", lambda *_: None)
|
||||
bm = BaselineManager(tmp_path)
|
||||
with patch("scanner.ai_analyzer._openrouter_chat", return_value=None):
|
||||
res = run_ai_analysis(_cfg(), bm, _snap(_LONG), {})
|
||||
assert res["findings"] == []
|
||||
assert res["api_calls"] == 0
|
||||
assert res["skipped"] # ein Hinweis wurde gesetzt, aber keine Exception
|
||||
# Inhalt bleibt UNGEPRÜFT — darf nicht still als clean durchgehen
|
||||
assert len(res["unchecked"]) == 1
|
||||
assert res["unchecked"][0]["kind"] == "text"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
|
|
@ -248,6 +251,21 @@ class TestScoring:
|
|||
out = score_ai_findings({"findings": findings}, _cfg())
|
||||
assert out["score"] == 40
|
||||
|
||||
def test_unchecked_level_yellow_raises_from_green(self):
|
||||
cfg = _cfg()
|
||||
cfg["ai_analysis"]["unchecked_level"] = "yellow"
|
||||
out = score_ai_findings(
|
||||
{"findings": [], "unchecked": [{"kind": "text", "url": "u"}]}, cfg)
|
||||
assert out["level"] == "yellow"
|
||||
assert out["exit_code"] == 1
|
||||
|
||||
def test_unchecked_level_warn_stays_green_but_notes_reason(self):
|
||||
cfg = _cfg() # default "warn"
|
||||
out = score_ai_findings(
|
||||
{"findings": [], "unchecked": [{"kind": "text", "url": "u"}]}, cfg)
|
||||
assert out["level"] == "green"
|
||||
assert any("nicht geprüft" in r for r in out["reasons"])
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Model escalation chain
|
||||
|
|
@ -291,18 +309,35 @@ class TestEscalation:
|
|||
entry = next(iter(bm.load_ai_ledger()["entries"].values()))
|
||||
assert entry["model"] == "paid-3"
|
||||
|
||||
def test_all_stages_fail_is_graceful(self, tmp_path, monkeypatch):
|
||||
def test_all_stages_fail_tracks_unchecked(self, tmp_path, monkeypatch):
|
||||
monkeypatch.setenv("OPENROUTER_API_KEY", "test")
|
||||
monkeypatch.setattr("scanner.ai_analyzer.time.sleep", lambda *_: None)
|
||||
bm = BaselineManager(tmp_path)
|
||||
cfg = _cfg()
|
||||
cfg["ai_analysis"]["text"]["models"] = ["free-1", "free-2", "paid-3"]
|
||||
with patch("scanner.ai_analyzer._openrouter_chat", side_effect=[None, None, None]):
|
||||
with patch("scanner.ai_analyzer._openrouter_chat", return_value=None):
|
||||
res = run_ai_analysis(cfg, bm, _snap(_LONG), {})
|
||||
assert res["findings"] == []
|
||||
assert res["api_calls"] == 0
|
||||
assert res["skipped"] # Hinweis gesetzt
|
||||
assert len(res["unchecked"]) == 1 # ungeprüft erfasst
|
||||
assert bm.load_ai_ledger()["entries"] == {} # nichts gespeichert
|
||||
|
||||
def test_retry_recovers_after_full_chain_failure(self, tmp_path, monkeypatch):
|
||||
"""Erster Durchlauf der Kette scheitert komplett → Wiederholung rettet das Verdikt."""
|
||||
monkeypatch.setenv("OPENROUTER_API_KEY", "test")
|
||||
monkeypatch.setattr("scanner.ai_analyzer.time.sleep", lambda *_: None)
|
||||
bm = BaselineManager(tmp_path)
|
||||
cfg = _cfg()
|
||||
cfg["ai_analysis"]["text"]["models"] = ["m1", "m2"]
|
||||
cfg["ai_analysis"]["max_retries"] = 1
|
||||
v = _verdict("clean", "none", 0.9)
|
||||
# Durchlauf 1: m1,m2 → None,None. Wiederholung: m1 → verdict.
|
||||
with patch("scanner.ai_analyzer._openrouter_chat", side_effect=[None, None, v]) as m:
|
||||
res = run_ai_analysis(cfg, bm, _snap(_LONG), {})
|
||||
assert m.call_count == 3
|
||||
assert res["unchecked"] == []
|
||||
assert res["api_calls"] == 1
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Fingerprint stability
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue