feat: schwerwiegende KI-Funde dürfen ROT auslösen (konfigurierbar)
Bisher war das KI-Level hart auf Gelb gedeckelt — auch injizierte
Pornografie oder strafbare/diffamierende Inhalte lösten nie Rot aus.
Das unterschätzt echte Notfälle (Sicherheits- + Rechts-/Reputationsrisiko).
Neu: kategoriebasierter Grave-Override in score_ai_findings. Nur Kategorien
aus red_categories (default ["pornography","defamation_illegal"]) lösen Rot
aus, und nur bei Schwere ≥ red_min_severity (high) UND Konfidenz ≥
red_min_confidence (0.9). Alle anderen Kategorien (Spam, Off-Topic,
Propaganda, Widerspruch) bleiben höchstens Gelb. Die Punktsumme allein
ergibt weiterhin nie Rot. red_categories: [] stellt das alte Verhalten her.
Begründung für die konservative Schwelle: LLM-Konfidenzen sind nicht
perfekt kalibriert — nur die gravierendsten Fälle dürfen die volle
ROT-Dringlichkeit ("Dienstleister informieren, nichts freigeben") tragen.
Rot fließt über die bestehende _worst_level-Verdrahtung automatisch ins
Gesamt-Level + Exit-Code 2; gilt auch für Bildfunde (OCR).
Tests: 237 grün (+7 Rot-Gate, bestehender Cap-Test auf Punktsumme-Logik
umgestellt). Config + Template + README + Bedienungsanleitung aktualisiert.
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
parent
d06e3d3dfd
commit
a8dcaabf7c
6 changed files with 98 additions and 15 deletions
|
|
@ -233,15 +233,15 @@ class TestScoring:
|
|||
out = score_ai_findings({"findings": findings}, _cfg())
|
||||
assert out["level"] == "yellow"
|
||||
|
||||
def test_score_capped_at_yellow_never_red(self):
|
||||
# Zwei schwere Funde → Summe ≥ rot-Schwelle, aber Cap hält es bei gelb.
|
||||
def test_score_sum_alone_never_red(self):
|
||||
# Zwei nicht-gravierende Funde → Summe ≥ rot-Schwelle, aber Punktsumme macht nie Rot.
|
||||
findings = [
|
||||
{"kind": "text", "url": "u1", "category": "pornography", "severity": "high", "confidence": 0.95},
|
||||
{"kind": "text", "url": "u2", "category": "defamation_illegal", "severity": "high", "confidence": 0.95},
|
||||
{"kind": "text", "url": "u1", "category": "off_topic_commercial", "severity": "high", "confidence": 0.95},
|
||||
{"kind": "text", "url": "u2", "category": "off_topic_commercial", "severity": "high", "confidence": 0.95},
|
||||
]
|
||||
out = score_ai_findings({"findings": findings}, _cfg())
|
||||
assert out["score"] >= 60 # Punktsumme über rot-Schwelle
|
||||
assert out["level"] == "yellow" # trotzdem nur gelb
|
||||
assert out["level"] == "yellow" # trotzdem nur gelb (kein Grave-Override)
|
||||
assert out["exit_code"] == 1
|
||||
|
||||
def test_image_gets_at_least_suspicious_image_points(self):
|
||||
|
|
@ -267,6 +267,43 @@ class TestScoring:
|
|||
assert any("nicht geprüft" in r for r in out["reasons"])
|
||||
|
||||
|
||||
class TestRedGate:
|
||||
def _find(self, cat, sev, conf, kind="text"):
|
||||
return {"kind": kind, "url": "u", "category": cat, "severity": sev, "confidence": conf}
|
||||
|
||||
def test_pornography_high_conf_is_red(self):
|
||||
out = score_ai_findings({"findings": [self._find("pornography", "high", 0.95)]}, _cfg())
|
||||
assert out["level"] == "red"
|
||||
assert out["exit_code"] == 2
|
||||
|
||||
def test_defamation_illegal_high_conf_is_red(self):
|
||||
out = score_ai_findings({"findings": [self._find("defamation_illegal", "high", 0.95)]}, _cfg())
|
||||
assert out["level"] == "red"
|
||||
|
||||
def test_pornography_image_is_red(self):
|
||||
out = score_ai_findings(
|
||||
{"findings": [self._find("pornography", "high", 0.95, kind="image")]}, _cfg())
|
||||
assert out["level"] == "red"
|
||||
|
||||
def test_severity_below_high_stays_yellow(self):
|
||||
out = score_ai_findings({"findings": [self._find("pornography", "medium", 0.95)]}, _cfg())
|
||||
assert out["level"] == "yellow"
|
||||
|
||||
def test_confidence_below_threshold_stays_yellow(self):
|
||||
out = score_ai_findings({"findings": [self._find("pornography", "high", 0.8)]}, _cfg())
|
||||
assert out["level"] == "yellow"
|
||||
|
||||
def test_non_grave_category_stays_yellow(self):
|
||||
out = score_ai_findings({"findings": [self._find("off_topic_commercial", "high", 1.0)]}, _cfg())
|
||||
assert out["level"] == "yellow"
|
||||
|
||||
def test_empty_red_categories_restores_old_capping(self):
|
||||
cfg = _cfg()
|
||||
cfg["ai_analysis"]["red_categories"] = []
|
||||
out = score_ai_findings({"findings": [self._find("pornography", "high", 1.0)]}, cfg)
|
||||
assert out["level"] == "yellow"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Model escalation chain
|
||||
# ---------------------------------------------------------------------------
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue