""" KI-gestützte semantische Inhaltsanalyse via OpenRouter. Ergänzung zum Integritäts-Kern, niemals dessen Ersatz. Erkennt problematische *Inhalte ohne Link-Signal*: Pornografie, Propaganda, diffamierende/strafbare Texte, versteckten Spam, thematisch unpassende Werbung, widersprüchliche Aussagen — in Text und in Bildern (inkl. eingebettetem Text via OCR). Kostengate: Jeder Kandidat bekommt einen Fingerprint (SHA-256). Ein bereits geprüfter Fingerprint liegt mit seinem Verdikt im Ledger (data/ai_ledger.json) → Cache-Treffer → KEIN API-Call. Nur neue/geänderte Inhalte kosten etwas. Robustheit: Fehlt der API-Key oder schlägt ein Call fehl, wird die Analyse übersprungen und der Scan läuft unverändert weiter (graceful degradation). KI darf den Kern-Scan nie brechen. """ import json import logging import os from datetime import datetime, timezone import requests from .checker import fetch_asset_hashes from .differ import normalize_text logger = logging.getLogger(__name__) _OPENROUTER_URL = "https://openrouter.ai/api/v1/chat/completions" # Erlaubte Kategorien, die das Modell zurückgeben darf. _CATEGORIES = [ "clean", "pornography", "propaganda", "defamation_illegal", "hidden_spam", "off_topic_commercial", "contradiction", ] # Kategorie → Scoring-Schlüssel (Fallback-Punkte, falls cfg sie nicht liefert). _CATEGORY_SCORE_KEY = { "pornography": ("ai_pornography", 50), "defamation_illegal": ("ai_defamation_illegal", 50), "propaganda": ("ai_propaganda", 40), "hidden_spam": ("ai_hidden_spam", 40), "off_topic_commercial": ("ai_off_topic_commercial", 30), "contradiction": ("ai_contradiction", 20), } _SEVERITY_ORDER = {"none": 0, "low": 1, "medium": 2, "high": 3} # JSON-Schema für die strukturierte Modell-Antwort. _RESPONSE_SCHEMA = { "type": "json_schema", "json_schema": { "name": "content_verdict", "strict": True, "schema": { "type": "object", "properties": { "category": {"type": "string", "enum": _CATEGORIES}, "severity": {"type": "string", "enum": ["none", "low", "medium", "high"]}, "confidence": {"type": "number"}, "explanation": {"type": "string"}, }, "required": ["category", "severity", "confidence", "explanation"], "additionalProperties": False, }, }, } # --------------------------------------------------------------------------- # Public API # --------------------------------------------------------------------------- def run_ai_analysis(cfg: dict, bm, snap: dict, diff: dict) -> dict: """ Hash-gegate KI-Analyse über neue/geänderte Texte und Bilder. Returns {"findings": [...], "checked": int, "cache_hits": int, "api_calls": int, "skipped": str|None}. Bricht nie mit einer Exception — bei Problemen wird "skipped" gesetzt. """ result = {"findings": [], "checked": 0, "cache_hits": 0, "api_calls": 0, "skipped": None} ai_cfg = cfg.get("ai_analysis", {}) if not ai_cfg.get("enabled"): result["skipped"] = "deaktiviert" return result api_key = os.environ.get(ai_cfg.get("api_key_env", "OPENROUTER_API_KEY"), "") if not api_key: result["skipped"] = f"kein API-Key ({ai_cfg.get('api_key_env', 'OPENROUTER_API_KEY')})" logger.warning("KI-Analyse übersprungen: %s", result["skipped"]) return result try: ledger = bm.load_ai_ledger() entries = ledger.setdefault("entries", {}) dirty = False if ai_cfg.get("text", {}).get("enabled", True): dirty |= _analyze_text(cfg, ai_cfg, api_key, snap, diff, entries, result) if ai_cfg.get("image", {}).get("enabled", True): dirty |= _analyze_images(cfg, ai_cfg, api_key, snap, diff, entries, result) # Audio/Video: vorbereitet, default aus (siehe _collect_audio/video_candidates). if dirty: bm.save_ai_ledger(ledger) except Exception as exc: # pragma: no cover - Schutzschirm, darf Scan nie brechen logger.error("KI-Analyse mit unerwartetem Fehler abgebrochen: %s", exc) result["skipped"] = f"interner Fehler: {exc}" return result def score_ai_findings(ai_result: dict, cfg: dict) -> dict: """ Bewertet KI-Funde additiv. Das Level ist hart auf 'yellow' gedeckelt — KI allein löst niemals ROT aus (ROT bleibt harten Integritäts-Signalen). Returns {score, level, reasons, exit_code}. """ sc = cfg.get("scoring", {}) thr = cfg.get("thresholds", {"yellow": 20, "red": 60}) yellow = thr.get("yellow", 20) score = 0 reasons: list[str] = [] for f in ai_result.get("findings", []): score += (pts := _finding_points(f, sc)) conf = f.get("confidence", 0.0) reasons.append( f"KI [{f.get('kind', '?')}] {f.get('url', '?')}: " f"{f.get('category', '?')} ({f.get('severity', '?')}, {conf:.0%}) (+{pts})" ) # Deckelung auf gelb: auch bei Score ≥ rot-Schwelle bleibt es gelb. if score >= yellow: level, exit_code = "yellow", 1 else: level, exit_code = "green", 0 return {"score": score, "level": level, "reasons": reasons, "exit_code": exit_code} # --------------------------------------------------------------------------- # Text analysis # --------------------------------------------------------------------------- def _analyze_text(cfg, ai_cfg, api_key, snap, diff, entries, result) -> bool: """Prüft neue/geänderte Seitentexte. Returns True wenn Ledger verändert.""" text_cfg = ai_cfg.get("text", {}) min_chars = text_cfg.get("min_chars", 200) models = _models_for(text_cfg) site_context = ai_cfg.get("site_context", "") # Fingerprint je Seite (lokal, gratis). Geänderte/neue Seiten zuerst (Risiko). changed = {pd["url"] for pd in diff.get("page_diffs", [])} | set(diff.get("new_internal_urls", [])) candidates: list[tuple[str, str, str]] = [] for url, page in snap.get("pages", {}).items(): if page.get("status") != 200: continue text = page.get("text", "") or "" if len(text) < min_chars: continue candidates.append((url, text, _text_fingerprint(text, cfg))) candidates.sort(key=lambda c: c[0] not in changed) # changed first (False < True) budget = text_cfg.get("max_pages_per_scan", 20) dirty = False for url, text, fp in candidates: result["checked"] += 1 entry = entries.get(fp) if entry is None: if budget <= 0: continue # Kostendeckel erreicht — nächster Scan holt den Rest nach classified = _classify_text(text, site_context, models, ai_cfg, api_key) if classified is None: result["skipped"] = result["skipped"] or "API-Fehler (Text)" continue verdict, used_model = classified entry = _make_entry("text", url, verdict, used_model) entries[fp] = entry result["api_calls"] += 1 budget -= 1 dirty = True else: result["cache_hits"] += 1 _maybe_finding(result, entry, fp, url, ai_cfg) return dirty # --------------------------------------------------------------------------- # Image analysis # --------------------------------------------------------------------------- def _analyze_images(cfg, ai_cfg, api_key, snap, diff, entries, result) -> bool: """Prüft neue/geänderte Bilder. Returns True wenn Ledger verändert.""" img_cfg = ai_cfg.get("image", {}) models = _models_for(img_cfg) site_context = ai_cfg.get("site_context", "") current_img_urls = _all_image_urls(snap) if not current_img_urls: return False has_image_entries = any(e.get("kind") == "image" for e in entries.values()) changed = {pd["url"] for pd in diff.get("page_diffs", [])} | set(diff.get("new_internal_urls", [])) changed_img_urls = _all_image_urls(snap, only_pages=changed) # Markierte Bilder erneut holen: ersetztes Bild → neue Bytes → Neubewertung # (verhindert Dauer-Fehlalarm nach Bereinigung); unverändert → Cache-Treffer. flagged_img_urls = { e["url"] for e in entries.values() if e.get("kind") == "image" and e.get("category", "clean") != "clean" and not e.get("dismissed") and e.get("url") in current_img_urls } if has_image_entries: fetch_urls = sorted((changed_img_urls | flagged_img_urls) & current_img_urls) else: fetch_urls = sorted(current_img_urls) # Erstlauf: vollständiger Sweep fetch_urls = fetch_urls[: img_cfg.get("max_images_per_scan", 15)] if not fetch_urls: return False # url → Seite(n) für den Report url_to_page = _image_url_sources(snap) hashes = fetch_asset_hashes(fetch_urls, timeout=cfg.get("request_timeout", 15)) dirty = False for url in fetch_urls: h = hashes.get(url, {}) fp = h.get("sha256") if not fp: continue # nicht abrufbar — überspringen result["checked"] += 1 page_url = url_to_page.get(url, url) entry = entries.get(fp) if entry is None: classified = _classify_image(url, site_context, models, ai_cfg, api_key) if classified is None: result["skipped"] = result["skipped"] or "API-Fehler (Bild)" continue verdict, used_model = classified entry = _make_entry("image", url, verdict, used_model) entries[fp] = entry result["api_calls"] += 1 dirty = True else: result["cache_hits"] += 1 _maybe_finding(result, entry, fp, page_url, ai_cfg, asset_url=url) return dirty # --------------------------------------------------------------------------- # Audio / Video — vorbereitete Erweiterungspunkte (default deaktiviert) # --------------------------------------------------------------------------- def _collect_audio_candidates(snap: dict, cfg: dict) -> list[str]: """TODO: Audio-Quellen sammeln. Erfordert