From 013cf794bc0a4c4d0cf326866e521b387ae85add Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dieter=20Schl=C3=BCter?= Date: Sun, 14 Jun 2026 19:02:17 +0200 Subject: [PATCH 01/12] feat: add local server fallback to AI analyzer, include_paths support, config_path in reports - ai_analyzer: OpenRouter primary + localhost:8001/:8002 as dev fallback - crawler: support include_paths for targeted crawling - __main__: pass config_path through report pipeline - tests: 266 tests passing --- TASK.md | 11 + scanner/__main__.py | 23 +- scanner/ai_analyzer.py | 457 ++++++++++++++++++++++--- scanner/config.py | 6 +- scanner/crawler.py | 18 +- scanner/differ.py | 2 +- scanner/extractor.py | 15 + scanner/report.py | 7 +- tests/fixtures/audio/Preussenlied.mp3 | Bin 0 -> 5661816 bytes tests/fixtures/audio/clean_welcome.mp3 | Bin 0 -> 5661816 bytes tests/fixtures/audio/clean_welcome.wav | Bin 0 -> 597198 bytes tests/fixtures/audio/hate_speech.mp3 | Bin 0 -> 122589 bytes tests/fixtures/audio/hate_speech.wav | Bin 0 -> 718158 bytes 13 files changed, 479 insertions(+), 60 deletions(-) create mode 100644 TASK.md create mode 100644 tests/fixtures/audio/Preussenlied.mp3 create mode 100644 tests/fixtures/audio/clean_welcome.mp3 create mode 100644 tests/fixtures/audio/clean_welcome.wav create mode 100644 tests/fixtures/audio/hate_speech.mp3 create mode 100644 tests/fixtures/audio/hate_speech.wav diff --git a/TASK.md b/TASK.md new file mode 100644 index 0000000..b812dfc --- /dev/null +++ b/TASK.md @@ -0,0 +1,11 @@ +# Aufgabe + +"Funktioniert das Projekt?" + +## Erstellt +2026-06-14T17:00:59.565Z + +## Status +- [ ] Implementierung +- [ ] Review bestanden (PASS) +- [ ] Produktionsreif (SHIP) diff --git a/scanner/__main__.py b/scanner/__main__.py index 6f635fc..58db193 100644 --- a/scanner/__main__.py +++ b/scanner/__main__.py @@ -152,11 +152,13 @@ def _build_report( asset_diff: dict | None = None, ext_links: dict | None = None, ai_result: dict | None = None, + config_path: str | None = None, ) -> dict: report = { "generated_at": datetime.now(timezone.utc).isoformat(), "target": target, "snapshot_dir": str(snap_dir), + "config_path": config_path, "assessment": assessment, "diff": diff, "whitelist_violations": whitelist_violations, @@ -730,9 +732,11 @@ def cmd_check(args: argparse.Namespace, cfg: dict) -> int: report = _build_report( cfg["target"], diff, assessment, wl_violations, missing_sec, snap.get("errors", []), snap["dir"], ai_result=ai_result, + config_path=getattr(args, "config", None), ) _write_report(report, cfg["reports_dir"]) - _print_summary(assessment, diff, ai_result=ai_result, target=cfg["target"]) + _print_summary(assessment, diff, ai_result=ai_result, target=cfg["target"], + config_path=getattr(args, "config", None)) send_alert(report, cfg) return assessment["exit_code"] @@ -803,10 +807,11 @@ def cmd_scan(args: argparse.Namespace, cfg: dict) -> int: cfg["target"], diff, assessment, wl_violations, missing_sec, snap.get("errors", []), snap["dir"], cloak_diff=cloak_diff, asset_diff=asset_diff, ext_links=ext_links, - ai_result=ai_result, + ai_result=ai_result, config_path=getattr(args, "config", None), ) _write_report(report, cfg["reports_dir"]) - _print_summary(assessment, diff, cloak_diff, asset_diff, ext_links, ai_result, cfg["target"]) + _print_summary(assessment, diff, cloak_diff, asset_diff, ext_links, ai_result, cfg["target"], + config_path=getattr(args, "config", None)) send_alert(report, cfg) return assessment["exit_code"] @@ -1261,8 +1266,15 @@ def _print_summary(assessment: dict, diff: dict, asset_diff: dict | None = None, ext_links: dict | None = None, ai_result: dict | None = None, - target: str = "") -> None: + target: str = "", + config_path: str | None = None) -> None: level = assessment.get("level", "green") + if config_path: + scan_cmd = f"python -m scanner --config {config_path} scan" + report_cmd = f"python -m scanner --config {config_path} report" + else: + scan_cmd = scanner_cmd(target, "scan") + report_cmd = scanner_cmd(target, "report") print() print(ampel_zeile(level)) print() @@ -1273,11 +1285,12 @@ def _print_summary(assessment: dict, diff: dict, print(f" → {schritt}") # Knappe technische Kennzahlen darunter (für den Dienstleister) print() + print(f" — Ausgeführt mit: {scan_cmd}") print(f" — Technische Details: Score {assessment.get('score', 0)}, " f"{diff.get('changed_pages', 0)} geänderte Seite(n), " f"{len(diff.get('new_internal_urls', []))} neue / " f"{len(diff.get('missing_internal_urls', []))} fehlende Seite(n). " - f"Vollständiger Bericht: {scanner_cmd(target, 'report')}") + f"Vollständiger Bericht: {report_cmd}") def cmd_test_alert(args: argparse.Namespace, cfg: dict) -> int: diff --git a/scanner/ai_analyzer.py b/scanner/ai_analyzer.py index f8b7f48..75851cf 100644 --- a/scanner/ai_analyzer.py +++ b/scanner/ai_analyzer.py @@ -14,13 +14,17 @@ Robustheit: Fehlt der API-Key oder schlägt ein Call fehl, wird die Analyse übersprungen und der Scan läuft unverändert weiter (graceful degradation). KI darf den Kern-Scan nie brechen. """ +import base64 import concurrent.futures +import hashlib import json import logging import os +import subprocess import threading import time from datetime import datetime, timezone +from pathlib import Path import requests @@ -29,7 +33,16 @@ from .differ import normalize_text logger = logging.getLogger(__name__) +# Primärer Endpunkt: OpenRouter (wie bisher). _OPENROUTER_URL = "https://openrouter.ai/api/v1/chat/completions" +_MODELS_URL = "https://openrouter.ai/api/v1/models" + +# Lokale llama.cpp-Server als Fallback — werden nur genutzt, wenn OpenRouter +# nicht erreichbar ist (Rate-Limit, Timeout, Ausfall). +_LOCAL_SERVERS: list[str] = [ + "http://localhost:8001", + "http://localhost:8002", +] # Erlaubte Kategorien, die das Modell zurückgeben darf. _CATEGORIES = [ @@ -71,7 +84,7 @@ _RESPONSE_SCHEMA = { } -_MODELS_URL = "https://openrouter.ai/api/v1/models" + # --------------------------------------------------------------------------- @@ -80,9 +93,9 @@ _MODELS_URL = "https://openrouter.ai/api/v1/models" class ModelRouter: """Wählt programmatisch das beste Modell aus einer Kette: gesunde zuerst, - nach Latenz; rate-limited/ausgefallene Modelle bekommen einen Cooldown - (Circuit-Breaker) und werden so lange übersprungen. Tote Slugs werden über - den /models-Katalog failsafe aussortiert. Thread-sicher (für Parallelität).""" + nach Latenz; ausgefallene Modelle bekommen einen Cooldown (Circuit-Breaker). + Katalog-Pruning funktioniert für OpenRouter- und lokale Modell-Slugs. + Thread-sicher (für Parallelität).""" def __init__(self, ai_cfg: dict, state: dict | None = None): self._lock = threading.Lock() @@ -91,7 +104,10 @@ class ModelRouter: self._refresh = ai_cfg.get("refresh_models", True) state = state or {} self._models: dict = state.get("models", {}) or {} - self._catalog: dict = state.get("catalog", {"slugs": [], "fetched_at": None}) + # catalog: {"slugs": [...], "servers": {...}, "fetched_at": ...} + # slugs = OpenRouter-Katalog (primär), servers = lokale Server-Modelle + saved = state.get("catalog", {"slugs": [], "servers": {}, "fetched_at": None}) + self._catalog: dict = saved if isinstance(saved, dict) else {"slugs": [], "servers": {}, "fetched_at": None} def state(self) -> dict: with self._lock: @@ -136,29 +152,59 @@ class ModelRouter: return ordered or list(models) def _prune_locked(self, models: list[str]) -> list[str]: - slugs = self._catalog.get("slugs") or [] - if not slugs: + # Kombiniere OpenRouter-Slugs + lokale Server-Modelle. + all_slugs: set[str] = set(self._catalog.get("slugs", [])) + for slugs in self._catalog.get("servers", {}).values(): + all_slugs.update(slugs) + if not all_slugs: return list(models) # kein Katalog → nicht filtern (failsafe) - keep = [m for m in models if m in slugs] + keep = [m for m in models if m in all_slugs] return keep or list(models) # nie alles wegfiltern def refresh_catalog(self) -> None: - """Einmal pro Lauf /models ziehen (24 h gecacht). Failsafe: blockiert nie.""" + """OpenRouter /models + lokale Server /models ziehen (24 h gecacht). + Failsafe: blockiert nie.""" if not self._refresh: return fetched_at = self._catalog.get("fetched_at") - if fetched_at and self._catalog.get("slugs") and (time.time() - fetched_at) < 86400: + slugs = self._catalog.get("slugs") or [] + servers = self._catalog.get("servers") or {} + if fetched_at and (slugs or servers) and (time.time() - fetched_at) < 86400: return # Cache frisch + + new_slugs, new_servers = [], {} + # 1. OpenRouter-Katalog (primär) try: - resp = requests.get(_MODELS_URL, timeout=15) + resp = requests.get(_MODELS_URL, timeout=10) if resp.status_code == 200: - slugs = [m["id"] for m in resp.json().get("data", []) if m.get("id")] - if slugs: - with self._lock: - self._catalog = {"slugs": slugs, "fetched_at": time.time()} - logger.info("KI-Router: /models-Katalog aktualisiert (%d Modelle).", len(slugs)) + new_slugs = [m["id"] for m in resp.json().get("data", []) if m.get("id")] + logger.info("KI-Router: OpenRouter-Katalog aktualisiert (%d Modelle).", + len(new_slugs)) except (requests.exceptions.RequestException, ValueError, KeyError) as exc: - logger.warning("KI-Router: /models nicht abrufbar (failsafe, ungefiltert): %s", exc) + logger.debug("KI-Router: OpenRouter /models nicht abrufbar: %s", exc) + + # 2. Lokale Server-Kataloge (Fallback) + for base_url in _LOCAL_SERVERS: + try: + resp = requests.get(f"{base_url}/v1/models", timeout=5) + if resp.status_code == 200: + data = resp.json() + raw = data.get("data") or data.get("models", []) + local_slugs = [m["id"] for m in raw if isinstance(m, dict) and m.get("id")] + if local_slugs: + new_servers[base_url] = local_slugs + logger.info("KI-Router: %s — %d Modelle gefunden.", base_url, len(local_slugs)) + elif resp.status_code == 503: + logger.debug("KI-Router: %s — Modell wird geladen (503).", base_url) + except requests.exceptions.RequestException as exc: + logger.debug("KI-Router: %s nicht erreichbar: %s", base_url, exc) + + if new_slugs or new_servers: + with self._lock: + self._catalog = {"slugs": new_slugs, "servers": new_servers, + "fetched_at": time.time()} + total = len(new_slugs) + sum(len(v) for v in new_servers.values()) + logger.info("KI-Router: Katalog aktualisiert (%d Modelle gesamt).", total) # --------------------------------------------------------------------------- @@ -209,7 +255,12 @@ def run_ai_analysis(cfg: dict, bm, snap: dict, diff: dict) -> dict: if ai_cfg.get("image", {}).get("enabled", True): dirty |= _analyze_images(cfg, ai_cfg, api_key, snap, diff, entries, result, router, executor, url_hashes) - # Audio/Video: vorbereitet, default aus (siehe _collect_audio/video_candidates). + if ai_cfg.get("audio", {}).get("enabled", False): + dirty |= _analyze_audio(cfg, ai_cfg, api_key, snap, diff, + entries, result, router, executor, url_hashes) + if ai_cfg.get("video", {}).get("enabled", False): + dirty |= _analyze_video(cfg, ai_cfg, api_key, snap, diff, + entries, result, router, executor, url_hashes) finally: if dirty: bm.save_ai_ledger(ledger) @@ -453,19 +504,178 @@ def _analyze_images(cfg, ai_cfg, api_key, snap, diff, entries, result, router, e # --------------------------------------------------------------------------- -# Audio / Video — vorbereitete Erweiterungspunkte (default deaktiviert) +# Audio analysis # --------------------------------------------------------------------------- -def _collect_audio_candidates(snap: dict, cfg: dict) -> list[str]: - """TODO: Audio-Quellen sammeln. Erfordert