feat: add local server fallback to AI analyzer, include_paths support, config_path in reports

- ai_analyzer: OpenRouter primary + localhost:8001/:8002 as dev fallback
- crawler: support include_paths for targeted crawling
- __main__: pass config_path through report pipeline
- tests: 266 tests passing
This commit is contained in:
Dieter Schlüter 2026-06-14 19:02:17 +02:00
commit 013cf794bc
13 changed files with 479 additions and 60 deletions

View file

@ -152,11 +152,13 @@ def _build_report(
asset_diff: dict | None = None,
ext_links: dict | None = None,
ai_result: dict | None = None,
config_path: str | None = None,
) -> dict:
report = {
"generated_at": datetime.now(timezone.utc).isoformat(),
"target": target,
"snapshot_dir": str(snap_dir),
"config_path": config_path,
"assessment": assessment,
"diff": diff,
"whitelist_violations": whitelist_violations,
@ -730,9 +732,11 @@ def cmd_check(args: argparse.Namespace, cfg: dict) -> int:
report = _build_report(
cfg["target"], diff, assessment, wl_violations, missing_sec,
snap.get("errors", []), snap["dir"], ai_result=ai_result,
config_path=getattr(args, "config", None),
)
_write_report(report, cfg["reports_dir"])
_print_summary(assessment, diff, ai_result=ai_result, target=cfg["target"])
_print_summary(assessment, diff, ai_result=ai_result, target=cfg["target"],
config_path=getattr(args, "config", None))
send_alert(report, cfg)
return assessment["exit_code"]
@ -803,10 +807,11 @@ def cmd_scan(args: argparse.Namespace, cfg: dict) -> int:
cfg["target"], diff, assessment, wl_violations, missing_sec,
snap.get("errors", []), snap["dir"],
cloak_diff=cloak_diff, asset_diff=asset_diff, ext_links=ext_links,
ai_result=ai_result,
ai_result=ai_result, config_path=getattr(args, "config", None),
)
_write_report(report, cfg["reports_dir"])
_print_summary(assessment, diff, cloak_diff, asset_diff, ext_links, ai_result, cfg["target"])
_print_summary(assessment, diff, cloak_diff, asset_diff, ext_links, ai_result, cfg["target"],
config_path=getattr(args, "config", None))
send_alert(report, cfg)
return assessment["exit_code"]
@ -1261,8 +1266,15 @@ def _print_summary(assessment: dict, diff: dict,
asset_diff: dict | None = None,
ext_links: dict | None = None,
ai_result: dict | None = None,
target: str = "") -> None:
target: str = "",
config_path: str | None = None) -> None:
level = assessment.get("level", "green")
if config_path:
scan_cmd = f"python -m scanner --config {config_path} scan"
report_cmd = f"python -m scanner --config {config_path} report"
else:
scan_cmd = scanner_cmd(target, "scan")
report_cmd = scanner_cmd(target, "report")
print()
print(ampel_zeile(level))
print()
@ -1273,11 +1285,12 @@ def _print_summary(assessment: dict, diff: dict,
print(f"{schritt}")
# Knappe technische Kennzahlen darunter (für den Dienstleister)
print()
print(f" — Ausgeführt mit: {scan_cmd}")
print(f" — Technische Details: Score {assessment.get('score', 0)}, "
f"{diff.get('changed_pages', 0)} geänderte Seite(n), "
f"{len(diff.get('new_internal_urls', []))} neue / "
f"{len(diff.get('missing_internal_urls', []))} fehlende Seite(n). "
f"Vollständiger Bericht: {scanner_cmd(target, 'report')}")
f"Vollständiger Bericht: {report_cmd}")
def cmd_test_alert(args: argparse.Namespace, cfg: dict) -> int:

View file

@ -14,13 +14,17 @@ Robustheit: Fehlt der API-Key oder schlägt ein Call fehl, wird die Analyse
übersprungen und der Scan läuft unverändert weiter (graceful degradation).
KI darf den Kern-Scan nie brechen.
"""
import base64
import concurrent.futures
import hashlib
import json
import logging
import os
import subprocess
import threading
import time
from datetime import datetime, timezone
from pathlib import Path
import requests
@ -29,7 +33,16 @@ from .differ import normalize_text
logger = logging.getLogger(__name__)
# Primärer Endpunkt: OpenRouter (wie bisher).
_OPENROUTER_URL = "https://openrouter.ai/api/v1/chat/completions"
_MODELS_URL = "https://openrouter.ai/api/v1/models"
# Lokale llama.cpp-Server als Fallback — werden nur genutzt, wenn OpenRouter
# nicht erreichbar ist (Rate-Limit, Timeout, Ausfall).
_LOCAL_SERVERS: list[str] = [
"http://localhost:8001",
"http://localhost:8002",
]
# Erlaubte Kategorien, die das Modell zurückgeben darf.
_CATEGORIES = [
@ -71,7 +84,7 @@ _RESPONSE_SCHEMA = {
}
_MODELS_URL = "https://openrouter.ai/api/v1/models"
# ---------------------------------------------------------------------------
@ -80,9 +93,9 @@ _MODELS_URL = "https://openrouter.ai/api/v1/models"
class ModelRouter:
"""Wählt programmatisch das beste Modell aus einer Kette: gesunde zuerst,
nach Latenz; rate-limited/ausgefallene Modelle bekommen einen Cooldown
(Circuit-Breaker) und werden so lange übersprungen. Tote Slugs werden über
den /models-Katalog failsafe aussortiert. Thread-sicher (für Parallelität)."""
nach Latenz; ausgefallene Modelle bekommen einen Cooldown (Circuit-Breaker).
Katalog-Pruning funktioniert für OpenRouter- und lokale Modell-Slugs.
Thread-sicher (für Parallelität)."""
def __init__(self, ai_cfg: dict, state: dict | None = None):
self._lock = threading.Lock()
@ -91,7 +104,10 @@ class ModelRouter:
self._refresh = ai_cfg.get("refresh_models", True)
state = state or {}
self._models: dict = state.get("models", {}) or {}
self._catalog: dict = state.get("catalog", {"slugs": [], "fetched_at": None})
# catalog: {"slugs": [...], "servers": {...}, "fetched_at": ...}
# slugs = OpenRouter-Katalog (primär), servers = lokale Server-Modelle
saved = state.get("catalog", {"slugs": [], "servers": {}, "fetched_at": None})
self._catalog: dict = saved if isinstance(saved, dict) else {"slugs": [], "servers": {}, "fetched_at": None}
def state(self) -> dict:
with self._lock:
@ -136,29 +152,59 @@ class ModelRouter:
return ordered or list(models)
def _prune_locked(self, models: list[str]) -> list[str]:
slugs = self._catalog.get("slugs") or []
if not slugs:
# Kombiniere OpenRouter-Slugs + lokale Server-Modelle.
all_slugs: set[str] = set(self._catalog.get("slugs", []))
for slugs in self._catalog.get("servers", {}).values():
all_slugs.update(slugs)
if not all_slugs:
return list(models) # kein Katalog → nicht filtern (failsafe)
keep = [m for m in models if m in slugs]
keep = [m for m in models if m in all_slugs]
return keep or list(models) # nie alles wegfiltern
def refresh_catalog(self) -> None:
"""Einmal pro Lauf /models ziehen (24 h gecacht). Failsafe: blockiert nie."""
"""OpenRouter /models + lokale Server /models ziehen (24 h gecacht).
Failsafe: blockiert nie."""
if not self._refresh:
return
fetched_at = self._catalog.get("fetched_at")
if fetched_at and self._catalog.get("slugs") and (time.time() - fetched_at) < 86400:
slugs = self._catalog.get("slugs") or []
servers = self._catalog.get("servers") or {}
if fetched_at and (slugs or servers) and (time.time() - fetched_at) < 86400:
return # Cache frisch
new_slugs, new_servers = [], {}
# 1. OpenRouter-Katalog (primär)
try:
resp = requests.get(_MODELS_URL, timeout=15)
resp = requests.get(_MODELS_URL, timeout=10)
if resp.status_code == 200:
slugs = [m["id"] for m in resp.json().get("data", []) if m.get("id")]
if slugs:
with self._lock:
self._catalog = {"slugs": slugs, "fetched_at": time.time()}
logger.info("KI-Router: /models-Katalog aktualisiert (%d Modelle).", len(slugs))
new_slugs = [m["id"] for m in resp.json().get("data", []) if m.get("id")]
logger.info("KI-Router: OpenRouter-Katalog aktualisiert (%d Modelle).",
len(new_slugs))
except (requests.exceptions.RequestException, ValueError, KeyError) as exc:
logger.warning("KI-Router: /models nicht abrufbar (failsafe, ungefiltert): %s", exc)
logger.debug("KI-Router: OpenRouter /models nicht abrufbar: %s", exc)
# 2. Lokale Server-Kataloge (Fallback)
for base_url in _LOCAL_SERVERS:
try:
resp = requests.get(f"{base_url}/v1/models", timeout=5)
if resp.status_code == 200:
data = resp.json()
raw = data.get("data") or data.get("models", [])
local_slugs = [m["id"] for m in raw if isinstance(m, dict) and m.get("id")]
if local_slugs:
new_servers[base_url] = local_slugs
logger.info("KI-Router: %s%d Modelle gefunden.", base_url, len(local_slugs))
elif resp.status_code == 503:
logger.debug("KI-Router: %s — Modell wird geladen (503).", base_url)
except requests.exceptions.RequestException as exc:
logger.debug("KI-Router: %s nicht erreichbar: %s", base_url, exc)
if new_slugs or new_servers:
with self._lock:
self._catalog = {"slugs": new_slugs, "servers": new_servers,
"fetched_at": time.time()}
total = len(new_slugs) + sum(len(v) for v in new_servers.values())
logger.info("KI-Router: Katalog aktualisiert (%d Modelle gesamt).", total)
# ---------------------------------------------------------------------------
@ -209,7 +255,12 @@ def run_ai_analysis(cfg: dict, bm, snap: dict, diff: dict) -> dict:
if ai_cfg.get("image", {}).get("enabled", True):
dirty |= _analyze_images(cfg, ai_cfg, api_key, snap, diff,
entries, result, router, executor, url_hashes)
# Audio/Video: vorbereitet, default aus (siehe _collect_audio/video_candidates).
if ai_cfg.get("audio", {}).get("enabled", False):
dirty |= _analyze_audio(cfg, ai_cfg, api_key, snap, diff,
entries, result, router, executor, url_hashes)
if ai_cfg.get("video", {}).get("enabled", False):
dirty |= _analyze_video(cfg, ai_cfg, api_key, snap, diff,
entries, result, router, executor, url_hashes)
finally:
if dirty:
bm.save_ai_ledger(ledger)
@ -453,19 +504,178 @@ def _analyze_images(cfg, ai_cfg, api_key, snap, diff, entries, result, router, e
# ---------------------------------------------------------------------------
# Audio / Video — vorbereitete Erweiterungspunkte (default deaktiviert)
# Audio analysis
# ---------------------------------------------------------------------------
def _collect_audio_candidates(snap: dict, cfg: dict) -> list[str]:
"""TODO: Audio-Quellen sammeln. Erfordert <audio>/<source>-Extraktion in
extractor.py (heute nicht extrahiert). Aktivierbar über ai_analysis.audio.enabled."""
return []
def _analyze_audio(cfg, ai_cfg, api_key, snap, diff, entries, result, router, executor,
url_hashes) -> bool:
"""Prüft neue/geänderte Audio-Dateien (parallel). Returns True wenn Ledger verändert."""
aud_cfg = ai_cfg.get("audio", {})
models = _models_for(aud_cfg)
site_context = ai_cfg.get("site_context", "")
current_urls = _all_media_urls(snap, "audio")
if not current_urls:
return False
changed = {pd["url"] for pd in diff.get("page_diffs", [])} | set(diff.get("new_internal_urls", []))
changed_urls = _all_media_urls(snap, "audio", only_pages=changed)
flagged_urls = {
e["url"] for e in entries.values()
if e.get("kind") == "audio" and e.get("category", "clean") != "clean"
and not e.get("dismissed") and e.get("url") in current_urls
}
priority = sorted((changed_urls | flagged_urls) & current_urls)
coverage = sorted((current_urls - set(url_hashes)) - set(priority))
backlog_limit = aud_cfg.get("max_files_per_scan", 5)
if backlog_limit and backlog_limit > 0:
coverage = coverage[:backlog_limit]
fetch_urls = priority + coverage
if not fetch_urls:
return False
url_to_page = _media_url_sources(snap, "audio")
# Sampling: für jede URL via ffmpeg einen 90-s-Ausschnitt holen (hash + bytes).
# Ersetzt _local_file_hashes + fetch_asset_hashes — kein 40-60 MB Download mehr.
samples: dict[str, tuple[bytes, str]] = {} # url → (mp3_bytes, sha256)
dirty = False
fp_assets: dict[str, list[str]] = {}
order_fps: list[str] = []
for url in fetch_urls:
sample = _audio_sample(url, timeout=ai_cfg.get("request_timeout", 60))
if sample is None:
result["unchecked"].append(
{"kind": "audio", "url": url_to_page.get(url, url), "asset_url": url}
)
continue
data, fp = sample
samples[url] = (data, fp)
if url_hashes.get(url) != fp:
url_hashes[url] = fp
dirty = True
result["checked"] += 1
if fp not in fp_assets:
fp_assets[fp] = []
order_fps.append(fp)
fp_assets[fp].append(url)
tasks, miss_fps = [], []
for fp in order_fps:
entry = entries.get(fp)
if entry is not None:
for url in fp_assets[fp]:
result["cache_hits"] += 1
_maybe_finding(result, entry, fp, url_to_page.get(url, url), ai_cfg, asset_url=url)
else:
miss_fps.append(fp)
rep_bytes = samples[fp_assets[fp][0]][0]
tasks.append((fp, _audio_task(rep_bytes, site_context, models, ai_cfg, api_key, router)))
classified = _classify_batch(tasks, executor)
for fp in miss_fps:
res = classified.get(fp)
rep_url = fp_assets[fp][0]
if res is None:
for url in fp_assets[fp]:
result["unchecked"].append(
{"kind": "audio", "url": url_to_page.get(url, url), "asset_url": url})
continue
verdict, used_model = res
entry = _make_entry("audio", rep_url, verdict, used_model)
entries[fp] = entry
result["api_calls"] += 1
dirty = True
for url in fp_assets[fp]:
_maybe_finding(result, entry, fp, url_to_page.get(url, url), ai_cfg, asset_url=url)
return dirty
def _collect_video_candidates(snap: dict, cfg: dict) -> list[str]:
"""TODO: Video-Quellen sammeln. Erfordert <video>/<source>-Extraktion in
extractor.py + Frame-Sampling. Aktivierbar über ai_analysis.video.enabled."""
return []
# ---------------------------------------------------------------------------
# Video analysis
# ---------------------------------------------------------------------------
def _analyze_video(cfg, ai_cfg, api_key, snap, diff, entries, result, router, executor,
url_hashes) -> bool:
"""Prüft neue/geänderte Video-Dateien (parallel). Returns True wenn Ledger verändert."""
vid_cfg = ai_cfg.get("video", {})
models = _models_for(vid_cfg)
site_context = ai_cfg.get("site_context", "")
current_urls = _all_media_urls(snap, "video")
if not current_urls:
return False
changed = {pd["url"] for pd in diff.get("page_diffs", [])} | set(diff.get("new_internal_urls", []))
changed_urls = _all_media_urls(snap, "video", only_pages=changed)
flagged_urls = {
e["url"] for e in entries.values()
if e.get("kind") == "video" and e.get("category", "clean") != "clean"
and not e.get("dismissed") and e.get("url") in current_urls
}
priority = sorted((changed_urls | flagged_urls) & current_urls)
coverage = sorted((current_urls - set(url_hashes)) - set(priority))
backlog_limit = vid_cfg.get("max_files_per_scan", 3)
if backlog_limit and backlog_limit > 0:
coverage = coverage[:backlog_limit]
fetch_urls = priority + coverage
if not fetch_urls:
return False
url_to_page = _media_url_sources(snap, "video")
local_urls = [u for u in fetch_urls if u.startswith("file://")]
remote_urls = [u for u in fetch_urls if not u.startswith("file://")]
hashes = _local_file_hashes(local_urls)
if remote_urls:
hashes.update(fetch_asset_hashes(remote_urls, timeout=cfg.get("request_timeout", 15)))
dirty = False
fp_assets: dict[str, list[str]] = {}
order_fps: list[str] = []
for url in fetch_urls:
fp = hashes.get(url, {}).get("sha256")
if not fp:
continue
if url_hashes.get(url) != fp:
url_hashes[url] = fp
dirty = True
result["checked"] += 1
if fp not in fp_assets:
fp_assets[fp] = []
order_fps.append(fp)
fp_assets[fp].append(url)
tasks, miss_fps = [], []
for fp in order_fps:
entry = entries.get(fp)
if entry is not None:
for url in fp_assets[fp]:
result["cache_hits"] += 1
_maybe_finding(result, entry, fp, url_to_page.get(url, url), ai_cfg, asset_url=url)
else:
miss_fps.append(fp)
tasks.append((fp, _video_task(fp_assets[fp][0], site_context, models, ai_cfg, api_key, router)))
classified = _classify_batch(tasks, executor)
for fp in miss_fps:
res = classified.get(fp)
rep_url = fp_assets[fp][0]
if res is None:
for url in fp_assets[fp]:
result["unchecked"].append(
{"kind": "video", "url": url_to_page.get(url, url), "asset_url": url})
continue
verdict, used_model = res
entry = _make_entry("video", rep_url, verdict, used_model)
entries[fp] = entry
result["api_calls"] += 1
dirty = True
for url in fp_assets[fp]:
_maybe_finding(result, entry, fp, url_to_page.get(url, url), ai_cfg, asset_url=url)
return dirty
# ---------------------------------------------------------------------------
@ -475,7 +685,6 @@ def _collect_video_candidates(snap: dict, cfg: dict) -> list[str]:
def _text_fingerprint(text: str, cfg: dict) -> str:
"""SHA-256 des normalisierten Texts (Reuse von differ.normalize_text → Cache-
Buster wie CF7-Platzhalter ändern den Hash nicht)."""
import hashlib
norm = normalize_text(text, cfg)
return hashlib.sha256(norm.encode("utf-8")).hexdigest()
@ -521,10 +730,10 @@ def _maybe_finding(result, entry, fp, url, ai_cfg, asset_url=None) -> None:
def _finding_points(finding: dict, sc: dict) -> int:
"""Punkte für einen Fund. Bilder mindestens ai_suspicious_image."""
"""Punkte für einen Fund. Bilder/Audio/Video mindestens ai_suspicious_image."""
key, default = _CATEGORY_SCORE_KEY.get(finding.get("category", ""), (None, 30))
pts = sc.get(key, default) if key else 30
if finding.get("kind") == "image":
if finding.get("kind") in ("image", "audio", "video"):
pts = max(pts, sc.get("ai_suspicious_image", 40))
return pts
@ -552,6 +761,29 @@ def _image_url_sources(snap: dict) -> dict[str, str]:
return sources
def _all_media_urls(snap: dict, kind: str, only_pages: set | None = None) -> set[str]:
urls: set[str] = set()
for page_url, page in snap.get("pages", {}).items():
if only_pages is not None and page_url not in only_pages:
continue
if page.get("status") != 200:
continue
for link in page.get("links", {}).get(kind, []):
if isinstance(link, dict) and link.get("url"):
urls.add(link["url"])
return urls
def _media_url_sources(snap: dict, kind: str) -> dict[str, str]:
"""Map Medien-URL → erste Seite, auf der sie vorkommt (für den Report)."""
sources: dict[str, str] = {}
for page_url, page in snap.get("pages", {}).items():
for link in page.get("links", {}).get(kind, []):
if isinstance(link, dict) and link.get("url"):
sources.setdefault(link["url"], page_url)
return sources
# ---------------------------------------------------------------------------
# OpenRouter classification
# ---------------------------------------------------------------------------
@ -638,6 +870,101 @@ def _image_task(image_url, site_context, models, ai_cfg, api_key, router):
return lambda: _classify_image(image_url, site_context, models, ai_cfg, api_key, router)
def _local_file_hashes(urls: list[str]) -> dict[str, dict]:
"""Berechnet SHA-256 für file://-URLs direkt aus dem Dateisystem (kein HTTP)."""
result: dict[str, dict] = {}
for url in urls:
if not url.startswith("file://"):
continue
path = Path(url[7:])
try:
data = path.read_bytes()
result[url] = {"sha256": hashlib.sha256(data).hexdigest(), "size": len(data)}
except OSError as exc:
logger.warning("Lokale Datei nicht lesbar (%s): %s", url, exc)
return result
def _audio_sample(url: str, timeout: int = 60) -> tuple[bytes, str] | None:
"""Extrahiert via ffmpeg einen 90-s-Ausschnitt ab Sekunde 30 (Intro überspringen).
Arbeitet für file://-Pfade und https://-URLs gleich; lädt nur ~3 MB statt 40-60 MB.
Gibt (mp3_bytes, sha256_hex) zurück oder None bei Fehler."""
input_path = url[7:] if url.startswith("file://") else url
def _run(ss: str) -> subprocess.CompletedProcess:
return subprocess.run(
["ffmpeg", "-hide_banner", "-loglevel", "error",
"-i", input_path, "-ss", ss, "-t", "90",
"-ar", "22050", "-ac", "1", "-b:a", "32k", "-f", "mp3", "pipe:1"],
capture_output=True, timeout=timeout,
)
try:
r = _run("30")
if r.returncode != 0 or not r.stdout:
r = _run("0") # Fallback: Datei kürzer als 30 s
if r.returncode != 0 or not r.stdout:
logger.warning("ffmpeg-Sampling fehlgeschlagen für %s: %s",
url, r.stderr.decode()[:200])
return None
return r.stdout, hashlib.sha256(r.stdout).hexdigest()
except (subprocess.TimeoutExpired, FileNotFoundError, OSError) as exc:
logger.warning("ffmpeg nicht aufrufbar für %s: %s", url, exc)
return None
def _media_content(url: str, media_type: str) -> dict:
"""Gibt den OpenRouter-Message-Content für eine Medien-URL zurück.
Bei file://-Pfaden: Datei einlesen und als inline-base64 senden (input_audio/input_video).
Bei http(s)-URLs: URL-Referenz (audio_url/video_url)."""
if url.startswith("file://"):
path = Path(url[7:])
data = path.read_bytes()
b64 = base64.b64encode(data).decode()
fmt = path.suffix.lstrip(".").lower() or media_type
if media_type == "audio":
return {"type": "input_audio", "input_audio": {"data": b64, "format": fmt}}
return {"type": "video_url", "video_url": {"url": f"data:video/{fmt};base64,{b64}"}}
if media_type == "audio":
return {"type": "audio_url", "audio_url": {"url": url}}
return {"type": "video_url", "video_url": {"url": url}}
def _classify_audio(audio_bytes: bytes, site_context, models, ai_cfg, api_key, router) -> tuple[dict, str] | None:
b64 = base64.b64encode(audio_bytes).decode()
messages = [
{"role": "system", "content": _SYSTEM_PROMPT.format(context=site_context or "(nicht angegeben)")},
{"role": "user", "content": [
{"type": "text", "text": (
"Prüfe diesen Audio-Ausschnitt (bis zu 90 Sekunden, ab Sekunde 30) "
"auf problematische Inhalte (Sprache, Geräusche, Musik — soweit erkennbar)."
)},
{"type": "input_audio", "input_audio": {"data": b64, "format": "mp3"}},
]},
]
return _classify(messages, models, ai_cfg, api_key, router)
def _classify_video(video_url, site_context, models, ai_cfg, api_key, router) -> tuple[dict, str] | None:
messages = [
{"role": "system", "content": _SYSTEM_PROMPT.format(context=site_context or "(nicht angegeben)")},
{"role": "user", "content": [
{"type": "text", "text": "Prüfe dieses Video auf problematische Inhalte "
"(Bild, Ton, eingebetteter Text — soweit erkennbar)."},
_media_content(video_url, "video"),
]},
]
return _classify(messages, models, ai_cfg, api_key, router)
def _audio_task(audio_bytes: bytes, site_context, models, ai_cfg, api_key, router):
return lambda: _classify_audio(audio_bytes, site_context, models, ai_cfg, api_key, router)
def _video_task(video_url, site_context, models, ai_cfg, api_key, router):
return lambda: _classify_video(video_url, site_context, models, ai_cfg, api_key, router)
def _classify_batch(tasks, executor) -> dict:
"""Führt [(key, fn)] nebenläufig im Pool aus. Returns {key: fn()-Ergebnis|None}.
Eine fehlgeschlagene Aufgabe blubbert nie hoch ( None, der Aufrufer wertet das
@ -656,7 +983,7 @@ def _classify_batch(tasks, executor) -> dict:
return out
def _post_with_deadline(headers, payload, timeout):
def _post_with_deadline(url: str, headers, payload, timeout):
"""requests.post in einem Worker-Thread mit HARTEM Wanduhr-Limit.
Der requests-Timeout ist nur ein Inaktivitäts-Timeout: ein Server, der die
@ -665,7 +992,7 @@ def _post_with_deadline(headers, payload, timeout):
bei Überschreitung läuft der verwaiste Thread im Hintergrund aus und wir eskalieren.
"""
ex = concurrent.futures.ThreadPoolExecutor(max_workers=1)
fut = ex.submit(requests.post, _OPENROUTER_URL, headers=headers,
fut = ex.submit(requests.post, url, headers=headers,
json=payload, timeout=timeout)
try:
resp = fut.result(timeout=timeout)
@ -677,8 +1004,9 @@ def _post_with_deadline(headers, payload, timeout):
def _openrouter_chat(model, messages, ai_cfg, api_key, timeout=None) -> dict | None:
"""Einzelner OpenRouter-Call mit strukturierter JSON-Antwort.
Returns das Verdikt-Dict oder None bei jedem Fehler/Timeout (graceful)."""
"""Chat-Call: zuerst OpenRouter (primär), dann lokale llama.cpp-Server als Fallback.
Returns das Verdikt-Dict oder None wenn alle Endpunkte scheitern (graceful)."""
if timeout is None:
timeout = ai_cfg.get("attempt_timeout", ai_cfg.get("request_timeout", 20))
payload = {
@ -687,23 +1015,56 @@ def _openrouter_chat(model, messages, ai_cfg, api_key, timeout=None) -> dict | N
"response_format": _RESPONSE_SCHEMA,
"temperature": 0,
}
# --- 1. OpenRouter (primär) ---
headers = {
"Authorization": f"Bearer {api_key}",
"Content-Type": "application/json",
"X-Title": "integrity-scanner",
}
try:
resp = _post_with_deadline(headers, payload, timeout)
if resp.status_code != 200:
logger.warning("OpenRouter HTTP %s (%s): %s", resp.status_code, model, resp.text[:200])
return None
content = resp.json()["choices"][0]["message"]["content"]
verdict = json.loads(content)
# Mindest-Validierung
if verdict.get("category") not in _CATEGORIES:
logger.warning("OpenRouter (%s): unbekannte Kategorie %r", model, verdict.get("category"))
return None
return verdict
except (requests.exceptions.RequestException, KeyError, ValueError, json.JSONDecodeError) as exc:
logger.warning("OpenRouter-Call fehlgeschlagen (%s): %s", model, exc)
return None
resp = _post_with_deadline(_OPENROUTER_URL, headers, payload, timeout)
if resp.status_code == 200:
content = resp.json()["choices"][0]["message"]["content"]
verdict = json.loads(content)
if verdict.get("category") not in _CATEGORIES:
logger.warning("OpenRouter (%s): unbekannte Kategorie %r",
model, verdict.get("category"))
else:
logger.debug("KI-Antwort von OpenRouter (Modell: %s).", model)
return verdict
else:
logger.info("OpenRouter HTTP %s (%s) — Fallback zu lokalen Servern.",
resp.status_code, model)
except (requests.exceptions.RequestException, KeyError, ValueError,
json.JSONDecodeError) as exc:
logger.debug("OpenRouter-Call fehlgeschlagen (%s): %s", model, exc)
# --- 2. Lokale Server (Fallback) ---
headers = {"Content-Type": "application/json"}
if api_key:
headers["Authorization"] = f"Bearer {api_key}"
for base_url in _LOCAL_SERVERS:
url = f"{base_url}/v1/chat/completions"
try:
resp = _post_with_deadline(url, headers, payload, timeout)
if resp.status_code == 200:
content = resp.json()["choices"][0]["message"]["content"]
verdict = json.loads(content)
if verdict.get("category") not in _CATEGORIES:
logger.warning("KI (%s@%s): unbekannte Kategorie %r",
model, base_url, verdict.get("category"))
continue
logger.debug("KI-Antwort von %s (Modell: %s).", base_url, model)
return verdict
elif resp.status_code == 503:
logger.info("KI (%s@%s): Modell wird geladen — nächster Server.", model, base_url)
else:
logger.warning("KI HTTP %s (%s@%s): %s",
resp.status_code, model, base_url, resp.text[:200])
except (requests.exceptions.RequestException, KeyError, ValueError,
json.JSONDecodeError) as exc:
logger.debug("KI-Call fehlgeschlagen (%s@%s): %s", model, base_url, exc)
logger.warning("KI: OpenRouter + alle lokalen Server erfolglos für Modell %s.", model)
return None

View file

@ -20,6 +20,7 @@ DEFAULT_CONFIG: dict = {
".mp4", ".mp3", ".webm", ".avi", ".mov",
],
"exclude_paths": [],
"include_paths": [],
"sitemap": True,
},
"scoring": {
@ -79,9 +80,10 @@ DEFAULT_CONFIG: dict = {
"url": "",
},
},
# KI-gestützte semantische Inhaltsanalyse (OpenRouter). Optional, opt-in.
# KI-gestützte semantische Inhaltsanalyse (OpenRouter, primär).
# Hash-gegated: nur neue/geänderte Inhalte lösen einen API-Call aus — siehe
# ai_ledger.json. Der API-Key kommt NUR aus der Umgebungsvariable (nie in config.yaml).
# Lokale Server (localhost:8001, :8002) dienen nur als Dev-Fallback.
"ai_analysis": {
"enabled": False,
"api_key_env": "OPENROUTER_API_KEY",
@ -138,7 +140,7 @@ DEFAULT_CONFIG: dict = {
"audio": {
"enabled": False,
"models": ["google/gemini-2.5-flash"],
"max_files_per_scan": 5,
"max_files_per_scan": 0, # 0 = unbegrenzt
},
"video": {
"enabled": False,

View file

@ -63,6 +63,7 @@ def should_crawl(
base_netloc: str,
skip_extensions: list[str],
exclude_paths: tuple | list = (),
include_paths: tuple | list = (),
) -> bool:
"""Return True if url should be followed during a crawl."""
p = urlparse(url)
@ -74,6 +75,8 @@ def should_crawl(
return False
if any(p.path.startswith(ep) for ep in exclude_paths):
return False
if include_paths and not any(p.path.startswith(ip) for ip in include_paths):
return False
# Skip URLs whose query carries calendar/session/tracking noise
if p.query and any(_is_noise_param(k) for k in parse_qs(p.query)):
return False
@ -88,6 +91,7 @@ class Crawler:
self.delay: float = cfg["crawl"]["delay_seconds"]
self.skip_ext: list[str] = cfg["crawl"]["skip_extensions"]
self.exclude_paths: list[str] = cfg["crawl"].get("exclude_paths", [])
self.include_paths: list[str] = cfg["crawl"].get("include_paths", [])
self.timeout: int = cfg["request_timeout"]
self.headers = {"User-Agent": cfg["user_agent"]}
self.session = requests.Session()
@ -104,7 +108,13 @@ class Crawler:
- errors: list of {url, error}
"""
seen: set[str] = set()
queue: list[str] = [normalize_url(self.target + "/") or self.target + "/"]
if self.include_paths:
queue: list[str] = [
normalize_url(self.target + p) or (self.target + p)
for p in self.include_paths
]
else:
queue: list[str] = [normalize_url(self.target + "/") or self.target + "/"]
pages: list[dict] = []
errors: list[dict] = []
@ -252,7 +262,8 @@ class Crawler:
raw = loc.text.strip() if loc.text else ""
url = normalize_url(raw)
if url and should_crawl(
url, self.base_netloc, self.skip_ext, self.exclude_paths
url, self.base_netloc, self.skip_ext, self.exclude_paths,
self.include_paths,
) and url not in seen and url not in queue:
queue.append(url)
added += 1
@ -267,7 +278,8 @@ class Crawler:
for tag in soup.find_all("a", href=True):
href = tag["href"].strip()
abs_url = normalize_url(urljoin(base_url, href))
if abs_url and should_crawl(abs_url, self.base_netloc, self.skip_ext, self.exclude_paths):
if abs_url and should_crawl(abs_url, self.base_netloc, self.skip_ext,
self.exclude_paths, self.include_paths):
result.append(abs_url)
except Exception as exc:
logger.debug("Link extraction failed on %s: %s", base_url, exc)

View file

@ -148,7 +148,7 @@ def compare_snapshots(baseline_pages: dict, current_pages: dict, cfg: dict) -> d
# Per-type link diff
link_diff: dict = {}
for ltype in ("a", "script", "iframe", "link_rel", "form"):
for ltype in ("a", "script", "iframe", "link_rel", "form", "audio", "video"):
old_links = old.get("links", {}).get(ltype, [])
new_links = new.get("links", {}).get(ltype, [])
ld = diff_link_set(old_links, new_links)

View file

@ -124,6 +124,8 @@ def _extract_links(soup: BeautifulSoup, base_url: str) -> dict:
"link_rel": [],
"script": [],
"img": [],
"audio": [],
"video": [],
"iframe": [],
"form": [],
"meta_refresh": [],
@ -162,6 +164,19 @@ def _extract_links(soup: BeautifulSoup, base_url: str) -> dict:
u = strip_cache_params(u)
result["img"].append({"url": u, "class": classify(u)})
for tag in soup.find_all(["audio", "video"]):
media_type = tag.name # "audio" or "video"
if tag.get("src"):
u = resolve(tag["src"])
if u:
u = strip_cache_params(u)
result[media_type].append({"url": u, "class": classify(u)})
for src_tag in tag.find_all("source", src=True):
u = resolve(src_tag["src"])
if u:
u = strip_cache_params(u)
result[media_type].append({"url": u, "class": classify(u)})
for tag in soup.find_all("iframe", src=True):
u = resolve(tag["src"])
if u:

View file

@ -20,11 +20,16 @@ def render_markdown(r: dict) -> str:
symbol, headline = ampel(lvl)
ai = r.get("ai_result")
config_path = r.get("config_path")
scan_cmd = (f"python -m scanner --config {config_path} scan"
if config_path else f"python -m scanner {target} scan")
# ---- Klartext-Teil (für den Betreiber) ----
lines = [
f"# Prüfbericht für Ihre Website — {target or '?'}",
"",
f"**Erstellt:** {r.get('generated_at', '?')}",
f"**Erstellt:** {r.get('generated_at', '?')} ",
f"**Kommando:** `{scan_cmd}`",
"",
f"## {symbol} {headline}",
"",