feat: initial implementation of website integrity scanner

Python-Package zur unabhängigen Überwachung statischer Websites gegen
SEO-Spam und unbefugte Manipulationen. Läuft außerhalb des Hosters.

Kernfunktionen:
- Reiner Python-Crawler (bs4+lxml), kein wget
- Baseline-Management mit manuellem Freigabe-Workflow (niemals automatisch)
- Text-/Link-/Meta-Diff mit Risiko-Scoring (grün/gelb/rot)
- Hidden-Content-Erkennung (CSS inline, noscript, Kommentar-Links)
- Whitelist für externe Domains und interne Pfade
- E-Mail + Webhook Alarmierung
- CLI: init, crawl, check, scan, approve, report, status
- 79 Unit-Tests (pytest), alle grün

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

scanner/checker.py

@@ -0,0 +1,164 @@
+"""
+Additional integrity checks beyond baseline diff:
+- link whitelist enforcement
+- security-header audit
+- canonical hijack detection
+- suspicious URL filenames (from v4 heritage)
+"""
+import re
+from pathlib import Path
+from urllib.parse import urlparse
+
+import yaml
+
+_SUSPICIOUS_FILENAME_RE = re.compile(
+    r"(shell|backdoor|cmd|upload|gate|mailer|wso|c99|r57|b374k)",
+    re.I,
+)
+
+
+# ---------------------------------------------------------------------------
+# Whitelist checks
+# ---------------------------------------------------------------------------
+
+def load_whitelist(path: str | Path) -> set[str]:
+    """Load a YAML list file into a set. Returns empty set if missing."""
+    p = Path(path)
+    if not p.exists():
+        return set()
+    data = yaml.safe_load(p.read_text(encoding="utf-8")) or []
+    if isinstance(data, list):
+        return {str(s).strip() for s in data if s}
+    return set()
+
+
+def check_links_against_whitelist(
+    page: dict,
+    allowed_external_domains: set[str],
+    base_netloc: str,
+) -> list[dict]:
+    """
+    Return violations: external links to domains not in allowed_external_domains.
+    Also flags noscript links and comment links to external domains.
+    """
+    violations: list[dict] = []
+
+    for link_type, links in page.get("links", {}).items():
+        if not isinstance(links, list):
+            continue
+        for link in links:
+            if not isinstance(link, dict):
+                continue
+            if link.get("class") != "external":
+                continue
+            domain = urlparse(link.get("url", "")).netloc
+            if domain and domain not in allowed_external_domains:
+                violations.append({
+                    "type": "unlisted_external_domain",
+                    "link_type": link_type,
+                    "url": link.get("url", ""),
+                    "domain": domain,
+                })
+
+    # Comment links
+    for href in page.get("comment_links", []):
+        domain = urlparse(href).netloc
+        if domain and domain != base_netloc and domain not in allowed_external_domains:
+            violations.append({
+                "type": "comment_link_to_unlisted_domain",
+                "link_type": "comment",
+                "url": href,
+                "domain": domain,
+            })
+
+    # noscript links
+    for hc in page.get("hidden_content", []):
+        if hc.get("type") == "noscript_links":
+            for href in hc.get("links", []):
+                domain = urlparse(href).netloc
+                if domain and domain != base_netloc and domain not in allowed_external_domains:
+                    violations.append({
+                        "type": "noscript_link_to_unlisted_domain",
+                        "link_type": "noscript",
+                        "url": href,
+                        "domain": domain,
+                    })
+
+    return violations
+
+
+def check_internal_paths(
+    page: dict,
+    allowed_paths: set[str],
+) -> list[dict]:
+    """
+    Flag internal links to paths not in the allowed_paths whitelist.
+    If allowed_paths is empty, the check is skipped.
+    """
+    if not allowed_paths:
+        return []
+    violations: list[dict] = []
+    for link in page.get("links", {}).get("a", []):
+        if link.get("class") != "internal":
+            continue
+        path = urlparse(link.get("url", "")).path
+        if path and path not in allowed_paths:
+            violations.append({
+                "type": "unlisted_internal_path",
+                "url": link.get("url", ""),
+                "path": path,
+            })
+    return violations
+
+
+# ---------------------------------------------------------------------------
+# Security headers
+# ---------------------------------------------------------------------------
+
+def check_security_headers(response_headers: dict, required: list[str]) -> list[str]:
+    """Return list of missing recommended security headers."""
+    # Header lookup is case-insensitive
+    present = {k.lower() for k in response_headers}
+    return [h for h in required if h.lower() not in present]
+
+
+# ---------------------------------------------------------------------------
+# Canonical hijack
+# ---------------------------------------------------------------------------
+
+def canonical_is_hijacked(page: dict, base_netloc: str) -> bool:
+    """
+    Return True if the canonical URL points to a different domain.
+    Canonical appearing in both <link rel=canonical> and metadata is checked.
+    """
+    canonical = (
+        page.get("links", {}).get("canonical")
+        or page.get("metadata", {}).get("canonical")
+    )
+    if not canonical:
+        return False
+    netloc = urlparse(canonical).netloc
+    return bool(netloc) and netloc != base_netloc
+
+
+# ---------------------------------------------------------------------------
+# Suspicious filenames
+# ---------------------------------------------------------------------------
+
+def is_suspicious_url(url: str) -> bool:
+    """Flag URLs with names typical for webshells / backdoors."""
+    filename = urlparse(url).path.rsplit("/", 1)[-1]
+    return bool(_SUSPICIOUS_FILENAME_RE.search(filename))
+
+
+# ---------------------------------------------------------------------------
+# JSON-LD type check
+# ---------------------------------------------------------------------------
+
+def check_jsonld_types(page: dict, allowed_types: list[str]) -> list[dict]:
+    """Return JSON-LD entries whose @type is not in the allowed list."""
+    allowed = set(allowed_types)
+    return [
+        j for j in page.get("jsonld", [])
+        if j.get("type") not in allowed
+    ]