feat: report broken links (4xx/5xx) in every scan

Neue 4xx/5xx-Seiten (nicht in Baseline) werden bewertet (+20 Pkt) und
lösen ab Schwelle einen Alert aus. Bekannte kaputte Links (in Baseline
vorhanden) erscheinen im Report als [bekannt] ohne Score-Beitrag.

Motivation: erster Crawl fand bredelar.info/www.stadtmarketing-marsberg.de
(404) — CMS-Fehler durch Schema-losen href. Soll dauerhaft sichtbar sein.

Änderungen: checker.find_broken_pages(), differ.compare_snapshots() mit
new_broken_urls/known_broken_urls, score_diff(), Report-Abschnitt,
config-Default new_broken_link:20, 4 neue Tests.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

config.yaml

@@ -50,6 +50,7 @@ scoring:
   comment_links: 25
   noscript_links: 25
   new_external_link_unlisted: 40
+  new_broken_link: 20          # neue 4xx/5xx-Seite, die in Baseline nicht vorhanden war
 
 thresholds:
   yellow: 20   # ab diesem Score: Warnung (gelb)

scanner/__main__.py

@@ -152,6 +152,8 @@ def _render_markdown(r: dict) -> str:
         f"| Neue interne URLs | {len(d.get('new_internal_urls', []))} |",
         f"| Fehlende URLs | {len(d.get('missing_internal_urls', []))} |",
         f"| Neue externe Domains | {len(d.get('new_external_domains', []))} |",
+        f"| Kaputte Links (neu) | {len(d.get('new_broken_urls', []))} |",
+        f"| Kaputte Links (bekannt) | {len(d.get('known_broken_urls', []))} |",
         f"| Whitelist-Verstösse | {len(r.get('whitelist_violations', []))} |",
         f"| Crawl-Fehler | {len(r.get('crawl_errors', []))} |",
         "",
@@ -233,6 +235,15 @@ def _render_markdown(r: dict) -> str:
             flag = " [VERDAECHTIG]" if problems else ""
             lines.append(f"- {npa['url']}{flag}")
 
+    new_broken = d.get("new_broken_urls", [])
+    known_broken = d.get("known_broken_urls", [])
+    if new_broken or known_broken:
+        lines += ["", "## Kaputte Links", ""]
+        for entry in new_broken:
+            lines.append(f"- [NEU] HTTP {entry['status']} `{entry['url']}`")
+        for entry in known_broken:
+            lines.append(f"- [bekannt] HTTP {entry['status']} `{entry['url']}`")
+
     if r.get("whitelist_violations"):
         lines += ["", "## Whitelist-Verstösse", ""]
         for v in r["whitelist_violations"][:50]:

scanner/checker.py

@@ -122,6 +122,19 @@ def check_security_headers(response_headers: dict, required: list[str]) -> list[
     return [h for h in required if h.lower() not in present]
 
 
+# ---------------------------------------------------------------------------
+# Broken links
+# ---------------------------------------------------------------------------
+
+def find_broken_pages(pages: dict) -> list[dict]:
+    """Return all pages whose HTTP status is >= 400 (broken / server error)."""
+    return [
+        {"url": p["url"], "status": p.get("status", 0)}
+        for p in pages.values()
+        if p.get("status", 0) >= 400
+    ]
+
+
 # ---------------------------------------------------------------------------
 # Canonical hijack
 # ---------------------------------------------------------------------------

scanner/config.py

@@ -35,6 +35,7 @@ DEFAULT_CONFIG: dict = {
         "comment_links": 25,
         "noscript_links": 25,
         "new_external_link_unlisted": 40,
+        "new_broken_link": 20,
     },
     "thresholds": {
         "yellow": 20,

scanner/differ.py

@@ -202,6 +202,25 @@ def compare_snapshots(baseline_pages: dict, current_pages: dict, cfg: dict) -> d
     old_ext = _all_external_domains(baseline_pages)
     new_ext = _all_external_domains(current_pages)
 
+    # Broken link diff (4xx/5xx pages)
+    baseline_broken = {
+        url for url, p in baseline_pages.items() if p.get("status", 0) >= 400
+    }
+    current_broken_map = {
+        url: p.get("status", 0)
+        for url, p in current_pages.items()
+        if p.get("status", 0) >= 400
+    }
+    current_broken = set(current_broken_map)
+    new_broken = [
+        {"url": url, "status": current_broken_map[url]}
+        for url in sorted(current_broken - baseline_broken)
+    ]
+    known_broken = [
+        {"url": url, "status": current_broken_map[url]}
+        for url in sorted(current_broken & baseline_broken)
+    ]
+
     # Analysis of brand-new pages (no baseline to compare against)
     new_pages_analysis = [
         _quick_check_new_page(current_pages[url])
@@ -216,6 +235,8 @@ def compare_snapshots(baseline_pages: dict, current_pages: dict, cfg: dict) -> d
         "removed_external_domains": sorted(old_ext - new_ext),
         "page_diffs": page_diffs,
         "new_pages_analysis": new_pages_analysis,
+        "new_broken_urls": new_broken,
+        "known_broken_urls": known_broken,
         "total_pages_checked": len(common_urls),
         "changed_pages": len(page_diffs),
     }
@@ -300,6 +321,11 @@ def score_diff(diff: dict, cfg: dict) -> dict:
             add(sc.get("large_text_addition", 20),
                 f"{u}: +{pd['added_chars']} Zeichen neuer Text")
 
+    for entry in diff.get("new_broken_urls", []):
+        add(sc.get("new_broken_link", 20),
+            f"Neuer kaputte Link: {entry['url']} (HTTP {entry['status']})")
+    # known_broken_urls are reported but not scored — they were already in the baseline
+
     # New pages: immediate suspicious content counts too
     for npa in diff.get("new_pages_analysis", []):
         u = npa["url"]

tests/test_differ.py

@@ -153,6 +153,55 @@ class TestCompareSnapshots:
         assert "spam.example.com" in result["new_external_domains"]
 
 
+class TestBrokenLinks:
+    def _make_page(self, url, text="Inhalt", status=200):
+        return {
+            "url": url, "status": status, "text": text,
+            "links": {"a": [], "script": [], "iframe": [], "link_rel": [], "form": [], "meta_refresh": [], "canonical": None},
+            "metadata": {"title": "T", "h1": [], "h2": []},
+            "hidden_content": [], "inline_scripts": [], "jsonld": [], "comment_links": [],
+        }
+
+    def test_new_broken_url_detected(self):
+        baseline = {"https://b.info/": self._make_page("https://b.info/")}
+        current = {
+            "https://b.info/": self._make_page("https://b.info/"),
+            "https://b.info/missing/": self._make_page("https://b.info/missing/", status=404),
+        }
+        result = compare_snapshots(baseline, current, BASE_CFG)
+        assert len(result["new_broken_urls"]) == 1
+        assert result["new_broken_urls"][0]["url"] == "https://b.info/missing/"
+        assert result["new_broken_urls"][0]["status"] == 404
+
+    def test_known_broken_not_in_new(self):
+        broken_page = {"https://b.info/old/": self._make_page("https://b.info/old/", status=404)}
+        baseline = {**{"https://b.info/": self._make_page("https://b.info/")}, **broken_page}
+        current = {**{"https://b.info/": self._make_page("https://b.info/")}, **broken_page}
+        result = compare_snapshots(baseline, current, BASE_CFG)
+        assert result["new_broken_urls"] == []
+        assert len(result["known_broken_urls"]) == 1
+
+    def test_broken_url_scores(self):
+        result = score_diff({
+            "new_internal_urls": [], "missing_internal_urls": [],
+            "new_external_domains": [], "page_diffs": [], "new_pages_analysis": [],
+            "new_broken_urls": [{"url": "https://b.info/gone/", "status": 404}],
+            "known_broken_urls": [],
+        }, BASE_CFG)
+        assert result["score"] == 20
+        assert result["level"] == "yellow"
+
+    def test_known_broken_not_scored(self):
+        result = score_diff({
+            "new_internal_urls": [], "missing_internal_urls": [],
+            "new_external_domains": [], "page_diffs": [], "new_pages_analysis": [],
+            "new_broken_urls": [],
+            "known_broken_urls": [{"url": "https://b.info/old/", "status": 404}],
+        }, BASE_CFG)
+        assert result["score"] == 0
+        assert result["level"] == "green"
+
+
 class TestScoreDiff:
     def test_green_for_no_changes(self):
         result = score_diff({