diff --git a/config.yaml b/config.yaml index 25ec957..a9a29d1 100644 --- a/config.yaml +++ b/config.yaml @@ -50,6 +50,7 @@ scoring: comment_links: 25 noscript_links: 25 new_external_link_unlisted: 40 + new_broken_link: 20 # neue 4xx/5xx-Seite, die in Baseline nicht vorhanden war thresholds: yellow: 20 # ab diesem Score: Warnung (gelb) diff --git a/scanner/__main__.py b/scanner/__main__.py index 82e3035..3369246 100644 --- a/scanner/__main__.py +++ b/scanner/__main__.py @@ -152,6 +152,8 @@ def _render_markdown(r: dict) -> str: f"| Neue interne URLs | {len(d.get('new_internal_urls', []))} |", f"| Fehlende URLs | {len(d.get('missing_internal_urls', []))} |", f"| Neue externe Domains | {len(d.get('new_external_domains', []))} |", + f"| Kaputte Links (neu) | {len(d.get('new_broken_urls', []))} |", + f"| Kaputte Links (bekannt) | {len(d.get('known_broken_urls', []))} |", f"| Whitelist-Verstösse | {len(r.get('whitelist_violations', []))} |", f"| Crawl-Fehler | {len(r.get('crawl_errors', []))} |", "", @@ -233,6 +235,15 @@ def _render_markdown(r: dict) -> str: flag = " [VERDAECHTIG]" if problems else "" lines.append(f"- {npa['url']}{flag}") + new_broken = d.get("new_broken_urls", []) + known_broken = d.get("known_broken_urls", []) + if new_broken or known_broken: + lines += ["", "## Kaputte Links", ""] + for entry in new_broken: + lines.append(f"- [NEU] HTTP {entry['status']} `{entry['url']}`") + for entry in known_broken: + lines.append(f"- [bekannt] HTTP {entry['status']} `{entry['url']}`") + if r.get("whitelist_violations"): lines += ["", "## Whitelist-Verstösse", ""] for v in r["whitelist_violations"][:50]: diff --git a/scanner/checker.py b/scanner/checker.py index 1fe4afe..5b801cf 100644 --- a/scanner/checker.py +++ b/scanner/checker.py @@ -122,6 +122,19 @@ def check_security_headers(response_headers: dict, required: list[str]) -> list[ return [h for h in required if h.lower() not in present] +# --------------------------------------------------------------------------- +# Broken links +# --------------------------------------------------------------------------- + +def find_broken_pages(pages: dict) -> list[dict]: + """Return all pages whose HTTP status is >= 400 (broken / server error).""" + return [ + {"url": p["url"], "status": p.get("status", 0)} + for p in pages.values() + if p.get("status", 0) >= 400 + ] + + # --------------------------------------------------------------------------- # Canonical hijack # --------------------------------------------------------------------------- diff --git a/scanner/config.py b/scanner/config.py index 0042cc3..b8e5009 100644 --- a/scanner/config.py +++ b/scanner/config.py @@ -35,6 +35,7 @@ DEFAULT_CONFIG: dict = { "comment_links": 25, "noscript_links": 25, "new_external_link_unlisted": 40, + "new_broken_link": 20, }, "thresholds": { "yellow": 20, diff --git a/scanner/differ.py b/scanner/differ.py index 13be939..f6fa857 100644 --- a/scanner/differ.py +++ b/scanner/differ.py @@ -202,6 +202,25 @@ def compare_snapshots(baseline_pages: dict, current_pages: dict, cfg: dict) -> d old_ext = _all_external_domains(baseline_pages) new_ext = _all_external_domains(current_pages) + # Broken link diff (4xx/5xx pages) + baseline_broken = { + url for url, p in baseline_pages.items() if p.get("status", 0) >= 400 + } + current_broken_map = { + url: p.get("status", 0) + for url, p in current_pages.items() + if p.get("status", 0) >= 400 + } + current_broken = set(current_broken_map) + new_broken = [ + {"url": url, "status": current_broken_map[url]} + for url in sorted(current_broken - baseline_broken) + ] + known_broken = [ + {"url": url, "status": current_broken_map[url]} + for url in sorted(current_broken & baseline_broken) + ] + # Analysis of brand-new pages (no baseline to compare against) new_pages_analysis = [ _quick_check_new_page(current_pages[url]) @@ -216,6 +235,8 @@ def compare_snapshots(baseline_pages: dict, current_pages: dict, cfg: dict) -> d "removed_external_domains": sorted(old_ext - new_ext), "page_diffs": page_diffs, "new_pages_analysis": new_pages_analysis, + "new_broken_urls": new_broken, + "known_broken_urls": known_broken, "total_pages_checked": len(common_urls), "changed_pages": len(page_diffs), } @@ -300,6 +321,11 @@ def score_diff(diff: dict, cfg: dict) -> dict: add(sc.get("large_text_addition", 20), f"{u}: +{pd['added_chars']} Zeichen neuer Text") + for entry in diff.get("new_broken_urls", []): + add(sc.get("new_broken_link", 20), + f"Neuer kaputte Link: {entry['url']} (HTTP {entry['status']})") + # known_broken_urls are reported but not scored — they were already in the baseline + # New pages: immediate suspicious content counts too for npa in diff.get("new_pages_analysis", []): u = npa["url"] diff --git a/tests/test_differ.py b/tests/test_differ.py index fb59f57..a8675b6 100644 --- a/tests/test_differ.py +++ b/tests/test_differ.py @@ -153,6 +153,55 @@ class TestCompareSnapshots: assert "spam.example.com" in result["new_external_domains"] +class TestBrokenLinks: + def _make_page(self, url, text="Inhalt", status=200): + return { + "url": url, "status": status, "text": text, + "links": {"a": [], "script": [], "iframe": [], "link_rel": [], "form": [], "meta_refresh": [], "canonical": None}, + "metadata": {"title": "T", "h1": [], "h2": []}, + "hidden_content": [], "inline_scripts": [], "jsonld": [], "comment_links": [], + } + + def test_new_broken_url_detected(self): + baseline = {"https://b.info/": self._make_page("https://b.info/")} + current = { + "https://b.info/": self._make_page("https://b.info/"), + "https://b.info/missing/": self._make_page("https://b.info/missing/", status=404), + } + result = compare_snapshots(baseline, current, BASE_CFG) + assert len(result["new_broken_urls"]) == 1 + assert result["new_broken_urls"][0]["url"] == "https://b.info/missing/" + assert result["new_broken_urls"][0]["status"] == 404 + + def test_known_broken_not_in_new(self): + broken_page = {"https://b.info/old/": self._make_page("https://b.info/old/", status=404)} + baseline = {**{"https://b.info/": self._make_page("https://b.info/")}, **broken_page} + current = {**{"https://b.info/": self._make_page("https://b.info/")}, **broken_page} + result = compare_snapshots(baseline, current, BASE_CFG) + assert result["new_broken_urls"] == [] + assert len(result["known_broken_urls"]) == 1 + + def test_broken_url_scores(self): + result = score_diff({ + "new_internal_urls": [], "missing_internal_urls": [], + "new_external_domains": [], "page_diffs": [], "new_pages_analysis": [], + "new_broken_urls": [{"url": "https://b.info/gone/", "status": 404}], + "known_broken_urls": [], + }, BASE_CFG) + assert result["score"] == 20 + assert result["level"] == "yellow" + + def test_known_broken_not_scored(self): + result = score_diff({ + "new_internal_urls": [], "missing_internal_urls": [], + "new_external_domains": [], "page_diffs": [], "new_pages_analysis": [], + "new_broken_urls": [], + "known_broken_urls": [{"url": "https://b.info/old/", "status": 404}], + }, BASE_CFG) + assert result["score"] == 0 + assert result["level"] == "green" + + class TestScoreDiff: def test_green_for_no_changes(self): result = score_diff({