fix: Bugfixes, toten Code entfernt, Cache-Normalisierung & Session

Echte Bugs:
- extractor: JSON-LD-Crash bei Nicht-Dict-Items (isinstance-Guard)
- baseline/__main__: Crawl-Fehler ins Snapshot-Manifest -> erscheinen im Report
- __main__: Whitelist-/Header-/Webshell-Checks nur noch auf status==200
- crawler: Noise-Param-Regex auf Voll-Key (view=/value= nicht mehr verworfen)
- differ/__main__: unerwartetes JSON-LD via eigenem Kanal, auch auf
  unveraenderten Seiten erkannt, kein Re-Alert auf bereits genehmigte Typen

Aufgeraeumt:
- checker: check_internal_paths, find_broken_pages, canonical_is_hijacked,
  check_jsonld_types entfernt (nicht verdrahtet)
- allowed_paths.yaml-Logik und tote Scoring-Keys entfernt
- tote Imports entfernt

Neuer aktiver Schutz:
- Webshell-/Backdoor-Dateinamen-Check verdrahtet (suspicious_filename: 60).
  Regex wortgrenzen-verankert gegen Fehlalarm auf PSEMailerAntispam.js

Effizienz/Struktur:
- crawler nutzt requests.Session (keep-alive)
- Cache-Buster-Normalisierung an einer Stelle (Extraktion) -> stabile
  Snapshots, differ wieder reiner Set-Diff

Tests: 111 gruen (neu: test_checker.py + Regressionstests)
CLAUDE.md aktualisiert

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

tests/test_checker.py

@@ -0,0 +1,63 @@
+"""Tests for stateless checker helpers."""
+import pytest
+from scanner.checker import (
+    check_links_against_whitelist,
+    check_security_headers,
+    is_suspicious_url,
+)
+
+
+class TestIsSuspiciousUrl:
+    @pytest.mark.parametrize("url", [
+        "https://b.info/uploads/shell.php",
+        "https://b.info/c99.php",
+        "https://b.info/wso.aspx",
+        "https://b.info/admin/backdoor.php",
+        "https://b.info/webshell.php",
+    ])
+    def test_flags_webshell_names(self, url):
+        assert is_suspicious_url(url) is True
+
+    @pytest.mark.parametrize("url", [
+        "https://b.info/index.html",
+        "https://b.info/css/app.css",
+        "https://b.info/impressum",
+        "https://b.info/team/photo.jpg",
+        # Regression: legitimate site assets must NOT be flagged
+        "https://b.info/assets/js/PSEMailerAntispam.js",
+        "https://b.info/navigate.php",
+        "https://b.info/uploads/document.pdf",
+    ])
+    def test_ignores_normal_names(self, url):
+        assert is_suspicious_url(url) is False
+
+
+class TestSecurityHeaders:
+    def test_reports_missing(self):
+        present = {"Content-Type": "text/html"}
+        missing = check_security_headers(present, ["Content-Security-Policy", "X-Content-Type-Options"])
+        assert "Content-Security-Policy" in missing
+
+    def test_case_insensitive(self):
+        present = {"content-security-policy": "default-src 'self'"}
+        missing = check_security_headers(present, ["Content-Security-Policy"])
+        assert missing == []
+
+
+class TestWhitelist:
+    def _page(self, ext_url):
+        return {
+            "links": {"a": [{"url": ext_url, "class": "external"}]},
+            "comment_links": [],
+            "hidden_content": [],
+        }
+
+    def test_flags_unlisted_domain(self):
+        page = self._page("https://spam.example.com/x")
+        violations = check_links_against_whitelist(page, set(), "b.info")
+        assert any(v["domain"] == "spam.example.com" for v in violations)
+
+    def test_allows_listed_domain(self):
+        page = self._page("https://good.example.com/x")
+        violations = check_links_against_whitelist(page, {"good.example.com"}, "b.info")
+        assert violations == []

tests/test_crawler.py

@@ -32,6 +32,16 @@ class TestNormalizeUrl:
         assert "sid" not in url
         assert "id=5" in url
 
+    def test_strips_cache_buster_v(self):
+        url = normalize_url("https://example.com/?v=1781223996&id=5")
+        assert "v=1781223996" not in url
+        assert "id=5" in url
+
+    def test_keeps_non_noise_v_prefixed_param(self):
+        # 'view' must NOT be dropped just because it starts with 'v'
+        url = normalize_url("https://example.com/?view=gallery")
+        assert "view=gallery" in url
+
     def test_adds_default_path(self):
         result = normalize_url("https://example.com")
         assert result.endswith("/") or "example.com" in result

tests/test_differ.py

@@ -79,6 +79,15 @@ class TestDiffLinkSet:
         result = diff_link_set(links, links)
         assert result == {"added": [], "removed": []}
 
+    def test_truly_new_url_detected(self):
+        # Cache-buster stripping happens at extraction time; diff_link_set is a
+        # plain set comparison on already-normalised URLs.
+        old = [{"url": "https://example.com/style.css"}]
+        new = [{"url": "https://example.com/style.css"},
+               {"url": "https://example.com/extra.js"}]
+        result = diff_link_set(old, new)
+        assert "https://example.com/extra.js" in result["added"]
+
 
 class TestDiffMetadata:
     def test_detects_title_change(self):
@@ -153,6 +162,59 @@ class TestCompareSnapshots:
         assert "spam.example.com" in result["new_external_domains"]
 
 
+class TestUnexpectedJsonLd:
+    CFG = {**BASE_CFG, "allowed_jsonld_types": ["Organization", "WebPage"]}
+
+    def _page(self, url, jsonld):
+        return {
+            "url": url, "status": 200, "text": "Inhalt",
+            "links": {"a": [], "script": [], "iframe": [], "link_rel": [], "form": [], "meta_refresh": [], "canonical": None},
+            "metadata": {"title": "T", "h1": [], "h2": []},
+            "hidden_content": [], "inline_scripts": [], "jsonld": jsonld, "comment_links": [],
+        }
+
+    def test_unexpected_type_flagged_without_page_diff(self):
+        # Page is otherwise unchanged (same text/links) but carries a bad @type.
+        base = {"https://b.info/": self._page("https://b.info/", [{"type": "Organization"}])}
+        cur = {"https://b.info/": self._page("https://b.info/", [{"type": "Pharmacy"}])}
+        result = compare_snapshots(base, cur, self.CFG)
+        types = [e["type"] for e in result["unexpected_jsonld"]]
+        assert "Pharmacy" in types
+
+    def test_allowed_type_not_flagged(self):
+        base = {"https://b.info/": self._page("https://b.info/", [])}
+        cur = {"https://b.info/": self._page("https://b.info/", [{"type": "WebPage"}])}
+        result = compare_snapshots(base, cur, self.CFG)
+        assert result["unexpected_jsonld"] == []
+
+    def test_preexisting_unexpected_type_not_realerted(self):
+        # Bad type already in baseline → approved → should not re-alert.
+        base = {"https://b.info/": self._page("https://b.info/", [{"type": "Pharmacy"}])}
+        cur = {"https://b.info/": self._page("https://b.info/", [{"type": "Pharmacy"}])}
+        result = compare_snapshots(base, cur, self.CFG)
+        assert result["unexpected_jsonld"] == []
+
+    def test_unexpected_type_scores(self):
+        result = score_diff({
+            "new_internal_urls": [], "missing_internal_urls": [],
+            "new_external_domains": [], "page_diffs": [], "new_pages_analysis": [],
+            "unexpected_jsonld": [{"url": "https://b.info/", "type": "Pharmacy"}],
+        }, BASE_CFG)
+        assert result["score"] == 35
+        assert result["level"] == "yellow"
+
+
+class TestSuspiciousFilenames:
+    def test_suspicious_filename_scores_red(self):
+        result = score_diff({
+            "new_internal_urls": [], "missing_internal_urls": [],
+            "new_external_domains": [], "page_diffs": [], "new_pages_analysis": [],
+            "suspicious_filenames": [{"page": "https://b.info/", "url": "https://b.info/shell.php"}],
+        }, BASE_CFG)
+        assert result["score"] == 60
+        assert result["level"] == "red"
+
+
 class TestBrokenLinks:
     def _make_page(self, url, text="Inhalt", status=200):
         return {

tests/test_extractor.py

@@ -150,3 +150,32 @@ class TestJsonLd:
         )
         result = _extract(html)
         assert any(j["type"] == "Pharmacy" for j in result["jsonld"])
+
+    def test_non_dict_jsonld_does_not_crash(self):
+        # Valid JSON that is not an object (scalar / array of scalars) must be
+        # tolerated, not raise AttributeError.
+        for body in ("[1, 2, 3]", '"just a string"', "42"):
+            html = f'<script type="application/ld+json">{body}</script>'
+            result = _extract(html)
+            assert isinstance(result["jsonld"], list)
+
+
+class TestCacheBusterNormalization:
+    def test_link_rel_cache_param_stripped(self):
+        html = '<head><link rel="stylesheet" href="/css/app.css?v=1781223996"/></head>'
+        result = _extract(html)
+        urls = [l["url"] for l in result["links"]["link_rel"]]
+        assert any(u.endswith("/css/app.css") for u in urls)
+        assert all("v=" not in u for u in urls)
+
+    def test_script_cache_param_stripped(self):
+        html = '<body><script src="/js/app.js?ver=42"></script></body>'
+        result = _extract(html)
+        urls = [l["url"] for l in result["links"]["script"]]
+        assert any(u.endswith("/js/app.js") for u in urls)
+
+    def test_non_cache_param_kept_on_asset(self):
+        html = '<body><img src="/img/pic.png?size=large"/></body>'
+        result = _extract(html)
+        urls = [l["url"] for l in result["links"]["img"]]
+        assert any("size=large" in u for u in urls)