feat: initial implementation of website integrity scanner

Python-Package zur unabhängigen Überwachung statischer Websites gegen
SEO-Spam und unbefugte Manipulationen. Läuft außerhalb des Hosters.

Kernfunktionen:
- Reiner Python-Crawler (bs4+lxml), kein wget
- Baseline-Management mit manuellem Freigabe-Workflow (niemals automatisch)
- Text-/Link-/Meta-Diff mit Risiko-Scoring (grün/gelb/rot)
- Hidden-Content-Erkennung (CSS inline, noscript, Kommentar-Links)
- Whitelist für externe Domains und interne Pfade
- E-Mail + Webhook Alarmierung
- CLI: init, crawl, check, scan, approve, report, status
- 79 Unit-Tests (pytest), alle grün

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

tests/test_crawler.py

@@ -0,0 +1,63 @@
+"""Tests for crawler URL normalization and crawl filtering."""
+import pytest
+from scanner.crawler import normalize_url, should_crawl
+
+SKIP_EXT = [".jpg", ".jpeg", ".png", ".pdf", ".zip", ".css", ".js"]
+
+
+class TestNormalizeUrl:
+    def test_strips_fragment(self):
+        assert normalize_url("https://example.com/page#section") == "https://example.com/page"
+
+    def test_lowercase_host(self):
+        assert normalize_url("https://EXAMPLE.COM/path") == "https://example.com/path"
+
+    def test_non_http_returns_none(self):
+        assert normalize_url("javascript:void(0)") is None
+        assert normalize_url("mailto:foo@bar.com") is None
+        assert normalize_url("ftp://example.com") is None
+
+    def test_sorts_query_params(self):
+        a = normalize_url("https://example.com/?z=1&a=2")
+        b = normalize_url("https://example.com/?a=2&z=1")
+        assert a == b
+
+    def test_strips_tracking_params(self):
+        url = normalize_url("https://example.com/?utm_source=google&page=1")
+        assert "utm_source" not in url
+        assert "page=1" in url
+
+    def test_strips_session_params(self):
+        url = normalize_url("https://example.com/?sid=abc123&id=5")
+        assert "sid" not in url
+        assert "id=5" in url
+
+    def test_adds_default_path(self):
+        result = normalize_url("https://example.com")
+        assert result.endswith("/") or "example.com" in result
+
+    def test_empty_string_returns_none(self):
+        assert normalize_url("") is None
+
+
+class TestShouldCrawl:
+    def test_same_host_plain_page(self):
+        assert should_crawl("https://example.com/about", "example.com", SKIP_EXT)
+
+    def test_different_host_rejected(self):
+        assert not should_crawl("https://other.com/page", "example.com", SKIP_EXT)
+
+    def test_image_rejected(self):
+        assert not should_crawl("https://example.com/photo.jpg", "example.com", SKIP_EXT)
+
+    def test_pdf_rejected(self):
+        assert not should_crawl("https://example.com/doc.pdf", "example.com", SKIP_EXT)
+
+    def test_html_accepted(self):
+        assert should_crawl("https://example.com/page.html", "example.com", SKIP_EXT)
+
+    def test_trailing_slash_accepted(self):
+        assert should_crawl("https://example.com/section/", "example.com", SKIP_EXT)
+
+    def test_utm_query_param_rejected(self):
+        assert not should_crawl("https://example.com/?utm_source=x", "example.com", SKIP_EXT)