feat: initial implementation of website integrity scanner

Python-Package zur unabhängigen Überwachung statischer Websites gegen
SEO-Spam und unbefugte Manipulationen. Läuft außerhalb des Hosters.

Kernfunktionen:
- Reiner Python-Crawler (bs4+lxml), kein wget
- Baseline-Management mit manuellem Freigabe-Workflow (niemals automatisch)
- Text-/Link-/Meta-Diff mit Risiko-Scoring (grün/gelb/rot)
- Hidden-Content-Erkennung (CSS inline, noscript, Kommentar-Links)
- Whitelist für externe Domains und interne Pfade
- E-Mail + Webhook Alarmierung
- CLI: init, crawl, check, scan, approve, report, status
- 79 Unit-Tests (pytest), alle grün

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Your Name 2026-06-12 00:51:41 +02:00
commit 98e8d11eb5
24 changed files with 4295 additions and 0 deletions

0
tests/__init__.py Normal file
View file

161
tests/test_baseline.py Normal file
View file

@ -0,0 +1,161 @@
"""Tests for snapshot saving, baseline management and approval workflow."""
import json
import pytest
from pathlib import Path
from scanner.baseline import BaselineManager, url_key
def _sample_page(url: str, text: str = "Inhalt") -> dict:
return {
"url": url,
"final_url": url,
"status": 200,
"content_type": "text/html",
"response_headers": {},
"text": text,
"links": {"a": [], "canonical": None, "meta_refresh": []},
"metadata": {"title": "T", "h1": [], "h2": []},
"hidden_content": [],
"inline_scripts": [],
"jsonld": [],
"comment_links": [],
}
@pytest.fixture
def bm(tmp_path: Path) -> BaselineManager:
return BaselineManager(tmp_path / "data")
class TestUrlKey:
def test_deterministic(self):
assert url_key("https://example.com/") == url_key("https://example.com/")
def test_different_urls_different_keys(self):
assert url_key("https://a.com/") != url_key("https://b.com/")
def test_16_hex_chars(self):
k = url_key("https://example.com/")
assert len(k) == 16
assert all(c in "0123456789abcdef" for c in k)
class TestSaveSnapshot:
def test_creates_manifest(self, bm, tmp_path):
pages = [_sample_page("https://b.info/")]
snap_dir = bm.save_snapshot(pages)
assert (snap_dir / "manifest.json").exists()
def test_manifest_content(self, bm):
pages = [_sample_page("https://b.info/"), _sample_page("https://b.info/p/")]
snap_dir = bm.save_snapshot(pages)
manifest = json.loads((snap_dir / "manifest.json").read_text())
assert manifest["page_count"] == 2
assert "https://b.info/" in manifest["urls"]
def test_creates_page_files(self, bm):
pages = [_sample_page("https://b.info/")]
snap_dir = bm.save_snapshot(pages)
page_files = list((snap_dir / "pages").glob("*.json"))
assert len(page_files) == 1
def test_latest_snapshot_dir(self, bm):
assert bm.latest_snapshot_dir() is None
bm.save_snapshot([_sample_page("https://b.info/")])
assert bm.latest_snapshot_dir() is not None
class TestLoadSnapshot:
def test_loads_saved_snapshot(self, bm):
pages = [_sample_page("https://b.info/"), _sample_page("https://b.info/p/")]
bm.save_snapshot(pages)
snap = bm.load_snapshot()
assert "https://b.info/" in snap["pages"]
assert "https://b.info/p/" in snap["pages"]
def test_returns_empty_when_no_snapshot(self, bm):
result = bm.load_snapshot()
assert result == {}
class TestBaselineExists:
def test_false_before_approve(self, bm):
assert not bm.baseline_exists()
def test_true_after_approve(self, bm):
bm.save_snapshot([_sample_page("https://b.info/")])
bm.approve_all(note="test")
assert bm.baseline_exists()
class TestApproveAll:
def test_approve_all_creates_baseline(self, bm):
pages = [_sample_page("https://b.info/"), _sample_page("https://b.info/p/")]
bm.save_snapshot(pages)
approved = bm.approve_all(note="Initial baseline")
assert "https://b.info/" in approved
assert "https://b.info/p/" in approved
assert bm.baseline_exists()
def test_manifest_contains_approval_metadata(self, bm):
bm.save_snapshot([_sample_page("https://b.info/")])
bm.approve_all(note="test note", approved_by="testuser")
manifest = json.loads((bm.baseline_dir / "manifest.json").read_text())
assert manifest["note"] == "test note"
assert manifest["approved_by"] == "testuser"
assert "approved_at" in manifest
def test_approve_all_raises_without_snapshot(self, bm):
with pytest.raises(RuntimeError):
bm.approve_all()
class TestApproveUrl:
def test_approve_single_url(self, bm):
pages = [_sample_page("https://b.info/"), _sample_page("https://b.info/p/")]
bm.save_snapshot(pages)
result = bm.approve_url("https://b.info/")
assert result is True
assert bm.baseline_exists()
baseline = bm.load_baseline()
assert "https://b.info/" in baseline["pages"]
assert "https://b.info/p/" not in baseline["pages"]
def test_approve_missing_url_returns_false(self, bm):
bm.save_snapshot([_sample_page("https://b.info/")])
result = bm.approve_url("https://b.info/does-not-exist/")
assert result is False
class TestLoadBaseline:
def test_loads_approved_pages(self, bm):
pages = [_sample_page("https://b.info/", "Text A")]
bm.save_snapshot(pages)
bm.approve_all()
baseline = bm.load_baseline()
assert "https://b.info/" in baseline["pages"]
assert baseline["pages"]["https://b.info/"]["text"] == "Text A"
def test_never_auto_updates(self, bm):
"""Baseline must not change unless approve is explicitly called."""
bm.save_snapshot([_sample_page("https://b.info/", "Original")])
bm.approve_all()
# Second snapshot with different content
bm.save_snapshot([_sample_page("https://b.info/", "Manipuliert")])
# Baseline must still contain original content
baseline = bm.load_baseline()
assert baseline["pages"]["https://b.info/"]["text"] == "Original"
class TestRebuildBaseline:
def test_rebuild_replaces_baseline(self, bm):
bm.save_snapshot([_sample_page("https://b.info/", "Alt")])
bm.approve_all()
bm.save_snapshot([_sample_page("https://b.info/", "Neu")])
bm.rebuild_baseline(note="Major update")
baseline = bm.load_baseline()
assert baseline["pages"]["https://b.info/"]["text"] == "Neu"

63
tests/test_crawler.py Normal file
View file

@ -0,0 +1,63 @@
"""Tests for crawler URL normalization and crawl filtering."""
import pytest
from scanner.crawler import normalize_url, should_crawl
SKIP_EXT = [".jpg", ".jpeg", ".png", ".pdf", ".zip", ".css", ".js"]
class TestNormalizeUrl:
def test_strips_fragment(self):
assert normalize_url("https://example.com/page#section") == "https://example.com/page"
def test_lowercase_host(self):
assert normalize_url("https://EXAMPLE.COM/path") == "https://example.com/path"
def test_non_http_returns_none(self):
assert normalize_url("javascript:void(0)") is None
assert normalize_url("mailto:foo@bar.com") is None
assert normalize_url("ftp://example.com") is None
def test_sorts_query_params(self):
a = normalize_url("https://example.com/?z=1&a=2")
b = normalize_url("https://example.com/?a=2&z=1")
assert a == b
def test_strips_tracking_params(self):
url = normalize_url("https://example.com/?utm_source=google&page=1")
assert "utm_source" not in url
assert "page=1" in url
def test_strips_session_params(self):
url = normalize_url("https://example.com/?sid=abc123&id=5")
assert "sid" not in url
assert "id=5" in url
def test_adds_default_path(self):
result = normalize_url("https://example.com")
assert result.endswith("/") or "example.com" in result
def test_empty_string_returns_none(self):
assert normalize_url("") is None
class TestShouldCrawl:
def test_same_host_plain_page(self):
assert should_crawl("https://example.com/about", "example.com", SKIP_EXT)
def test_different_host_rejected(self):
assert not should_crawl("https://other.com/page", "example.com", SKIP_EXT)
def test_image_rejected(self):
assert not should_crawl("https://example.com/photo.jpg", "example.com", SKIP_EXT)
def test_pdf_rejected(self):
assert not should_crawl("https://example.com/doc.pdf", "example.com", SKIP_EXT)
def test_html_accepted(self):
assert should_crawl("https://example.com/page.html", "example.com", SKIP_EXT)
def test_trailing_slash_accepted(self):
assert should_crawl("https://example.com/section/", "example.com", SKIP_EXT)
def test_utm_query_param_rejected(self):
assert not should_crawl("https://example.com/?utm_source=x", "example.com", SKIP_EXT)

250
tests/test_differ.py Normal file
View file

@ -0,0 +1,250 @@
"""Tests for diff logic and scoring."""
import pytest
from scanner.differ import (
compare_snapshots,
diff_link_set,
diff_metadata,
diff_text,
normalize_text,
score_diff,
text_hash,
)
BASE_CFG = {
"scoring": {
"new_external_domain": 50,
"new_internal_url": 30,
"missing_internal_url": 20,
"hidden_content": 40,
"unexpected_canonical": 35,
"meta_refresh": 40,
"unexpected_jsonld": 35,
"large_text_addition": 20,
"new_inline_script_suspicious": 30,
"comment_links": 25,
"noscript_links": 25,
"missing_security_header": 5,
"suspicious_content_pattern": 50,
},
"thresholds": {"yellow": 20, "red": 60, "large_text_block_chars": 200},
"normalization": {"collapse_whitespace": True, "ignore_patterns": []},
}
class TestNormalizeText:
def test_collapses_whitespace(self):
result = normalize_text("Hello World\n\nFoo", BASE_CFG)
assert result == "Hello World Foo"
def test_applies_ignore_patterns(self):
cfg = dict(BASE_CFG)
cfg["normalization"] = {
"collapse_whitespace": True,
"ignore_patterns": [r"\d{2}\.\d{2}\.\d{4}"],
}
result = normalize_text("Stand: 12.06.2026 Text", cfg)
assert "[IGNORED]" in result
assert "12.06.2026" not in result
class TestDiffText:
def test_identical_texts_produce_empty_diff(self):
assert diff_text("Hello", "Hello") == []
def test_addition_shows_plus_line(self):
lines = diff_text("Hello", "Hello\nWorld")
assert any(l.startswith("+") and "World" in l for l in lines)
def test_removal_shows_minus_line(self):
lines = diff_text("Hello\nWorld", "Hello")
assert any(l.startswith("-") and "World" in l for l in lines)
class TestDiffLinkSet:
def test_detects_added_link(self):
old = [{"url": "https://example.com/a"}]
new = [{"url": "https://example.com/a"}, {"url": "https://example.com/b"}]
result = diff_link_set(old, new)
assert "https://example.com/b" in result["added"]
assert result["removed"] == []
def test_detects_removed_link(self):
old = [{"url": "https://example.com/a"}, {"url": "https://example.com/b"}]
new = [{"url": "https://example.com/a"}]
result = diff_link_set(old, new)
assert "https://example.com/b" in result["removed"]
def test_no_change(self):
links = [{"url": "https://example.com/a"}]
result = diff_link_set(links, links)
assert result == {"added": [], "removed": []}
class TestDiffMetadata:
def test_detects_title_change(self):
old = {"title": "Alte Seite", "h1": [], "h2": []}
new = {"title": "Neue Seite", "h1": [], "h2": []}
result = diff_metadata(old, new)
assert "title" in result
assert result["title"]["old"] == "Alte Seite"
def test_no_change(self):
meta = {"title": "Seite", "description": "Desc", "h1": [], "h2": []}
assert diff_metadata(meta, meta) == {}
def test_detects_canonical_change(self):
old = {"canonical": "https://bredelar.info/", "h1": [], "h2": []}
new = {"canonical": "https://spam.example.com/", "h1": [], "h2": []}
result = diff_metadata(old, new)
assert "canonical" in result
class TestCompareSnapshots:
def _make_page(self, url, text="Inhalt", links=None, hidden=None):
return {
"url": url,
"text": text,
"links": links or {"a": [], "script": [], "iframe": [], "link_rel": [], "form": [], "meta_refresh": [], "canonical": None},
"metadata": {"title": "T", "h1": [], "h2": []},
"hidden_content": hidden or [],
"inline_scripts": [],
"jsonld": [],
"comment_links": [],
}
def test_new_url_detected(self):
baseline = {"https://b.info/": self._make_page("https://b.info/")}
current = {
"https://b.info/": self._make_page("https://b.info/"),
"https://b.info/new/": self._make_page("https://b.info/new/"),
}
result = compare_snapshots(baseline, current, BASE_CFG)
assert "https://b.info/new/" in result["new_internal_urls"]
def test_missing_url_detected(self):
baseline = {
"https://b.info/": self._make_page("https://b.info/"),
"https://b.info/old/": self._make_page("https://b.info/old/"),
}
current = {"https://b.info/": self._make_page("https://b.info/")}
result = compare_snapshots(baseline, current, BASE_CFG)
assert "https://b.info/old/" in result["missing_internal_urls"]
def test_unchanged_page_not_in_diffs(self):
page = self._make_page("https://b.info/", "Gleicher Inhalt")
result = compare_snapshots({"https://b.info/": page}, {"https://b.info/": page}, BASE_CFG)
assert result["changed_pages"] == 0
def test_text_change_detected(self):
old = self._make_page("https://b.info/", "Alter Text")
new = self._make_page("https://b.info/", "Alter Text plus neuer Spam-Block im SEO-Kontext")
result = compare_snapshots({"https://b.info/": old}, {"https://b.info/": new}, BASE_CFG)
assert result["changed_pages"] == 1
def test_new_external_domain_detected(self):
def _page_with_ext(url, ext_url):
p = self._make_page(url)
p["links"]["a"] = [{"url": ext_url, "class": "external", "text": "x", "rel": ""}]
return p
baseline = {"https://b.info/": self._make_page("https://b.info/")}
current = {"https://b.info/": _page_with_ext("https://b.info/", "https://spam.example.com/")}
result = compare_snapshots(baseline, current, BASE_CFG)
assert "spam.example.com" in result["new_external_domains"]
class TestScoreDiff:
def test_green_for_no_changes(self):
result = score_diff({
"new_internal_urls": [],
"missing_internal_urls": [],
"new_external_domains": [],
"page_diffs": [],
"new_pages_analysis": [],
}, BASE_CFG)
assert result["level"] == "green"
assert result["exit_code"] == 0
def test_yellow_for_new_url(self):
result = score_diff({
"new_internal_urls": ["https://b.info/new/"],
"missing_internal_urls": [],
"new_external_domains": [],
"page_diffs": [],
"new_pages_analysis": [],
}, BASE_CFG)
assert result["level"] == "yellow"
assert result["exit_code"] == 1
def test_yellow_for_single_new_external_domain(self):
# One new external domain = 50 pts → yellow (red threshold is 60)
result = score_diff({
"new_internal_urls": [],
"missing_internal_urls": [],
"new_external_domains": ["pharmacy.example.com"],
"page_diffs": [],
"new_pages_analysis": [],
}, BASE_CFG)
assert result["level"] == "yellow"
assert result["exit_code"] == 1
def test_red_for_new_external_domain_plus_hidden_content(self):
# External domain (50) + hidden content (40) = 90 pts → red
result = score_diff({
"new_internal_urls": [],
"missing_internal_urls": [],
"new_external_domains": ["pharmacy.example.com"],
"page_diffs": [{
"url": "https://b.info/",
"text_diff": [],
"added_chars": 0,
"link_diff": {},
"meta_diff": {},
"new_hidden_content": [{"type": "inline_style_hidden", "indicators": ["display:none"]}],
"new_inline_scripts": [],
"new_jsonld": [],
"new_meta_refresh": [],
"canonical_changed": False,
"new_comment_links": [],
"old_canonical": None,
"new_canonical": None,
}],
"new_pages_analysis": [],
}, BASE_CFG)
assert result["level"] == "red"
assert result["exit_code"] == 2
def test_yellow_for_hidden_content_alone(self):
# One hidden content finding = 40 pts → yellow (below red threshold of 60)
result = score_diff({
"new_internal_urls": [],
"missing_internal_urls": [],
"new_external_domains": [],
"page_diffs": [{
"url": "https://b.info/",
"text_diff": [],
"added_chars": 0,
"link_diff": {},
"meta_diff": {},
"new_hidden_content": [{"type": "inline_style_hidden", "indicators": ["display:none"]}],
"new_inline_scripts": [],
"new_jsonld": [],
"new_meta_refresh": [],
"canonical_changed": False,
"new_comment_links": [],
"old_canonical": None,
"new_canonical": None,
}],
"new_pages_analysis": [],
}, BASE_CFG)
assert result["level"] == "yellow"
def test_reasons_include_domain(self):
result = score_diff({
"new_internal_urls": [],
"missing_internal_urls": [],
"new_external_domains": ["spam.com"],
"page_diffs": [],
"new_pages_analysis": [],
}, BASE_CFG)
assert any("spam.com" in r for r in result["reasons"])

152
tests/test_extractor.py Normal file
View file

@ -0,0 +1,152 @@
"""Tests for HTML extraction: links, metadata, hidden content, JSON-LD."""
import pytest
from scanner.extractor import extract_page
BASE_URL = "https://bredelar.info/"
def _extract(html: str, url: str = BASE_URL) -> dict:
return extract_page(html, url)
class TestVisibleText:
def test_strips_scripts(self):
result = _extract("<html><script>var x=1;</script><body>Hallo Welt</body></html>")
assert "var x" not in result["text"]
assert "Hallo Welt" in result["text"]
def test_strips_style(self):
result = _extract("<html><style>.cls{color:red}</style><body>Text</body></html>")
assert ".cls" not in result["text"]
assert "Text" in result["text"]
def test_collapses_whitespace(self):
result = _extract("<body><p>Hello World</p></body>")
assert "Hello World" in result["text"]
class TestLinkExtraction:
def test_extracts_anchor_links(self):
result = _extract('<body><a href="/page">Link</a></body>')
urls = [l["url"] for l in result["links"]["a"]]
assert any("/page" in u for u in urls)
def test_classifies_internal_vs_external(self):
html = (
'<body>'
'<a href="/internal">intern</a>'
'<a href="https://spam.example.com/x">extern</a>'
'</body>'
)
result = _extract(html)
classes = {l["url"]: l["class"] for l in result["links"]["a"]}
assert any(v == "internal" for v in classes.values())
assert any(v == "external" for v in classes.values())
def test_extracts_canonical(self):
html = '<head><link rel="canonical" href="https://bredelar.info/page"/></head>'
result = _extract(html)
assert result["links"]["canonical"] == "https://bredelar.info/page"
def test_extracts_meta_refresh(self):
html = '<head><meta http-equiv="refresh" content="0; url=https://malware.example.com"/></head>'
result = _extract(html)
assert any("malware.example.com" in u for u in result["links"]["meta_refresh"])
def test_ignores_javascript_hrefs(self):
result = _extract('<body><a href="javascript:void(0)">JS</a></body>')
assert result["links"]["a"] == []
def test_extracts_iframe_src(self):
result = _extract('<body><iframe src="https://evil.com/x"></iframe></body>')
iframe_urls = [l["url"] for l in result["links"]["iframe"]]
assert any("evil.com" in u for u in iframe_urls)
class TestMetadata:
def test_extracts_title(self):
result = _extract("<html><head><title>Bredelar</title></head></html>")
assert result["metadata"]["title"] == "Bredelar"
def test_extracts_description(self):
html = '<head><meta name="description" content="Ein Dorf in NRW"/></head>'
result = _extract(html)
assert result["metadata"]["description"] == "Ein Dorf in NRW"
def test_extracts_og_tags(self):
html = '<head><meta property="og:title" content="OG-Titel"/></head>'
result = _extract(html)
assert result["metadata"].get("og:title") == "OG-Titel"
def test_extracts_h1(self):
result = _extract("<body><h1>Hauptüberschrift</h1></body>")
assert "Hauptüberschrift" in result["metadata"]["h1"]
class TestHiddenContent:
def test_detects_display_none(self):
html = '<body><div style="display:none"><a href="https://spam.com">Spam</a></div></body>'
result = _extract(html)
assert len(result["hidden_content"]) > 0
assert any("display:none" in str(h.get("indicators", [])) for h in result["hidden_content"])
def test_detects_visibility_hidden(self):
html = '<body><span style="visibility:hidden">Spam Text</span></body>'
result = _extract(html)
assert len(result["hidden_content"]) > 0
def test_detects_offscreen_position(self):
html = '<body><div style="position:absolute; left:-9999px">Hidden</div></body>'
result = _extract(html)
assert len(result["hidden_content"]) > 0
def test_detects_noscript_links(self):
html = '<body><noscript><a href="https://spam.com/x">Spam</a></noscript></body>'
result = _extract(html)
noscript = [h for h in result["hidden_content"] if h.get("type") == "noscript_links"]
assert len(noscript) > 0
def test_visible_element_not_flagged(self):
html = '<body><div style="color:red">Sichtbarer Text</div></body>'
result = _extract(html)
# color:red should NOT be flagged (not a hiding technique)
assert not any(
"red" in str(h) for h in result["hidden_content"]
)
class TestCommentLinks:
def test_finds_link_in_comment(self):
html = '<!-- <a href="https://hidden-spam.com/x">click</a> --><body></body>'
result = _extract(html)
assert any("hidden-spam.com" in u for u in result["comment_links"])
def test_no_false_positives(self):
html = '<!-- This is a normal comment --><body>Text</body>'
result = _extract(html)
assert result["comment_links"] == []
class TestJsonLd:
def test_extracts_known_type(self):
html = (
'<script type="application/ld+json">'
'{"@type": "Organization", "name": "Bredelar"}'
'</script>'
)
result = _extract(html)
assert any(j["type"] == "Organization" for j in result["jsonld"])
def test_handles_parse_error(self):
html = '<script type="application/ld+json">{broken json}</script>'
result = _extract(html)
assert any(j["type"] == "PARSE_ERROR" for j in result["jsonld"])
def test_extracts_unexpected_type(self):
html = (
'<script type="application/ld+json">'
'{"@type": "Pharmacy", "name": "SpamShop"}'
'</script>'
)
result = _extract(html)
assert any(j["type"] == "Pharmacy" for j in result["jsonld"])