68243a97b9
Echte Bugs: - extractor: JSON-LD-Crash bei Nicht-Dict-Items (isinstance-Guard) - baseline/__main__: Crawl-Fehler ins Snapshot-Manifest -> erscheinen im Report - __main__: Whitelist-/Header-/Webshell-Checks nur noch auf status==200 - crawler: Noise-Param-Regex auf Voll-Key (view=/value= nicht mehr verworfen) - differ/__main__: unerwartetes JSON-LD via eigenem Kanal, auch auf unveraenderten Seiten erkannt, kein Re-Alert auf bereits genehmigte Typen Aufgeraeumt: - checker: check_internal_paths, find_broken_pages, canonical_is_hijacked, check_jsonld_types entfernt (nicht verdrahtet) - allowed_paths.yaml-Logik und tote Scoring-Keys entfernt - tote Imports entfernt Neuer aktiver Schutz: - Webshell-/Backdoor-Dateinamen-Check verdrahtet (suspicious_filename: 60). Regex wortgrenzen-verankert gegen Fehlalarm auf PSEMailerAntispam.js Effizienz/Struktur: - crawler nutzt requests.Session (keep-alive) - Cache-Buster-Normalisierung an einer Stelle (Extraktion) -> stabile Snapshots, differ wieder reiner Set-Diff Tests: 111 gruen (neu: test_checker.py + Regressionstests) CLAUDE.md aktualisiert Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
361 lines
15 KiB
Python
361 lines
15 KiB
Python
"""Tests for diff logic and scoring."""
|
|
import pytest
|
|
from scanner.differ import (
|
|
compare_snapshots,
|
|
diff_link_set,
|
|
diff_metadata,
|
|
diff_text,
|
|
normalize_text,
|
|
score_diff,
|
|
text_hash,
|
|
)
|
|
|
|
BASE_CFG = {
|
|
"scoring": {
|
|
"new_external_domain": 50,
|
|
"new_internal_url": 30,
|
|
"missing_internal_url": 20,
|
|
"hidden_content": 40,
|
|
"unexpected_canonical": 35,
|
|
"meta_refresh": 40,
|
|
"unexpected_jsonld": 35,
|
|
"large_text_addition": 20,
|
|
"new_inline_script_suspicious": 30,
|
|
"comment_links": 25,
|
|
"noscript_links": 25,
|
|
"missing_security_header": 5,
|
|
"suspicious_content_pattern": 50,
|
|
},
|
|
"thresholds": {"yellow": 20, "red": 60, "large_text_block_chars": 200},
|
|
"normalization": {"collapse_whitespace": True, "ignore_patterns": []},
|
|
}
|
|
|
|
|
|
class TestNormalizeText:
|
|
def test_collapses_whitespace(self):
|
|
result = normalize_text("Hello World\n\nFoo", BASE_CFG)
|
|
assert result == "Hello World Foo"
|
|
|
|
def test_applies_ignore_patterns(self):
|
|
cfg = dict(BASE_CFG)
|
|
cfg["normalization"] = {
|
|
"collapse_whitespace": True,
|
|
"ignore_patterns": [r"\d{2}\.\d{2}\.\d{4}"],
|
|
}
|
|
result = normalize_text("Stand: 12.06.2026 Text", cfg)
|
|
assert "[IGNORED]" in result
|
|
assert "12.06.2026" not in result
|
|
|
|
|
|
class TestDiffText:
|
|
def test_identical_texts_produce_empty_diff(self):
|
|
assert diff_text("Hello", "Hello") == []
|
|
|
|
def test_addition_shows_plus_line(self):
|
|
lines = diff_text("Hello", "Hello\nWorld")
|
|
assert any(l.startswith("+") and "World" in l for l in lines)
|
|
|
|
def test_removal_shows_minus_line(self):
|
|
lines = diff_text("Hello\nWorld", "Hello")
|
|
assert any(l.startswith("-") and "World" in l for l in lines)
|
|
|
|
|
|
class TestDiffLinkSet:
|
|
def test_detects_added_link(self):
|
|
old = [{"url": "https://example.com/a"}]
|
|
new = [{"url": "https://example.com/a"}, {"url": "https://example.com/b"}]
|
|
result = diff_link_set(old, new)
|
|
assert "https://example.com/b" in result["added"]
|
|
assert result["removed"] == []
|
|
|
|
def test_detects_removed_link(self):
|
|
old = [{"url": "https://example.com/a"}, {"url": "https://example.com/b"}]
|
|
new = [{"url": "https://example.com/a"}]
|
|
result = diff_link_set(old, new)
|
|
assert "https://example.com/b" in result["removed"]
|
|
|
|
def test_no_change(self):
|
|
links = [{"url": "https://example.com/a"}]
|
|
result = diff_link_set(links, links)
|
|
assert result == {"added": [], "removed": []}
|
|
|
|
def test_truly_new_url_detected(self):
|
|
# Cache-buster stripping happens at extraction time; diff_link_set is a
|
|
# plain set comparison on already-normalised URLs.
|
|
old = [{"url": "https://example.com/style.css"}]
|
|
new = [{"url": "https://example.com/style.css"},
|
|
{"url": "https://example.com/extra.js"}]
|
|
result = diff_link_set(old, new)
|
|
assert "https://example.com/extra.js" in result["added"]
|
|
|
|
|
|
class TestDiffMetadata:
|
|
def test_detects_title_change(self):
|
|
old = {"title": "Alte Seite", "h1": [], "h2": []}
|
|
new = {"title": "Neue Seite", "h1": [], "h2": []}
|
|
result = diff_metadata(old, new)
|
|
assert "title" in result
|
|
assert result["title"]["old"] == "Alte Seite"
|
|
|
|
def test_no_change(self):
|
|
meta = {"title": "Seite", "description": "Desc", "h1": [], "h2": []}
|
|
assert diff_metadata(meta, meta) == {}
|
|
|
|
def test_detects_canonical_change(self):
|
|
old = {"canonical": "https://bredelar.info/", "h1": [], "h2": []}
|
|
new = {"canonical": "https://spam.example.com/", "h1": [], "h2": []}
|
|
result = diff_metadata(old, new)
|
|
assert "canonical" in result
|
|
|
|
|
|
class TestCompareSnapshots:
|
|
def _make_page(self, url, text="Inhalt", links=None, hidden=None):
|
|
return {
|
|
"url": url,
|
|
"text": text,
|
|
"links": links or {"a": [], "script": [], "iframe": [], "link_rel": [], "form": [], "meta_refresh": [], "canonical": None},
|
|
"metadata": {"title": "T", "h1": [], "h2": []},
|
|
"hidden_content": hidden or [],
|
|
"inline_scripts": [],
|
|
"jsonld": [],
|
|
"comment_links": [],
|
|
}
|
|
|
|
def test_new_url_detected(self):
|
|
baseline = {"https://b.info/": self._make_page("https://b.info/")}
|
|
current = {
|
|
"https://b.info/": self._make_page("https://b.info/"),
|
|
"https://b.info/new/": self._make_page("https://b.info/new/"),
|
|
}
|
|
result = compare_snapshots(baseline, current, BASE_CFG)
|
|
assert "https://b.info/new/" in result["new_internal_urls"]
|
|
|
|
def test_missing_url_detected(self):
|
|
baseline = {
|
|
"https://b.info/": self._make_page("https://b.info/"),
|
|
"https://b.info/old/": self._make_page("https://b.info/old/"),
|
|
}
|
|
current = {"https://b.info/": self._make_page("https://b.info/")}
|
|
result = compare_snapshots(baseline, current, BASE_CFG)
|
|
assert "https://b.info/old/" in result["missing_internal_urls"]
|
|
|
|
def test_unchanged_page_not_in_diffs(self):
|
|
page = self._make_page("https://b.info/", "Gleicher Inhalt")
|
|
result = compare_snapshots({"https://b.info/": page}, {"https://b.info/": page}, BASE_CFG)
|
|
assert result["changed_pages"] == 0
|
|
|
|
def test_text_change_detected(self):
|
|
old = self._make_page("https://b.info/", "Alter Text")
|
|
new = self._make_page("https://b.info/", "Alter Text plus neuer Spam-Block im SEO-Kontext")
|
|
result = compare_snapshots({"https://b.info/": old}, {"https://b.info/": new}, BASE_CFG)
|
|
assert result["changed_pages"] == 1
|
|
|
|
def test_new_external_domain_detected(self):
|
|
def _page_with_ext(url, ext_url):
|
|
p = self._make_page(url)
|
|
p["links"]["a"] = [{"url": ext_url, "class": "external", "text": "x", "rel": ""}]
|
|
return p
|
|
|
|
baseline = {"https://b.info/": self._make_page("https://b.info/")}
|
|
current = {"https://b.info/": _page_with_ext("https://b.info/", "https://spam.example.com/")}
|
|
result = compare_snapshots(baseline, current, BASE_CFG)
|
|
assert "spam.example.com" in result["new_external_domains"]
|
|
|
|
|
|
class TestUnexpectedJsonLd:
|
|
CFG = {**BASE_CFG, "allowed_jsonld_types": ["Organization", "WebPage"]}
|
|
|
|
def _page(self, url, jsonld):
|
|
return {
|
|
"url": url, "status": 200, "text": "Inhalt",
|
|
"links": {"a": [], "script": [], "iframe": [], "link_rel": [], "form": [], "meta_refresh": [], "canonical": None},
|
|
"metadata": {"title": "T", "h1": [], "h2": []},
|
|
"hidden_content": [], "inline_scripts": [], "jsonld": jsonld, "comment_links": [],
|
|
}
|
|
|
|
def test_unexpected_type_flagged_without_page_diff(self):
|
|
# Page is otherwise unchanged (same text/links) but carries a bad @type.
|
|
base = {"https://b.info/": self._page("https://b.info/", [{"type": "Organization"}])}
|
|
cur = {"https://b.info/": self._page("https://b.info/", [{"type": "Pharmacy"}])}
|
|
result = compare_snapshots(base, cur, self.CFG)
|
|
types = [e["type"] for e in result["unexpected_jsonld"]]
|
|
assert "Pharmacy" in types
|
|
|
|
def test_allowed_type_not_flagged(self):
|
|
base = {"https://b.info/": self._page("https://b.info/", [])}
|
|
cur = {"https://b.info/": self._page("https://b.info/", [{"type": "WebPage"}])}
|
|
result = compare_snapshots(base, cur, self.CFG)
|
|
assert result["unexpected_jsonld"] == []
|
|
|
|
def test_preexisting_unexpected_type_not_realerted(self):
|
|
# Bad type already in baseline → approved → should not re-alert.
|
|
base = {"https://b.info/": self._page("https://b.info/", [{"type": "Pharmacy"}])}
|
|
cur = {"https://b.info/": self._page("https://b.info/", [{"type": "Pharmacy"}])}
|
|
result = compare_snapshots(base, cur, self.CFG)
|
|
assert result["unexpected_jsonld"] == []
|
|
|
|
def test_unexpected_type_scores(self):
|
|
result = score_diff({
|
|
"new_internal_urls": [], "missing_internal_urls": [],
|
|
"new_external_domains": [], "page_diffs": [], "new_pages_analysis": [],
|
|
"unexpected_jsonld": [{"url": "https://b.info/", "type": "Pharmacy"}],
|
|
}, BASE_CFG)
|
|
assert result["score"] == 35
|
|
assert result["level"] == "yellow"
|
|
|
|
|
|
class TestSuspiciousFilenames:
|
|
def test_suspicious_filename_scores_red(self):
|
|
result = score_diff({
|
|
"new_internal_urls": [], "missing_internal_urls": [],
|
|
"new_external_domains": [], "page_diffs": [], "new_pages_analysis": [],
|
|
"suspicious_filenames": [{"page": "https://b.info/", "url": "https://b.info/shell.php"}],
|
|
}, BASE_CFG)
|
|
assert result["score"] == 60
|
|
assert result["level"] == "red"
|
|
|
|
|
|
class TestBrokenLinks:
|
|
def _make_page(self, url, text="Inhalt", status=200):
|
|
return {
|
|
"url": url, "status": status, "text": text,
|
|
"links": {"a": [], "script": [], "iframe": [], "link_rel": [], "form": [], "meta_refresh": [], "canonical": None},
|
|
"metadata": {"title": "T", "h1": [], "h2": []},
|
|
"hidden_content": [], "inline_scripts": [], "jsonld": [], "comment_links": [],
|
|
}
|
|
|
|
def test_new_broken_url_detected(self):
|
|
baseline = {"https://b.info/": self._make_page("https://b.info/")}
|
|
current = {
|
|
"https://b.info/": self._make_page("https://b.info/"),
|
|
"https://b.info/missing/": self._make_page("https://b.info/missing/", status=404),
|
|
}
|
|
result = compare_snapshots(baseline, current, BASE_CFG)
|
|
assert len(result["new_broken_urls"]) == 1
|
|
assert result["new_broken_urls"][0]["url"] == "https://b.info/missing/"
|
|
assert result["new_broken_urls"][0]["status"] == 404
|
|
|
|
def test_known_broken_not_in_new(self):
|
|
broken_page = {"https://b.info/old/": self._make_page("https://b.info/old/", status=404)}
|
|
baseline = {**{"https://b.info/": self._make_page("https://b.info/")}, **broken_page}
|
|
current = {**{"https://b.info/": self._make_page("https://b.info/")}, **broken_page}
|
|
result = compare_snapshots(baseline, current, BASE_CFG)
|
|
assert result["new_broken_urls"] == []
|
|
assert len(result["known_broken_urls"]) == 1
|
|
|
|
def test_broken_url_scores(self):
|
|
result = score_diff({
|
|
"new_internal_urls": [], "missing_internal_urls": [],
|
|
"new_external_domains": [], "page_diffs": [], "new_pages_analysis": [],
|
|
"new_broken_urls": [{"url": "https://b.info/gone/", "status": 404}],
|
|
"known_broken_urls": [],
|
|
}, BASE_CFG)
|
|
assert result["score"] == 20
|
|
assert result["level"] == "yellow"
|
|
|
|
def test_known_broken_not_scored(self):
|
|
result = score_diff({
|
|
"new_internal_urls": [], "missing_internal_urls": [],
|
|
"new_external_domains": [], "page_diffs": [], "new_pages_analysis": [],
|
|
"new_broken_urls": [],
|
|
"known_broken_urls": [{"url": "https://b.info/old/", "status": 404}],
|
|
}, BASE_CFG)
|
|
assert result["score"] == 0
|
|
assert result["level"] == "green"
|
|
|
|
|
|
class TestScoreDiff:
|
|
def test_green_for_no_changes(self):
|
|
result = score_diff({
|
|
"new_internal_urls": [],
|
|
"missing_internal_urls": [],
|
|
"new_external_domains": [],
|
|
"page_diffs": [],
|
|
"new_pages_analysis": [],
|
|
}, BASE_CFG)
|
|
assert result["level"] == "green"
|
|
assert result["exit_code"] == 0
|
|
|
|
def test_yellow_for_new_url(self):
|
|
result = score_diff({
|
|
"new_internal_urls": ["https://b.info/new/"],
|
|
"missing_internal_urls": [],
|
|
"new_external_domains": [],
|
|
"page_diffs": [],
|
|
"new_pages_analysis": [],
|
|
}, BASE_CFG)
|
|
assert result["level"] == "yellow"
|
|
assert result["exit_code"] == 1
|
|
|
|
def test_yellow_for_single_new_external_domain(self):
|
|
# One new external domain = 50 pts → yellow (red threshold is 60)
|
|
result = score_diff({
|
|
"new_internal_urls": [],
|
|
"missing_internal_urls": [],
|
|
"new_external_domains": ["pharmacy.example.com"],
|
|
"page_diffs": [],
|
|
"new_pages_analysis": [],
|
|
}, BASE_CFG)
|
|
assert result["level"] == "yellow"
|
|
assert result["exit_code"] == 1
|
|
|
|
def test_red_for_new_external_domain_plus_hidden_content(self):
|
|
# External domain (50) + hidden content (40) = 90 pts → red
|
|
result = score_diff({
|
|
"new_internal_urls": [],
|
|
"missing_internal_urls": [],
|
|
"new_external_domains": ["pharmacy.example.com"],
|
|
"page_diffs": [{
|
|
"url": "https://b.info/",
|
|
"text_diff": [],
|
|
"added_chars": 0,
|
|
"link_diff": {},
|
|
"meta_diff": {},
|
|
"new_hidden_content": [{"type": "inline_style_hidden", "indicators": ["display:none"]}],
|
|
"new_inline_scripts": [],
|
|
"new_jsonld": [],
|
|
"new_meta_refresh": [],
|
|
"canonical_changed": False,
|
|
"new_comment_links": [],
|
|
"old_canonical": None,
|
|
"new_canonical": None,
|
|
}],
|
|
"new_pages_analysis": [],
|
|
}, BASE_CFG)
|
|
assert result["level"] == "red"
|
|
assert result["exit_code"] == 2
|
|
|
|
def test_yellow_for_hidden_content_alone(self):
|
|
# One hidden content finding = 40 pts → yellow (below red threshold of 60)
|
|
result = score_diff({
|
|
"new_internal_urls": [],
|
|
"missing_internal_urls": [],
|
|
"new_external_domains": [],
|
|
"page_diffs": [{
|
|
"url": "https://b.info/",
|
|
"text_diff": [],
|
|
"added_chars": 0,
|
|
"link_diff": {},
|
|
"meta_diff": {},
|
|
"new_hidden_content": [{"type": "inline_style_hidden", "indicators": ["display:none"]}],
|
|
"new_inline_scripts": [],
|
|
"new_jsonld": [],
|
|
"new_meta_refresh": [],
|
|
"canonical_changed": False,
|
|
"new_comment_links": [],
|
|
"old_canonical": None,
|
|
"new_canonical": None,
|
|
}],
|
|
"new_pages_analysis": [],
|
|
}, BASE_CFG)
|
|
assert result["level"] == "yellow"
|
|
|
|
def test_reasons_include_domain(self):
|
|
result = score_diff({
|
|
"new_internal_urls": [],
|
|
"missing_internal_urls": [],
|
|
"new_external_domains": ["spam.com"],
|
|
"page_diffs": [],
|
|
"new_pages_analysis": [],
|
|
}, BASE_CFG)
|
|
assert any("spam.com" in r for r in result["reasons"])
|