feat: initial implementation of website integrity scanner
Python-Package zur unabhängigen Überwachung statischer Websites gegen SEO-Spam und unbefugte Manipulationen. Läuft außerhalb des Hosters. Kernfunktionen: - Reiner Python-Crawler (bs4+lxml), kein wget - Baseline-Management mit manuellem Freigabe-Workflow (niemals automatisch) - Text-/Link-/Meta-Diff mit Risiko-Scoring (grün/gelb/rot) - Hidden-Content-Erkennung (CSS inline, noscript, Kommentar-Links) - Whitelist für externe Domains und interne Pfade - E-Mail + Webhook Alarmierung - CLI: init, crawl, check, scan, approve, report, status - 79 Unit-Tests (pytest), alle grün Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
commit
98e8d11eb5
24 changed files with 4295 additions and 0 deletions
0
tests/__init__.py
Normal file
0
tests/__init__.py
Normal file
161
tests/test_baseline.py
Normal file
161
tests/test_baseline.py
Normal file
|
|
@ -0,0 +1,161 @@
|
|||
"""Tests for snapshot saving, baseline management and approval workflow."""
|
||||
import json
|
||||
import pytest
|
||||
from pathlib import Path
|
||||
from scanner.baseline import BaselineManager, url_key
|
||||
|
||||
|
||||
def _sample_page(url: str, text: str = "Inhalt") -> dict:
|
||||
return {
|
||||
"url": url,
|
||||
"final_url": url,
|
||||
"status": 200,
|
||||
"content_type": "text/html",
|
||||
"response_headers": {},
|
||||
"text": text,
|
||||
"links": {"a": [], "canonical": None, "meta_refresh": []},
|
||||
"metadata": {"title": "T", "h1": [], "h2": []},
|
||||
"hidden_content": [],
|
||||
"inline_scripts": [],
|
||||
"jsonld": [],
|
||||
"comment_links": [],
|
||||
}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def bm(tmp_path: Path) -> BaselineManager:
|
||||
return BaselineManager(tmp_path / "data")
|
||||
|
||||
|
||||
class TestUrlKey:
|
||||
def test_deterministic(self):
|
||||
assert url_key("https://example.com/") == url_key("https://example.com/")
|
||||
|
||||
def test_different_urls_different_keys(self):
|
||||
assert url_key("https://a.com/") != url_key("https://b.com/")
|
||||
|
||||
def test_16_hex_chars(self):
|
||||
k = url_key("https://example.com/")
|
||||
assert len(k) == 16
|
||||
assert all(c in "0123456789abcdef" for c in k)
|
||||
|
||||
|
||||
class TestSaveSnapshot:
|
||||
def test_creates_manifest(self, bm, tmp_path):
|
||||
pages = [_sample_page("https://b.info/")]
|
||||
snap_dir = bm.save_snapshot(pages)
|
||||
assert (snap_dir / "manifest.json").exists()
|
||||
|
||||
def test_manifest_content(self, bm):
|
||||
pages = [_sample_page("https://b.info/"), _sample_page("https://b.info/p/")]
|
||||
snap_dir = bm.save_snapshot(pages)
|
||||
manifest = json.loads((snap_dir / "manifest.json").read_text())
|
||||
assert manifest["page_count"] == 2
|
||||
assert "https://b.info/" in manifest["urls"]
|
||||
|
||||
def test_creates_page_files(self, bm):
|
||||
pages = [_sample_page("https://b.info/")]
|
||||
snap_dir = bm.save_snapshot(pages)
|
||||
page_files = list((snap_dir / "pages").glob("*.json"))
|
||||
assert len(page_files) == 1
|
||||
|
||||
def test_latest_snapshot_dir(self, bm):
|
||||
assert bm.latest_snapshot_dir() is None
|
||||
bm.save_snapshot([_sample_page("https://b.info/")])
|
||||
assert bm.latest_snapshot_dir() is not None
|
||||
|
||||
|
||||
class TestLoadSnapshot:
|
||||
def test_loads_saved_snapshot(self, bm):
|
||||
pages = [_sample_page("https://b.info/"), _sample_page("https://b.info/p/")]
|
||||
bm.save_snapshot(pages)
|
||||
snap = bm.load_snapshot()
|
||||
assert "https://b.info/" in snap["pages"]
|
||||
assert "https://b.info/p/" in snap["pages"]
|
||||
|
||||
def test_returns_empty_when_no_snapshot(self, bm):
|
||||
result = bm.load_snapshot()
|
||||
assert result == {}
|
||||
|
||||
|
||||
class TestBaselineExists:
|
||||
def test_false_before_approve(self, bm):
|
||||
assert not bm.baseline_exists()
|
||||
|
||||
def test_true_after_approve(self, bm):
|
||||
bm.save_snapshot([_sample_page("https://b.info/")])
|
||||
bm.approve_all(note="test")
|
||||
assert bm.baseline_exists()
|
||||
|
||||
|
||||
class TestApproveAll:
|
||||
def test_approve_all_creates_baseline(self, bm):
|
||||
pages = [_sample_page("https://b.info/"), _sample_page("https://b.info/p/")]
|
||||
bm.save_snapshot(pages)
|
||||
approved = bm.approve_all(note="Initial baseline")
|
||||
assert "https://b.info/" in approved
|
||||
assert "https://b.info/p/" in approved
|
||||
assert bm.baseline_exists()
|
||||
|
||||
def test_manifest_contains_approval_metadata(self, bm):
|
||||
bm.save_snapshot([_sample_page("https://b.info/")])
|
||||
bm.approve_all(note="test note", approved_by="testuser")
|
||||
manifest = json.loads((bm.baseline_dir / "manifest.json").read_text())
|
||||
assert manifest["note"] == "test note"
|
||||
assert manifest["approved_by"] == "testuser"
|
||||
assert "approved_at" in manifest
|
||||
|
||||
def test_approve_all_raises_without_snapshot(self, bm):
|
||||
with pytest.raises(RuntimeError):
|
||||
bm.approve_all()
|
||||
|
||||
|
||||
class TestApproveUrl:
|
||||
def test_approve_single_url(self, bm):
|
||||
pages = [_sample_page("https://b.info/"), _sample_page("https://b.info/p/")]
|
||||
bm.save_snapshot(pages)
|
||||
result = bm.approve_url("https://b.info/")
|
||||
assert result is True
|
||||
assert bm.baseline_exists()
|
||||
baseline = bm.load_baseline()
|
||||
assert "https://b.info/" in baseline["pages"]
|
||||
assert "https://b.info/p/" not in baseline["pages"]
|
||||
|
||||
def test_approve_missing_url_returns_false(self, bm):
|
||||
bm.save_snapshot([_sample_page("https://b.info/")])
|
||||
result = bm.approve_url("https://b.info/does-not-exist/")
|
||||
assert result is False
|
||||
|
||||
|
||||
class TestLoadBaseline:
|
||||
def test_loads_approved_pages(self, bm):
|
||||
pages = [_sample_page("https://b.info/", "Text A")]
|
||||
bm.save_snapshot(pages)
|
||||
bm.approve_all()
|
||||
baseline = bm.load_baseline()
|
||||
assert "https://b.info/" in baseline["pages"]
|
||||
assert baseline["pages"]["https://b.info/"]["text"] == "Text A"
|
||||
|
||||
def test_never_auto_updates(self, bm):
|
||||
"""Baseline must not change unless approve is explicitly called."""
|
||||
bm.save_snapshot([_sample_page("https://b.info/", "Original")])
|
||||
bm.approve_all()
|
||||
|
||||
# Second snapshot with different content
|
||||
bm.save_snapshot([_sample_page("https://b.info/", "Manipuliert")])
|
||||
|
||||
# Baseline must still contain original content
|
||||
baseline = bm.load_baseline()
|
||||
assert baseline["pages"]["https://b.info/"]["text"] == "Original"
|
||||
|
||||
|
||||
class TestRebuildBaseline:
|
||||
def test_rebuild_replaces_baseline(self, bm):
|
||||
bm.save_snapshot([_sample_page("https://b.info/", "Alt")])
|
||||
bm.approve_all()
|
||||
|
||||
bm.save_snapshot([_sample_page("https://b.info/", "Neu")])
|
||||
bm.rebuild_baseline(note="Major update")
|
||||
|
||||
baseline = bm.load_baseline()
|
||||
assert baseline["pages"]["https://b.info/"]["text"] == "Neu"
|
||||
63
tests/test_crawler.py
Normal file
63
tests/test_crawler.py
Normal file
|
|
@ -0,0 +1,63 @@
|
|||
"""Tests for crawler URL normalization and crawl filtering."""
|
||||
import pytest
|
||||
from scanner.crawler import normalize_url, should_crawl
|
||||
|
||||
SKIP_EXT = [".jpg", ".jpeg", ".png", ".pdf", ".zip", ".css", ".js"]
|
||||
|
||||
|
||||
class TestNormalizeUrl:
|
||||
def test_strips_fragment(self):
|
||||
assert normalize_url("https://example.com/page#section") == "https://example.com/page"
|
||||
|
||||
def test_lowercase_host(self):
|
||||
assert normalize_url("https://EXAMPLE.COM/path") == "https://example.com/path"
|
||||
|
||||
def test_non_http_returns_none(self):
|
||||
assert normalize_url("javascript:void(0)") is None
|
||||
assert normalize_url("mailto:foo@bar.com") is None
|
||||
assert normalize_url("ftp://example.com") is None
|
||||
|
||||
def test_sorts_query_params(self):
|
||||
a = normalize_url("https://example.com/?z=1&a=2")
|
||||
b = normalize_url("https://example.com/?a=2&z=1")
|
||||
assert a == b
|
||||
|
||||
def test_strips_tracking_params(self):
|
||||
url = normalize_url("https://example.com/?utm_source=google&page=1")
|
||||
assert "utm_source" not in url
|
||||
assert "page=1" in url
|
||||
|
||||
def test_strips_session_params(self):
|
||||
url = normalize_url("https://example.com/?sid=abc123&id=5")
|
||||
assert "sid" not in url
|
||||
assert "id=5" in url
|
||||
|
||||
def test_adds_default_path(self):
|
||||
result = normalize_url("https://example.com")
|
||||
assert result.endswith("/") or "example.com" in result
|
||||
|
||||
def test_empty_string_returns_none(self):
|
||||
assert normalize_url("") is None
|
||||
|
||||
|
||||
class TestShouldCrawl:
|
||||
def test_same_host_plain_page(self):
|
||||
assert should_crawl("https://example.com/about", "example.com", SKIP_EXT)
|
||||
|
||||
def test_different_host_rejected(self):
|
||||
assert not should_crawl("https://other.com/page", "example.com", SKIP_EXT)
|
||||
|
||||
def test_image_rejected(self):
|
||||
assert not should_crawl("https://example.com/photo.jpg", "example.com", SKIP_EXT)
|
||||
|
||||
def test_pdf_rejected(self):
|
||||
assert not should_crawl("https://example.com/doc.pdf", "example.com", SKIP_EXT)
|
||||
|
||||
def test_html_accepted(self):
|
||||
assert should_crawl("https://example.com/page.html", "example.com", SKIP_EXT)
|
||||
|
||||
def test_trailing_slash_accepted(self):
|
||||
assert should_crawl("https://example.com/section/", "example.com", SKIP_EXT)
|
||||
|
||||
def test_utm_query_param_rejected(self):
|
||||
assert not should_crawl("https://example.com/?utm_source=x", "example.com", SKIP_EXT)
|
||||
250
tests/test_differ.py
Normal file
250
tests/test_differ.py
Normal file
|
|
@ -0,0 +1,250 @@
|
|||
"""Tests for diff logic and scoring."""
|
||||
import pytest
|
||||
from scanner.differ import (
|
||||
compare_snapshots,
|
||||
diff_link_set,
|
||||
diff_metadata,
|
||||
diff_text,
|
||||
normalize_text,
|
||||
score_diff,
|
||||
text_hash,
|
||||
)
|
||||
|
||||
BASE_CFG = {
|
||||
"scoring": {
|
||||
"new_external_domain": 50,
|
||||
"new_internal_url": 30,
|
||||
"missing_internal_url": 20,
|
||||
"hidden_content": 40,
|
||||
"unexpected_canonical": 35,
|
||||
"meta_refresh": 40,
|
||||
"unexpected_jsonld": 35,
|
||||
"large_text_addition": 20,
|
||||
"new_inline_script_suspicious": 30,
|
||||
"comment_links": 25,
|
||||
"noscript_links": 25,
|
||||
"missing_security_header": 5,
|
||||
"suspicious_content_pattern": 50,
|
||||
},
|
||||
"thresholds": {"yellow": 20, "red": 60, "large_text_block_chars": 200},
|
||||
"normalization": {"collapse_whitespace": True, "ignore_patterns": []},
|
||||
}
|
||||
|
||||
|
||||
class TestNormalizeText:
|
||||
def test_collapses_whitespace(self):
|
||||
result = normalize_text("Hello World\n\nFoo", BASE_CFG)
|
||||
assert result == "Hello World Foo"
|
||||
|
||||
def test_applies_ignore_patterns(self):
|
||||
cfg = dict(BASE_CFG)
|
||||
cfg["normalization"] = {
|
||||
"collapse_whitespace": True,
|
||||
"ignore_patterns": [r"\d{2}\.\d{2}\.\d{4}"],
|
||||
}
|
||||
result = normalize_text("Stand: 12.06.2026 Text", cfg)
|
||||
assert "[IGNORED]" in result
|
||||
assert "12.06.2026" not in result
|
||||
|
||||
|
||||
class TestDiffText:
|
||||
def test_identical_texts_produce_empty_diff(self):
|
||||
assert diff_text("Hello", "Hello") == []
|
||||
|
||||
def test_addition_shows_plus_line(self):
|
||||
lines = diff_text("Hello", "Hello\nWorld")
|
||||
assert any(l.startswith("+") and "World" in l for l in lines)
|
||||
|
||||
def test_removal_shows_minus_line(self):
|
||||
lines = diff_text("Hello\nWorld", "Hello")
|
||||
assert any(l.startswith("-") and "World" in l for l in lines)
|
||||
|
||||
|
||||
class TestDiffLinkSet:
|
||||
def test_detects_added_link(self):
|
||||
old = [{"url": "https://example.com/a"}]
|
||||
new = [{"url": "https://example.com/a"}, {"url": "https://example.com/b"}]
|
||||
result = diff_link_set(old, new)
|
||||
assert "https://example.com/b" in result["added"]
|
||||
assert result["removed"] == []
|
||||
|
||||
def test_detects_removed_link(self):
|
||||
old = [{"url": "https://example.com/a"}, {"url": "https://example.com/b"}]
|
||||
new = [{"url": "https://example.com/a"}]
|
||||
result = diff_link_set(old, new)
|
||||
assert "https://example.com/b" in result["removed"]
|
||||
|
||||
def test_no_change(self):
|
||||
links = [{"url": "https://example.com/a"}]
|
||||
result = diff_link_set(links, links)
|
||||
assert result == {"added": [], "removed": []}
|
||||
|
||||
|
||||
class TestDiffMetadata:
|
||||
def test_detects_title_change(self):
|
||||
old = {"title": "Alte Seite", "h1": [], "h2": []}
|
||||
new = {"title": "Neue Seite", "h1": [], "h2": []}
|
||||
result = diff_metadata(old, new)
|
||||
assert "title" in result
|
||||
assert result["title"]["old"] == "Alte Seite"
|
||||
|
||||
def test_no_change(self):
|
||||
meta = {"title": "Seite", "description": "Desc", "h1": [], "h2": []}
|
||||
assert diff_metadata(meta, meta) == {}
|
||||
|
||||
def test_detects_canonical_change(self):
|
||||
old = {"canonical": "https://bredelar.info/", "h1": [], "h2": []}
|
||||
new = {"canonical": "https://spam.example.com/", "h1": [], "h2": []}
|
||||
result = diff_metadata(old, new)
|
||||
assert "canonical" in result
|
||||
|
||||
|
||||
class TestCompareSnapshots:
|
||||
def _make_page(self, url, text="Inhalt", links=None, hidden=None):
|
||||
return {
|
||||
"url": url,
|
||||
"text": text,
|
||||
"links": links or {"a": [], "script": [], "iframe": [], "link_rel": [], "form": [], "meta_refresh": [], "canonical": None},
|
||||
"metadata": {"title": "T", "h1": [], "h2": []},
|
||||
"hidden_content": hidden or [],
|
||||
"inline_scripts": [],
|
||||
"jsonld": [],
|
||||
"comment_links": [],
|
||||
}
|
||||
|
||||
def test_new_url_detected(self):
|
||||
baseline = {"https://b.info/": self._make_page("https://b.info/")}
|
||||
current = {
|
||||
"https://b.info/": self._make_page("https://b.info/"),
|
||||
"https://b.info/new/": self._make_page("https://b.info/new/"),
|
||||
}
|
||||
result = compare_snapshots(baseline, current, BASE_CFG)
|
||||
assert "https://b.info/new/" in result["new_internal_urls"]
|
||||
|
||||
def test_missing_url_detected(self):
|
||||
baseline = {
|
||||
"https://b.info/": self._make_page("https://b.info/"),
|
||||
"https://b.info/old/": self._make_page("https://b.info/old/"),
|
||||
}
|
||||
current = {"https://b.info/": self._make_page("https://b.info/")}
|
||||
result = compare_snapshots(baseline, current, BASE_CFG)
|
||||
assert "https://b.info/old/" in result["missing_internal_urls"]
|
||||
|
||||
def test_unchanged_page_not_in_diffs(self):
|
||||
page = self._make_page("https://b.info/", "Gleicher Inhalt")
|
||||
result = compare_snapshots({"https://b.info/": page}, {"https://b.info/": page}, BASE_CFG)
|
||||
assert result["changed_pages"] == 0
|
||||
|
||||
def test_text_change_detected(self):
|
||||
old = self._make_page("https://b.info/", "Alter Text")
|
||||
new = self._make_page("https://b.info/", "Alter Text plus neuer Spam-Block im SEO-Kontext")
|
||||
result = compare_snapshots({"https://b.info/": old}, {"https://b.info/": new}, BASE_CFG)
|
||||
assert result["changed_pages"] == 1
|
||||
|
||||
def test_new_external_domain_detected(self):
|
||||
def _page_with_ext(url, ext_url):
|
||||
p = self._make_page(url)
|
||||
p["links"]["a"] = [{"url": ext_url, "class": "external", "text": "x", "rel": ""}]
|
||||
return p
|
||||
|
||||
baseline = {"https://b.info/": self._make_page("https://b.info/")}
|
||||
current = {"https://b.info/": _page_with_ext("https://b.info/", "https://spam.example.com/")}
|
||||
result = compare_snapshots(baseline, current, BASE_CFG)
|
||||
assert "spam.example.com" in result["new_external_domains"]
|
||||
|
||||
|
||||
class TestScoreDiff:
|
||||
def test_green_for_no_changes(self):
|
||||
result = score_diff({
|
||||
"new_internal_urls": [],
|
||||
"missing_internal_urls": [],
|
||||
"new_external_domains": [],
|
||||
"page_diffs": [],
|
||||
"new_pages_analysis": [],
|
||||
}, BASE_CFG)
|
||||
assert result["level"] == "green"
|
||||
assert result["exit_code"] == 0
|
||||
|
||||
def test_yellow_for_new_url(self):
|
||||
result = score_diff({
|
||||
"new_internal_urls": ["https://b.info/new/"],
|
||||
"missing_internal_urls": [],
|
||||
"new_external_domains": [],
|
||||
"page_diffs": [],
|
||||
"new_pages_analysis": [],
|
||||
}, BASE_CFG)
|
||||
assert result["level"] == "yellow"
|
||||
assert result["exit_code"] == 1
|
||||
|
||||
def test_yellow_for_single_new_external_domain(self):
|
||||
# One new external domain = 50 pts → yellow (red threshold is 60)
|
||||
result = score_diff({
|
||||
"new_internal_urls": [],
|
||||
"missing_internal_urls": [],
|
||||
"new_external_domains": ["pharmacy.example.com"],
|
||||
"page_diffs": [],
|
||||
"new_pages_analysis": [],
|
||||
}, BASE_CFG)
|
||||
assert result["level"] == "yellow"
|
||||
assert result["exit_code"] == 1
|
||||
|
||||
def test_red_for_new_external_domain_plus_hidden_content(self):
|
||||
# External domain (50) + hidden content (40) = 90 pts → red
|
||||
result = score_diff({
|
||||
"new_internal_urls": [],
|
||||
"missing_internal_urls": [],
|
||||
"new_external_domains": ["pharmacy.example.com"],
|
||||
"page_diffs": [{
|
||||
"url": "https://b.info/",
|
||||
"text_diff": [],
|
||||
"added_chars": 0,
|
||||
"link_diff": {},
|
||||
"meta_diff": {},
|
||||
"new_hidden_content": [{"type": "inline_style_hidden", "indicators": ["display:none"]}],
|
||||
"new_inline_scripts": [],
|
||||
"new_jsonld": [],
|
||||
"new_meta_refresh": [],
|
||||
"canonical_changed": False,
|
||||
"new_comment_links": [],
|
||||
"old_canonical": None,
|
||||
"new_canonical": None,
|
||||
}],
|
||||
"new_pages_analysis": [],
|
||||
}, BASE_CFG)
|
||||
assert result["level"] == "red"
|
||||
assert result["exit_code"] == 2
|
||||
|
||||
def test_yellow_for_hidden_content_alone(self):
|
||||
# One hidden content finding = 40 pts → yellow (below red threshold of 60)
|
||||
result = score_diff({
|
||||
"new_internal_urls": [],
|
||||
"missing_internal_urls": [],
|
||||
"new_external_domains": [],
|
||||
"page_diffs": [{
|
||||
"url": "https://b.info/",
|
||||
"text_diff": [],
|
||||
"added_chars": 0,
|
||||
"link_diff": {},
|
||||
"meta_diff": {},
|
||||
"new_hidden_content": [{"type": "inline_style_hidden", "indicators": ["display:none"]}],
|
||||
"new_inline_scripts": [],
|
||||
"new_jsonld": [],
|
||||
"new_meta_refresh": [],
|
||||
"canonical_changed": False,
|
||||
"new_comment_links": [],
|
||||
"old_canonical": None,
|
||||
"new_canonical": None,
|
||||
}],
|
||||
"new_pages_analysis": [],
|
||||
}, BASE_CFG)
|
||||
assert result["level"] == "yellow"
|
||||
|
||||
def test_reasons_include_domain(self):
|
||||
result = score_diff({
|
||||
"new_internal_urls": [],
|
||||
"missing_internal_urls": [],
|
||||
"new_external_domains": ["spam.com"],
|
||||
"page_diffs": [],
|
||||
"new_pages_analysis": [],
|
||||
}, BASE_CFG)
|
||||
assert any("spam.com" in r for r in result["reasons"])
|
||||
152
tests/test_extractor.py
Normal file
152
tests/test_extractor.py
Normal file
|
|
@ -0,0 +1,152 @@
|
|||
"""Tests for HTML extraction: links, metadata, hidden content, JSON-LD."""
|
||||
import pytest
|
||||
from scanner.extractor import extract_page
|
||||
|
||||
BASE_URL = "https://bredelar.info/"
|
||||
|
||||
|
||||
def _extract(html: str, url: str = BASE_URL) -> dict:
|
||||
return extract_page(html, url)
|
||||
|
||||
|
||||
class TestVisibleText:
|
||||
def test_strips_scripts(self):
|
||||
result = _extract("<html><script>var x=1;</script><body>Hallo Welt</body></html>")
|
||||
assert "var x" not in result["text"]
|
||||
assert "Hallo Welt" in result["text"]
|
||||
|
||||
def test_strips_style(self):
|
||||
result = _extract("<html><style>.cls{color:red}</style><body>Text</body></html>")
|
||||
assert ".cls" not in result["text"]
|
||||
assert "Text" in result["text"]
|
||||
|
||||
def test_collapses_whitespace(self):
|
||||
result = _extract("<body><p>Hello World</p></body>")
|
||||
assert "Hello World" in result["text"]
|
||||
|
||||
|
||||
class TestLinkExtraction:
|
||||
def test_extracts_anchor_links(self):
|
||||
result = _extract('<body><a href="/page">Link</a></body>')
|
||||
urls = [l["url"] for l in result["links"]["a"]]
|
||||
assert any("/page" in u for u in urls)
|
||||
|
||||
def test_classifies_internal_vs_external(self):
|
||||
html = (
|
||||
'<body>'
|
||||
'<a href="/internal">intern</a>'
|
||||
'<a href="https://spam.example.com/x">extern</a>'
|
||||
'</body>'
|
||||
)
|
||||
result = _extract(html)
|
||||
classes = {l["url"]: l["class"] for l in result["links"]["a"]}
|
||||
assert any(v == "internal" for v in classes.values())
|
||||
assert any(v == "external" for v in classes.values())
|
||||
|
||||
def test_extracts_canonical(self):
|
||||
html = '<head><link rel="canonical" href="https://bredelar.info/page"/></head>'
|
||||
result = _extract(html)
|
||||
assert result["links"]["canonical"] == "https://bredelar.info/page"
|
||||
|
||||
def test_extracts_meta_refresh(self):
|
||||
html = '<head><meta http-equiv="refresh" content="0; url=https://malware.example.com"/></head>'
|
||||
result = _extract(html)
|
||||
assert any("malware.example.com" in u for u in result["links"]["meta_refresh"])
|
||||
|
||||
def test_ignores_javascript_hrefs(self):
|
||||
result = _extract('<body><a href="javascript:void(0)">JS</a></body>')
|
||||
assert result["links"]["a"] == []
|
||||
|
||||
def test_extracts_iframe_src(self):
|
||||
result = _extract('<body><iframe src="https://evil.com/x"></iframe></body>')
|
||||
iframe_urls = [l["url"] for l in result["links"]["iframe"]]
|
||||
assert any("evil.com" in u for u in iframe_urls)
|
||||
|
||||
|
||||
class TestMetadata:
|
||||
def test_extracts_title(self):
|
||||
result = _extract("<html><head><title>Bredelar</title></head></html>")
|
||||
assert result["metadata"]["title"] == "Bredelar"
|
||||
|
||||
def test_extracts_description(self):
|
||||
html = '<head><meta name="description" content="Ein Dorf in NRW"/></head>'
|
||||
result = _extract(html)
|
||||
assert result["metadata"]["description"] == "Ein Dorf in NRW"
|
||||
|
||||
def test_extracts_og_tags(self):
|
||||
html = '<head><meta property="og:title" content="OG-Titel"/></head>'
|
||||
result = _extract(html)
|
||||
assert result["metadata"].get("og:title") == "OG-Titel"
|
||||
|
||||
def test_extracts_h1(self):
|
||||
result = _extract("<body><h1>Hauptüberschrift</h1></body>")
|
||||
assert "Hauptüberschrift" in result["metadata"]["h1"]
|
||||
|
||||
|
||||
class TestHiddenContent:
|
||||
def test_detects_display_none(self):
|
||||
html = '<body><div style="display:none"><a href="https://spam.com">Spam</a></div></body>'
|
||||
result = _extract(html)
|
||||
assert len(result["hidden_content"]) > 0
|
||||
assert any("display:none" in str(h.get("indicators", [])) for h in result["hidden_content"])
|
||||
|
||||
def test_detects_visibility_hidden(self):
|
||||
html = '<body><span style="visibility:hidden">Spam Text</span></body>'
|
||||
result = _extract(html)
|
||||
assert len(result["hidden_content"]) > 0
|
||||
|
||||
def test_detects_offscreen_position(self):
|
||||
html = '<body><div style="position:absolute; left:-9999px">Hidden</div></body>'
|
||||
result = _extract(html)
|
||||
assert len(result["hidden_content"]) > 0
|
||||
|
||||
def test_detects_noscript_links(self):
|
||||
html = '<body><noscript><a href="https://spam.com/x">Spam</a></noscript></body>'
|
||||
result = _extract(html)
|
||||
noscript = [h for h in result["hidden_content"] if h.get("type") == "noscript_links"]
|
||||
assert len(noscript) > 0
|
||||
|
||||
def test_visible_element_not_flagged(self):
|
||||
html = '<body><div style="color:red">Sichtbarer Text</div></body>'
|
||||
result = _extract(html)
|
||||
# color:red should NOT be flagged (not a hiding technique)
|
||||
assert not any(
|
||||
"red" in str(h) for h in result["hidden_content"]
|
||||
)
|
||||
|
||||
|
||||
class TestCommentLinks:
|
||||
def test_finds_link_in_comment(self):
|
||||
html = '<!-- <a href="https://hidden-spam.com/x">click</a> --><body></body>'
|
||||
result = _extract(html)
|
||||
assert any("hidden-spam.com" in u for u in result["comment_links"])
|
||||
|
||||
def test_no_false_positives(self):
|
||||
html = '<!-- This is a normal comment --><body>Text</body>'
|
||||
result = _extract(html)
|
||||
assert result["comment_links"] == []
|
||||
|
||||
|
||||
class TestJsonLd:
|
||||
def test_extracts_known_type(self):
|
||||
html = (
|
||||
'<script type="application/ld+json">'
|
||||
'{"@type": "Organization", "name": "Bredelar"}'
|
||||
'</script>'
|
||||
)
|
||||
result = _extract(html)
|
||||
assert any(j["type"] == "Organization" for j in result["jsonld"])
|
||||
|
||||
def test_handles_parse_error(self):
|
||||
html = '<script type="application/ld+json">{broken json}</script>'
|
||||
result = _extract(html)
|
||||
assert any(j["type"] == "PARSE_ERROR" for j in result["jsonld"])
|
||||
|
||||
def test_extracts_unexpected_type(self):
|
||||
html = (
|
||||
'<script type="application/ld+json">'
|
||||
'{"@type": "Pharmacy", "name": "SpamShop"}'
|
||||
'</script>'
|
||||
)
|
||||
result = _extract(html)
|
||||
assert any(j["type"] == "Pharmacy" for j in result["jsonld"])
|
||||
Loading…
Add table
Add a link
Reference in a new issue