feat: Cloaking-Erkennung per Doppel-Crawl (Browser-UA vs. Googlebot-UA)
Neuer Befehl `python -m scanner cloak-check`: - Crawlt die Website zweimal: einmal mit Browser-UA, einmal als Googlebot - Vergleicht Text, Links und Response-Header (Vary: User-Agent) - Scoring: Bot-Only-Link 60 Pkt (RED), extra Text >100 Zeichen 40 Pkt, Vary: User-Agent ohne Inhaltsdiff 25 Pkt - Eigener Report unter reports/<ts>_cloak/ Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
parent
ad939f3a31
commit
0302d469b1
5 changed files with 336 additions and 1 deletions
106
tests/test_cloak.py
Normal file
106
tests/test_cloak.py
Normal file
|
|
@ -0,0 +1,106 @@
|
|||
"""Tests for cloaking detection."""
|
||||
import pytest
|
||||
from scanner.differ import compare_cloaking, score_cloak_diff
|
||||
|
||||
|
||||
def _page(url, text="Hallo Welt", links=None, vary_ua=False):
|
||||
hdrs = {"Vary": "User-Agent"} if vary_ua else {}
|
||||
return {
|
||||
"url": url,
|
||||
"text": text,
|
||||
"links": {"a": links or [], "script": [], "iframe": []},
|
||||
"response_headers": hdrs,
|
||||
"status": 200,
|
||||
}
|
||||
|
||||
|
||||
CFG = {"normalization": {"collapse_whitespace": True, "ignore_patterns": []}, "scoring": {}, "thresholds": {}}
|
||||
|
||||
|
||||
class TestCompareCloaking:
|
||||
def test_identical_pages_no_findings(self):
|
||||
page = _page("https://x.de/")
|
||||
result = compare_cloaking({"https://x.de/": page}, {"https://x.de/": page}, CFG)
|
||||
assert result["findings"] == []
|
||||
assert result["pages_checked"] == 1
|
||||
|
||||
def test_extra_link_in_bot_version_flagged(self):
|
||||
normal = _page("https://x.de/", links=[{"url": "https://x.de/a", "class": "internal"}])
|
||||
bot = _page("https://x.de/", links=[
|
||||
{"url": "https://x.de/a", "class": "internal"},
|
||||
{"url": "https://spam.example.com/evil", "class": "external"},
|
||||
])
|
||||
result = compare_cloaking({"https://x.de/": normal}, {"https://x.de/": bot}, CFG)
|
||||
assert len(result["findings"]) == 1
|
||||
assert "https://spam.example.com/evil" in result["findings"][0]["extra_links"]
|
||||
|
||||
def test_extra_text_in_bot_version_flagged(self):
|
||||
normal = _page("https://x.de/", text="Normaler Text")
|
||||
bot = _page("https://x.de/", text="Normaler Text " + "x" * 200)
|
||||
result = compare_cloaking({"https://x.de/": normal}, {"https://x.de/": bot}, CFG)
|
||||
assert len(result["findings"]) == 1
|
||||
assert result["findings"][0]["extra_text_chars"] > 100
|
||||
|
||||
def test_vary_ua_header_flagged(self):
|
||||
normal = _page("https://x.de/")
|
||||
bot = _page("https://x.de/", vary_ua=True)
|
||||
result = compare_cloaking({"https://x.de/": normal}, {"https://x.de/": bot}, CFG)
|
||||
assert len(result["findings"]) == 1
|
||||
assert result["findings"][0]["vary_ua"] is True
|
||||
|
||||
def test_url_only_in_one_crawl_ignored(self):
|
||||
"""URLs not seen in both crawls are not compared."""
|
||||
normal = _page("https://x.de/only-normal")
|
||||
bot = _page("https://x.de/only-bot")
|
||||
result = compare_cloaking(
|
||||
{"https://x.de/only-normal": normal},
|
||||
{"https://x.de/only-bot": bot},
|
||||
CFG,
|
||||
)
|
||||
assert result["findings"] == []
|
||||
assert result["pages_checked"] == 0
|
||||
|
||||
|
||||
class TestScoreCloakDiff:
|
||||
def test_no_findings_green(self):
|
||||
result = score_cloak_diff({"findings": [], "pages_checked": 5}, CFG)
|
||||
assert result["level"] == "green"
|
||||
assert result["score"] == 0
|
||||
assert result["exit_code"] == 0
|
||||
|
||||
def test_extra_link_scores_60_red(self):
|
||||
diff = {
|
||||
"findings": [{"url": "https://x.de/", "vary_ua": False, "extra_links": ["https://spam.com/x"], "extra_text_chars": 0}],
|
||||
"pages_checked": 1,
|
||||
}
|
||||
result = score_cloak_diff(diff, CFG)
|
||||
assert result["score"] == 60
|
||||
assert result["level"] == "red"
|
||||
assert result["exit_code"] == 2
|
||||
|
||||
def test_extra_text_scores_40_yellow(self):
|
||||
diff = {
|
||||
"findings": [{"url": "https://x.de/", "vary_ua": False, "extra_links": [], "extra_text_chars": 150}],
|
||||
"pages_checked": 1,
|
||||
}
|
||||
result = score_cloak_diff(diff, CFG)
|
||||
assert result["score"] == 40
|
||||
assert result["level"] == "yellow"
|
||||
|
||||
def test_vary_ua_only_scores_25_yellow(self):
|
||||
diff = {
|
||||
"findings": [{"url": "https://x.de/", "vary_ua": True, "extra_links": [], "extra_text_chars": 0}],
|
||||
"pages_checked": 1,
|
||||
}
|
||||
result = score_cloak_diff(diff, CFG)
|
||||
assert result["score"] == 25
|
||||
assert result["level"] == "yellow"
|
||||
|
||||
def test_vary_ua_with_extra_link_not_double_scored(self):
|
||||
"""vary_ua surcharge is suppressed when extra_links are present (link is already 60)."""
|
||||
diff = {
|
||||
"findings": [{"url": "https://x.de/", "vary_ua": True, "extra_links": ["https://spam.com/x"], "extra_text_chars": 0}],
|
||||
"pages_checked": 1,
|
||||
}
|
||||
result = score_cloak_diff(diff, CFG)
|
||||
assert result["score"] == 60 # only link score, not 60+25
|
||||
Loading…
Add table
Add a link
Reference in a new issue