feat: Cloaking-Erkennung per Doppel-Crawl (Browser-UA vs. Googlebot-UA)
Neuer Befehl `python -m scanner cloak-check`: - Crawlt die Website zweimal: einmal mit Browser-UA, einmal als Googlebot - Vergleicht Text, Links und Response-Header (Vary: User-Agent) - Scoring: Bot-Only-Link 60 Pkt (RED), extra Text >100 Zeichen 40 Pkt, Vary: User-Agent ohne Inhaltsdiff 25 Pkt - Eigener Report unter reports/<ts>_cloak/ Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
parent
ad939f3a31
commit
0302d469b1
5 changed files with 336 additions and 1 deletions
|
|
@ -377,6 +377,96 @@ def score_diff(diff: dict, cfg: dict) -> dict:
|
|||
return {"score": score, "level": level, "reasons": reasons, "exit_code": exit_code}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Cloaking detection
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def compare_cloaking(normal_pages: dict, bot_pages: dict, cfg: dict) -> dict:
|
||||
"""
|
||||
Compare pages crawled with a normal browser UA vs. Googlebot UA.
|
||||
Returns findings where the bot version contains extra content.
|
||||
"""
|
||||
findings: list[dict] = []
|
||||
common = sorted(set(normal_pages) & set(bot_pages))
|
||||
|
||||
for url in common:
|
||||
normal = normal_pages[url]
|
||||
bot = bot_pages[url]
|
||||
|
||||
# Vary: User-Agent signals server-side content negotiation
|
||||
vary = bot.get("response_headers", {}).get("Vary", "")
|
||||
has_vary_ua = "user-agent" in vary.lower()
|
||||
|
||||
# Text diff
|
||||
normal_text = normalize_text(normal.get("text", ""), cfg)
|
||||
bot_text = normalize_text(bot.get("text", ""), cfg)
|
||||
text_changed = text_hash(normal_text) != text_hash(bot_text)
|
||||
text_diff = diff_text(normal_text, bot_text) if text_changed else []
|
||||
extra_chars = sum(
|
||||
len(line[1:]) for line in text_diff
|
||||
if line.startswith("+") and not line.startswith("+++")
|
||||
)
|
||||
|
||||
# Links only in bot version
|
||||
extra_links: list[str] = []
|
||||
for ltype in ("a", "script", "iframe"):
|
||||
ld = diff_link_set(
|
||||
normal.get("links", {}).get(ltype, []),
|
||||
bot.get("links", {}).get(ltype, []),
|
||||
)
|
||||
extra_links.extend(ld["added"])
|
||||
|
||||
if extra_links or extra_chars > 50 or has_vary_ua:
|
||||
findings.append({
|
||||
"url": url,
|
||||
"vary_ua": has_vary_ua,
|
||||
"extra_links": extra_links,
|
||||
"extra_text_chars": extra_chars,
|
||||
"text_diff": text_diff[:40],
|
||||
})
|
||||
|
||||
return {
|
||||
"findings": findings,
|
||||
"pages_checked": len(common),
|
||||
}
|
||||
|
||||
|
||||
def score_cloak_diff(cloak_diff: dict, cfg: dict) -> dict:
|
||||
"""Score the cloaking check result."""
|
||||
sc = cfg.get("scoring", {})
|
||||
thr = cfg.get("thresholds", {"yellow": 20, "red": 60})
|
||||
score = 0
|
||||
reasons: list[str] = []
|
||||
|
||||
def add(pts: int, msg: str) -> None:
|
||||
nonlocal score
|
||||
score += pts
|
||||
reasons.append(f"{msg} (+{pts})")
|
||||
|
||||
for page in cloak_diff.get("findings", []):
|
||||
url = page["url"]
|
||||
for link in page.get("extra_links", []):
|
||||
add(sc.get("cloaking_extra_link", 60), f"{url}: Bot-Only-Link: {link}")
|
||||
if page.get("extra_text_chars", 0) > 100:
|
||||
add(sc.get("cloaking_extra_text", 40),
|
||||
f"{url}: Bot sieht +{page['extra_text_chars']} Zeichen mehr Text")
|
||||
# Vary: User-Agent ohne sonstigen Unterschied — informativer Hinweis
|
||||
if page.get("vary_ua") and not page.get("extra_links") and page.get("extra_text_chars", 0) <= 100:
|
||||
add(sc.get("cloaking_vary_ua", 25), f"{url}: Vary: User-Agent Header")
|
||||
|
||||
yellow = thr.get("yellow", 20)
|
||||
red = thr.get("red", 60)
|
||||
|
||||
if score >= red:
|
||||
level, exit_code = "red", 2
|
||||
elif score >= yellow:
|
||||
level, exit_code = "yellow", 1
|
||||
else:
|
||||
level, exit_code = "green", 0
|
||||
|
||||
return {"score": score, "level": level, "reasons": reasons, "exit_code": exit_code}
|
||||
|
||||
|
||||
def _fingerprint(obj: dict | list | str) -> str:
|
||||
"""Stable hash for deduplicating diff entries."""
|
||||
import json as _json
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue