feat: Cloaking-Erkennung per Doppel-Crawl (Browser-UA vs. Googlebot-UA)

Neuer Befehl `python -m scanner cloak-check`:
- Crawlt die Website zweimal: einmal mit Browser-UA, einmal als Googlebot
- Vergleicht Text, Links und Response-Header (Vary: User-Agent)
- Scoring: Bot-Only-Link 60 Pkt (RED), extra Text >100 Zeichen 40 Pkt,
  Vary: User-Agent ohne Inhaltsdiff 25 Pkt
- Eigener Report unter reports/<ts>_cloak/

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Your Name 2026-06-12 04:02:24 +02:00
commit 0302d469b1
5 changed files with 336 additions and 1 deletions

View file

@ -377,6 +377,96 @@ def score_diff(diff: dict, cfg: dict) -> dict:
return {"score": score, "level": level, "reasons": reasons, "exit_code": exit_code}
# ---------------------------------------------------------------------------
# Cloaking detection
# ---------------------------------------------------------------------------
def compare_cloaking(normal_pages: dict, bot_pages: dict, cfg: dict) -> dict:
"""
Compare pages crawled with a normal browser UA vs. Googlebot UA.
Returns findings where the bot version contains extra content.
"""
findings: list[dict] = []
common = sorted(set(normal_pages) & set(bot_pages))
for url in common:
normal = normal_pages[url]
bot = bot_pages[url]
# Vary: User-Agent signals server-side content negotiation
vary = bot.get("response_headers", {}).get("Vary", "")
has_vary_ua = "user-agent" in vary.lower()
# Text diff
normal_text = normalize_text(normal.get("text", ""), cfg)
bot_text = normalize_text(bot.get("text", ""), cfg)
text_changed = text_hash(normal_text) != text_hash(bot_text)
text_diff = diff_text(normal_text, bot_text) if text_changed else []
extra_chars = sum(
len(line[1:]) for line in text_diff
if line.startswith("+") and not line.startswith("+++")
)
# Links only in bot version
extra_links: list[str] = []
for ltype in ("a", "script", "iframe"):
ld = diff_link_set(
normal.get("links", {}).get(ltype, []),
bot.get("links", {}).get(ltype, []),
)
extra_links.extend(ld["added"])
if extra_links or extra_chars > 50 or has_vary_ua:
findings.append({
"url": url,
"vary_ua": has_vary_ua,
"extra_links": extra_links,
"extra_text_chars": extra_chars,
"text_diff": text_diff[:40],
})
return {
"findings": findings,
"pages_checked": len(common),
}
def score_cloak_diff(cloak_diff: dict, cfg: dict) -> dict:
"""Score the cloaking check result."""
sc = cfg.get("scoring", {})
thr = cfg.get("thresholds", {"yellow": 20, "red": 60})
score = 0
reasons: list[str] = []
def add(pts: int, msg: str) -> None:
nonlocal score
score += pts
reasons.append(f"{msg} (+{pts})")
for page in cloak_diff.get("findings", []):
url = page["url"]
for link in page.get("extra_links", []):
add(sc.get("cloaking_extra_link", 60), f"{url}: Bot-Only-Link: {link}")
if page.get("extra_text_chars", 0) > 100:
add(sc.get("cloaking_extra_text", 40),
f"{url}: Bot sieht +{page['extra_text_chars']} Zeichen mehr Text")
# Vary: User-Agent ohne sonstigen Unterschied — informativer Hinweis
if page.get("vary_ua") and not page.get("extra_links") and page.get("extra_text_chars", 0) <= 100:
add(sc.get("cloaking_vary_ua", 25), f"{url}: Vary: User-Agent Header")
yellow = thr.get("yellow", 20)
red = thr.get("red", 60)
if score >= red:
level, exit_code = "red", 2
elif score >= yellow:
level, exit_code = "yellow", 1
else:
level, exit_code = "green", 0
return {"score": score, "level": level, "reasons": reasons, "exit_code": exit_code}
def _fingerprint(obj: dict | list | str) -> str:
"""Stable hash for deduplicating diff entries."""
import json as _json