feat: Cloaking-Erkennung per Doppel-Crawl (Browser-UA vs. Googlebot-UA)
Neuer Befehl `python -m scanner cloak-check`: - Crawlt die Website zweimal: einmal mit Browser-UA, einmal als Googlebot - Vergleicht Text, Links und Response-Header (Vary: User-Agent) - Scoring: Bot-Only-Link 60 Pkt (RED), extra Text >100 Zeichen 40 Pkt, Vary: User-Agent ohne Inhaltsdiff 25 Pkt - Eigener Report unter reports/<ts>_cloak/ Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Your Name
parent
ad939f3a31
commit
0302d469b1
5 changed files with 336 additions and 1 deletions
|
|
@ -31,7 +31,7 @@ from .checker import (
|
|||
)
|
||||
from .config import load_config, resolve_paths
|
||||
from .crawler import Crawler
|
||||
from .differ import compare_snapshots, score_diff
|
||||
from .differ import compare_cloaking, compare_snapshots, score_cloak_diff, score_diff
|
||||
from .extractor import extract_page
|
||||
|
||||
|
||||
|
|
@ -39,6 +39,9 @@ from .extractor import extract_page
|
|||
# Logging
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
GOOGLEBOT_UA = "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"
|
||||
|
||||
|
||||
def _setup_logging(logs_dir: str, verbose: bool) -> None:
|
||||
level = logging.DEBUG if verbose else logging.INFO
|
||||
logs_path = Path(logs_dir)
|
||||
|
|
@ -294,6 +297,99 @@ def _render_markdown(r: dict) -> str:
|
|||
return "\n".join(lines) + "\n"
|
||||
|
||||
|
||||
def _write_cloak_report(report: dict, reports_dir: str) -> tuple[Path, Path]:
|
||||
ts = datetime.now(timezone.utc).strftime("%Y%m%d_%H%M%S")
|
||||
out = Path(reports_dir) / f"{ts}_cloak"
|
||||
out.mkdir(parents=True, exist_ok=True)
|
||||
json_path = out / "report.json"
|
||||
md_path = out / "report.md"
|
||||
json_path.write_text(json.dumps(report, indent=2, ensure_ascii=False), encoding="utf-8")
|
||||
md_path.write_text(_render_cloak_markdown(report), encoding="utf-8")
|
||||
return json_path, md_path
|
||||
|
||||
|
||||
def _render_cloak_markdown(r: dict) -> str:
|
||||
a = r.get("assessment", {})
|
||||
cd = r.get("cloak_diff", {})
|
||||
level = a.get("level", "?").upper()
|
||||
score = a.get("score", 0)
|
||||
level_badge = {"GREEN": "[OK]", "YELLOW": "[WARNUNG]", "RED": "[ALARM]"}.get(level, level)
|
||||
|
||||
lines = [
|
||||
f"# Cloaking-Check — {r.get('target', '?')}",
|
||||
"",
|
||||
f"**Erstellt:** {r.get('generated_at', '?')} ",
|
||||
f"**Level:** {level_badge} ",
|
||||
f"**Score:** {score}",
|
||||
"",
|
||||
"## Zusammenfassung",
|
||||
"",
|
||||
"| Kennzahl | Wert |",
|
||||
"|---|---|",
|
||||
f"| Geprüfte Seiten | {cd.get('pages_checked', '?')} |",
|
||||
f"| Seiten mit Unterschieden | {len(cd.get('findings', []))} |",
|
||||
"",
|
||||
"## Risikobewertung",
|
||||
"",
|
||||
]
|
||||
for reason in a.get("reasons", []):
|
||||
lines.append(f"- {reason}")
|
||||
if not a.get("reasons"):
|
||||
lines.append("- Keine Cloaking-Anzeichen gefunden.")
|
||||
|
||||
if cd.get("findings"):
|
||||
lines += ["", "## Auffällige Seiten", ""]
|
||||
for page in cd["findings"]:
|
||||
lines += [f"### {page['url']}", ""]
|
||||
if page.get("vary_ua"):
|
||||
lines.append("- `Vary: User-Agent` Header gesetzt")
|
||||
if page.get("extra_links"):
|
||||
lines.append(f"- Bot-Only-Links ({len(page['extra_links'])}):")
|
||||
for link in page["extra_links"]:
|
||||
lines.append(f" - `{link}`")
|
||||
if page.get("extra_text_chars", 0) > 50:
|
||||
lines.append(f"- Bot sieht +{page['extra_text_chars']} Zeichen mehr Text")
|
||||
if page.get("text_diff"):
|
||||
lines += ["", "```diff"]
|
||||
lines += page["text_diff"][:20]
|
||||
if len(page["text_diff"]) > 20:
|
||||
lines.append(f"... ({len(page['text_diff']) - 20} weitere Zeilen)")
|
||||
lines.append("```")
|
||||
lines.append("")
|
||||
|
||||
lines += ["", "---", "", "## Nächste Schritte", ""]
|
||||
lev = a.get("level", "green")
|
||||
if lev == "red":
|
||||
lines += [
|
||||
"1. **Sofort**: Seiten manuell vergleichen: `curl -A 'Googlebot' <URL> | grep -i hidden`",
|
||||
"2. Server-Logs auf Bot-IP-Ranges prüfen.",
|
||||
"3. CMS-Plugins / Theme-Dateien auf injizierte Cloaking-Logik untersuchen.",
|
||||
"4. Nach Bereinigung: `python -m scanner cloak-check` erneut ausführen.",
|
||||
]
|
||||
elif lev == "yellow":
|
||||
lines += [
|
||||
"1. Markierte Seiten prüfen: `curl -s -A 'Googlebot' <URL> > /tmp/bot.html && diff <(curl -s <URL>) /tmp/bot.html`",
|
||||
"2. Bei bestätigtem Cloaking: Hoster kontaktieren.",
|
||||
]
|
||||
else:
|
||||
lines += ["Keine Aktion erforderlich."]
|
||||
|
||||
return "\n".join(lines) + "\n"
|
||||
|
||||
|
||||
def _print_cloak_summary(assessment: dict, cloak_diff: dict) -> None:
|
||||
level = assessment.get("level", "?").upper()
|
||||
score = assessment.get("score", 0)
|
||||
print()
|
||||
print(f"=== Cloaking-Check: {level} (Score {score}) ===")
|
||||
for reason in assessment.get("reasons", []):
|
||||
print(f" * {reason}")
|
||||
if not assessment.get("reasons"):
|
||||
print(" Keine Cloaking-Anzeichen gefunden.")
|
||||
print(f" Geprüfte Seiten: {cloak_diff.get('pages_checked', 0)}")
|
||||
print(f" Seiten mit Unterschied: {len(cloak_diff.get('findings', []))}")
|
||||
|
||||
|
||||
def _latest_report_path(reports_dir: str) -> Path | None:
|
||||
rd = Path(reports_dir)
|
||||
if not rd.exists():
|
||||
|
|
@ -600,6 +696,40 @@ def cmd_status(args: argparse.Namespace, cfg: dict) -> int:
|
|||
return 0
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Cloaking check
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def cmd_cloak_check(args: argparse.Namespace, cfg: dict) -> int:
|
||||
logger = logging.getLogger("scanner.cloak")
|
||||
print(f"Cloaking-Check: {cfg['target']}")
|
||||
print(" Schritt 1/2: Crawl mit Browser-UA ...")
|
||||
pages_normal, _ = _crawl_and_extract(cfg)
|
||||
|
||||
print(" Schritt 2/2: Crawl mit Googlebot-UA ...")
|
||||
bot_cfg = {**cfg, "user_agent": GOOGLEBOT_UA}
|
||||
pages_bot, _ = _crawl_and_extract(bot_cfg)
|
||||
|
||||
normal_map = {p["url"]: p for p in pages_normal}
|
||||
bot_map = {p["url"]: p for p in pages_bot}
|
||||
|
||||
cloak_diff = compare_cloaking(normal_map, bot_map, cfg)
|
||||
assessment = score_cloak_diff(cloak_diff, cfg)
|
||||
|
||||
report = {
|
||||
"generated_at": datetime.now(timezone.utc).isoformat(),
|
||||
"target": cfg["target"],
|
||||
"type": "cloak-check",
|
||||
"assessment": assessment,
|
||||
"cloak_diff": cloak_diff,
|
||||
}
|
||||
json_path, md_path = _write_cloak_report(report, cfg["reports_dir"])
|
||||
logger.info("Cloak-Report: %s", md_path)
|
||||
|
||||
_print_cloak_summary(assessment, cloak_diff)
|
||||
return assessment["exit_code"]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Output helper
|
||||
# ---------------------------------------------------------------------------
|
||||
|
|
@ -653,6 +783,8 @@ def _build_parser() -> argparse.ArgumentParser:
|
|||
|
||||
sub.add_parser("status", help="Übersichtsstatus anzeigen")
|
||||
|
||||
sub.add_parser("cloak-check", help="Doppel-Crawl: Browser-UA vs. Googlebot-UA (Cloaking-Erkennung)")
|
||||
|
||||
return p
|
||||
|
||||
|
||||
|
|
@ -678,6 +810,7 @@ def main() -> None:
|
|||
"approve": cmd_approve,
|
||||
"report": cmd_report,
|
||||
"status": cmd_status,
|
||||
"cloak-check": cmd_cloak_check,
|
||||
}
|
||||
handler = commands.get(args.command)
|
||||
if not handler:
|
||||
|
|
|
|||
|
|
@ -33,6 +33,9 @@ DEFAULT_CONFIG: dict = {
|
|||
"comment_links": 25,
|
||||
"new_broken_link": 20,
|
||||
"suspicious_filename": 60,
|
||||
"cloaking_extra_link": 60,
|
||||
"cloaking_extra_text": 40,
|
||||
"cloaking_vary_ua": 25,
|
||||
},
|
||||
"thresholds": {
|
||||
"yellow": 20,
|
||||
|
|
|
|||
|
|
@ -377,6 +377,96 @@ def score_diff(diff: dict, cfg: dict) -> dict:
|
|||
return {"score": score, "level": level, "reasons": reasons, "exit_code": exit_code}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Cloaking detection
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def compare_cloaking(normal_pages: dict, bot_pages: dict, cfg: dict) -> dict:
|
||||
"""
|
||||
Compare pages crawled with a normal browser UA vs. Googlebot UA.
|
||||
Returns findings where the bot version contains extra content.
|
||||
"""
|
||||
findings: list[dict] = []
|
||||
common = sorted(set(normal_pages) & set(bot_pages))
|
||||
|
||||
for url in common:
|
||||
normal = normal_pages[url]
|
||||
bot = bot_pages[url]
|
||||
|
||||
# Vary: User-Agent signals server-side content negotiation
|
||||
vary = bot.get("response_headers", {}).get("Vary", "")
|
||||
has_vary_ua = "user-agent" in vary.lower()
|
||||
|
||||
# Text diff
|
||||
normal_text = normalize_text(normal.get("text", ""), cfg)
|
||||
bot_text = normalize_text(bot.get("text", ""), cfg)
|
||||
text_changed = text_hash(normal_text) != text_hash(bot_text)
|
||||
text_diff = diff_text(normal_text, bot_text) if text_changed else []
|
||||
extra_chars = sum(
|
||||
len(line[1:]) for line in text_diff
|
||||
if line.startswith("+") and not line.startswith("+++")
|
||||
)
|
||||
|
||||
# Links only in bot version
|
||||
extra_links: list[str] = []
|
||||
for ltype in ("a", "script", "iframe"):
|
||||
ld = diff_link_set(
|
||||
normal.get("links", {}).get(ltype, []),
|
||||
bot.get("links", {}).get(ltype, []),
|
||||
)
|
||||
extra_links.extend(ld["added"])
|
||||
|
||||
if extra_links or extra_chars > 50 or has_vary_ua:
|
||||
findings.append({
|
||||
"url": url,
|
||||
"vary_ua": has_vary_ua,
|
||||
"extra_links": extra_links,
|
||||
"extra_text_chars": extra_chars,
|
||||
"text_diff": text_diff[:40],
|
||||
})
|
||||
|
||||
return {
|
||||
"findings": findings,
|
||||
"pages_checked": len(common),
|
||||
}
|
||||
|
||||
|
||||
def score_cloak_diff(cloak_diff: dict, cfg: dict) -> dict:
|
||||
"""Score the cloaking check result."""
|
||||
sc = cfg.get("scoring", {})
|
||||
thr = cfg.get("thresholds", {"yellow": 20, "red": 60})
|
||||
score = 0
|
||||
reasons: list[str] = []
|
||||
|
||||
def add(pts: int, msg: str) -> None:
|
||||
nonlocal score
|
||||
score += pts
|
||||
reasons.append(f"{msg} (+{pts})")
|
||||
|
||||
for page in cloak_diff.get("findings", []):
|
||||
url = page["url"]
|
||||
for link in page.get("extra_links", []):
|
||||
add(sc.get("cloaking_extra_link", 60), f"{url}: Bot-Only-Link: {link}")
|
||||
if page.get("extra_text_chars", 0) > 100:
|
||||
add(sc.get("cloaking_extra_text", 40),
|
||||
f"{url}: Bot sieht +{page['extra_text_chars']} Zeichen mehr Text")
|
||||
# Vary: User-Agent ohne sonstigen Unterschied — informativer Hinweis
|
||||
if page.get("vary_ua") and not page.get("extra_links") and page.get("extra_text_chars", 0) <= 100:
|
||||
add(sc.get("cloaking_vary_ua", 25), f"{url}: Vary: User-Agent Header")
|
||||
|
||||
yellow = thr.get("yellow", 20)
|
||||
red = thr.get("red", 60)
|
||||
|
||||
if score >= red:
|
||||
level, exit_code = "red", 2
|
||||
elif score >= yellow:
|
||||
level, exit_code = "yellow", 1
|
||||
else:
|
||||
level, exit_code = "green", 0
|
||||
|
||||
return {"score": score, "level": level, "reasons": reasons, "exit_code": exit_code}
|
||||
|
||||
|
||||
def _fingerprint(obj: dict | list | str) -> str:
|
||||
"""Stable hash for deduplicating diff entries."""
|
||||
import json as _json
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue