From 0302d469b1ecfeffffc51f145e95fa7e9d5b7f0a Mon Sep 17 00:00:00 2001 From: Your Name Date: Fri, 12 Jun 2026 04:02:24 +0200 Subject: [PATCH] feat: Cloaking-Erkennung per Doppel-Crawl (Browser-UA vs. Googlebot-UA) Neuer Befehl `python -m scanner cloak-check`: - Crawlt die Website zweimal: einmal mit Browser-UA, einmal als Googlebot - Vergleicht Text, Links und Response-Header (Vary: User-Agent) - Scoring: Bot-Only-Link 60 Pkt (RED), extra Text >100 Zeichen 40 Pkt, Vary: User-Agent ohne Inhaltsdiff 25 Pkt - Eigener Report unter reports/_cloak/ Co-Authored-By: Claude Sonnet 4.6 --- config.yaml | 3 + scanner/__main__.py | 135 +++++++++++++++++++++++++++++++++++++++++++- scanner/config.py | 3 + scanner/differ.py | 90 +++++++++++++++++++++++++++++ tests/test_cloak.py | 106 ++++++++++++++++++++++++++++++++++ 5 files changed, 336 insertions(+), 1 deletion(-) create mode 100644 tests/test_cloak.py diff --git a/config.yaml b/config.yaml index 8f15abd..ceff972 100644 --- a/config.yaml +++ b/config.yaml @@ -48,6 +48,9 @@ scoring: comment_links: 25 new_broken_link: 20 # neue 4xx/5xx-Seite, die in Baseline nicht vorhanden war suspicious_filename: 60 # Link auf Webshell-/Backdoor-typischen Dateinamen + cloaking_extra_link: 60 # Link nur im Bot-Crawl sichtbar + cloaking_extra_text: 40 # signifikant mehr Text im Bot-Crawl (>100 Zeichen) + cloaking_vary_ua: 25 # Vary: User-Agent Header (ohne sonstigen Unterschied) thresholds: yellow: 20 # ab diesem Score: Warnung (gelb) diff --git a/scanner/__main__.py b/scanner/__main__.py index e111209..c07f8d3 100644 --- a/scanner/__main__.py +++ b/scanner/__main__.py @@ -31,7 +31,7 @@ from .checker import ( ) from .config import load_config, resolve_paths from .crawler import Crawler -from .differ import compare_snapshots, score_diff +from .differ import compare_cloaking, compare_snapshots, score_cloak_diff, score_diff from .extractor import extract_page @@ -39,6 +39,9 @@ from .extractor import extract_page # Logging # --------------------------------------------------------------------------- +GOOGLEBOT_UA = "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)" + + def _setup_logging(logs_dir: str, verbose: bool) -> None: level = logging.DEBUG if verbose else logging.INFO logs_path = Path(logs_dir) @@ -294,6 +297,99 @@ def _render_markdown(r: dict) -> str: return "\n".join(lines) + "\n" +def _write_cloak_report(report: dict, reports_dir: str) -> tuple[Path, Path]: + ts = datetime.now(timezone.utc).strftime("%Y%m%d_%H%M%S") + out = Path(reports_dir) / f"{ts}_cloak" + out.mkdir(parents=True, exist_ok=True) + json_path = out / "report.json" + md_path = out / "report.md" + json_path.write_text(json.dumps(report, indent=2, ensure_ascii=False), encoding="utf-8") + md_path.write_text(_render_cloak_markdown(report), encoding="utf-8") + return json_path, md_path + + +def _render_cloak_markdown(r: dict) -> str: + a = r.get("assessment", {}) + cd = r.get("cloak_diff", {}) + level = a.get("level", "?").upper() + score = a.get("score", 0) + level_badge = {"GREEN": "[OK]", "YELLOW": "[WARNUNG]", "RED": "[ALARM]"}.get(level, level) + + lines = [ + f"# Cloaking-Check — {r.get('target', '?')}", + "", + f"**Erstellt:** {r.get('generated_at', '?')} ", + f"**Level:** {level_badge} ", + f"**Score:** {score}", + "", + "## Zusammenfassung", + "", + "| Kennzahl | Wert |", + "|---|---|", + f"| Geprüfte Seiten | {cd.get('pages_checked', '?')} |", + f"| Seiten mit Unterschieden | {len(cd.get('findings', []))} |", + "", + "## Risikobewertung", + "", + ] + for reason in a.get("reasons", []): + lines.append(f"- {reason}") + if not a.get("reasons"): + lines.append("- Keine Cloaking-Anzeichen gefunden.") + + if cd.get("findings"): + lines += ["", "## Auffällige Seiten", ""] + for page in cd["findings"]: + lines += [f"### {page['url']}", ""] + if page.get("vary_ua"): + lines.append("- `Vary: User-Agent` Header gesetzt") + if page.get("extra_links"): + lines.append(f"- Bot-Only-Links ({len(page['extra_links'])}):") + for link in page["extra_links"]: + lines.append(f" - `{link}`") + if page.get("extra_text_chars", 0) > 50: + lines.append(f"- Bot sieht +{page['extra_text_chars']} Zeichen mehr Text") + if page.get("text_diff"): + lines += ["", "```diff"] + lines += page["text_diff"][:20] + if len(page["text_diff"]) > 20: + lines.append(f"... ({len(page['text_diff']) - 20} weitere Zeilen)") + lines.append("```") + lines.append("") + + lines += ["", "---", "", "## Nächste Schritte", ""] + lev = a.get("level", "green") + if lev == "red": + lines += [ + "1. **Sofort**: Seiten manuell vergleichen: `curl -A 'Googlebot' | grep -i hidden`", + "2. Server-Logs auf Bot-IP-Ranges prüfen.", + "3. CMS-Plugins / Theme-Dateien auf injizierte Cloaking-Logik untersuchen.", + "4. Nach Bereinigung: `python -m scanner cloak-check` erneut ausführen.", + ] + elif lev == "yellow": + lines += [ + "1. Markierte Seiten prüfen: `curl -s -A 'Googlebot' > /tmp/bot.html && diff <(curl -s ) /tmp/bot.html`", + "2. Bei bestätigtem Cloaking: Hoster kontaktieren.", + ] + else: + lines += ["Keine Aktion erforderlich."] + + return "\n".join(lines) + "\n" + + +def _print_cloak_summary(assessment: dict, cloak_diff: dict) -> None: + level = assessment.get("level", "?").upper() + score = assessment.get("score", 0) + print() + print(f"=== Cloaking-Check: {level} (Score {score}) ===") + for reason in assessment.get("reasons", []): + print(f" * {reason}") + if not assessment.get("reasons"): + print(" Keine Cloaking-Anzeichen gefunden.") + print(f" Geprüfte Seiten: {cloak_diff.get('pages_checked', 0)}") + print(f" Seiten mit Unterschied: {len(cloak_diff.get('findings', []))}") + + def _latest_report_path(reports_dir: str) -> Path | None: rd = Path(reports_dir) if not rd.exists(): @@ -600,6 +696,40 @@ def cmd_status(args: argparse.Namespace, cfg: dict) -> int: return 0 +# --------------------------------------------------------------------------- +# Cloaking check +# --------------------------------------------------------------------------- + +def cmd_cloak_check(args: argparse.Namespace, cfg: dict) -> int: + logger = logging.getLogger("scanner.cloak") + print(f"Cloaking-Check: {cfg['target']}") + print(" Schritt 1/2: Crawl mit Browser-UA ...") + pages_normal, _ = _crawl_and_extract(cfg) + + print(" Schritt 2/2: Crawl mit Googlebot-UA ...") + bot_cfg = {**cfg, "user_agent": GOOGLEBOT_UA} + pages_bot, _ = _crawl_and_extract(bot_cfg) + + normal_map = {p["url"]: p for p in pages_normal} + bot_map = {p["url"]: p for p in pages_bot} + + cloak_diff = compare_cloaking(normal_map, bot_map, cfg) + assessment = score_cloak_diff(cloak_diff, cfg) + + report = { + "generated_at": datetime.now(timezone.utc).isoformat(), + "target": cfg["target"], + "type": "cloak-check", + "assessment": assessment, + "cloak_diff": cloak_diff, + } + json_path, md_path = _write_cloak_report(report, cfg["reports_dir"]) + logger.info("Cloak-Report: %s", md_path) + + _print_cloak_summary(assessment, cloak_diff) + return assessment["exit_code"] + + # --------------------------------------------------------------------------- # Output helper # --------------------------------------------------------------------------- @@ -653,6 +783,8 @@ def _build_parser() -> argparse.ArgumentParser: sub.add_parser("status", help="Übersichtsstatus anzeigen") + sub.add_parser("cloak-check", help="Doppel-Crawl: Browser-UA vs. Googlebot-UA (Cloaking-Erkennung)") + return p @@ -678,6 +810,7 @@ def main() -> None: "approve": cmd_approve, "report": cmd_report, "status": cmd_status, + "cloak-check": cmd_cloak_check, } handler = commands.get(args.command) if not handler: diff --git a/scanner/config.py b/scanner/config.py index 96aab38..3adcb87 100644 --- a/scanner/config.py +++ b/scanner/config.py @@ -33,6 +33,9 @@ DEFAULT_CONFIG: dict = { "comment_links": 25, "new_broken_link": 20, "suspicious_filename": 60, + "cloaking_extra_link": 60, + "cloaking_extra_text": 40, + "cloaking_vary_ua": 25, }, "thresholds": { "yellow": 20, diff --git a/scanner/differ.py b/scanner/differ.py index f3e15a0..b2afa97 100644 --- a/scanner/differ.py +++ b/scanner/differ.py @@ -377,6 +377,96 @@ def score_diff(diff: dict, cfg: dict) -> dict: return {"score": score, "level": level, "reasons": reasons, "exit_code": exit_code} +# --------------------------------------------------------------------------- +# Cloaking detection +# --------------------------------------------------------------------------- + +def compare_cloaking(normal_pages: dict, bot_pages: dict, cfg: dict) -> dict: + """ + Compare pages crawled with a normal browser UA vs. Googlebot UA. + Returns findings where the bot version contains extra content. + """ + findings: list[dict] = [] + common = sorted(set(normal_pages) & set(bot_pages)) + + for url in common: + normal = normal_pages[url] + bot = bot_pages[url] + + # Vary: User-Agent signals server-side content negotiation + vary = bot.get("response_headers", {}).get("Vary", "") + has_vary_ua = "user-agent" in vary.lower() + + # Text diff + normal_text = normalize_text(normal.get("text", ""), cfg) + bot_text = normalize_text(bot.get("text", ""), cfg) + text_changed = text_hash(normal_text) != text_hash(bot_text) + text_diff = diff_text(normal_text, bot_text) if text_changed else [] + extra_chars = sum( + len(line[1:]) for line in text_diff + if line.startswith("+") and not line.startswith("+++") + ) + + # Links only in bot version + extra_links: list[str] = [] + for ltype in ("a", "script", "iframe"): + ld = diff_link_set( + normal.get("links", {}).get(ltype, []), + bot.get("links", {}).get(ltype, []), + ) + extra_links.extend(ld["added"]) + + if extra_links or extra_chars > 50 or has_vary_ua: + findings.append({ + "url": url, + "vary_ua": has_vary_ua, + "extra_links": extra_links, + "extra_text_chars": extra_chars, + "text_diff": text_diff[:40], + }) + + return { + "findings": findings, + "pages_checked": len(common), + } + + +def score_cloak_diff(cloak_diff: dict, cfg: dict) -> dict: + """Score the cloaking check result.""" + sc = cfg.get("scoring", {}) + thr = cfg.get("thresholds", {"yellow": 20, "red": 60}) + score = 0 + reasons: list[str] = [] + + def add(pts: int, msg: str) -> None: + nonlocal score + score += pts + reasons.append(f"{msg} (+{pts})") + + for page in cloak_diff.get("findings", []): + url = page["url"] + for link in page.get("extra_links", []): + add(sc.get("cloaking_extra_link", 60), f"{url}: Bot-Only-Link: {link}") + if page.get("extra_text_chars", 0) > 100: + add(sc.get("cloaking_extra_text", 40), + f"{url}: Bot sieht +{page['extra_text_chars']} Zeichen mehr Text") + # Vary: User-Agent ohne sonstigen Unterschied — informativer Hinweis + if page.get("vary_ua") and not page.get("extra_links") and page.get("extra_text_chars", 0) <= 100: + add(sc.get("cloaking_vary_ua", 25), f"{url}: Vary: User-Agent Header") + + yellow = thr.get("yellow", 20) + red = thr.get("red", 60) + + if score >= red: + level, exit_code = "red", 2 + elif score >= yellow: + level, exit_code = "yellow", 1 + else: + level, exit_code = "green", 0 + + return {"score": score, "level": level, "reasons": reasons, "exit_code": exit_code} + + def _fingerprint(obj: dict | list | str) -> str: """Stable hash for deduplicating diff entries.""" import json as _json diff --git a/tests/test_cloak.py b/tests/test_cloak.py new file mode 100644 index 0000000..7ca1f91 --- /dev/null +++ b/tests/test_cloak.py @@ -0,0 +1,106 @@ +"""Tests for cloaking detection.""" +import pytest +from scanner.differ import compare_cloaking, score_cloak_diff + + +def _page(url, text="Hallo Welt", links=None, vary_ua=False): + hdrs = {"Vary": "User-Agent"} if vary_ua else {} + return { + "url": url, + "text": text, + "links": {"a": links or [], "script": [], "iframe": []}, + "response_headers": hdrs, + "status": 200, + } + + +CFG = {"normalization": {"collapse_whitespace": True, "ignore_patterns": []}, "scoring": {}, "thresholds": {}} + + +class TestCompareCloaking: + def test_identical_pages_no_findings(self): + page = _page("https://x.de/") + result = compare_cloaking({"https://x.de/": page}, {"https://x.de/": page}, CFG) + assert result["findings"] == [] + assert result["pages_checked"] == 1 + + def test_extra_link_in_bot_version_flagged(self): + normal = _page("https://x.de/", links=[{"url": "https://x.de/a", "class": "internal"}]) + bot = _page("https://x.de/", links=[ + {"url": "https://x.de/a", "class": "internal"}, + {"url": "https://spam.example.com/evil", "class": "external"}, + ]) + result = compare_cloaking({"https://x.de/": normal}, {"https://x.de/": bot}, CFG) + assert len(result["findings"]) == 1 + assert "https://spam.example.com/evil" in result["findings"][0]["extra_links"] + + def test_extra_text_in_bot_version_flagged(self): + normal = _page("https://x.de/", text="Normaler Text") + bot = _page("https://x.de/", text="Normaler Text " + "x" * 200) + result = compare_cloaking({"https://x.de/": normal}, {"https://x.de/": bot}, CFG) + assert len(result["findings"]) == 1 + assert result["findings"][0]["extra_text_chars"] > 100 + + def test_vary_ua_header_flagged(self): + normal = _page("https://x.de/") + bot = _page("https://x.de/", vary_ua=True) + result = compare_cloaking({"https://x.de/": normal}, {"https://x.de/": bot}, CFG) + assert len(result["findings"]) == 1 + assert result["findings"][0]["vary_ua"] is True + + def test_url_only_in_one_crawl_ignored(self): + """URLs not seen in both crawls are not compared.""" + normal = _page("https://x.de/only-normal") + bot = _page("https://x.de/only-bot") + result = compare_cloaking( + {"https://x.de/only-normal": normal}, + {"https://x.de/only-bot": bot}, + CFG, + ) + assert result["findings"] == [] + assert result["pages_checked"] == 0 + + +class TestScoreCloakDiff: + def test_no_findings_green(self): + result = score_cloak_diff({"findings": [], "pages_checked": 5}, CFG) + assert result["level"] == "green" + assert result["score"] == 0 + assert result["exit_code"] == 0 + + def test_extra_link_scores_60_red(self): + diff = { + "findings": [{"url": "https://x.de/", "vary_ua": False, "extra_links": ["https://spam.com/x"], "extra_text_chars": 0}], + "pages_checked": 1, + } + result = score_cloak_diff(diff, CFG) + assert result["score"] == 60 + assert result["level"] == "red" + assert result["exit_code"] == 2 + + def test_extra_text_scores_40_yellow(self): + diff = { + "findings": [{"url": "https://x.de/", "vary_ua": False, "extra_links": [], "extra_text_chars": 150}], + "pages_checked": 1, + } + result = score_cloak_diff(diff, CFG) + assert result["score"] == 40 + assert result["level"] == "yellow" + + def test_vary_ua_only_scores_25_yellow(self): + diff = { + "findings": [{"url": "https://x.de/", "vary_ua": True, "extra_links": [], "extra_text_chars": 0}], + "pages_checked": 1, + } + result = score_cloak_diff(diff, CFG) + assert result["score"] == 25 + assert result["level"] == "yellow" + + def test_vary_ua_with_extra_link_not_double_scored(self): + """vary_ua surcharge is suppressed when extra_links are present (link is already 60).""" + diff = { + "findings": [{"url": "https://x.de/", "vary_ua": True, "extra_links": ["https://spam.com/x"], "extra_text_chars": 0}], + "pages_checked": 1, + } + result = score_cloak_diff(diff, CFG) + assert result["score"] == 60 # only link score, not 60+25