feat: Cloaking-Erkennung per Doppel-Crawl (Browser-UA vs. Googlebot-UA)
Neuer Befehl `python -m scanner cloak-check`: - Crawlt die Website zweimal: einmal mit Browser-UA, einmal als Googlebot - Vergleicht Text, Links und Response-Header (Vary: User-Agent) - Scoring: Bot-Only-Link 60 Pkt (RED), extra Text >100 Zeichen 40 Pkt, Vary: User-Agent ohne Inhaltsdiff 25 Pkt - Eigener Report unter reports/<ts>_cloak/ Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Your Name
parent
ad939f3a31
commit
0302d469b1
5 changed files with 336 additions and 1 deletions
|
|
@ -48,6 +48,9 @@ scoring:
|
|||
comment_links: 25
|
||||
new_broken_link: 20 # neue 4xx/5xx-Seite, die in Baseline nicht vorhanden war
|
||||
suspicious_filename: 60 # Link auf Webshell-/Backdoor-typischen Dateinamen
|
||||
cloaking_extra_link: 60 # Link nur im Bot-Crawl sichtbar
|
||||
cloaking_extra_text: 40 # signifikant mehr Text im Bot-Crawl (>100 Zeichen)
|
||||
cloaking_vary_ua: 25 # Vary: User-Agent Header (ohne sonstigen Unterschied)
|
||||
|
||||
thresholds:
|
||||
yellow: 20 # ab diesem Score: Warnung (gelb)
|
||||
|
|
|
|||
|
|
@ -31,7 +31,7 @@ from .checker import (
|
|||
)
|
||||
from .config import load_config, resolve_paths
|
||||
from .crawler import Crawler
|
||||
from .differ import compare_snapshots, score_diff
|
||||
from .differ import compare_cloaking, compare_snapshots, score_cloak_diff, score_diff
|
||||
from .extractor import extract_page
|
||||
|
||||
|
||||
|
|
@ -39,6 +39,9 @@ from .extractor import extract_page
|
|||
# Logging
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
GOOGLEBOT_UA = "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"
|
||||
|
||||
|
||||
def _setup_logging(logs_dir: str, verbose: bool) -> None:
|
||||
level = logging.DEBUG if verbose else logging.INFO
|
||||
logs_path = Path(logs_dir)
|
||||
|
|
@ -294,6 +297,99 @@ def _render_markdown(r: dict) -> str:
|
|||
return "\n".join(lines) + "\n"
|
||||
|
||||
|
||||
def _write_cloak_report(report: dict, reports_dir: str) -> tuple[Path, Path]:
|
||||
ts = datetime.now(timezone.utc).strftime("%Y%m%d_%H%M%S")
|
||||
out = Path(reports_dir) / f"{ts}_cloak"
|
||||
out.mkdir(parents=True, exist_ok=True)
|
||||
json_path = out / "report.json"
|
||||
md_path = out / "report.md"
|
||||
json_path.write_text(json.dumps(report, indent=2, ensure_ascii=False), encoding="utf-8")
|
||||
md_path.write_text(_render_cloak_markdown(report), encoding="utf-8")
|
||||
return json_path, md_path
|
||||
|
||||
|
||||
def _render_cloak_markdown(r: dict) -> str:
|
||||
a = r.get("assessment", {})
|
||||
cd = r.get("cloak_diff", {})
|
||||
level = a.get("level", "?").upper()
|
||||
score = a.get("score", 0)
|
||||
level_badge = {"GREEN": "[OK]", "YELLOW": "[WARNUNG]", "RED": "[ALARM]"}.get(level, level)
|
||||
|
||||
lines = [
|
||||
f"# Cloaking-Check — {r.get('target', '?')}",
|
||||
"",
|
||||
f"**Erstellt:** {r.get('generated_at', '?')} ",
|
||||
f"**Level:** {level_badge} ",
|
||||
f"**Score:** {score}",
|
||||
"",
|
||||
"## Zusammenfassung",
|
||||
"",
|
||||
"| Kennzahl | Wert |",
|
||||
"|---|---|",
|
||||
f"| Geprüfte Seiten | {cd.get('pages_checked', '?')} |",
|
||||
f"| Seiten mit Unterschieden | {len(cd.get('findings', []))} |",
|
||||
"",
|
||||
"## Risikobewertung",
|
||||
"",
|
||||
]
|
||||
for reason in a.get("reasons", []):
|
||||
lines.append(f"- {reason}")
|
||||
if not a.get("reasons"):
|
||||
lines.append("- Keine Cloaking-Anzeichen gefunden.")
|
||||
|
||||
if cd.get("findings"):
|
||||
lines += ["", "## Auffällige Seiten", ""]
|
||||
for page in cd["findings"]:
|
||||
lines += [f"### {page['url']}", ""]
|
||||
if page.get("vary_ua"):
|
||||
lines.append("- `Vary: User-Agent` Header gesetzt")
|
||||
if page.get("extra_links"):
|
||||
lines.append(f"- Bot-Only-Links ({len(page['extra_links'])}):")
|
||||
for link in page["extra_links"]:
|
||||
lines.append(f" - `{link}`")
|
||||
if page.get("extra_text_chars", 0) > 50:
|
||||
lines.append(f"- Bot sieht +{page['extra_text_chars']} Zeichen mehr Text")
|
||||
if page.get("text_diff"):
|
||||
lines += ["", "```diff"]
|
||||
lines += page["text_diff"][:20]
|
||||
if len(page["text_diff"]) > 20:
|
||||
lines.append(f"... ({len(page['text_diff']) - 20} weitere Zeilen)")
|
||||
lines.append("```")
|
||||
lines.append("")
|
||||
|
||||
lines += ["", "---", "", "## Nächste Schritte", ""]
|
||||
lev = a.get("level", "green")
|
||||
if lev == "red":
|
||||
lines += [
|
||||
"1. **Sofort**: Seiten manuell vergleichen: `curl -A 'Googlebot' <URL> | grep -i hidden`",
|
||||
"2. Server-Logs auf Bot-IP-Ranges prüfen.",
|
||||
"3. CMS-Plugins / Theme-Dateien auf injizierte Cloaking-Logik untersuchen.",
|
||||
"4. Nach Bereinigung: `python -m scanner cloak-check` erneut ausführen.",
|
||||
]
|
||||
elif lev == "yellow":
|
||||
lines += [
|
||||
"1. Markierte Seiten prüfen: `curl -s -A 'Googlebot' <URL> > /tmp/bot.html && diff <(curl -s <URL>) /tmp/bot.html`",
|
||||
"2. Bei bestätigtem Cloaking: Hoster kontaktieren.",
|
||||
]
|
||||
else:
|
||||
lines += ["Keine Aktion erforderlich."]
|
||||
|
||||
return "\n".join(lines) + "\n"
|
||||
|
||||
|
||||
def _print_cloak_summary(assessment: dict, cloak_diff: dict) -> None:
|
||||
level = assessment.get("level", "?").upper()
|
||||
score = assessment.get("score", 0)
|
||||
print()
|
||||
print(f"=== Cloaking-Check: {level} (Score {score}) ===")
|
||||
for reason in assessment.get("reasons", []):
|
||||
print(f" * {reason}")
|
||||
if not assessment.get("reasons"):
|
||||
print(" Keine Cloaking-Anzeichen gefunden.")
|
||||
print(f" Geprüfte Seiten: {cloak_diff.get('pages_checked', 0)}")
|
||||
print(f" Seiten mit Unterschied: {len(cloak_diff.get('findings', []))}")
|
||||
|
||||
|
||||
def _latest_report_path(reports_dir: str) -> Path | None:
|
||||
rd = Path(reports_dir)
|
||||
if not rd.exists():
|
||||
|
|
@ -600,6 +696,40 @@ def cmd_status(args: argparse.Namespace, cfg: dict) -> int:
|
|||
return 0
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Cloaking check
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def cmd_cloak_check(args: argparse.Namespace, cfg: dict) -> int:
|
||||
logger = logging.getLogger("scanner.cloak")
|
||||
print(f"Cloaking-Check: {cfg['target']}")
|
||||
print(" Schritt 1/2: Crawl mit Browser-UA ...")
|
||||
pages_normal, _ = _crawl_and_extract(cfg)
|
||||
|
||||
print(" Schritt 2/2: Crawl mit Googlebot-UA ...")
|
||||
bot_cfg = {**cfg, "user_agent": GOOGLEBOT_UA}
|
||||
pages_bot, _ = _crawl_and_extract(bot_cfg)
|
||||
|
||||
normal_map = {p["url"]: p for p in pages_normal}
|
||||
bot_map = {p["url"]: p for p in pages_bot}
|
||||
|
||||
cloak_diff = compare_cloaking(normal_map, bot_map, cfg)
|
||||
assessment = score_cloak_diff(cloak_diff, cfg)
|
||||
|
||||
report = {
|
||||
"generated_at": datetime.now(timezone.utc).isoformat(),
|
||||
"target": cfg["target"],
|
||||
"type": "cloak-check",
|
||||
"assessment": assessment,
|
||||
"cloak_diff": cloak_diff,
|
||||
}
|
||||
json_path, md_path = _write_cloak_report(report, cfg["reports_dir"])
|
||||
logger.info("Cloak-Report: %s", md_path)
|
||||
|
||||
_print_cloak_summary(assessment, cloak_diff)
|
||||
return assessment["exit_code"]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Output helper
|
||||
# ---------------------------------------------------------------------------
|
||||
|
|
@ -653,6 +783,8 @@ def _build_parser() -> argparse.ArgumentParser:
|
|||
|
||||
sub.add_parser("status", help="Übersichtsstatus anzeigen")
|
||||
|
||||
sub.add_parser("cloak-check", help="Doppel-Crawl: Browser-UA vs. Googlebot-UA (Cloaking-Erkennung)")
|
||||
|
||||
return p
|
||||
|
||||
|
||||
|
|
@ -678,6 +810,7 @@ def main() -> None:
|
|||
"approve": cmd_approve,
|
||||
"report": cmd_report,
|
||||
"status": cmd_status,
|
||||
"cloak-check": cmd_cloak_check,
|
||||
}
|
||||
handler = commands.get(args.command)
|
||||
if not handler:
|
||||
|
|
|
|||
|
|
@ -33,6 +33,9 @@ DEFAULT_CONFIG: dict = {
|
|||
"comment_links": 25,
|
||||
"new_broken_link": 20,
|
||||
"suspicious_filename": 60,
|
||||
"cloaking_extra_link": 60,
|
||||
"cloaking_extra_text": 40,
|
||||
"cloaking_vary_ua": 25,
|
||||
},
|
||||
"thresholds": {
|
||||
"yellow": 20,
|
||||
|
|
|
|||
|
|
@ -377,6 +377,96 @@ def score_diff(diff: dict, cfg: dict) -> dict:
|
|||
return {"score": score, "level": level, "reasons": reasons, "exit_code": exit_code}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Cloaking detection
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def compare_cloaking(normal_pages: dict, bot_pages: dict, cfg: dict) -> dict:
|
||||
"""
|
||||
Compare pages crawled with a normal browser UA vs. Googlebot UA.
|
||||
Returns findings where the bot version contains extra content.
|
||||
"""
|
||||
findings: list[dict] = []
|
||||
common = sorted(set(normal_pages) & set(bot_pages))
|
||||
|
||||
for url in common:
|
||||
normal = normal_pages[url]
|
||||
bot = bot_pages[url]
|
||||
|
||||
# Vary: User-Agent signals server-side content negotiation
|
||||
vary = bot.get("response_headers", {}).get("Vary", "")
|
||||
has_vary_ua = "user-agent" in vary.lower()
|
||||
|
||||
# Text diff
|
||||
normal_text = normalize_text(normal.get("text", ""), cfg)
|
||||
bot_text = normalize_text(bot.get("text", ""), cfg)
|
||||
text_changed = text_hash(normal_text) != text_hash(bot_text)
|
||||
text_diff = diff_text(normal_text, bot_text) if text_changed else []
|
||||
extra_chars = sum(
|
||||
len(line[1:]) for line in text_diff
|
||||
if line.startswith("+") and not line.startswith("+++")
|
||||
)
|
||||
|
||||
# Links only in bot version
|
||||
extra_links: list[str] = []
|
||||
for ltype in ("a", "script", "iframe"):
|
||||
ld = diff_link_set(
|
||||
normal.get("links", {}).get(ltype, []),
|
||||
bot.get("links", {}).get(ltype, []),
|
||||
)
|
||||
extra_links.extend(ld["added"])
|
||||
|
||||
if extra_links or extra_chars > 50 or has_vary_ua:
|
||||
findings.append({
|
||||
"url": url,
|
||||
"vary_ua": has_vary_ua,
|
||||
"extra_links": extra_links,
|
||||
"extra_text_chars": extra_chars,
|
||||
"text_diff": text_diff[:40],
|
||||
})
|
||||
|
||||
return {
|
||||
"findings": findings,
|
||||
"pages_checked": len(common),
|
||||
}
|
||||
|
||||
|
||||
def score_cloak_diff(cloak_diff: dict, cfg: dict) -> dict:
|
||||
"""Score the cloaking check result."""
|
||||
sc = cfg.get("scoring", {})
|
||||
thr = cfg.get("thresholds", {"yellow": 20, "red": 60})
|
||||
score = 0
|
||||
reasons: list[str] = []
|
||||
|
||||
def add(pts: int, msg: str) -> None:
|
||||
nonlocal score
|
||||
score += pts
|
||||
reasons.append(f"{msg} (+{pts})")
|
||||
|
||||
for page in cloak_diff.get("findings", []):
|
||||
url = page["url"]
|
||||
for link in page.get("extra_links", []):
|
||||
add(sc.get("cloaking_extra_link", 60), f"{url}: Bot-Only-Link: {link}")
|
||||
if page.get("extra_text_chars", 0) > 100:
|
||||
add(sc.get("cloaking_extra_text", 40),
|
||||
f"{url}: Bot sieht +{page['extra_text_chars']} Zeichen mehr Text")
|
||||
# Vary: User-Agent ohne sonstigen Unterschied — informativer Hinweis
|
||||
if page.get("vary_ua") and not page.get("extra_links") and page.get("extra_text_chars", 0) <= 100:
|
||||
add(sc.get("cloaking_vary_ua", 25), f"{url}: Vary: User-Agent Header")
|
||||
|
||||
yellow = thr.get("yellow", 20)
|
||||
red = thr.get("red", 60)
|
||||
|
||||
if score >= red:
|
||||
level, exit_code = "red", 2
|
||||
elif score >= yellow:
|
||||
level, exit_code = "yellow", 1
|
||||
else:
|
||||
level, exit_code = "green", 0
|
||||
|
||||
return {"score": score, "level": level, "reasons": reasons, "exit_code": exit_code}
|
||||
|
||||
|
||||
def _fingerprint(obj: dict | list | str) -> str:
|
||||
"""Stable hash for deduplicating diff entries."""
|
||||
import json as _json
|
||||
|
|
|
|||
106
tests/test_cloak.py
Normal file
106
tests/test_cloak.py
Normal file
|
|
@ -0,0 +1,106 @@
|
|||
"""Tests for cloaking detection."""
|
||||
import pytest
|
||||
from scanner.differ import compare_cloaking, score_cloak_diff
|
||||
|
||||
|
||||
def _page(url, text="Hallo Welt", links=None, vary_ua=False):
|
||||
hdrs = {"Vary": "User-Agent"} if vary_ua else {}
|
||||
return {
|
||||
"url": url,
|
||||
"text": text,
|
||||
"links": {"a": links or [], "script": [], "iframe": []},
|
||||
"response_headers": hdrs,
|
||||
"status": 200,
|
||||
}
|
||||
|
||||
|
||||
CFG = {"normalization": {"collapse_whitespace": True, "ignore_patterns": []}, "scoring": {}, "thresholds": {}}
|
||||
|
||||
|
||||
class TestCompareCloaking:
|
||||
def test_identical_pages_no_findings(self):
|
||||
page = _page("https://x.de/")
|
||||
result = compare_cloaking({"https://x.de/": page}, {"https://x.de/": page}, CFG)
|
||||
assert result["findings"] == []
|
||||
assert result["pages_checked"] == 1
|
||||
|
||||
def test_extra_link_in_bot_version_flagged(self):
|
||||
normal = _page("https://x.de/", links=[{"url": "https://x.de/a", "class": "internal"}])
|
||||
bot = _page("https://x.de/", links=[
|
||||
{"url": "https://x.de/a", "class": "internal"},
|
||||
{"url": "https://spam.example.com/evil", "class": "external"},
|
||||
])
|
||||
result = compare_cloaking({"https://x.de/": normal}, {"https://x.de/": bot}, CFG)
|
||||
assert len(result["findings"]) == 1
|
||||
assert "https://spam.example.com/evil" in result["findings"][0]["extra_links"]
|
||||
|
||||
def test_extra_text_in_bot_version_flagged(self):
|
||||
normal = _page("https://x.de/", text="Normaler Text")
|
||||
bot = _page("https://x.de/", text="Normaler Text " + "x" * 200)
|
||||
result = compare_cloaking({"https://x.de/": normal}, {"https://x.de/": bot}, CFG)
|
||||
assert len(result["findings"]) == 1
|
||||
assert result["findings"][0]["extra_text_chars"] > 100
|
||||
|
||||
def test_vary_ua_header_flagged(self):
|
||||
normal = _page("https://x.de/")
|
||||
bot = _page("https://x.de/", vary_ua=True)
|
||||
result = compare_cloaking({"https://x.de/": normal}, {"https://x.de/": bot}, CFG)
|
||||
assert len(result["findings"]) == 1
|
||||
assert result["findings"][0]["vary_ua"] is True
|
||||
|
||||
def test_url_only_in_one_crawl_ignored(self):
|
||||
"""URLs not seen in both crawls are not compared."""
|
||||
normal = _page("https://x.de/only-normal")
|
||||
bot = _page("https://x.de/only-bot")
|
||||
result = compare_cloaking(
|
||||
{"https://x.de/only-normal": normal},
|
||||
{"https://x.de/only-bot": bot},
|
||||
CFG,
|
||||
)
|
||||
assert result["findings"] == []
|
||||
assert result["pages_checked"] == 0
|
||||
|
||||
|
||||
class TestScoreCloakDiff:
|
||||
def test_no_findings_green(self):
|
||||
result = score_cloak_diff({"findings": [], "pages_checked": 5}, CFG)
|
||||
assert result["level"] == "green"
|
||||
assert result["score"] == 0
|
||||
assert result["exit_code"] == 0
|
||||
|
||||
def test_extra_link_scores_60_red(self):
|
||||
diff = {
|
||||
"findings": [{"url": "https://x.de/", "vary_ua": False, "extra_links": ["https://spam.com/x"], "extra_text_chars": 0}],
|
||||
"pages_checked": 1,
|
||||
}
|
||||
result = score_cloak_diff(diff, CFG)
|
||||
assert result["score"] == 60
|
||||
assert result["level"] == "red"
|
||||
assert result["exit_code"] == 2
|
||||
|
||||
def test_extra_text_scores_40_yellow(self):
|
||||
diff = {
|
||||
"findings": [{"url": "https://x.de/", "vary_ua": False, "extra_links": [], "extra_text_chars": 150}],
|
||||
"pages_checked": 1,
|
||||
}
|
||||
result = score_cloak_diff(diff, CFG)
|
||||
assert result["score"] == 40
|
||||
assert result["level"] == "yellow"
|
||||
|
||||
def test_vary_ua_only_scores_25_yellow(self):
|
||||
diff = {
|
||||
"findings": [{"url": "https://x.de/", "vary_ua": True, "extra_links": [], "extra_text_chars": 0}],
|
||||
"pages_checked": 1,
|
||||
}
|
||||
result = score_cloak_diff(diff, CFG)
|
||||
assert result["score"] == 25
|
||||
assert result["level"] == "yellow"
|
||||
|
||||
def test_vary_ua_with_extra_link_not_double_scored(self):
|
||||
"""vary_ua surcharge is suppressed when extra_links are present (link is already 60)."""
|
||||
diff = {
|
||||
"findings": [{"url": "https://x.de/", "vary_ua": True, "extra_links": ["https://spam.com/x"], "extra_text_chars": 0}],
|
||||
"pages_checked": 1,
|
||||
}
|
||||
result = score_cloak_diff(diff, CFG)
|
||||
assert result["score"] == 60 # only link score, not 60+25
|
||||
Loading…
Add table
Add a link
Reference in a new issue