feat: initial implementation of website integrity scanner
Python-Package zur unabhängigen Überwachung statischer Websites gegen SEO-Spam und unbefugte Manipulationen. Läuft außerhalb des Hosters. Kernfunktionen: - Reiner Python-Crawler (bs4+lxml), kein wget - Baseline-Management mit manuellem Freigabe-Workflow (niemals automatisch) - Text-/Link-/Meta-Diff mit Risiko-Scoring (grün/gelb/rot) - Hidden-Content-Erkennung (CSS inline, noscript, Kommentar-Links) - Whitelist für externe Domains und interne Pfade - E-Mail + Webhook Alarmierung - CLI: init, crawl, check, scan, approve, report, status - 79 Unit-Tests (pytest), alle grün Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Your Name
commit
98e8d11eb5
24 changed files with 4295 additions and 0 deletions
0
scanner/__init__.py
Normal file
0
scanner/__init__.py
Normal file
672
scanner/__main__.py
Normal file
672
scanner/__main__.py
Normal file
|
|
@ -0,0 +1,672 @@
|
|||
"""
|
||||
integrity-scanner — CLI entry point.
|
||||
|
||||
Usage:
|
||||
python -m scanner <subcommand> [options]
|
||||
|
||||
Subcommands:
|
||||
init First-time crawl + baseline setup workflow
|
||||
crawl Crawl site, save snapshot (no comparison)
|
||||
check Compare latest snapshot against baseline, report + alert
|
||||
scan crawl + check in one step (use this for cron)
|
||||
approve Approve snapshot as new baseline
|
||||
report Display / export latest report
|
||||
status Show baseline age, last scan, open changes
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import logging
|
||||
import sys
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from .alerter import send_alert
|
||||
from .baseline import BaselineManager
|
||||
from .checker import (
|
||||
canonical_is_hijacked,
|
||||
check_jsonld_types,
|
||||
check_links_against_whitelist,
|
||||
check_security_headers,
|
||||
is_suspicious_url,
|
||||
load_whitelist,
|
||||
)
|
||||
from .config import load_config, resolve_paths
|
||||
from .crawler import Crawler
|
||||
from .differ import compare_snapshots, normalize_text, score_diff
|
||||
from .extractor import extract_page
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Logging
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _setup_logging(logs_dir: str, verbose: bool) -> None:
|
||||
level = logging.DEBUG if verbose else logging.INFO
|
||||
logs_path = Path(logs_dir)
|
||||
logs_path.mkdir(parents=True, exist_ok=True)
|
||||
log_file = logs_path / "scanner.log"
|
||||
|
||||
fmt = "%(asctime)s %(levelname)-8s %(name)s: %(message)s"
|
||||
handlers: list[logging.Handler] = [
|
||||
logging.StreamHandler(sys.stderr),
|
||||
logging.FileHandler(log_file, encoding="utf-8"),
|
||||
]
|
||||
logging.basicConfig(level=level, format=fmt, handlers=handlers, force=True)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Crawl + extract helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _crawl_and_extract(cfg: dict) -> tuple[list[dict], list[dict]]:
|
||||
"""Run crawler, extract each HTML page. Returns (pages, errors)."""
|
||||
logger = logging.getLogger(__name__)
|
||||
crawler = Crawler(cfg)
|
||||
raw = crawler.crawl()
|
||||
|
||||
pages: list[dict] = []
|
||||
for raw_page in raw["pages"]:
|
||||
if raw_page.get("html"):
|
||||
extracted = extract_page(raw_page["html"], raw_page["url"], cfg)
|
||||
page = {**raw_page, **extracted}
|
||||
else:
|
||||
page = dict(raw_page)
|
||||
pages.append(page)
|
||||
|
||||
logger.info("Extracted %d pages", len(pages))
|
||||
return pages, raw["errors"]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Whitelist loading
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _load_whitelists(cfg: dict) -> tuple[set[str], set[str]]:
|
||||
config_dir = Path(cfg["config_dir"])
|
||||
allowed_ext = load_whitelist(config_dir / "allowed_external.yaml")
|
||||
allowed_int = load_whitelist(config_dir / "allowed_paths.yaml")
|
||||
return allowed_ext, allowed_int
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Report building
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _build_report(
|
||||
target: str,
|
||||
diff: dict,
|
||||
assessment: dict,
|
||||
whitelist_violations: list[dict],
|
||||
missing_sec_headers: dict,
|
||||
crawl_errors: list[dict],
|
||||
snap_dir: Path,
|
||||
) -> dict:
|
||||
return {
|
||||
"generated_at": datetime.now(timezone.utc).isoformat(),
|
||||
"target": target,
|
||||
"snapshot_dir": str(snap_dir),
|
||||
"assessment": assessment,
|
||||
"diff": diff,
|
||||
"whitelist_violations": whitelist_violations,
|
||||
"missing_security_headers": missing_sec_headers,
|
||||
"crawl_errors": crawl_errors,
|
||||
}
|
||||
|
||||
|
||||
def _write_report(report: dict, reports_dir: str) -> tuple[Path, Path]:
|
||||
ts = datetime.now(timezone.utc).strftime("%Y%m%d_%H%M%S")
|
||||
out = Path(reports_dir) / ts
|
||||
out.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
json_path = out / "report.json"
|
||||
md_path = out / "report.md"
|
||||
|
||||
json_path.write_text(
|
||||
json.dumps(report, indent=2, ensure_ascii=False), encoding="utf-8"
|
||||
)
|
||||
md_path.write_text(_render_markdown(report), encoding="utf-8")
|
||||
return json_path, md_path
|
||||
|
||||
|
||||
def _render_markdown(r: dict) -> str:
|
||||
a = r.get("assessment", {})
|
||||
d = r.get("diff", {})
|
||||
level = a.get("level", "?").upper()
|
||||
score = a.get("score", 0)
|
||||
level_badge = {"GREEN": "[OK]", "YELLOW": "[WARNUNG]", "RED": "[ALARM]"}.get(level, level)
|
||||
|
||||
lines = [
|
||||
f"# Integrity Report — {r.get('target', '?')}",
|
||||
"",
|
||||
f"**Erstellt:** {r.get('generated_at', '?')} ",
|
||||
f"**Level:** {level_badge} ",
|
||||
f"**Score:** {score}",
|
||||
"",
|
||||
"## Zusammenfassung",
|
||||
"",
|
||||
f"| Kennzahl | Wert |",
|
||||
f"|---|---|",
|
||||
f"| Geprüfte Seiten | {d.get('total_pages_checked', '?')} |",
|
||||
f"| Geänderte Seiten | {d.get('changed_pages', 0)} |",
|
||||
f"| Neue interne URLs | {len(d.get('new_internal_urls', []))} |",
|
||||
f"| Fehlende URLs | {len(d.get('missing_internal_urls', []))} |",
|
||||
f"| Neue externe Domains | {len(d.get('new_external_domains', []))} |",
|
||||
f"| Whitelist-Verstösse | {len(r.get('whitelist_violations', []))} |",
|
||||
f"| Crawl-Fehler | {len(r.get('crawl_errors', []))} |",
|
||||
"",
|
||||
"## Risikobewertung",
|
||||
"",
|
||||
]
|
||||
for reason in a.get("reasons", []):
|
||||
lines.append(f"- {reason}")
|
||||
if not a.get("reasons"):
|
||||
lines.append("- Keine Auffälligkeiten.")
|
||||
|
||||
if d.get("new_external_domains"):
|
||||
lines += ["", "## Neue externe Domains", ""]
|
||||
for dom in d["new_external_domains"]:
|
||||
lines.append(f"- `{dom}`")
|
||||
|
||||
if d.get("new_internal_urls"):
|
||||
lines += ["", "## Neue interne URLs", ""]
|
||||
for url in d["new_internal_urls"]:
|
||||
lines.append(f"- {url}")
|
||||
|
||||
if d.get("missing_internal_urls"):
|
||||
lines += ["", "## Fehlende URLs (waren in Baseline)", ""]
|
||||
for url in d["missing_internal_urls"]:
|
||||
lines.append(f"- {url}")
|
||||
|
||||
if d.get("page_diffs"):
|
||||
lines += ["", "## Geänderte Seiten", ""]
|
||||
for pd in d["page_diffs"]:
|
||||
lines += [f"### {pd['url']}", ""]
|
||||
if pd.get("added_chars"):
|
||||
lines.append(f"- Neuer Text: +{pd['added_chars']} Zeichen")
|
||||
if pd.get("meta_diff"):
|
||||
for field, change in pd["meta_diff"].items():
|
||||
if isinstance(change, dict) and "old" in change:
|
||||
lines.append(f"- {field}: `{change['old']}` → `{change['new']}`")
|
||||
else:
|
||||
lines.append(f"- {field}: {change}")
|
||||
if pd.get("canonical_changed"):
|
||||
lines.append(
|
||||
f"- Canonical geändert: `{pd.get('old_canonical')}` → `{pd.get('new_canonical')}`"
|
||||
)
|
||||
if pd.get("new_hidden_content"):
|
||||
for h in pd["new_hidden_content"]:
|
||||
lines.append(f"- Hidden Content ({h.get('type')}): `{', '.join(h.get('indicators', []))}`")
|
||||
if pd.get("new_meta_refresh"):
|
||||
for mr in pd["new_meta_refresh"]:
|
||||
lines.append(f"- Meta-Refresh → `{mr}`")
|
||||
if pd.get("new_comment_links"):
|
||||
for cl in pd["new_comment_links"]:
|
||||
lines.append(f"- Kommentar-Link: `{cl}`")
|
||||
if pd.get("new_inline_scripts"):
|
||||
for s in pd["new_inline_scripts"]:
|
||||
lines.append(f"- Inline-Script: `{', '.join(s.get('patterns', []))}`")
|
||||
if pd.get("link_diff"):
|
||||
for ltype, ld in pd["link_diff"].items():
|
||||
for added in ld.get("added", []):
|
||||
lines.append(f"- {ltype} hinzugefügt: `{added}`")
|
||||
for removed in ld.get("removed", []):
|
||||
lines.append(f"- {ltype} entfernt: `{removed}`")
|
||||
if pd.get("text_diff"):
|
||||
lines += ["", "```diff"]
|
||||
lines += pd["text_diff"][:40]
|
||||
if len(pd["text_diff"]) > 40:
|
||||
lines.append(f"... ({len(pd['text_diff']) - 40} weitere Zeilen)")
|
||||
lines.append("```")
|
||||
lines.append("")
|
||||
|
||||
if d.get("new_pages_analysis"):
|
||||
lines += ["", "## Neue Seiten (Erstanalyse)", ""]
|
||||
for npa in d["new_pages_analysis"]:
|
||||
problems = (
|
||||
npa.get("hidden_content")
|
||||
or npa.get("inline_scripts")
|
||||
or npa.get("meta_refresh")
|
||||
or npa.get("unexpected_jsonld")
|
||||
or npa.get("comment_links")
|
||||
)
|
||||
flag = " [VERDAECHTIG]" if problems else ""
|
||||
lines.append(f"- {npa['url']}{flag}")
|
||||
|
||||
if r.get("whitelist_violations"):
|
||||
lines += ["", "## Whitelist-Verstösse", ""]
|
||||
for v in r["whitelist_violations"][:50]:
|
||||
lines.append(f"- [{v.get('type')}] {v.get('url', '')} (Domain: {v.get('domain', '')})")
|
||||
|
||||
if r.get("missing_security_headers"):
|
||||
lines += ["", "## Fehlende Security-Header", ""]
|
||||
for url, hdrs in r["missing_security_headers"].items():
|
||||
for h in hdrs:
|
||||
lines.append(f"- `{url}`: {h}")
|
||||
|
||||
if r.get("crawl_errors"):
|
||||
lines += ["", "## Crawl-Fehler", ""]
|
||||
for e in r["crawl_errors"][:20]:
|
||||
lines.append(f"- `{e.get('url', '?')}`: {e.get('error', '?')}")
|
||||
|
||||
lines += [
|
||||
"",
|
||||
"---",
|
||||
"",
|
||||
"## Nächste Schritte",
|
||||
"",
|
||||
]
|
||||
lev = a.get("level", "green")
|
||||
if lev == "red":
|
||||
lines += [
|
||||
"1. **Sofort**: Website-Backend überprüfen, Dateien mit Backup vergleichen.",
|
||||
"2. Dienstleister kontaktieren.",
|
||||
"3. Nach Bereinigung: `python -m scanner approve --rebuild`",
|
||||
]
|
||||
elif lev == "yellow":
|
||||
lines += [
|
||||
"1. Alle markierten URLs manuell prüfen.",
|
||||
"2. Legitime Änderungen freigeben: `python -m scanner approve --url <URL>`",
|
||||
"3. Alle freigeben (wenn geprüft): `python -m scanner approve --all`",
|
||||
]
|
||||
else:
|
||||
lines += ["Keine Aktion erforderlich."]
|
||||
|
||||
return "\n".join(lines) + "\n"
|
||||
|
||||
|
||||
def _latest_report_path(reports_dir: str) -> Path | None:
|
||||
rd = Path(reports_dir)
|
||||
if not rd.exists():
|
||||
return None
|
||||
dirs = sorted(d for d in rd.iterdir() if d.is_dir())
|
||||
if not dirs:
|
||||
return None
|
||||
return dirs[-1] / "report.md"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Subcommands
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def cmd_init(args: argparse.Namespace, cfg: dict) -> int:
|
||||
logger = logging.getLogger("scanner.init")
|
||||
print("=== Initialisierung: Erster Crawl ===")
|
||||
print(f"Ziel: {cfg['target']}")
|
||||
|
||||
pages, errors = _crawl_and_extract(cfg)
|
||||
if not pages:
|
||||
print("FEHLER: Kein Seiten gecrawlt.", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
bm = BaselineManager(cfg["data_dir"])
|
||||
snap_dir = bm.save_snapshot(pages)
|
||||
print(f"Snapshot gespeichert: {snap_dir}")
|
||||
print(f" Seiten: {len(pages)}")
|
||||
print(f" Fehler: {len(errors)}")
|
||||
print()
|
||||
|
||||
allowed_ext, allowed_int = _load_whitelists(cfg)
|
||||
base_netloc = urlparse(cfg["target"]).netloc
|
||||
|
||||
print("=== Gefundene interne URLs ===")
|
||||
for p in pages:
|
||||
print(f" {p['url']}")
|
||||
|
||||
print()
|
||||
print("=== Externe Links (alle) ===")
|
||||
seen_domains: set[str] = set()
|
||||
for p in pages:
|
||||
for ltype, links in p.get("links", {}).items():
|
||||
if not isinstance(links, list):
|
||||
continue
|
||||
for l in links:
|
||||
if isinstance(l, dict) and l.get("class") == "external":
|
||||
domain = urlparse(l.get("url", "")).netloc
|
||||
if domain and domain not in seen_domains:
|
||||
seen_domains.add(domain)
|
||||
in_wl = " [in Whitelist]" if domain in allowed_ext else " [NICHT in Whitelist]"
|
||||
print(f" {domain}{in_wl}")
|
||||
|
||||
print()
|
||||
auffaelligkeiten = 0
|
||||
for p in pages:
|
||||
if p.get("hidden_content"):
|
||||
print(f" [!] Hidden Content auf {p['url']}: {len(p['hidden_content'])} Fund(e)")
|
||||
auffaelligkeiten += len(p["hidden_content"])
|
||||
if p.get("inline_scripts"):
|
||||
print(f" [!] Verdächtige Inline-Scripts auf {p['url']}: {len(p['inline_scripts'])} Fund(e)")
|
||||
auffaelligkeiten += len(p["inline_scripts"])
|
||||
if p.get("links", {}).get("meta_refresh"):
|
||||
print(f" [!] Meta-Refresh auf {p['url']}")
|
||||
auffaelligkeiten += 1
|
||||
if p.get("comment_links"):
|
||||
print(f" [!] Links in HTML-Kommentaren auf {p['url']}")
|
||||
auffaelligkeiten += len(p["comment_links"])
|
||||
|
||||
if errors:
|
||||
print()
|
||||
print("=== Crawl-Fehler ===")
|
||||
for e in errors:
|
||||
print(f" {e['url']}: {e['error']}")
|
||||
|
||||
print()
|
||||
if auffaelligkeiten:
|
||||
print(f"[WARNUNG] {auffaelligkeiten} potenzielle Auffälligkeit(en) gefunden.")
|
||||
print(" Bitte alle URLs oben manuell prüfen, bevor Sie freigeben.")
|
||||
else:
|
||||
print("[OK] Keine offensichtlichen Auffälligkeiten gefunden.")
|
||||
|
||||
print()
|
||||
print("=== Naechste Schritte ===")
|
||||
print(" 1. Prüfen Sie alle oben aufgelisteten externen Links und Auffälligkeiten.")
|
||||
print(" 2. Wenn alles sauber ist, Baseline freigeben:")
|
||||
print(" python -m scanner approve --all")
|
||||
print(" 3. Danach ist der tägliche Betrieb möglich:")
|
||||
print(" python -m scanner scan")
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_crawl(args: argparse.Namespace, cfg: dict) -> int:
|
||||
logger = logging.getLogger("scanner.crawl")
|
||||
print(f"Crawle {cfg['target']} ...")
|
||||
pages, errors = _crawl_and_extract(cfg)
|
||||
bm = BaselineManager(cfg["data_dir"])
|
||||
snap_dir = bm.save_snapshot(pages)
|
||||
print(f"Snapshot: {snap_dir}")
|
||||
print(f" Seiten: {len(pages)}")
|
||||
print(f" Fehler: {len(errors)}")
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_check(args: argparse.Namespace, cfg: dict) -> int:
|
||||
logger = logging.getLogger("scanner.check")
|
||||
bm = BaselineManager(cfg["data_dir"])
|
||||
|
||||
if not bm.baseline_exists():
|
||||
print(
|
||||
"FEHLER: Keine Baseline vorhanden. Zuerst `python -m scanner init` ausführen.",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 1
|
||||
|
||||
snap = bm.load_snapshot()
|
||||
if not snap:
|
||||
print(
|
||||
"FEHLER: Kein Snapshot vorhanden. Zuerst `python -m scanner crawl` ausführen.",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 1
|
||||
|
||||
baseline = bm.load_baseline()
|
||||
logger.info(
|
||||
"Vergleiche %d Baseline-Seiten mit %d Snapshot-Seiten ...",
|
||||
len(baseline["pages"]),
|
||||
len(snap["pages"]),
|
||||
)
|
||||
|
||||
diff = compare_snapshots(baseline["pages"], snap["pages"], cfg)
|
||||
|
||||
# Whitelist checks on all current pages
|
||||
allowed_ext, allowed_int = _load_whitelists(cfg)
|
||||
base_netloc = urlparse(cfg["target"]).netloc
|
||||
wl_violations: list[dict] = []
|
||||
missing_sec: dict = {}
|
||||
|
||||
for page in snap["pages"].values():
|
||||
violations = check_links_against_whitelist(page, allowed_ext, base_netloc)
|
||||
if violations:
|
||||
wl_violations.extend({"page": page["url"], **v} for v in violations)
|
||||
|
||||
hdrs = page.get("response_headers", {})
|
||||
missing = check_security_headers(hdrs, cfg.get("security_headers", []))
|
||||
if missing:
|
||||
missing_sec[page["url"]] = missing
|
||||
|
||||
# Unexpected JSON-LD types (absolute check, not just diff-based)
|
||||
unexp = check_jsonld_types(page, cfg.get("allowed_jsonld_types", []))
|
||||
for j in unexp:
|
||||
if not any(
|
||||
pd.get("url") == page["url"] and j in pd.get("new_jsonld", [])
|
||||
for pd in diff.get("page_diffs", [])
|
||||
):
|
||||
# Add as new_jsonld to the page_diff if not already there
|
||||
pd_for_url = next(
|
||||
(pd for pd in diff.get("page_diffs", []) if pd["url"] == page["url"]),
|
||||
None,
|
||||
)
|
||||
if pd_for_url:
|
||||
pd_for_url.setdefault("new_jsonld", []).append({**j, "unexpected": True})
|
||||
|
||||
# Score with whitelist-violation contribution
|
||||
for v in wl_violations:
|
||||
if v.get("type") == "unlisted_external_domain":
|
||||
existing = diff.get("new_external_domains", [])
|
||||
domain = v.get("domain", "")
|
||||
if domain and domain not in existing:
|
||||
existing.append(domain)
|
||||
diff["new_external_domains"] = existing
|
||||
|
||||
assessment = score_diff(diff, cfg)
|
||||
|
||||
crawl_errors: list[dict] = []
|
||||
if getattr(args, "with_errors", False):
|
||||
# errors were not persisted in snapshot, skip
|
||||
pass
|
||||
|
||||
report = _build_report(
|
||||
cfg["target"], diff, assessment, wl_violations, missing_sec,
|
||||
crawl_errors, snap["dir"],
|
||||
)
|
||||
json_path, md_path = _write_report(report, cfg["reports_dir"])
|
||||
logger.info("Report: %s", md_path)
|
||||
|
||||
_print_summary(assessment, diff)
|
||||
|
||||
send_alert(report, cfg)
|
||||
|
||||
return assessment["exit_code"]
|
||||
|
||||
|
||||
def cmd_scan(args: argparse.Namespace, cfg: dict) -> int:
|
||||
"""crawl + check."""
|
||||
logger = logging.getLogger("scanner.scan")
|
||||
print(f"Scanne {cfg['target']} ...")
|
||||
pages, errors = _crawl_and_extract(cfg)
|
||||
bm = BaselineManager(cfg["data_dir"])
|
||||
snap_dir = bm.save_snapshot(pages)
|
||||
logger.info("Snapshot: %s (%d Seiten, %d Fehler)", snap_dir, len(pages), len(errors))
|
||||
|
||||
if not bm.baseline_exists():
|
||||
print()
|
||||
print("Noch keine Baseline vorhanden — Scan abgeschlossen, kein Vergleich.")
|
||||
print("Führen Sie aus: python -m scanner approve --all")
|
||||
return 0
|
||||
|
||||
return cmd_check(args, cfg)
|
||||
|
||||
|
||||
def cmd_approve(args: argparse.Namespace, cfg: dict) -> int:
|
||||
logger = logging.getLogger("scanner.approve")
|
||||
bm = BaselineManager(cfg["data_dir"])
|
||||
snap_dir = Path(args.snapshot) if getattr(args, "snapshot", None) else None
|
||||
note = getattr(args, "note", "") or ""
|
||||
|
||||
if getattr(args, "rebuild", False):
|
||||
approved = bm.rebuild_baseline(snap_dir=snap_dir, note=note)
|
||||
print(f"Baseline komplett neu aufgebaut: {len(approved)} URLs freigegeben.")
|
||||
return 0
|
||||
|
||||
if getattr(args, "all", False):
|
||||
approved = bm.approve_all(snap_dir=snap_dir, note=note)
|
||||
print(f"Alle {len(approved)} URLs freigegeben.")
|
||||
for u in approved:
|
||||
print(f" + {u}")
|
||||
return 0
|
||||
|
||||
url = getattr(args, "url", None)
|
||||
if url:
|
||||
ok = bm.approve_url(url, snap_dir=snap_dir, note=note)
|
||||
if ok:
|
||||
print(f"Freigegeben: {url}")
|
||||
return 0
|
||||
else:
|
||||
print(f"FEHLER: URL nicht im Snapshot gefunden: {url}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
print("FEHLER: --url, --all oder --rebuild angeben.", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
def cmd_report(args: argparse.Namespace, cfg: dict) -> int:
|
||||
fmt = getattr(args, "format", "md")
|
||||
rd = Path(cfg["reports_dir"])
|
||||
if not rd.exists():
|
||||
print("Noch keine Reports vorhanden.")
|
||||
return 0
|
||||
|
||||
dirs = sorted(d for d in rd.iterdir() if d.is_dir())
|
||||
if not dirs:
|
||||
print("Noch keine Reports vorhanden.")
|
||||
return 0
|
||||
|
||||
latest = dirs[-1]
|
||||
if fmt == "json":
|
||||
p = latest / "report.json"
|
||||
else:
|
||||
p = latest / "report.md"
|
||||
|
||||
if not p.exists():
|
||||
print(f"Report-Datei nicht gefunden: {p}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
print(p.read_text(encoding="utf-8"))
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_status(args: argparse.Namespace, cfg: dict) -> int:
|
||||
bm = BaselineManager(cfg["data_dir"])
|
||||
|
||||
print(f"Ziel: {cfg['target']}")
|
||||
|
||||
if bm.baseline_exists():
|
||||
manifest = json.loads(
|
||||
(bm.baseline_dir / "manifest.json").read_text(encoding="utf-8")
|
||||
)
|
||||
print(f"Baseline: {manifest.get('approved_at', '?')} (von: {manifest.get('approved_by', '?')})")
|
||||
print(f" Seiten: {len(manifest.get('urls', []))}")
|
||||
if manifest.get("note"):
|
||||
print(f" Notiz: {manifest['note']}")
|
||||
else:
|
||||
print("Baseline: KEINE — 'python -m scanner init' ausführen")
|
||||
|
||||
snap_dir = bm.latest_snapshot_dir()
|
||||
if snap_dir:
|
||||
snap_manifest = json.loads((snap_dir / "manifest.json").read_text(encoding="utf-8"))
|
||||
print(f"Letzter Scan: {snap_manifest.get('timestamp', '?')} ({snap_manifest.get('page_count', '?')} Seiten)")
|
||||
else:
|
||||
print("Letzter Scan: KEINER")
|
||||
|
||||
rd = Path(cfg["reports_dir"])
|
||||
if rd.exists():
|
||||
dirs = sorted(d for d in rd.iterdir() if d.is_dir())
|
||||
if dirs:
|
||||
rp = dirs[-1] / "report.json"
|
||||
if rp.exists():
|
||||
r = json.loads(rp.read_text(encoding="utf-8"))
|
||||
a = r.get("assessment", {})
|
||||
print(f"Letzter Report: {dirs[-1].name} — Level: {a.get('level','?').upper()} (Score: {a.get('score','?')})")
|
||||
else:
|
||||
print(f"Letzter Report: {dirs[-1].name}")
|
||||
return 0
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Output helper
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _print_summary(assessment: dict, diff: dict) -> None:
|
||||
level = assessment.get("level", "?").upper()
|
||||
score = assessment.get("score", 0)
|
||||
print()
|
||||
print(f"=== Ergebnis: {level} (Score {score}) ===")
|
||||
for reason in assessment.get("reasons", []):
|
||||
print(f" * {reason}")
|
||||
if not assessment.get("reasons"):
|
||||
print(" Keine Auffälligkeiten.")
|
||||
print(f" Neue URLs: {len(diff.get('new_internal_urls', []))}")
|
||||
print(f" Fehlende URLs: {len(diff.get('missing_internal_urls', []))}")
|
||||
print(f" Neue externe Domains:{len(diff.get('new_external_domains', []))}")
|
||||
print(f" Geänderte Seiten: {diff.get('changed_pages', 0)}")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Argument parser
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _build_parser() -> argparse.ArgumentParser:
|
||||
p = argparse.ArgumentParser(
|
||||
prog="scanner",
|
||||
description="Website-Integrity-Scanner für statische Webseiten",
|
||||
)
|
||||
p.add_argument("--config", default="config.yaml", help="Pfad zur config.yaml")
|
||||
p.add_argument("--verbose", "-v", action="store_true", help="Debug-Ausgabe")
|
||||
|
||||
sub = p.add_subparsers(dest="command", required=True)
|
||||
|
||||
sub.add_parser("init", help="Erstcrawl + Baseline-Workflow starten")
|
||||
|
||||
sub.add_parser("crawl", help="Nur crawlen, Snapshot speichern")
|
||||
|
||||
sub.add_parser("check", help="Snapshot vs. Baseline vergleichen")
|
||||
|
||||
sub.add_parser("scan", help="crawl + check (für Cron)")
|
||||
|
||||
ap = sub.add_parser("approve", help="Snapshot als neue Baseline freigeben")
|
||||
ap.add_argument("--all", action="store_true", help="Alle URLs freigeben")
|
||||
ap.add_argument("--url", help="Einzelne URL freigeben")
|
||||
ap.add_argument("--rebuild", action="store_true", help="Baseline komplett ersetzen")
|
||||
ap.add_argument("--snapshot", help="Pfad zu einem bestimmten Snapshot-Verzeichnis")
|
||||
ap.add_argument("--note", default="", help="Freitext-Notiz für die Baseline-Versionierung")
|
||||
|
||||
rp = sub.add_parser("report", help="Letzten Report anzeigen")
|
||||
rp.add_argument("--format", choices=["md", "json"], default="md")
|
||||
|
||||
sub.add_parser("status", help="Übersichtsstatus anzeigen")
|
||||
|
||||
return p
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Main
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def main() -> None:
|
||||
parser = _build_parser()
|
||||
args = parser.parse_args()
|
||||
|
||||
cfg = load_config(args.config)
|
||||
base_dir = Path(args.config).parent.resolve()
|
||||
resolve_paths(cfg, base_dir)
|
||||
|
||||
_setup_logging(cfg["logs_dir"], args.verbose)
|
||||
|
||||
commands = {
|
||||
"init": cmd_init,
|
||||
"crawl": cmd_crawl,
|
||||
"check": cmd_check,
|
||||
"scan": cmd_scan,
|
||||
"approve": cmd_approve,
|
||||
"report": cmd_report,
|
||||
"status": cmd_status,
|
||||
}
|
||||
handler = commands.get(args.command)
|
||||
if not handler:
|
||||
parser.print_help()
|
||||
sys.exit(1)
|
||||
|
||||
sys.exit(handler(args, cfg))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
183
scanner/alerter.py
Normal file
183
scanner/alerter.py
Normal file
|
|
@ -0,0 +1,183 @@
|
|||
"""
|
||||
Alert dispatch: e-mail (smtplib) and HTTP webhook.
|
||||
|
||||
Alerts are only sent when the assessment level meets or exceeds min_level.
|
||||
"""
|
||||
import logging
|
||||
import os
|
||||
import smtplib
|
||||
from datetime import datetime, timezone
|
||||
from email.message import EmailMessage
|
||||
|
||||
import requests
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_LEVEL_ORDER = {"green": 0, "yellow": 1, "red": 2}
|
||||
|
||||
|
||||
def send_alert(report: dict, cfg: dict) -> None:
|
||||
"""Send alert if the report's level is at or above min_level."""
|
||||
alerting = cfg.get("alerting", {})
|
||||
level = report.get("assessment", {}).get("level", "green")
|
||||
min_level = alerting.get("min_level", "yellow")
|
||||
|
||||
if _LEVEL_ORDER.get(level, 0) < _LEVEL_ORDER.get(min_level, 1):
|
||||
logger.debug("Alert suppressed: level=%s < min_level=%s", level, min_level)
|
||||
return
|
||||
|
||||
subject = _make_subject(report)
|
||||
body = _format_body(report)
|
||||
|
||||
email_cfg = alerting.get("email", {})
|
||||
if email_cfg.get("enabled"):
|
||||
_send_email(subject, body, email_cfg)
|
||||
|
||||
webhook_cfg = alerting.get("webhook", {})
|
||||
if webhook_cfg.get("enabled") and webhook_cfg.get("url"):
|
||||
_send_webhook(webhook_cfg["url"], report, subject)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Formatting
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _make_subject(report: dict) -> str:
|
||||
level = report.get("assessment", {}).get("level", "?").upper()
|
||||
target = report.get("target", "?")
|
||||
ts = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M UTC")
|
||||
return f"[{level}] Integrity Alert: {target} — {ts}"
|
||||
|
||||
|
||||
def _format_body(report: dict) -> str:
|
||||
lines = [
|
||||
"=" * 60,
|
||||
"WEBSITE INTEGRITY ALERT",
|
||||
"=" * 60,
|
||||
f"Ziel: {report.get('target', '?')}",
|
||||
f"Zeitpunkt: {datetime.now(timezone.utc).isoformat()}",
|
||||
]
|
||||
|
||||
assessment = report.get("assessment", {})
|
||||
level = assessment.get("level", "?").upper()
|
||||
score = assessment.get("score", "?")
|
||||
lines += [
|
||||
f"Level: {level}",
|
||||
f"Score: {score}",
|
||||
"",
|
||||
"--- Gruende ---",
|
||||
]
|
||||
for reason in assessment.get("reasons", []):
|
||||
lines.append(f" * {reason}")
|
||||
|
||||
diff = report.get("diff", {})
|
||||
|
||||
_section(lines, "Neue externe Domains", diff.get("new_external_domains", []), "+")
|
||||
_section(lines, "Neue interne URLs", diff.get("new_internal_urls", []), "+")
|
||||
_section(lines, "Fehlende URLs", diff.get("missing_internal_urls", []), "-")
|
||||
|
||||
for pd in diff.get("page_diffs", [])[:10]:
|
||||
lines += ["", f"--- Aenderung: {pd['url']} ---"]
|
||||
if pd.get("text_diff"):
|
||||
lines.append(f" Text-Diff: {pd['added_chars']} Zeichen zugefügt")
|
||||
lines += [f" {l}" for l in pd["text_diff"][:30]]
|
||||
if pd.get("new_hidden_content"):
|
||||
lines.append(f" Hidden Content: {len(pd['new_hidden_content'])} neue(r) Fund(e)")
|
||||
if pd.get("new_meta_refresh"):
|
||||
lines.append(f" Meta-Refresh: {pd['new_meta_refresh']}")
|
||||
if pd.get("canonical_changed"):
|
||||
lines.append(f" Canonical geaendert: {pd.get('old_canonical')} -> {pd.get('new_canonical')}")
|
||||
if pd.get("new_comment_links"):
|
||||
lines.append(f" Kommentar-Links: {pd['new_comment_links']}")
|
||||
|
||||
lines += [
|
||||
"",
|
||||
"--- Empfehlung ---",
|
||||
]
|
||||
level_lower = assessment.get("level", "green")
|
||||
if level_lower == "red":
|
||||
lines += [
|
||||
" SOFORT HANDELN: Website moeglicherweise kompromittiert.",
|
||||
" Zugriff auf Hosting-Backend prüfen, Dateien vergleichen.",
|
||||
]
|
||||
elif level_lower == "yellow":
|
||||
lines += [
|
||||
" PRÜFEN: Unerwartete Aenderungen gefunden.",
|
||||
" Alle markierten URLs manuell kontrollieren.",
|
||||
" Falls legitim: python -m scanner approve --all",
|
||||
]
|
||||
else:
|
||||
lines.append(" OK — nur zur Information.")
|
||||
|
||||
lines += [
|
||||
"",
|
||||
"--- Workflow ---",
|
||||
" Details: python -m scanner report",
|
||||
" Freigabe: python -m scanner approve --url <URL>",
|
||||
" Rebuild: python -m scanner approve --rebuild",
|
||||
"=" * 60,
|
||||
]
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def _section(lines: list, title: str, items: list, prefix: str) -> None:
|
||||
if not items:
|
||||
return
|
||||
lines += ["", f"--- {title} ---"]
|
||||
for item in items[:50]:
|
||||
lines.append(f" {prefix} {item}")
|
||||
if len(items) > 50:
|
||||
lines.append(f" ... und {len(items) - 50} weitere")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Transport
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _send_email(subject: str, body: str, cfg: dict) -> None:
|
||||
password = os.environ.get(cfg.get("smtp_password_env", "SCANNER_SMTP_PASSWORD"), "")
|
||||
recipients = cfg.get("to", [])
|
||||
if not recipients:
|
||||
logger.warning("E-Mail alert enabled but no recipients configured.")
|
||||
return
|
||||
|
||||
msg = EmailMessage()
|
||||
msg["Subject"] = subject
|
||||
msg["From"] = cfg.get("from", "scanner@localhost")
|
||||
msg["To"] = ", ".join(recipients)
|
||||
msg.set_content(body)
|
||||
|
||||
host = cfg.get("smtp_host", "localhost")
|
||||
port = cfg.get("smtp_port", 587)
|
||||
use_tls = cfg.get("smtp_tls", True)
|
||||
|
||||
try:
|
||||
with smtplib.SMTP(host, port) as server:
|
||||
if use_tls:
|
||||
server.starttls()
|
||||
user = cfg.get("smtp_user", "")
|
||||
if user and password:
|
||||
server.login(user, password)
|
||||
server.send_message(msg)
|
||||
logger.info("Alert e-mail sent to %s", recipients)
|
||||
except Exception as exc:
|
||||
logger.error("E-mail alert failed: %s", exc)
|
||||
|
||||
|
||||
def _send_webhook(url: str, report: dict, subject: str) -> None:
|
||||
payload = {
|
||||
"text": subject,
|
||||
"target": report.get("target"),
|
||||
"level": report.get("assessment", {}).get("level"),
|
||||
"score": report.get("assessment", {}).get("score"),
|
||||
"reasons": report.get("assessment", {}).get("reasons", []),
|
||||
"new_external_domains": report.get("diff", {}).get("new_external_domains", []),
|
||||
"new_internal_urls": report.get("diff", {}).get("new_internal_urls", []),
|
||||
"missing_internal_urls": report.get("diff", {}).get("missing_internal_urls", []),
|
||||
"changed_pages": report.get("diff", {}).get("changed_pages", 0),
|
||||
}
|
||||
try:
|
||||
resp = requests.post(url, json=payload, timeout=10)
|
||||
logger.info("Webhook delivered: HTTP %s", resp.status_code)
|
||||
except Exception as exc:
|
||||
logger.error("Webhook alert failed: %s", exc)
|
||||
231
scanner/baseline.py
Normal file
231
scanner/baseline.py
Normal file
|
|
@ -0,0 +1,231 @@
|
|||
"""
|
||||
Snapshot and baseline management.
|
||||
|
||||
Snapshots are timestamped crawl results. The baseline is a manually
|
||||
approved reference state. Compromised content must never flow into the
|
||||
baseline automatically — every update requires an explicit `approve` call.
|
||||
"""
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def url_key(url: str) -> str:
|
||||
"""Stable filename-safe key for a URL (first 16 hex chars of SHA-256)."""
|
||||
return hashlib.sha256(url.encode("utf-8")).hexdigest()[:16]
|
||||
|
||||
|
||||
class BaselineManager:
|
||||
def __init__(self, data_dir: str | Path):
|
||||
self.data_dir = Path(data_dir)
|
||||
self.baseline_dir = self.data_dir / "baseline"
|
||||
self.pages_dir = self.baseline_dir / "pages"
|
||||
self.snapshots_dir = self.data_dir / "snapshots"
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Snapshot operations
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def save_snapshot(self, pages: list[dict]) -> Path:
|
||||
"""
|
||||
Persist a crawl+extraction result as a timestamped snapshot.
|
||||
Returns the snapshot directory path.
|
||||
"""
|
||||
ts = _utc_stamp()
|
||||
snap_dir = self.snapshots_dir / ts
|
||||
snap_pages_dir = snap_dir / "pages"
|
||||
snap_pages_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
manifest = {
|
||||
"timestamp": ts,
|
||||
"page_count": len(pages),
|
||||
"urls": [p["url"] for p in pages],
|
||||
}
|
||||
(snap_dir / "manifest.json").write_text(
|
||||
json.dumps(manifest, indent=2, ensure_ascii=False), encoding="utf-8"
|
||||
)
|
||||
for page in pages:
|
||||
fname = url_key(page["url"]) + ".json"
|
||||
(snap_pages_dir / fname).write_text(
|
||||
json.dumps(page, indent=2, ensure_ascii=False), encoding="utf-8"
|
||||
)
|
||||
|
||||
logger.info("Snapshot saved: %s (%d pages)", snap_dir, len(pages))
|
||||
return snap_dir
|
||||
|
||||
def latest_snapshot_dir(self) -> Optional[Path]:
|
||||
"""Return the most recent snapshot directory, or None."""
|
||||
if not self.snapshots_dir.exists():
|
||||
return None
|
||||
dirs = sorted(
|
||||
(d for d in self.snapshots_dir.iterdir() if d.is_dir()),
|
||||
key=lambda d: d.name,
|
||||
)
|
||||
return dirs[-1] if dirs else None
|
||||
|
||||
def load_snapshot(self, snap_dir: Optional[Path] = None) -> dict:
|
||||
"""
|
||||
Load a snapshot (default: latest).
|
||||
Returns {manifest, pages: {url: page_dict}, dir: Path}.
|
||||
"""
|
||||
if snap_dir is None:
|
||||
snap_dir = self.latest_snapshot_dir()
|
||||
if snap_dir is None or not snap_dir.exists():
|
||||
return {}
|
||||
|
||||
manifest = json.loads((snap_dir / "manifest.json").read_text(encoding="utf-8"))
|
||||
pages: dict[str, dict] = {}
|
||||
pages_dir = snap_dir / "pages"
|
||||
for f in pages_dir.glob("*.json"):
|
||||
page = json.loads(f.read_text(encoding="utf-8"))
|
||||
pages[page["url"]] = page
|
||||
|
||||
return {"manifest": manifest, "pages": pages, "dir": snap_dir}
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Baseline operations
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def baseline_exists(self) -> bool:
|
||||
return (self.baseline_dir / "manifest.json").exists()
|
||||
|
||||
def load_baseline(self) -> dict:
|
||||
"""
|
||||
Load the current approved baseline.
|
||||
Returns {manifest, pages: {url: page_dict}}.
|
||||
"""
|
||||
if not self.baseline_exists():
|
||||
return {"manifest": {}, "pages": {}}
|
||||
|
||||
manifest = json.loads(
|
||||
(self.baseline_dir / "manifest.json").read_text(encoding="utf-8")
|
||||
)
|
||||
pages: dict[str, dict] = {}
|
||||
if self.pages_dir.exists():
|
||||
for f in self.pages_dir.glob("*.json"):
|
||||
page = json.loads(f.read_text(encoding="utf-8"))
|
||||
pages[page["url"]] = page
|
||||
|
||||
return {"manifest": manifest, "pages": pages}
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Approve operations (the only way baseline content is ever updated)
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def approve_all(
|
||||
self,
|
||||
snap_dir: Optional[Path] = None,
|
||||
note: str = "",
|
||||
approved_by: str = "cli",
|
||||
) -> list[str]:
|
||||
"""
|
||||
Approve every page in the snapshot as the new baseline.
|
||||
Existing baseline pages NOT present in snapshot are kept unless
|
||||
--rebuild is requested (caller responsibility to wipe baseline_dir first).
|
||||
"""
|
||||
snap = self.load_snapshot(snap_dir)
|
||||
if not snap:
|
||||
raise RuntimeError("No snapshot found to approve.")
|
||||
|
||||
self.pages_dir.mkdir(parents=True, exist_ok=True)
|
||||
approved: list[str] = []
|
||||
|
||||
for url, page in snap["pages"].items():
|
||||
fname = url_key(url) + ".json"
|
||||
(self.pages_dir / fname).write_text(
|
||||
json.dumps(page, indent=2, ensure_ascii=False), encoding="utf-8"
|
||||
)
|
||||
approved.append(url)
|
||||
|
||||
self._write_manifest(approved, note, approved_by, snap.get("dir"))
|
||||
logger.info("Approved %d URLs into baseline.", len(approved))
|
||||
return approved
|
||||
|
||||
def approve_url(
|
||||
self,
|
||||
url: str,
|
||||
snap_dir: Optional[Path] = None,
|
||||
note: str = "",
|
||||
approved_by: str = "cli",
|
||||
) -> bool:
|
||||
"""Approve a single URL from snapshot into the baseline."""
|
||||
snap = self.load_snapshot(snap_dir)
|
||||
if url not in snap.get("pages", {}):
|
||||
logger.error("URL not found in snapshot: %s", url)
|
||||
return False
|
||||
|
||||
self.pages_dir.mkdir(parents=True, exist_ok=True)
|
||||
fname = url_key(url) + ".json"
|
||||
(self.pages_dir / fname).write_text(
|
||||
json.dumps(snap["pages"][url], indent=2, ensure_ascii=False),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
# Update manifest incrementally
|
||||
if self.baseline_exists():
|
||||
manifest = json.loads(
|
||||
(self.baseline_dir / "manifest.json").read_text(encoding="utf-8")
|
||||
)
|
||||
else:
|
||||
manifest = {"urls": []}
|
||||
|
||||
if url not in manifest.get("urls", []):
|
||||
manifest.setdefault("urls", []).append(url)
|
||||
|
||||
manifest["last_updated"] = datetime.now(timezone.utc).isoformat()
|
||||
manifest["last_note"] = note
|
||||
manifest["last_approved_by"] = approved_by
|
||||
|
||||
(self.baseline_dir / "manifest.json").write_text(
|
||||
json.dumps(manifest, indent=2, ensure_ascii=False), encoding="utf-8"
|
||||
)
|
||||
logger.info("Approved single URL into baseline: %s", url)
|
||||
return True
|
||||
|
||||
def rebuild_baseline(
|
||||
self,
|
||||
snap_dir: Optional[Path] = None,
|
||||
note: str = "",
|
||||
approved_by: str = "cli",
|
||||
) -> list[str]:
|
||||
"""
|
||||
Wipe the entire baseline and replace it with the given snapshot.
|
||||
Use when a major legitimate redesign has taken place.
|
||||
"""
|
||||
import shutil
|
||||
if self.baseline_dir.exists():
|
||||
shutil.rmtree(self.baseline_dir)
|
||||
self.pages_dir.mkdir(parents=True, exist_ok=True)
|
||||
return self.approve_all(snap_dir=snap_dir, note=note, approved_by=approved_by)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Internal helpers
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _write_manifest(
|
||||
self,
|
||||
urls: list[str],
|
||||
note: str,
|
||||
approved_by: str,
|
||||
source_snap: Optional[Path],
|
||||
) -> None:
|
||||
self.baseline_dir.mkdir(parents=True, exist_ok=True)
|
||||
manifest = {
|
||||
"approved_at": datetime.now(timezone.utc).isoformat(),
|
||||
"approved_by": approved_by,
|
||||
"note": note,
|
||||
"source_snapshot": str(source_snap) if source_snap else "",
|
||||
"urls": urls,
|
||||
}
|
||||
(self.baseline_dir / "manifest.json").write_text(
|
||||
json.dumps(manifest, indent=2, ensure_ascii=False), encoding="utf-8"
|
||||
)
|
||||
|
||||
|
||||
def _utc_stamp() -> str:
|
||||
return datetime.now(timezone.utc).strftime("%Y%m%d_%H%M%S")
|
||||
164
scanner/checker.py
Normal file
164
scanner/checker.py
Normal file
|
|
@ -0,0 +1,164 @@
|
|||
"""
|
||||
Additional integrity checks beyond baseline diff:
|
||||
- link whitelist enforcement
|
||||
- security-header audit
|
||||
- canonical hijack detection
|
||||
- suspicious URL filenames (from v4 heritage)
|
||||
"""
|
||||
import re
|
||||
from pathlib import Path
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import yaml
|
||||
|
||||
_SUSPICIOUS_FILENAME_RE = re.compile(
|
||||
r"(shell|backdoor|cmd|upload|gate|mailer|wso|c99|r57|b374k)",
|
||||
re.I,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Whitelist checks
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def load_whitelist(path: str | Path) -> set[str]:
|
||||
"""Load a YAML list file into a set. Returns empty set if missing."""
|
||||
p = Path(path)
|
||||
if not p.exists():
|
||||
return set()
|
||||
data = yaml.safe_load(p.read_text(encoding="utf-8")) or []
|
||||
if isinstance(data, list):
|
||||
return {str(s).strip() for s in data if s}
|
||||
return set()
|
||||
|
||||
|
||||
def check_links_against_whitelist(
|
||||
page: dict,
|
||||
allowed_external_domains: set[str],
|
||||
base_netloc: str,
|
||||
) -> list[dict]:
|
||||
"""
|
||||
Return violations: external links to domains not in allowed_external_domains.
|
||||
Also flags noscript links and comment links to external domains.
|
||||
"""
|
||||
violations: list[dict] = []
|
||||
|
||||
for link_type, links in page.get("links", {}).items():
|
||||
if not isinstance(links, list):
|
||||
continue
|
||||
for link in links:
|
||||
if not isinstance(link, dict):
|
||||
continue
|
||||
if link.get("class") != "external":
|
||||
continue
|
||||
domain = urlparse(link.get("url", "")).netloc
|
||||
if domain and domain not in allowed_external_domains:
|
||||
violations.append({
|
||||
"type": "unlisted_external_domain",
|
||||
"link_type": link_type,
|
||||
"url": link.get("url", ""),
|
||||
"domain": domain,
|
||||
})
|
||||
|
||||
# Comment links
|
||||
for href in page.get("comment_links", []):
|
||||
domain = urlparse(href).netloc
|
||||
if domain and domain != base_netloc and domain not in allowed_external_domains:
|
||||
violations.append({
|
||||
"type": "comment_link_to_unlisted_domain",
|
||||
"link_type": "comment",
|
||||
"url": href,
|
||||
"domain": domain,
|
||||
})
|
||||
|
||||
# noscript links
|
||||
for hc in page.get("hidden_content", []):
|
||||
if hc.get("type") == "noscript_links":
|
||||
for href in hc.get("links", []):
|
||||
domain = urlparse(href).netloc
|
||||
if domain and domain != base_netloc and domain not in allowed_external_domains:
|
||||
violations.append({
|
||||
"type": "noscript_link_to_unlisted_domain",
|
||||
"link_type": "noscript",
|
||||
"url": href,
|
||||
"domain": domain,
|
||||
})
|
||||
|
||||
return violations
|
||||
|
||||
|
||||
def check_internal_paths(
|
||||
page: dict,
|
||||
allowed_paths: set[str],
|
||||
) -> list[dict]:
|
||||
"""
|
||||
Flag internal links to paths not in the allowed_paths whitelist.
|
||||
If allowed_paths is empty, the check is skipped.
|
||||
"""
|
||||
if not allowed_paths:
|
||||
return []
|
||||
violations: list[dict] = []
|
||||
for link in page.get("links", {}).get("a", []):
|
||||
if link.get("class") != "internal":
|
||||
continue
|
||||
path = urlparse(link.get("url", "")).path
|
||||
if path and path not in allowed_paths:
|
||||
violations.append({
|
||||
"type": "unlisted_internal_path",
|
||||
"url": link.get("url", ""),
|
||||
"path": path,
|
||||
})
|
||||
return violations
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Security headers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def check_security_headers(response_headers: dict, required: list[str]) -> list[str]:
|
||||
"""Return list of missing recommended security headers."""
|
||||
# Header lookup is case-insensitive
|
||||
present = {k.lower() for k in response_headers}
|
||||
return [h for h in required if h.lower() not in present]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Canonical hijack
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def canonical_is_hijacked(page: dict, base_netloc: str) -> bool:
|
||||
"""
|
||||
Return True if the canonical URL points to a different domain.
|
||||
Canonical appearing in both <link rel=canonical> and metadata is checked.
|
||||
"""
|
||||
canonical = (
|
||||
page.get("links", {}).get("canonical")
|
||||
or page.get("metadata", {}).get("canonical")
|
||||
)
|
||||
if not canonical:
|
||||
return False
|
||||
netloc = urlparse(canonical).netloc
|
||||
return bool(netloc) and netloc != base_netloc
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Suspicious filenames
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def is_suspicious_url(url: str) -> bool:
|
||||
"""Flag URLs with names typical for webshells / backdoors."""
|
||||
filename = urlparse(url).path.rsplit("/", 1)[-1]
|
||||
return bool(_SUSPICIOUS_FILENAME_RE.search(filename))
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# JSON-LD type check
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def check_jsonld_types(page: dict, allowed_types: list[str]) -> list[dict]:
|
||||
"""Return JSON-LD entries whose @type is not in the allowed list."""
|
||||
allowed = set(allowed_types)
|
||||
return [
|
||||
j for j in page.get("jsonld", [])
|
||||
if j.get("type") not in allowed
|
||||
]
|
||||
110
scanner/config.py
Normal file
110
scanner/config.py
Normal file
|
|
@ -0,0 +1,110 @@
|
|||
"""Config loading with deep-merge against defaults."""
|
||||
from pathlib import Path
|
||||
import yaml
|
||||
|
||||
DEFAULT_CONFIG: dict = {
|
||||
"target": "https://bredelar.info",
|
||||
"user_agent": "integrity-scanner/1.0 (+security-monitoring)",
|
||||
"request_timeout": 20,
|
||||
"data_dir": "data",
|
||||
"reports_dir": "reports",
|
||||
"logs_dir": "logs",
|
||||
"config_dir": "config",
|
||||
"crawl": {
|
||||
"max_pages": 200,
|
||||
"delay_seconds": 1.0,
|
||||
"respect_robots_txt": False,
|
||||
"skip_extensions": [
|
||||
".jpg", ".jpeg", ".png", ".gif", ".webp", ".svg", ".ico",
|
||||
".pdf", ".zip", ".woff", ".woff2", ".ttf", ".otf", ".eot",
|
||||
".mp4", ".mp3", ".webm", ".avi", ".mov",
|
||||
],
|
||||
},
|
||||
"scoring": {
|
||||
"new_external_domain": 50,
|
||||
"new_internal_url": 30,
|
||||
"missing_internal_url": 20,
|
||||
"hidden_content": 40,
|
||||
"unexpected_canonical": 35,
|
||||
"meta_refresh": 40,
|
||||
"unexpected_jsonld": 35,
|
||||
"large_text_addition": 20,
|
||||
"new_inline_script_suspicious": 30,
|
||||
"missing_security_header": 5,
|
||||
"suspicious_content_pattern": 50,
|
||||
"comment_links": 25,
|
||||
"noscript_links": 25,
|
||||
"new_external_link_unlisted": 40,
|
||||
},
|
||||
"thresholds": {
|
||||
"yellow": 20,
|
||||
"red": 60,
|
||||
"large_text_block_chars": 200,
|
||||
},
|
||||
"alerting": {
|
||||
"min_level": "yellow",
|
||||
"email": {
|
||||
"enabled": False,
|
||||
"smtp_host": "localhost",
|
||||
"smtp_port": 587,
|
||||
"smtp_tls": True,
|
||||
"smtp_user": "",
|
||||
"smtp_password_env": "SCANNER_SMTP_PASSWORD",
|
||||
"from": "scanner@example.com",
|
||||
"to": [],
|
||||
},
|
||||
"webhook": {
|
||||
"enabled": False,
|
||||
"url": "",
|
||||
},
|
||||
},
|
||||
"normalization": {
|
||||
"strip_html_comments": True,
|
||||
"collapse_whitespace": True,
|
||||
"ignore_selectors": [],
|
||||
"ignore_patterns": [],
|
||||
},
|
||||
"allowed_jsonld_types": [
|
||||
"Organization", "WebSite", "WebPage", "Article", "BreadcrumbList",
|
||||
"ItemList", "LocalBusiness", "Place", "Person", "Event",
|
||||
"FAQPage", "Question", "Answer", "ImageObject", "SiteLinksSearchBox",
|
||||
"ContactPage", "AboutPage",
|
||||
],
|
||||
"security_headers": [
|
||||
"Content-Security-Policy",
|
||||
"Strict-Transport-Security",
|
||||
"X-Content-Type-Options",
|
||||
"Referrer-Policy",
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def load_config(config_file: str = "config.yaml") -> dict:
|
||||
"""Load config from YAML, deep-merged on top of defaults."""
|
||||
cfg = _deep_copy(DEFAULT_CONFIG)
|
||||
p = Path(config_file)
|
||||
if p.exists():
|
||||
with p.open(encoding="utf-8") as f:
|
||||
user_cfg = yaml.safe_load(f) or {}
|
||||
_deep_update(cfg, user_cfg)
|
||||
return cfg
|
||||
|
||||
|
||||
def resolve_paths(cfg: dict, base_dir: Path) -> dict:
|
||||
"""Resolve relative paths in config against base_dir."""
|
||||
for key in ("data_dir", "reports_dir", "logs_dir", "config_dir"):
|
||||
cfg[key] = str(base_dir / cfg[key])
|
||||
return cfg
|
||||
|
||||
|
||||
def _deep_copy(d: dict) -> dict:
|
||||
import copy
|
||||
return copy.deepcopy(d)
|
||||
|
||||
|
||||
def _deep_update(base: dict, override: dict) -> None:
|
||||
for k, v in override.items():
|
||||
if isinstance(v, dict) and isinstance(base.get(k), dict):
|
||||
_deep_update(base[k], v)
|
||||
else:
|
||||
base[k] = v
|
||||
150
scanner/crawler.py
Normal file
150
scanner/crawler.py
Normal file
|
|
@ -0,0 +1,150 @@
|
|||
"""Pure-Python crawler — no wget dependency."""
|
||||
import logging
|
||||
import re
|
||||
import time
|
||||
from urllib.parse import parse_qs, urlencode, urljoin, urlparse, urlunparse
|
||||
|
||||
import requests
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Query parameters that indicate calendar/session/tracking noise
|
||||
_NOISE_PARAM_RE = re.compile(
|
||||
r"^(year|month|day|date|week|session|token|sid|nonce|_|v|cache|utm_|fbclid)",
|
||||
re.I,
|
||||
)
|
||||
|
||||
|
||||
def normalize_url(url: str) -> str | None:
|
||||
"""
|
||||
Normalise a URL: lowercase scheme+host, drop fragment, sort and filter
|
||||
query params that look like cache-busters or tracking noise.
|
||||
Returns None for non-http(s) URLs.
|
||||
"""
|
||||
try:
|
||||
p = urlparse(url.strip())
|
||||
if p.scheme not in ("http", "https"):
|
||||
return None
|
||||
netloc = p.netloc.lower()
|
||||
path = p.path or "/"
|
||||
# Remove fragment
|
||||
p = p._replace(netloc=netloc, path=path, fragment="")
|
||||
# Keep only non-noise query params, sorted for stable comparison
|
||||
if p.query:
|
||||
clean_params = sorted(
|
||||
(k, v)
|
||||
for k, vs in parse_qs(p.query, keep_blank_values=True).items()
|
||||
for v in vs
|
||||
if not _NOISE_PARAM_RE.match(k)
|
||||
)
|
||||
p = p._replace(query=urlencode(clean_params))
|
||||
return urlunparse(p)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def should_crawl(url: str, base_netloc: str, skip_extensions: list[str]) -> bool:
|
||||
"""Return True if url should be followed during a crawl."""
|
||||
p = urlparse(url)
|
||||
if p.netloc != base_netloc:
|
||||
return False
|
||||
path_lower = p.path.lower()
|
||||
for ext in skip_extensions:
|
||||
if path_lower.endswith(ext):
|
||||
return False
|
||||
# Skip URLs whose path looks like calendar/session noise
|
||||
if _NOISE_PARAM_RE.search(p.query):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
class Crawler:
|
||||
def __init__(self, cfg: dict):
|
||||
self.target = cfg["target"].rstrip("/")
|
||||
self.base_netloc = urlparse(self.target).netloc
|
||||
self.max_pages: int = cfg["crawl"]["max_pages"]
|
||||
self.delay: float = cfg["crawl"]["delay_seconds"]
|
||||
self.skip_ext: list[str] = cfg["crawl"]["skip_extensions"]
|
||||
self.timeout: int = cfg["request_timeout"]
|
||||
self.headers = {"User-Agent": cfg["user_agent"]}
|
||||
|
||||
def crawl(self) -> dict:
|
||||
"""
|
||||
Crawl the target site breadth-first.
|
||||
|
||||
Returns a dict with:
|
||||
- pages: list of page-dicts (url, final_url, status, html, headers, ...)
|
||||
- errors: list of {url, error}
|
||||
"""
|
||||
seen: set[str] = set()
|
||||
queue: list[str] = [normalize_url(self.target + "/") or self.target + "/"]
|
||||
pages: list[dict] = []
|
||||
errors: list[dict] = []
|
||||
|
||||
while queue and len(pages) < self.max_pages:
|
||||
url = queue.pop(0)
|
||||
if url in seen:
|
||||
continue
|
||||
seen.add(url)
|
||||
|
||||
if self.delay > 0 and pages:
|
||||
time.sleep(self.delay)
|
||||
|
||||
logger.debug("Crawling %s", url)
|
||||
try:
|
||||
resp = requests.get(
|
||||
url,
|
||||
headers=self.headers,
|
||||
timeout=self.timeout,
|
||||
allow_redirects=True,
|
||||
)
|
||||
final_url = normalize_url(resp.url) or resp.url
|
||||
content_type = resp.headers.get("content-type", "")
|
||||
is_html = "text/html" in content_type and resp.status_code == 200
|
||||
|
||||
page: dict = {
|
||||
"url": url,
|
||||
"final_url": final_url,
|
||||
"status": resp.status_code,
|
||||
"content_type": content_type,
|
||||
"response_headers": dict(resp.headers),
|
||||
"html": resp.text if is_html else None,
|
||||
"redirected": final_url != url,
|
||||
}
|
||||
pages.append(page)
|
||||
|
||||
if is_html:
|
||||
for link_url in self._extract_follow_links(resp.text, final_url):
|
||||
if link_url not in seen and link_url not in queue:
|
||||
queue.append(link_url)
|
||||
|
||||
except requests.exceptions.RequestException as exc:
|
||||
errors.append({"url": url, "error": str(exc)})
|
||||
logger.warning("Error crawling %s: %s", url, exc)
|
||||
|
||||
logger.info(
|
||||
"Crawl finished: %d pages, %d errors, %d queued but skipped",
|
||||
len(pages),
|
||||
len(errors),
|
||||
len(queue),
|
||||
)
|
||||
return {
|
||||
"target": self.target,
|
||||
"pages": pages,
|
||||
"errors": errors,
|
||||
}
|
||||
|
||||
def _extract_follow_links(self, html: str, base_url: str) -> list[str]:
|
||||
"""Extract internal <a href> links worth following."""
|
||||
result = []
|
||||
try:
|
||||
soup = BeautifulSoup(html, "lxml")
|
||||
for tag in soup.find_all("a", href=True):
|
||||
href = tag["href"].strip()
|
||||
abs_url = normalize_url(urljoin(base_url, href))
|
||||
if abs_url and should_crawl(abs_url, self.base_netloc, self.skip_ext):
|
||||
result.append(abs_url)
|
||||
except Exception as exc:
|
||||
logger.debug("Link extraction failed on %s: %s", base_url, exc)
|
||||
return result
|
||||
338
scanner/differ.py
Normal file
338
scanner/differ.py
Normal file
|
|
@ -0,0 +1,338 @@
|
|||
"""
|
||||
Diff logic: text, link, metadata, and URL-set comparison.
|
||||
Scoring maps diff results to a risk level (green/yellow/red).
|
||||
"""
|
||||
import difflib
|
||||
import hashlib
|
||||
import re
|
||||
from urllib.parse import urlparse
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Normalisation
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def normalize_text(text: str, cfg: dict) -> str:
|
||||
"""Apply noise-reduction before diffing."""
|
||||
norm_cfg = cfg.get("normalization", {})
|
||||
if norm_cfg.get("collapse_whitespace", True):
|
||||
text = re.sub(r"\s+", " ", text).strip()
|
||||
for pattern in norm_cfg.get("ignore_patterns", []):
|
||||
text = re.sub(pattern, "[IGNORED]", text)
|
||||
return text
|
||||
|
||||
|
||||
def text_hash(text: str) -> str:
|
||||
return hashlib.sha256(text.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Atomic diffs
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def diff_text(old: str, new: str, context: int = 2) -> list[str]:
|
||||
"""Unified diff of two normalised texts."""
|
||||
return list(difflib.unified_diff(
|
||||
old.splitlines(),
|
||||
new.splitlines(),
|
||||
fromfile="baseline",
|
||||
tofile="current",
|
||||
lineterm="",
|
||||
n=context,
|
||||
))
|
||||
|
||||
|
||||
def diff_link_set(old_list: list[dict], new_list: list[dict]) -> dict:
|
||||
"""Compare two link lists by URL."""
|
||||
old_urls = {l["url"] for l in old_list if isinstance(l, dict) and "url" in l}
|
||||
new_urls = {l["url"] for l in new_list if isinstance(l, dict) and "url" in l}
|
||||
return {
|
||||
"added": sorted(new_urls - old_urls),
|
||||
"removed": sorted(old_urls - new_urls),
|
||||
}
|
||||
|
||||
|
||||
def diff_metadata(old: dict, new: dict) -> dict:
|
||||
"""Compare metadata fields that matter for integrity monitoring."""
|
||||
changes: dict = {}
|
||||
for key in ("title", "description", "canonical", "robots"):
|
||||
if old.get(key) != new.get(key):
|
||||
changes[key] = {"old": old.get(key), "new": new.get(key)}
|
||||
|
||||
for heading in ("h1", "h2"):
|
||||
old_set = set(old.get(heading, []))
|
||||
new_set = set(new.get(heading, []))
|
||||
if old_set != new_set:
|
||||
changes[heading] = {
|
||||
"added": sorted(new_set - old_set),
|
||||
"removed": sorted(old_set - new_set),
|
||||
}
|
||||
return changes
|
||||
|
||||
|
||||
def _all_external_domains(pages: dict) -> set[str]:
|
||||
"""Collect every external domain referenced across all pages."""
|
||||
domains: set[str] = set()
|
||||
for page in pages.values():
|
||||
for link_type, links in page.get("links", {}).items():
|
||||
if not isinstance(links, list):
|
||||
continue
|
||||
for l in links:
|
||||
if isinstance(l, dict) and l.get("class") == "external":
|
||||
netloc = urlparse(l.get("url", "")).netloc
|
||||
if netloc:
|
||||
domains.add(netloc)
|
||||
return domains
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Full snapshot comparison
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def compare_snapshots(baseline_pages: dict, current_pages: dict, cfg: dict) -> dict:
|
||||
"""
|
||||
Compare two page sets (url → page_dict) and produce a structured diff.
|
||||
|
||||
Returns a dict with:
|
||||
new_internal_urls, missing_internal_urls,
|
||||
new_external_domains, removed_external_domains,
|
||||
page_diffs (list of per-page change records),
|
||||
new_pages_analysis (freshly seen pages checked for immediate issues)
|
||||
"""
|
||||
b_urls = set(baseline_pages)
|
||||
c_urls = set(current_pages)
|
||||
|
||||
new_urls = sorted(c_urls - b_urls)
|
||||
missing_urls = sorted(b_urls - c_urls)
|
||||
common_urls = sorted(b_urls & c_urls)
|
||||
|
||||
page_diffs: list[dict] = []
|
||||
|
||||
for url in common_urls:
|
||||
old = baseline_pages[url]
|
||||
new = current_pages[url]
|
||||
|
||||
old_text = normalize_text(old.get("text", ""), cfg)
|
||||
new_text = normalize_text(new.get("text", ""), cfg)
|
||||
text_changed = text_hash(old_text) != text_hash(new_text)
|
||||
|
||||
# Quick-skip if nothing changed (text + links + meta all identical)
|
||||
if (
|
||||
not text_changed
|
||||
and old.get("links") == new.get("links")
|
||||
and old.get("metadata") == new.get("metadata")
|
||||
and old.get("hidden_content") == new.get("hidden_content")
|
||||
and old.get("inline_scripts") == new.get("inline_scripts")
|
||||
and old.get("jsonld") == new.get("jsonld")
|
||||
):
|
||||
continue
|
||||
|
||||
text_diff = diff_text(old_text, new_text) if text_changed else []
|
||||
|
||||
# Per-type link diff
|
||||
link_diff: dict = {}
|
||||
for ltype in ("a", "script", "iframe", "link_rel", "form"):
|
||||
old_links = old.get("links", {}).get(ltype, [])
|
||||
new_links = new.get("links", {}).get(ltype, [])
|
||||
ld = diff_link_set(old_links, new_links)
|
||||
if ld["added"] or ld["removed"]:
|
||||
link_diff[ltype] = ld
|
||||
|
||||
meta_diff = diff_metadata(old.get("metadata", {}), new.get("metadata", {}))
|
||||
|
||||
# New hidden content (compare by fingerprint to avoid dupe noise)
|
||||
old_hc_fps = {_fingerprint(h) for h in old.get("hidden_content", [])}
|
||||
new_hidden = [
|
||||
h for h in new.get("hidden_content", [])
|
||||
if _fingerprint(h) not in old_hc_fps
|
||||
]
|
||||
|
||||
# New suspicious inline scripts
|
||||
old_scripts_fps = {_fingerprint(s) for s in old.get("inline_scripts", [])}
|
||||
new_scripts = [
|
||||
s for s in new.get("inline_scripts", [])
|
||||
if _fingerprint(s) not in old_scripts_fps
|
||||
]
|
||||
|
||||
# New JSON-LD
|
||||
old_jld_fps = {_fingerprint(j) for j in old.get("jsonld", [])}
|
||||
new_jsonld = [
|
||||
j for j in new.get("jsonld", [])
|
||||
if _fingerprint(j) not in old_jld_fps
|
||||
]
|
||||
|
||||
# New meta-refresh targets
|
||||
old_mr = set(old.get("links", {}).get("meta_refresh", []))
|
||||
new_mr = [u for u in new.get("links", {}).get("meta_refresh", []) if u not in old_mr]
|
||||
|
||||
# New canonical
|
||||
old_canonical = old.get("links", {}).get("canonical") or old.get("metadata", {}).get("canonical")
|
||||
new_canonical = new.get("links", {}).get("canonical") or new.get("metadata", {}).get("canonical")
|
||||
canonical_changed = old_canonical != new_canonical
|
||||
|
||||
# New comment links
|
||||
old_cl = set(old.get("comment_links", []))
|
||||
new_cl = [u for u in new.get("comment_links", []) if u not in old_cl]
|
||||
|
||||
if any([
|
||||
text_diff, link_diff, meta_diff, new_hidden, new_scripts,
|
||||
new_jsonld, new_mr, canonical_changed, new_cl,
|
||||
]):
|
||||
added_chars = sum(
|
||||
len(line[1:]) for line in text_diff
|
||||
if line.startswith("+") and not line.startswith("+++")
|
||||
)
|
||||
page_diffs.append({
|
||||
"url": url,
|
||||
"text_diff": text_diff,
|
||||
"added_chars": added_chars,
|
||||
"link_diff": link_diff,
|
||||
"meta_diff": meta_diff,
|
||||
"new_hidden_content": new_hidden,
|
||||
"new_inline_scripts": new_scripts,
|
||||
"new_jsonld": new_jsonld,
|
||||
"new_meta_refresh": new_mr,
|
||||
"canonical_changed": canonical_changed,
|
||||
"old_canonical": old_canonical,
|
||||
"new_canonical": new_canonical,
|
||||
"new_comment_links": new_cl,
|
||||
})
|
||||
|
||||
# Site-wide external domain diff
|
||||
old_ext = _all_external_domains(baseline_pages)
|
||||
new_ext = _all_external_domains(current_pages)
|
||||
|
||||
# Analysis of brand-new pages (no baseline to compare against)
|
||||
new_pages_analysis = [
|
||||
_quick_check_new_page(current_pages[url])
|
||||
for url in new_urls
|
||||
if url in current_pages
|
||||
]
|
||||
|
||||
return {
|
||||
"new_internal_urls": new_urls,
|
||||
"missing_internal_urls": missing_urls,
|
||||
"new_external_domains": sorted(new_ext - old_ext),
|
||||
"removed_external_domains": sorted(old_ext - new_ext),
|
||||
"page_diffs": page_diffs,
|
||||
"new_pages_analysis": new_pages_analysis,
|
||||
"total_pages_checked": len(common_urls),
|
||||
"changed_pages": len(page_diffs),
|
||||
}
|
||||
|
||||
|
||||
def _quick_check_new_page(page: dict) -> dict:
|
||||
"""Summarise a new (not in baseline) page for immediate risk indicators."""
|
||||
return {
|
||||
"url": page["url"],
|
||||
"hidden_content": page.get("hidden_content", []),
|
||||
"inline_scripts": page.get("inline_scripts", []),
|
||||
"unexpected_jsonld": [j for j in page.get("jsonld", []) if j.get("unexpected")],
|
||||
"meta_refresh": page.get("links", {}).get("meta_refresh", []),
|
||||
"comment_links": page.get("comment_links", []),
|
||||
"external_link_count": sum(
|
||||
1 for links in page.get("links", {}).values()
|
||||
if isinstance(links, list)
|
||||
for l in links
|
||||
if isinstance(l, dict) and l.get("class") == "external"
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Scoring
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def score_diff(diff: dict, cfg: dict) -> dict:
|
||||
"""
|
||||
Map a diff result to a risk score and level.
|
||||
|
||||
Returns {score, level, reasons, exit_code}.
|
||||
"""
|
||||
sc = cfg.get("scoring", {})
|
||||
thr = cfg.get("thresholds", {"yellow": 20, "red": 60})
|
||||
large_text = thr.get("large_text_block_chars", 200)
|
||||
|
||||
score = 0
|
||||
reasons: list[str] = []
|
||||
|
||||
def add(pts: int, msg: str) -> None:
|
||||
nonlocal score
|
||||
score += pts
|
||||
reasons.append(f"{msg} (+{pts})")
|
||||
|
||||
for domain in diff.get("new_external_domains", []):
|
||||
add(sc.get("new_external_domain", 50), f"Neue externe Domain: {domain}")
|
||||
|
||||
for url in diff.get("new_internal_urls", []):
|
||||
add(sc.get("new_internal_url", 30), f"Neue interne URL: {url}")
|
||||
|
||||
for url in diff.get("missing_internal_urls", []):
|
||||
add(sc.get("missing_internal_url", 20), f"Fehlende URL: {url}")
|
||||
|
||||
for pd in diff.get("page_diffs", []):
|
||||
u = pd["url"]
|
||||
|
||||
for h in pd.get("new_hidden_content", []):
|
||||
add(sc.get("hidden_content", 40),
|
||||
f"{u}: Hidden Content [{', '.join(h.get('indicators', []))}]")
|
||||
|
||||
for mr in pd.get("new_meta_refresh", []):
|
||||
add(sc.get("meta_refresh", 40), f"{u}: Meta-Refresh → {mr}")
|
||||
|
||||
for s in pd.get("new_inline_scripts", []):
|
||||
add(sc.get("new_inline_script_suspicious", 30),
|
||||
f"{u}: Verdächtiges Inline-Script [{', '.join(s.get('patterns', []))}]")
|
||||
|
||||
for j in pd.get("new_jsonld", []):
|
||||
if j.get("unexpected"):
|
||||
add(sc.get("unexpected_jsonld", 35),
|
||||
f"{u}: Unerwartetes JSON-LD @type={j.get('type')}")
|
||||
|
||||
if pd.get("canonical_changed"):
|
||||
add(sc.get("unexpected_canonical", 35),
|
||||
f"{u}: Canonical geändert → {pd.get('new_canonical')}")
|
||||
|
||||
for cl in pd.get("new_comment_links", []):
|
||||
add(sc.get("comment_links", 25), f"{u}: Link in HTML-Kommentar: {cl}")
|
||||
|
||||
if pd.get("added_chars", 0) > large_text:
|
||||
add(sc.get("large_text_addition", 20),
|
||||
f"{u}: +{pd['added_chars']} Zeichen neuer Text")
|
||||
|
||||
# New pages: immediate suspicious content counts too
|
||||
for npa in diff.get("new_pages_analysis", []):
|
||||
u = npa["url"]
|
||||
if npa.get("hidden_content"):
|
||||
add(sc.get("hidden_content", 40), f"{u} (neu): Hidden Content")
|
||||
if npa.get("inline_scripts"):
|
||||
add(sc.get("new_inline_script_suspicious", 30), f"{u} (neu): Verdächtiges Script")
|
||||
if npa.get("meta_refresh"):
|
||||
add(sc.get("meta_refresh", 40), f"{u} (neu): Meta-Refresh")
|
||||
if npa.get("comment_links"):
|
||||
add(sc.get("comment_links", 25), f"{u} (neu): Kommentar-Link")
|
||||
if npa.get("unexpected_jsonld"):
|
||||
add(sc.get("unexpected_jsonld", 35), f"{u} (neu): Unerwartetes JSON-LD")
|
||||
|
||||
yellow = thr.get("yellow", 20)
|
||||
red = thr.get("red", 60)
|
||||
|
||||
if score >= red:
|
||||
level = "red"
|
||||
exit_code = 2
|
||||
elif score >= yellow:
|
||||
level = "yellow"
|
||||
exit_code = 1
|
||||
else:
|
||||
level = "green"
|
||||
exit_code = 0
|
||||
|
||||
return {"score": score, "level": level, "reasons": reasons, "exit_code": exit_code}
|
||||
|
||||
|
||||
def _fingerprint(obj: dict | list | str) -> str:
|
||||
"""Stable hash for deduplicating diff entries."""
|
||||
import json as _json
|
||||
return hashlib.md5( # noqa: S324 — not crypto, just dedup
|
||||
_json.dumps(obj, sort_keys=True, ensure_ascii=False).encode()
|
||||
).hexdigest()
|
||||
312
scanner/extractor.py
Normal file
312
scanner/extractor.py
Normal file
|
|
@ -0,0 +1,312 @@
|
|||
"""
|
||||
HTML extraction with BeautifulSoup4+lxml.
|
||||
|
||||
Extracts: visible text, links (all types), metadata, hidden-content indicators,
|
||||
inline script patterns, JSON-LD, and links buried in HTML comments.
|
||||
"""
|
||||
import json
|
||||
import re
|
||||
from urllib.parse import urljoin, urlparse
|
||||
|
||||
from bs4 import BeautifulSoup, Comment
|
||||
|
||||
# Inline-script patterns that warrant flagging
|
||||
_SUSPICIOUS_INLINE = [
|
||||
(r"eval\s*\(", "eval()"),
|
||||
(r"atob\s*\(", "atob()"),
|
||||
(r"fromCharCode", "fromCharCode"),
|
||||
(r"document\.createElement\s*\(\s*[\"']script[\"']", "createElement(script)"),
|
||||
(r"\bfetch\s*\(", "fetch()"),
|
||||
(r"XMLHttpRequest", "XMLHttpRequest"),
|
||||
(r"base64_decode", "base64_decode"),
|
||||
]
|
||||
|
||||
# CSS-based hiding patterns (applied to inline style attributes)
|
||||
_HIDDEN_STYLE_CHECKS = [
|
||||
(r"display\s*:\s*none", "display:none"),
|
||||
(r"visibility\s*:\s*hidden", "visibility:hidden"),
|
||||
(r"opacity\s*:\s*0(?:\.0+)?(?!\d)", "opacity:0"),
|
||||
(r"font-size\s*:\s*0", "font-size:0"),
|
||||
(r"color\s*:\s*(?:white|#fff(?:fff)?|rgba?\(\s*255\s*,\s*255\s*,\s*255)", "white-text"),
|
||||
(r"width\s*:\s*0(?:px)?[;\s]", "width:0"),
|
||||
(r"height\s*:\s*0(?:px)?[;\s]", "height:0"),
|
||||
(r"position\s*:\s*absolute", "position:absolute"), # combined with off-screen check below
|
||||
(r"(?:left|top)\s*:\s*-\d{3,}", "off-screen-position"),
|
||||
(r"clip\s*:\s*rect\s*\(\s*0\s*,?\s*0\s*,?\s*0\s*,?\s*0\s*\)", "clip:rect(0)"),
|
||||
(r"text-indent\s*:\s*-\d{3,}", "text-indent-offscreen"),
|
||||
]
|
||||
|
||||
|
||||
def extract_page(html: str, url: str, cfg: dict | None = None) -> dict:
|
||||
"""
|
||||
Full extraction from page HTML.
|
||||
|
||||
Returns a structured dict with all data needed for diffing and checking.
|
||||
"""
|
||||
cfg = cfg or {}
|
||||
soup = BeautifulSoup(html, "lxml")
|
||||
|
||||
# Remove ignored selectors before text extraction
|
||||
ignore_selectors = cfg.get("normalization", {}).get("ignore_selectors", [])
|
||||
for sel in ignore_selectors:
|
||||
for el in soup.select(sel):
|
||||
el.decompose()
|
||||
|
||||
return {
|
||||
"url": url,
|
||||
"text": _visible_text(soup),
|
||||
"links": _extract_links(soup, url),
|
||||
"metadata": _extract_metadata(soup),
|
||||
"hidden_content": _find_hidden_content(soup),
|
||||
"inline_scripts": _check_inline_scripts(soup),
|
||||
"jsonld": _extract_jsonld(soup),
|
||||
"comment_links": _find_comment_links(html, url),
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Text
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _visible_text(soup: BeautifulSoup) -> str:
|
||||
"""Extract visible text — strips scripts, styles, noscript, head."""
|
||||
work = soup.__copy__()
|
||||
for tag in work(["script", "style", "noscript", "head", "meta", "link"]):
|
||||
tag.decompose()
|
||||
text = work.get_text(separator=" ")
|
||||
return re.sub(r"\s+", " ", text).strip()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Links
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _extract_links(soup: BeautifulSoup, base_url: str) -> dict:
|
||||
"""
|
||||
Extract all link categories:
|
||||
a, link, script[src], img[src], iframe[src], form[action],
|
||||
meta_refresh, canonical
|
||||
Each entry: {url, class: 'internal'|'external', ...extra}
|
||||
"""
|
||||
base_netloc = urlparse(base_url).netloc
|
||||
|
||||
def classify(u: str) -> str:
|
||||
return "internal" if urlparse(u).netloc == base_netloc else "external"
|
||||
|
||||
def resolve(href: str | None) -> str | None:
|
||||
if not href:
|
||||
return None
|
||||
href = href.strip()
|
||||
if href.startswith(("javascript:", "mailto:", "tel:", "data:", "#")):
|
||||
return None
|
||||
abs_url = urljoin(base_url, href)
|
||||
p = urlparse(abs_url)
|
||||
if p.scheme not in ("http", "https"):
|
||||
return None
|
||||
# Drop fragment
|
||||
return abs_url.split("#")[0]
|
||||
|
||||
result: dict = {
|
||||
"a": [],
|
||||
"link_rel": [],
|
||||
"script": [],
|
||||
"img": [],
|
||||
"iframe": [],
|
||||
"form": [],
|
||||
"meta_refresh": [],
|
||||
"canonical": None,
|
||||
}
|
||||
|
||||
for tag in soup.find_all("a", href=True):
|
||||
u = resolve(tag["href"])
|
||||
if u:
|
||||
result["a"].append({
|
||||
"url": u,
|
||||
"text": tag.get_text(" ", strip=True)[:200],
|
||||
"class": classify(u),
|
||||
"rel": " ".join(tag.get("rel", [])),
|
||||
})
|
||||
|
||||
for tag in soup.find_all("link", href=True):
|
||||
u = resolve(tag["href"])
|
||||
if not u:
|
||||
continue
|
||||
rel = " ".join(tag.get("rel", []))
|
||||
if "canonical" in rel:
|
||||
result["canonical"] = u
|
||||
result["link_rel"].append({"url": u, "rel": rel, "class": classify(u)})
|
||||
|
||||
for tag in soup.find_all("script", src=True):
|
||||
u = resolve(tag["src"])
|
||||
if u:
|
||||
result["script"].append({"url": u, "class": classify(u)})
|
||||
|
||||
for tag in soup.find_all("img", src=True):
|
||||
u = resolve(tag["src"])
|
||||
if u:
|
||||
result["img"].append({"url": u, "class": classify(u)})
|
||||
|
||||
for tag in soup.find_all("iframe", src=True):
|
||||
u = resolve(tag["src"])
|
||||
if u:
|
||||
result["iframe"].append({"url": u, "class": classify(u)})
|
||||
|
||||
for tag in soup.find_all("form", action=True):
|
||||
u = resolve(tag["action"])
|
||||
if u:
|
||||
result["form"].append({"url": u, "class": classify(u)})
|
||||
|
||||
for tag in soup.find_all("meta"):
|
||||
if tag.get("http-equiv", "").lower() == "refresh":
|
||||
content = tag.get("content", "")
|
||||
m = re.search(r"url\s*=\s*(.+)", content, re.I)
|
||||
if m:
|
||||
u = resolve(m.group(1).strip().strip("\"'"))
|
||||
if u:
|
||||
result["meta_refresh"].append(u)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Metadata
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _extract_metadata(soup: BeautifulSoup) -> dict:
|
||||
"""Extract title, description, canonical, og-tags, robots, hreflang, H1/H2."""
|
||||
meta: dict = {}
|
||||
|
||||
t = soup.find("title")
|
||||
meta["title"] = t.get_text(strip=True) if t else None
|
||||
|
||||
canonical_tag = soup.find("link", rel="canonical")
|
||||
if canonical_tag:
|
||||
meta["canonical"] = canonical_tag.get("href", "").strip()
|
||||
|
||||
for tag in soup.find_all("meta"):
|
||||
name = (tag.get("name") or "").lower().strip()
|
||||
prop = (tag.get("property") or "").lower().strip()
|
||||
content = tag.get("content", "")
|
||||
|
||||
if name == "description":
|
||||
meta["description"] = content
|
||||
elif name == "robots":
|
||||
meta["robots"] = content
|
||||
elif name == "keywords":
|
||||
meta["keywords"] = content
|
||||
elif prop.startswith("og:"):
|
||||
meta[prop] = content
|
||||
elif prop.startswith("twitter:"):
|
||||
meta[prop] = content
|
||||
|
||||
hreflang = []
|
||||
for tag in soup.find_all("link", rel="alternate"):
|
||||
hl = tag.get("hreflang")
|
||||
if hl:
|
||||
hreflang.append({"lang": hl, "href": tag.get("href", "")})
|
||||
meta["hreflang"] = hreflang
|
||||
|
||||
meta["h1"] = [h.get_text(" ", strip=True) for h in soup.find_all("h1")]
|
||||
meta["h2"] = [h.get_text(" ", strip=True) for h in soup.find_all("h2")]
|
||||
|
||||
return meta
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Hidden content
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _find_hidden_content(soup: BeautifulSoup) -> list[dict]:
|
||||
"""
|
||||
Detect elements that are visually hidden but contain text or links.
|
||||
Catches common SEO-spam hiding techniques.
|
||||
"""
|
||||
findings: list[dict] = []
|
||||
|
||||
for tag in soup.find_all(style=True):
|
||||
style = tag.get("style", "")
|
||||
matched_indicators = []
|
||||
for pattern, label in _HIDDEN_STYLE_CHECKS:
|
||||
if re.search(pattern, style, re.I):
|
||||
matched_indicators.append(label)
|
||||
if not matched_indicators:
|
||||
continue
|
||||
|
||||
text = tag.get_text(" ", strip=True)[:300]
|
||||
links = [a.get("href", "") for a in tag.find_all("a", href=True)]
|
||||
if text or links:
|
||||
findings.append({
|
||||
"type": "inline_style_hidden",
|
||||
"indicators": matched_indicators,
|
||||
"tag": tag.name,
|
||||
"text_preview": text,
|
||||
"links": links,
|
||||
})
|
||||
|
||||
# <noscript> with links
|
||||
for tag in soup.find_all("noscript"):
|
||||
links = re.findall(r'href=["\']([^"\']+)["\']', str(tag), re.I)
|
||||
if links:
|
||||
findings.append({
|
||||
"type": "noscript_links",
|
||||
"links": links,
|
||||
"text_preview": tag.get_text(" ", strip=True)[:200],
|
||||
})
|
||||
|
||||
return findings
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Inline scripts
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _check_inline_scripts(soup: BeautifulSoup) -> list[dict]:
|
||||
"""Find inline <script> blocks that match suspicious patterns."""
|
||||
findings: list[dict] = []
|
||||
for i, tag in enumerate(soup.find_all("script"), 1):
|
||||
if tag.get("src"):
|
||||
continue
|
||||
code = tag.get_text() or ""
|
||||
hits = [label for pat, label in _SUSPICIOUS_INLINE if re.search(pat, code, re.I)]
|
||||
if hits:
|
||||
findings.append({
|
||||
"script_index": i,
|
||||
"patterns": hits,
|
||||
"preview": code[:400],
|
||||
})
|
||||
return findings
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# JSON-LD
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _extract_jsonld(soup: BeautifulSoup) -> list[dict]:
|
||||
"""Extract JSON-LD blocks and flag unexpected @types."""
|
||||
results: list[dict] = []
|
||||
for tag in soup.find_all("script", type="application/ld+json"):
|
||||
raw = tag.string or ""
|
||||
try:
|
||||
data = json.loads(raw)
|
||||
except (json.JSONDecodeError, ValueError):
|
||||
results.append({"type": "PARSE_ERROR", "unexpected": True, "preview": raw[:200]})
|
||||
continue
|
||||
items = data if isinstance(data, list) else [data]
|
||||
for item in items:
|
||||
t = item.get("@type", "Unknown")
|
||||
results.append({"type": t, "preview": json.dumps(item, ensure_ascii=False)[:500]})
|
||||
return results
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Comment links
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _find_comment_links(html: str, base_url: str) -> list[str]:
|
||||
"""Find URLs hidden inside HTML comments."""
|
||||
found: list[str] = []
|
||||
for comment in re.findall(r"<!--(.*?)-->", html, re.S):
|
||||
for href in re.findall(r'href=["\']([^"\']+)["\']', comment, re.I):
|
||||
if href.startswith(("http://", "https://", "/")):
|
||||
found.append(urljoin(base_url, href))
|
||||
return found
|
||||
Loading…
Add table
Add a link
Reference in a new issue