feat: initial implementation of website integrity scanner
Python-Package zur unabhängigen Überwachung statischer Websites gegen SEO-Spam und unbefugte Manipulationen. Läuft außerhalb des Hosters. Kernfunktionen: - Reiner Python-Crawler (bs4+lxml), kein wget - Baseline-Management mit manuellem Freigabe-Workflow (niemals automatisch) - Text-/Link-/Meta-Diff mit Risiko-Scoring (grün/gelb/rot) - Hidden-Content-Erkennung (CSS inline, noscript, Kommentar-Links) - Whitelist für externe Domains und interne Pfade - E-Mail + Webhook Alarmierung - CLI: init, crawl, check, scan, approve, report, status - 79 Unit-Tests (pytest), alle grün Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Your Name
commit
98e8d11eb5
24 changed files with 4295 additions and 0 deletions
231
scanner/baseline.py
Normal file
231
scanner/baseline.py
Normal file
|
|
@ -0,0 +1,231 @@
|
|||
"""
|
||||
Snapshot and baseline management.
|
||||
|
||||
Snapshots are timestamped crawl results. The baseline is a manually
|
||||
approved reference state. Compromised content must never flow into the
|
||||
baseline automatically — every update requires an explicit `approve` call.
|
||||
"""
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def url_key(url: str) -> str:
|
||||
"""Stable filename-safe key for a URL (first 16 hex chars of SHA-256)."""
|
||||
return hashlib.sha256(url.encode("utf-8")).hexdigest()[:16]
|
||||
|
||||
|
||||
class BaselineManager:
|
||||
def __init__(self, data_dir: str | Path):
|
||||
self.data_dir = Path(data_dir)
|
||||
self.baseline_dir = self.data_dir / "baseline"
|
||||
self.pages_dir = self.baseline_dir / "pages"
|
||||
self.snapshots_dir = self.data_dir / "snapshots"
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Snapshot operations
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def save_snapshot(self, pages: list[dict]) -> Path:
|
||||
"""
|
||||
Persist a crawl+extraction result as a timestamped snapshot.
|
||||
Returns the snapshot directory path.
|
||||
"""
|
||||
ts = _utc_stamp()
|
||||
snap_dir = self.snapshots_dir / ts
|
||||
snap_pages_dir = snap_dir / "pages"
|
||||
snap_pages_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
manifest = {
|
||||
"timestamp": ts,
|
||||
"page_count": len(pages),
|
||||
"urls": [p["url"] for p in pages],
|
||||
}
|
||||
(snap_dir / "manifest.json").write_text(
|
||||
json.dumps(manifest, indent=2, ensure_ascii=False), encoding="utf-8"
|
||||
)
|
||||
for page in pages:
|
||||
fname = url_key(page["url"]) + ".json"
|
||||
(snap_pages_dir / fname).write_text(
|
||||
json.dumps(page, indent=2, ensure_ascii=False), encoding="utf-8"
|
||||
)
|
||||
|
||||
logger.info("Snapshot saved: %s (%d pages)", snap_dir, len(pages))
|
||||
return snap_dir
|
||||
|
||||
def latest_snapshot_dir(self) -> Optional[Path]:
|
||||
"""Return the most recent snapshot directory, or None."""
|
||||
if not self.snapshots_dir.exists():
|
||||
return None
|
||||
dirs = sorted(
|
||||
(d for d in self.snapshots_dir.iterdir() if d.is_dir()),
|
||||
key=lambda d: d.name,
|
||||
)
|
||||
return dirs[-1] if dirs else None
|
||||
|
||||
def load_snapshot(self, snap_dir: Optional[Path] = None) -> dict:
|
||||
"""
|
||||
Load a snapshot (default: latest).
|
||||
Returns {manifest, pages: {url: page_dict}, dir: Path}.
|
||||
"""
|
||||
if snap_dir is None:
|
||||
snap_dir = self.latest_snapshot_dir()
|
||||
if snap_dir is None or not snap_dir.exists():
|
||||
return {}
|
||||
|
||||
manifest = json.loads((snap_dir / "manifest.json").read_text(encoding="utf-8"))
|
||||
pages: dict[str, dict] = {}
|
||||
pages_dir = snap_dir / "pages"
|
||||
for f in pages_dir.glob("*.json"):
|
||||
page = json.loads(f.read_text(encoding="utf-8"))
|
||||
pages[page["url"]] = page
|
||||
|
||||
return {"manifest": manifest, "pages": pages, "dir": snap_dir}
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Baseline operations
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def baseline_exists(self) -> bool:
|
||||
return (self.baseline_dir / "manifest.json").exists()
|
||||
|
||||
def load_baseline(self) -> dict:
|
||||
"""
|
||||
Load the current approved baseline.
|
||||
Returns {manifest, pages: {url: page_dict}}.
|
||||
"""
|
||||
if not self.baseline_exists():
|
||||
return {"manifest": {}, "pages": {}}
|
||||
|
||||
manifest = json.loads(
|
||||
(self.baseline_dir / "manifest.json").read_text(encoding="utf-8")
|
||||
)
|
||||
pages: dict[str, dict] = {}
|
||||
if self.pages_dir.exists():
|
||||
for f in self.pages_dir.glob("*.json"):
|
||||
page = json.loads(f.read_text(encoding="utf-8"))
|
||||
pages[page["url"]] = page
|
||||
|
||||
return {"manifest": manifest, "pages": pages}
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Approve operations (the only way baseline content is ever updated)
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def approve_all(
|
||||
self,
|
||||
snap_dir: Optional[Path] = None,
|
||||
note: str = "",
|
||||
approved_by: str = "cli",
|
||||
) -> list[str]:
|
||||
"""
|
||||
Approve every page in the snapshot as the new baseline.
|
||||
Existing baseline pages NOT present in snapshot are kept unless
|
||||
--rebuild is requested (caller responsibility to wipe baseline_dir first).
|
||||
"""
|
||||
snap = self.load_snapshot(snap_dir)
|
||||
if not snap:
|
||||
raise RuntimeError("No snapshot found to approve.")
|
||||
|
||||
self.pages_dir.mkdir(parents=True, exist_ok=True)
|
||||
approved: list[str] = []
|
||||
|
||||
for url, page in snap["pages"].items():
|
||||
fname = url_key(url) + ".json"
|
||||
(self.pages_dir / fname).write_text(
|
||||
json.dumps(page, indent=2, ensure_ascii=False), encoding="utf-8"
|
||||
)
|
||||
approved.append(url)
|
||||
|
||||
self._write_manifest(approved, note, approved_by, snap.get("dir"))
|
||||
logger.info("Approved %d URLs into baseline.", len(approved))
|
||||
return approved
|
||||
|
||||
def approve_url(
|
||||
self,
|
||||
url: str,
|
||||
snap_dir: Optional[Path] = None,
|
||||
note: str = "",
|
||||
approved_by: str = "cli",
|
||||
) -> bool:
|
||||
"""Approve a single URL from snapshot into the baseline."""
|
||||
snap = self.load_snapshot(snap_dir)
|
||||
if url not in snap.get("pages", {}):
|
||||
logger.error("URL not found in snapshot: %s", url)
|
||||
return False
|
||||
|
||||
self.pages_dir.mkdir(parents=True, exist_ok=True)
|
||||
fname = url_key(url) + ".json"
|
||||
(self.pages_dir / fname).write_text(
|
||||
json.dumps(snap["pages"][url], indent=2, ensure_ascii=False),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
# Update manifest incrementally
|
||||
if self.baseline_exists():
|
||||
manifest = json.loads(
|
||||
(self.baseline_dir / "manifest.json").read_text(encoding="utf-8")
|
||||
)
|
||||
else:
|
||||
manifest = {"urls": []}
|
||||
|
||||
if url not in manifest.get("urls", []):
|
||||
manifest.setdefault("urls", []).append(url)
|
||||
|
||||
manifest["last_updated"] = datetime.now(timezone.utc).isoformat()
|
||||
manifest["last_note"] = note
|
||||
manifest["last_approved_by"] = approved_by
|
||||
|
||||
(self.baseline_dir / "manifest.json").write_text(
|
||||
json.dumps(manifest, indent=2, ensure_ascii=False), encoding="utf-8"
|
||||
)
|
||||
logger.info("Approved single URL into baseline: %s", url)
|
||||
return True
|
||||
|
||||
def rebuild_baseline(
|
||||
self,
|
||||
snap_dir: Optional[Path] = None,
|
||||
note: str = "",
|
||||
approved_by: str = "cli",
|
||||
) -> list[str]:
|
||||
"""
|
||||
Wipe the entire baseline and replace it with the given snapshot.
|
||||
Use when a major legitimate redesign has taken place.
|
||||
"""
|
||||
import shutil
|
||||
if self.baseline_dir.exists():
|
||||
shutil.rmtree(self.baseline_dir)
|
||||
self.pages_dir.mkdir(parents=True, exist_ok=True)
|
||||
return self.approve_all(snap_dir=snap_dir, note=note, approved_by=approved_by)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Internal helpers
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _write_manifest(
|
||||
self,
|
||||
urls: list[str],
|
||||
note: str,
|
||||
approved_by: str,
|
||||
source_snap: Optional[Path],
|
||||
) -> None:
|
||||
self.baseline_dir.mkdir(parents=True, exist_ok=True)
|
||||
manifest = {
|
||||
"approved_at": datetime.now(timezone.utc).isoformat(),
|
||||
"approved_by": approved_by,
|
||||
"note": note,
|
||||
"source_snapshot": str(source_snap) if source_snap else "",
|
||||
"urls": urls,
|
||||
}
|
||||
(self.baseline_dir / "manifest.json").write_text(
|
||||
json.dumps(manifest, indent=2, ensure_ascii=False), encoding="utf-8"
|
||||
)
|
||||
|
||||
|
||||
def _utc_stamp() -> str:
|
||||
return datetime.now(timezone.utc).strftime("%Y%m%d_%H%M%S")
|
||||
Loading…
Add table
Add a link
Reference in a new issue