68243a97b9
Echte Bugs: - extractor: JSON-LD-Crash bei Nicht-Dict-Items (isinstance-Guard) - baseline/__main__: Crawl-Fehler ins Snapshot-Manifest -> erscheinen im Report - __main__: Whitelist-/Header-/Webshell-Checks nur noch auf status==200 - crawler: Noise-Param-Regex auf Voll-Key (view=/value= nicht mehr verworfen) - differ/__main__: unerwartetes JSON-LD via eigenem Kanal, auch auf unveraenderten Seiten erkannt, kein Re-Alert auf bereits genehmigte Typen Aufgeraeumt: - checker: check_internal_paths, find_broken_pages, canonical_is_hijacked, check_jsonld_types entfernt (nicht verdrahtet) - allowed_paths.yaml-Logik und tote Scoring-Keys entfernt - tote Imports entfernt Neuer aktiver Schutz: - Webshell-/Backdoor-Dateinamen-Check verdrahtet (suspicious_filename: 60). Regex wortgrenzen-verankert gegen Fehlalarm auf PSEMailerAntispam.js Effizienz/Struktur: - crawler nutzt requests.Session (keep-alive) - Cache-Buster-Normalisierung an einer Stelle (Extraktion) -> stabile Snapshots, differ wieder reiner Set-Diff Tests: 111 gruen (neu: test_checker.py + Regressionstests) CLAUDE.md aktualisiert Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
110 lines
3.3 KiB
Python
110 lines
3.3 KiB
Python
"""Config loading with deep-merge against defaults."""
|
|
from pathlib import Path
|
|
import yaml
|
|
|
|
DEFAULT_CONFIG: dict = {
|
|
"target": "https://bredelar.info",
|
|
"user_agent": "integrity-scanner/1.0 (+security-monitoring)",
|
|
"request_timeout": 20,
|
|
"data_dir": "data",
|
|
"reports_dir": "reports",
|
|
"logs_dir": "logs",
|
|
"config_dir": "config",
|
|
"crawl": {
|
|
"max_pages": 200,
|
|
"delay_seconds": 1.0,
|
|
"respect_robots_txt": False,
|
|
"skip_extensions": [
|
|
".jpg", ".jpeg", ".png", ".gif", ".webp", ".svg", ".ico",
|
|
".pdf", ".zip", ".woff", ".woff2", ".ttf", ".otf", ".eot",
|
|
".mp4", ".mp3", ".webm", ".avi", ".mov",
|
|
],
|
|
},
|
|
"scoring": {
|
|
"new_external_domain": 50,
|
|
"new_internal_url": 30,
|
|
"missing_internal_url": 20,
|
|
"hidden_content": 40,
|
|
"unexpected_canonical": 35,
|
|
"meta_refresh": 40,
|
|
"unexpected_jsonld": 35,
|
|
"large_text_addition": 20,
|
|
"new_inline_script_suspicious": 30,
|
|
"comment_links": 25,
|
|
"new_broken_link": 20,
|
|
"suspicious_filename": 60,
|
|
},
|
|
"thresholds": {
|
|
"yellow": 20,
|
|
"red": 60,
|
|
"large_text_block_chars": 200,
|
|
},
|
|
"alerting": {
|
|
"min_level": "yellow",
|
|
"email": {
|
|
"enabled": False,
|
|
"smtp_host": "localhost",
|
|
"smtp_port": 587,
|
|
"smtp_tls": True,
|
|
"smtp_user": "",
|
|
"smtp_password_env": "SCANNER_SMTP_PASSWORD",
|
|
"from": "scanner@example.com",
|
|
"to": [],
|
|
},
|
|
"webhook": {
|
|
"enabled": False,
|
|
"url": "",
|
|
},
|
|
},
|
|
"normalization": {
|
|
"strip_html_comments": True,
|
|
"collapse_whitespace": True,
|
|
"ignore_selectors": [],
|
|
"ignore_patterns": [],
|
|
},
|
|
"allowed_jsonld_types": [
|
|
"Organization", "WebSite", "WebPage", "Article", "BreadcrumbList",
|
|
"ItemList", "LocalBusiness", "Place", "Person", "Event",
|
|
"FAQPage", "Question", "Answer", "ImageObject", "SiteLinksSearchBox",
|
|
"ContactPage", "AboutPage",
|
|
],
|
|
"security_headers": [
|
|
"Content-Security-Policy",
|
|
"Strict-Transport-Security",
|
|
"X-Content-Type-Options",
|
|
"Referrer-Policy",
|
|
],
|
|
}
|
|
|
|
|
|
def load_config(config_file: str = "config.yaml") -> dict:
|
|
"""Load config from YAML, deep-merged on top of defaults."""
|
|
cfg = _deep_copy(DEFAULT_CONFIG)
|
|
p = Path(config_file)
|
|
if p.exists():
|
|
with p.open(encoding="utf-8") as f:
|
|
user_cfg = yaml.safe_load(f) or {}
|
|
_deep_update(cfg, user_cfg)
|
|
return cfg
|
|
|
|
|
|
def resolve_paths(cfg: dict, base_dir: Path) -> dict:
|
|
"""Resolve relative paths in config against base_dir."""
|
|
for key in ("data_dir", "reports_dir", "logs_dir", "config_dir"):
|
|
cfg[key] = str(base_dir / cfg[key])
|
|
return cfg
|
|
|
|
|
|
def _deep_copy(d: dict) -> dict:
|
|
import copy
|
|
return copy.deepcopy(d)
|
|
|
|
|
|
def _deep_update(base: dict, override: dict) -> None:
|
|
for k, v in override.items():
|
|
if isinstance(v, dict) and isinstance(base.get(k), dict):
|
|
_deep_update(base[k], v)
|
|
elif v is None and isinstance(base.get(k), list):
|
|
pass # YAML null on a list field = "not set" → keep default
|
|
else:
|
|
base[k] = v
|