b3ef3a90d4
Neue 4xx/5xx-Seiten (nicht in Baseline) werden bewertet (+20 Pkt) und lösen ab Schwelle einen Alert aus. Bekannte kaputte Links (in Baseline vorhanden) erscheinen im Report als [bekannt] ohne Score-Beitrag. Motivation: erster Crawl fand bredelar.info/www.stadtmarketing-marsberg.de (404) — CMS-Fehler durch Schema-losen href. Soll dauerhaft sichtbar sein. Änderungen: checker.find_broken_pages(), differ.compare_snapshots() mit new_broken_urls/known_broken_urls, score_diff(), Report-Abschnitt, config-Default new_broken_link:20, 4 neue Tests. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
113 lines
3.4 KiB
Python
113 lines
3.4 KiB
Python
"""Config loading with deep-merge against defaults."""
|
|
from pathlib import Path
|
|
import yaml
|
|
|
|
DEFAULT_CONFIG: dict = {
|
|
"target": "https://bredelar.info",
|
|
"user_agent": "integrity-scanner/1.0 (+security-monitoring)",
|
|
"request_timeout": 20,
|
|
"data_dir": "data",
|
|
"reports_dir": "reports",
|
|
"logs_dir": "logs",
|
|
"config_dir": "config",
|
|
"crawl": {
|
|
"max_pages": 200,
|
|
"delay_seconds": 1.0,
|
|
"respect_robots_txt": False,
|
|
"skip_extensions": [
|
|
".jpg", ".jpeg", ".png", ".gif", ".webp", ".svg", ".ico",
|
|
".pdf", ".zip", ".woff", ".woff2", ".ttf", ".otf", ".eot",
|
|
".mp4", ".mp3", ".webm", ".avi", ".mov",
|
|
],
|
|
},
|
|
"scoring": {
|
|
"new_external_domain": 50,
|
|
"new_internal_url": 30,
|
|
"missing_internal_url": 20,
|
|
"hidden_content": 40,
|
|
"unexpected_canonical": 35,
|
|
"meta_refresh": 40,
|
|
"unexpected_jsonld": 35,
|
|
"large_text_addition": 20,
|
|
"new_inline_script_suspicious": 30,
|
|
"missing_security_header": 5,
|
|
"suspicious_content_pattern": 50,
|
|
"comment_links": 25,
|
|
"noscript_links": 25,
|
|
"new_external_link_unlisted": 40,
|
|
"new_broken_link": 20,
|
|
},
|
|
"thresholds": {
|
|
"yellow": 20,
|
|
"red": 60,
|
|
"large_text_block_chars": 200,
|
|
},
|
|
"alerting": {
|
|
"min_level": "yellow",
|
|
"email": {
|
|
"enabled": False,
|
|
"smtp_host": "localhost",
|
|
"smtp_port": 587,
|
|
"smtp_tls": True,
|
|
"smtp_user": "",
|
|
"smtp_password_env": "SCANNER_SMTP_PASSWORD",
|
|
"from": "scanner@example.com",
|
|
"to": [],
|
|
},
|
|
"webhook": {
|
|
"enabled": False,
|
|
"url": "",
|
|
},
|
|
},
|
|
"normalization": {
|
|
"strip_html_comments": True,
|
|
"collapse_whitespace": True,
|
|
"ignore_selectors": [],
|
|
"ignore_patterns": [],
|
|
},
|
|
"allowed_jsonld_types": [
|
|
"Organization", "WebSite", "WebPage", "Article", "BreadcrumbList",
|
|
"ItemList", "LocalBusiness", "Place", "Person", "Event",
|
|
"FAQPage", "Question", "Answer", "ImageObject", "SiteLinksSearchBox",
|
|
"ContactPage", "AboutPage",
|
|
],
|
|
"security_headers": [
|
|
"Content-Security-Policy",
|
|
"Strict-Transport-Security",
|
|
"X-Content-Type-Options",
|
|
"Referrer-Policy",
|
|
],
|
|
}
|
|
|
|
|
|
def load_config(config_file: str = "config.yaml") -> dict:
|
|
"""Load config from YAML, deep-merged on top of defaults."""
|
|
cfg = _deep_copy(DEFAULT_CONFIG)
|
|
p = Path(config_file)
|
|
if p.exists():
|
|
with p.open(encoding="utf-8") as f:
|
|
user_cfg = yaml.safe_load(f) or {}
|
|
_deep_update(cfg, user_cfg)
|
|
return cfg
|
|
|
|
|
|
def resolve_paths(cfg: dict, base_dir: Path) -> dict:
|
|
"""Resolve relative paths in config against base_dir."""
|
|
for key in ("data_dir", "reports_dir", "logs_dir", "config_dir"):
|
|
cfg[key] = str(base_dir / cfg[key])
|
|
return cfg
|
|
|
|
|
|
def _deep_copy(d: dict) -> dict:
|
|
import copy
|
|
return copy.deepcopy(d)
|
|
|
|
|
|
def _deep_update(base: dict, override: dict) -> None:
|
|
for k, v in override.items():
|
|
if isinstance(v, dict) and isinstance(base.get(k), dict):
|
|
_deep_update(base[k], v)
|
|
elif v is None and isinstance(base.get(k), list):
|
|
pass # YAML null on a list field = "not set" → keep default
|
|
else:
|
|
base[k] = v
|