integrity_scanner_fuer_statische_Webseiten/scanner/config.py

"""Config loading with deep-merge against defaults."""
from pathlib import Path
import yaml

DEFAULT_CONFIG: dict = {
    "target": "https://bredelar.info",
    "user_agent": "integrity-scanner/1.0 (+security-monitoring)",
    "request_timeout": 20,
    "data_dir": "data",
    "reports_dir": "reports",
    "logs_dir": "logs",
    "config_dir": "config",
    "crawl": {
        "max_pages": 200,
        "delay_seconds": 1.0,
        "respect_robots_txt": False,
        "skip_extensions": [
            ".jpg", ".jpeg", ".png", ".gif", ".webp", ".svg", ".ico",
            ".pdf", ".zip", ".woff", ".woff2", ".ttf", ".otf", ".eot",
            ".mp4", ".mp3", ".webm", ".avi", ".mov",
        ],
    },
    "scoring": {
        "new_external_domain": 50,
        "new_internal_url": 30,
        "missing_internal_url": 20,
        "hidden_content": 40,
        "unexpected_canonical": 35,
        "meta_refresh": 40,
        "unexpected_jsonld": 35,
        "large_text_addition": 20,
        "new_inline_script_suspicious": 30,
        "comment_links": 25,
        "new_broken_link": 20,
        "suspicious_filename": 60,
        "cloaking_extra_link": 60,
        "cloaking_extra_text": 40,
        "cloaking_vary_ua": 25,
        "changed_script": 40,
        "changed_stylesheet": 30,
        "changed_asset": 20,
    },
    "thresholds": {
        "yellow": 20,
        "red": 60,
        "large_text_block_chars": 200,
    },
    "periodic_checks": {
        "enabled": True,
        "interval_days": 7,
        "cloak_check": True,
        "ext_links": True,
        "assets": True,
    },
    "alerting": {
        "min_level": "yellow",
        "email": {
            "enabled": False,
            "smtp_host": "localhost",
            "smtp_port": 587,
            "smtp_tls": True,
            "smtp_user": "",
            "smtp_password_env": "SCANNER_SMTP_PASSWORD",
            "from": "scanner@example.com",
            "to": [],
        },
        "webhook": {
            "enabled": False,
            "url": "",
        },
    },
    "normalization": {
        "strip_html_comments": True,
        "collapse_whitespace": True,
        "ignore_selectors": [],
        "ignore_patterns": [],
    },
    "allowed_jsonld_types": [
        "Organization", "WebSite", "WebPage", "Article", "BreadcrumbList",
        "ItemList", "LocalBusiness", "Place", "Person", "Event",
        "FAQPage", "Question", "Answer", "ImageObject", "SiteLinksSearchBox",
        "ContactPage", "AboutPage",
    ],
    "security_headers": [
        "Content-Security-Policy",
        "Strict-Transport-Security",
        "X-Content-Type-Options",
        "Referrer-Policy",
    ],
}


def load_config(config_file: str = "config.yaml") -> dict:
    """Load config from YAML, deep-merged on top of defaults."""
    cfg = _deep_copy(DEFAULT_CONFIG)
    p = Path(config_file)
    if p.exists():
        with p.open(encoding="utf-8") as f:
            user_cfg = yaml.safe_load(f) or {}
        _deep_update(cfg, user_cfg)
    return cfg


def resolve_paths(cfg: dict, base_dir: Path) -> dict:
    """Resolve relative paths in config against base_dir."""
    for key in ("data_dir", "reports_dir", "logs_dir", "config_dir"):
        cfg[key] = str(base_dir / cfg[key])
    return cfg


def _deep_copy(d: dict) -> dict:
    import copy
    return copy.deepcopy(d)


def _deep_update(base: dict, override: dict) -> None:
    for k, v in override.items():
        if isinstance(v, dict) and isinstance(base.get(k), dict):
            _deep_update(base[k], v)
        elif v is None and isinstance(base.get(k), list):
            pass  # YAML null on a list field = "not set" → keep default
        else:
            base[k] = v