"""Config loading with deep-merge against defaults.""" from pathlib import Path import yaml DEFAULT_CONFIG: dict = { "target": "https://bredelar.info", "user_agent": "integrity-scanner/1.0 (+security-monitoring)", "request_timeout": 20, "data_dir": "data", "reports_dir": "reports", "logs_dir": "logs", "config_dir": "config", "crawl": { "max_pages": 200, "delay_seconds": 1.0, "respect_robots_txt": False, "skip_extensions": [ ".jpg", ".jpeg", ".png", ".gif", ".webp", ".svg", ".ico", ".pdf", ".zip", ".woff", ".woff2", ".ttf", ".otf", ".eot", ".mp4", ".mp3", ".webm", ".avi", ".mov", ], }, "scoring": { "new_external_domain": 50, "new_internal_url": 30, "missing_internal_url": 20, "hidden_content": 40, "unexpected_canonical": 35, "meta_refresh": 40, "unexpected_jsonld": 35, "large_text_addition": 20, "new_inline_script_suspicious": 30, "missing_security_header": 5, "suspicious_content_pattern": 50, "comment_links": 25, "noscript_links": 25, "new_external_link_unlisted": 40, }, "thresholds": { "yellow": 20, "red": 60, "large_text_block_chars": 200, }, "alerting": { "min_level": "yellow", "email": { "enabled": False, "smtp_host": "localhost", "smtp_port": 587, "smtp_tls": True, "smtp_user": "", "smtp_password_env": "SCANNER_SMTP_PASSWORD", "from": "scanner@example.com", "to": [], }, "webhook": { "enabled": False, "url": "", }, }, "normalization": { "strip_html_comments": True, "collapse_whitespace": True, "ignore_selectors": [], "ignore_patterns": [], }, "allowed_jsonld_types": [ "Organization", "WebSite", "WebPage", "Article", "BreadcrumbList", "ItemList", "LocalBusiness", "Place", "Person", "Event", "FAQPage", "Question", "Answer", "ImageObject", "SiteLinksSearchBox", "ContactPage", "AboutPage", ], "security_headers": [ "Content-Security-Policy", "Strict-Transport-Security", "X-Content-Type-Options", "Referrer-Policy", ], } def load_config(config_file: str = "config.yaml") -> dict: """Load config from YAML, deep-merged on top of defaults.""" cfg = _deep_copy(DEFAULT_CONFIG) p = Path(config_file) if p.exists(): with p.open(encoding="utf-8") as f: user_cfg = yaml.safe_load(f) or {} _deep_update(cfg, user_cfg) return cfg def resolve_paths(cfg: dict, base_dir: Path) -> dict: """Resolve relative paths in config against base_dir.""" for key in ("data_dir", "reports_dir", "logs_dir", "config_dir"): cfg[key] = str(base_dir / cfg[key]) return cfg def _deep_copy(d: dict) -> dict: import copy return copy.deepcopy(d) def _deep_update(base: dict, override: dict) -> None: for k, v in override.items(): if isinstance(v, dict) and isinstance(base.get(k), dict): _deep_update(base[k], v) else: base[k] = v