diff --git a/scanner/config.py b/scanner/config.py index c9b56eb..0042cc3 100644 --- a/scanner/config.py +++ b/scanner/config.py @@ -106,5 +106,7 @@ def _deep_update(base: dict, override: dict) -> None: for k, v in override.items(): if isinstance(v, dict) and isinstance(base.get(k), dict): _deep_update(base[k], v) + elif v is None and isinstance(base.get(k), list): + pass # YAML null on a list field = "not set" → keep default else: base[k] = v diff --git a/scanner/differ.py b/scanner/differ.py index 7b6d100..13be939 100644 --- a/scanner/differ.py +++ b/scanner/differ.py @@ -17,7 +17,7 @@ def normalize_text(text: str, cfg: dict) -> str: norm_cfg = cfg.get("normalization", {}) if norm_cfg.get("collapse_whitespace", True): text = re.sub(r"\s+", " ", text).strip() - for pattern in norm_cfg.get("ignore_patterns", []): + for pattern in (norm_cfg.get("ignore_patterns") or []): text = re.sub(pattern, "[IGNORED]", text) return text diff --git a/scanner/extractor.py b/scanner/extractor.py index cde2d58..68aa669 100644 --- a/scanner/extractor.py +++ b/scanner/extractor.py @@ -47,7 +47,7 @@ def extract_page(html: str, url: str, cfg: dict | None = None) -> dict: soup = BeautifulSoup(html, "lxml") # Remove ignored selectors before text extraction - ignore_selectors = cfg.get("normalization", {}).get("ignore_selectors", []) + ignore_selectors = cfg.get("normalization", {}).get("ignore_selectors") or [] for sel in ignore_selectors: for el in soup.select(sel): el.decompose()