fix: handle YAML null values for list config fields

PyYAML parst auskommentierte Listen als None statt []. Da der Key
vorhanden ist, greift .get(key, []) nicht. Drei Fixes:
- extractor.py, differ.py: `or []` statt Default-Argument
- config.py: _deep_update überspringt None-Override auf List-Defaults

Tritt auf bei `ignore_selectors` und `ignore_patterns` in config.yaml.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Your Name 2026-06-12 01:12:09 +02:00
commit de90dbc19c
3 changed files with 4 additions and 2 deletions

View file

@ -106,5 +106,7 @@ def _deep_update(base: dict, override: dict) -> None:
for k, v in override.items():
if isinstance(v, dict) and isinstance(base.get(k), dict):
_deep_update(base[k], v)
elif v is None and isinstance(base.get(k), list):
pass # YAML null on a list field = "not set" → keep default
else:
base[k] = v

View file

@ -17,7 +17,7 @@ def normalize_text(text: str, cfg: dict) -> str:
norm_cfg = cfg.get("normalization", {})
if norm_cfg.get("collapse_whitespace", True):
text = re.sub(r"\s+", " ", text).strip()
for pattern in norm_cfg.get("ignore_patterns", []):
for pattern in (norm_cfg.get("ignore_patterns") or []):
text = re.sub(pattern, "[IGNORED]", text)
return text

View file

@ -47,7 +47,7 @@ def extract_page(html: str, url: str, cfg: dict | None = None) -> dict:
soup = BeautifulSoup(html, "lxml")
# Remove ignored selectors before text extraction
ignore_selectors = cfg.get("normalization", {}).get("ignore_selectors", [])
ignore_selectors = cfg.get("normalization", {}).get("ignore_selectors") or []
for sel in ignore_selectors:
for el in soup.select(sel):
el.decompose()