fix: handle YAML null values for list config fields
PyYAML parst auskommentierte Listen als None statt []. Da der Key vorhanden ist, greift .get(key, []) nicht. Drei Fixes: - extractor.py, differ.py: `or []` statt Default-Argument - config.py: _deep_update überspringt None-Override auf List-Defaults Tritt auf bei `ignore_selectors` und `ignore_patterns` in config.yaml. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
parent
98e8d11eb5
commit
de90dbc19c
3 changed files with 4 additions and 2 deletions
|
|
@ -106,5 +106,7 @@ def _deep_update(base: dict, override: dict) -> None:
|
|||
for k, v in override.items():
|
||||
if isinstance(v, dict) and isinstance(base.get(k), dict):
|
||||
_deep_update(base[k], v)
|
||||
elif v is None and isinstance(base.get(k), list):
|
||||
pass # YAML null on a list field = "not set" → keep default
|
||||
else:
|
||||
base[k] = v
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@ def normalize_text(text: str, cfg: dict) -> str:
|
|||
norm_cfg = cfg.get("normalization", {})
|
||||
if norm_cfg.get("collapse_whitespace", True):
|
||||
text = re.sub(r"\s+", " ", text).strip()
|
||||
for pattern in norm_cfg.get("ignore_patterns", []):
|
||||
for pattern in (norm_cfg.get("ignore_patterns") or []):
|
||||
text = re.sub(pattern, "[IGNORED]", text)
|
||||
return text
|
||||
|
||||
|
|
|
|||
|
|
@ -47,7 +47,7 @@ def extract_page(html: str, url: str, cfg: dict | None = None) -> dict:
|
|||
soup = BeautifulSoup(html, "lxml")
|
||||
|
||||
# Remove ignored selectors before text extraction
|
||||
ignore_selectors = cfg.get("normalization", {}).get("ignore_selectors", [])
|
||||
ignore_selectors = cfg.get("normalization", {}).get("ignore_selectors") or []
|
||||
for sel in ignore_selectors:
|
||||
for el in soup.select(sel):
|
||||
el.decompose()
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue