feat: exclude_paths — skip dynamic areas (RSS feeds, news archives) from scan

Adds crawl.exclude_paths config option (list of URL path prefixes).
Matching URLs are excluded at two levels:
- Crawler: not fetched at all (saves HTTP requests)
- Differ: filtered from both baseline and current before comparison,
  so legacy baseline entries under excluded paths don't appear as
  "missing URLs" after the option is added retroactively

Example config:
  crawl:
    exclude_paths: ["/rss/", "/feed/", "/news/archiv/"]

8 new tests (176 total).

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

scanner/config.py

@@ -19,6 +19,7 @@ DEFAULT_CONFIG: dict = {
             ".pdf", ".zip", ".woff", ".woff2", ".ttf", ".otf", ".eot",
             ".mp4", ".mp3", ".webm", ".avi", ".mov",
         ],
+        "exclude_paths": [],
     },
     "scoring": {
         "new_external_domain": 50,

scanner/crawler.py

@@ -52,7 +52,12 @@ def normalize_url(url: str) -> str | None:
         return None
 
 
-def should_crawl(url: str, base_netloc: str, skip_extensions: list[str]) -> bool:
+def should_crawl(
+    url: str,
+    base_netloc: str,
+    skip_extensions: list[str],
+    exclude_paths: tuple | list = (),
+) -> bool:
     """Return True if url should be followed during a crawl."""
     p = urlparse(url)
     if p.netloc != base_netloc:
@@ -61,6 +66,8 @@ def should_crawl(url: str, base_netloc: str, skip_extensions: list[str]) -> bool
     for ext in skip_extensions:
         if path_lower.endswith(ext):
             return False
+    if any(p.path.startswith(ep) for ep in exclude_paths):
+        return False
     # Skip URLs whose query carries calendar/session/tracking noise
     if p.query and any(_is_noise_param(k) for k in parse_qs(p.query)):
         return False
@@ -74,6 +81,7 @@ class Crawler:
         self.max_pages: int = cfg["crawl"]["max_pages"]
         self.delay: float = cfg["crawl"]["delay_seconds"]
         self.skip_ext: list[str] = cfg["crawl"]["skip_extensions"]
+        self.exclude_paths: list[str] = cfg["crawl"].get("exclude_paths", [])
         self.timeout: int = cfg["request_timeout"]
         self.headers = {"User-Agent": cfg["user_agent"]}
         self.session = requests.Session()
@@ -164,7 +172,7 @@ class Crawler:
             for tag in soup.find_all("a", href=True):
                 href = tag["href"].strip()
                 abs_url = normalize_url(urljoin(base_url, href))
-                if abs_url and should_crawl(abs_url, self.base_netloc, self.skip_ext):
+                if abs_url and should_crawl(abs_url, self.base_netloc, self.skip_ext, self.exclude_paths):
                     result.append(abs_url)
         except Exception as exc:
             logger.debug("Link extraction failed on %s: %s", base_url, exc)

scanner/differ.py

@@ -12,6 +12,14 @@ from urllib.parse import urlparse
 # Normalisation
 # ---------------------------------------------------------------------------
 
+def _is_excluded(url: str, exclude_paths: list[str]) -> bool:
+    """Return True if the URL's path starts with any of the configured exclude prefixes."""
+    if not exclude_paths:
+        return False
+    path = urlparse(url).path
+    return any(path.startswith(ep) for ep in exclude_paths)
+
+
 def normalize_text(text: str, cfg: dict) -> str:
     """Apply noise-reduction before diffing."""
     norm_cfg = cfg.get("normalization", {})
@@ -103,6 +111,11 @@ def compare_snapshots(baseline_pages: dict, current_pages: dict, cfg: dict) -> d
       page_diffs (list of per-page change records),
       new_pages_analysis (freshly seen pages checked for immediate issues)
     """
+    excl = cfg.get("crawl", {}).get("exclude_paths", [])
+    if excl:
+        baseline_pages = {u: p for u, p in baseline_pages.items() if not _is_excluded(u, excl)}
+        current_pages  = {u: p for u, p in current_pages.items()  if not _is_excluded(u, excl)}
+
     b_urls = set(baseline_pages)
     c_urls = set(current_pages)