feat: exclude_paths — skip dynamic areas (RSS feeds, news archives) from scan

Adds crawl.exclude_paths config option (list of URL path prefixes).
Matching URLs are excluded at two levels:
- Crawler: not fetched at all (saves HTTP requests)
- Differ: filtered from both baseline and current before comparison,
  so legacy baseline entries under excluded paths don't appear as
  "missing URLs" after the option is added retroactively

Example config:
  crawl:
    exclude_paths: ["/rss/", "/feed/", "/news/archiv/"]

8 new tests (176 total).

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

config.yaml

@@ -14,6 +14,12 @@ crawl:
   max_pages: 200
   delay_seconds: 1.0
   respect_robots_txt: false
+  # Pfade, die vom Crawl und vom Vergleich komplett ausgeschlossen werden.
+  # Nützlich für Bereiche mit gewollt häufig wechselnden Inhalten (RSS-Feeds, News-Archive).
+  # Beispiel: ["/rss/", "/feed/", "/news/archiv/"]
+  # Alle URLs, deren Pfad mit einem dieser Einträge beginnt, werden ignoriert.
+  exclude_paths: []
+
   skip_extensions:
     - ".jpg"
     - ".jpeg"

scanner/config.py

@@ -19,6 +19,7 @@ DEFAULT_CONFIG: dict = {
             ".pdf", ".zip", ".woff", ".woff2", ".ttf", ".otf", ".eot",
             ".mp4", ".mp3", ".webm", ".avi", ".mov",
         ],
+        "exclude_paths": [],
     },
     "scoring": {
         "new_external_domain": 50,

scanner/crawler.py

@@ -52,7 +52,12 @@ def normalize_url(url: str) -> str | None:
         return None
 
 
-def should_crawl(url: str, base_netloc: str, skip_extensions: list[str]) -> bool:
+def should_crawl(
+    url: str,
+    base_netloc: str,
+    skip_extensions: list[str],
+    exclude_paths: tuple | list = (),
+) -> bool:
     """Return True if url should be followed during a crawl."""
     p = urlparse(url)
     if p.netloc != base_netloc:
@@ -61,6 +66,8 @@ def should_crawl(url: str, base_netloc: str, skip_extensions: list[str]) -> bool
     for ext in skip_extensions:
         if path_lower.endswith(ext):
             return False
+    if any(p.path.startswith(ep) for ep in exclude_paths):
+        return False
     # Skip URLs whose query carries calendar/session/tracking noise
     if p.query and any(_is_noise_param(k) for k in parse_qs(p.query)):
         return False
@@ -74,6 +81,7 @@ class Crawler:
         self.max_pages: int = cfg["crawl"]["max_pages"]
         self.delay: float = cfg["crawl"]["delay_seconds"]
         self.skip_ext: list[str] = cfg["crawl"]["skip_extensions"]
+        self.exclude_paths: list[str] = cfg["crawl"].get("exclude_paths", [])
         self.timeout: int = cfg["request_timeout"]
         self.headers = {"User-Agent": cfg["user_agent"]}
         self.session = requests.Session()
@@ -164,7 +172,7 @@ class Crawler:
             for tag in soup.find_all("a", href=True):
                 href = tag["href"].strip()
                 abs_url = normalize_url(urljoin(base_url, href))
-                if abs_url and should_crawl(abs_url, self.base_netloc, self.skip_ext):
+                if abs_url and should_crawl(abs_url, self.base_netloc, self.skip_ext, self.exclude_paths):
                     result.append(abs_url)
         except Exception as exc:
             logger.debug("Link extraction failed on %s: %s", base_url, exc)

scanner/differ.py

@@ -12,6 +12,14 @@ from urllib.parse import urlparse
 # Normalisation
 # ---------------------------------------------------------------------------
 
+def _is_excluded(url: str, exclude_paths: list[str]) -> bool:
+    """Return True if the URL's path starts with any of the configured exclude prefixes."""
+    if not exclude_paths:
+        return False
+    path = urlparse(url).path
+    return any(path.startswith(ep) for ep in exclude_paths)
+
+
 def normalize_text(text: str, cfg: dict) -> str:
     """Apply noise-reduction before diffing."""
     norm_cfg = cfg.get("normalization", {})
@@ -103,6 +111,11 @@ def compare_snapshots(baseline_pages: dict, current_pages: dict, cfg: dict) -> d
       page_diffs (list of per-page change records),
       new_pages_analysis (freshly seen pages checked for immediate issues)
     """
+    excl = cfg.get("crawl", {}).get("exclude_paths", [])
+    if excl:
+        baseline_pages = {u: p for u, p in baseline_pages.items() if not _is_excluded(u, excl)}
+        current_pages  = {u: p for u, p in current_pages.items()  if not _is_excluded(u, excl)}
+
     b_urls = set(baseline_pages)
     c_urls = set(current_pages)
 

tests/test_crawler.py

@@ -71,3 +71,25 @@ class TestShouldCrawl:
 
     def test_utm_query_param_rejected(self):
         assert not should_crawl("https://example.com/?utm_source=x", "example.com", SKIP_EXT)
+
+    def test_excluded_path_prefix_rejected(self):
+        assert not should_crawl(
+            "https://example.com/rss/feed.xml", "example.com", SKIP_EXT,
+            exclude_paths=["/rss/"],
+        )
+
+    def test_excluded_path_subpage_rejected(self):
+        assert not should_crawl(
+            "https://example.com/news/archiv/2024/artikel", "example.com", SKIP_EXT,
+            exclude_paths=["/news/archiv/"],
+        )
+
+    def test_similar_path_not_excluded(self):
+        """'/rss-link/' must not be excluded when only '/rss/' is configured."""
+        assert should_crawl(
+            "https://example.com/rss-link/", "example.com", SKIP_EXT,
+            exclude_paths=["/rss/"],
+        )
+
+    def test_no_exclude_paths_unchanged(self):
+        assert should_crawl("https://example.com/page/", "example.com", SKIP_EXT, exclude_paths=[])

tests/test_differ.py

@@ -279,6 +279,47 @@ class TestBrokenLinks:
         assert result["level"] == "green"
 
 
+class TestExcludePaths:
+    def _page(self, url, status=200):
+        return {
+            "url": url, "status": status, "text": "x",
+            "links": {"a": [], "script": [], "iframe": [], "link_rel": [], "form": [],
+                      "meta_refresh": [], "canonical": None},
+            "metadata": {"title": "T", "h1": [], "h2": []},
+            "hidden_content": [], "inline_scripts": [], "jsonld": [], "comment_links": [],
+        }
+
+    def _cfg(self, exclude):
+        return {**BASE_CFG, "crawl": {"exclude_paths": exclude}}
+
+    def test_new_url_under_excluded_path_not_reported(self):
+        baseline = {"https://b.info/": self._page("https://b.info/")}
+        current = {
+            "https://b.info/": self._page("https://b.info/"),
+            "https://b.info/rss/feed.xml": self._page("https://b.info/rss/feed.xml"),
+        }
+        result = compare_snapshots(baseline, current, self._cfg(["/rss/"]))
+        assert "https://b.info/rss/feed.xml" not in result["new_internal_urls"]
+
+    def test_missing_url_under_excluded_path_not_reported(self):
+        baseline = {
+            "https://b.info/": self._page("https://b.info/"),
+            "https://b.info/rss/feed.xml": self._page("https://b.info/rss/feed.xml"),
+        }
+        current = {"https://b.info/": self._page("https://b.info/")}
+        result = compare_snapshots(baseline, current, self._cfg(["/rss/"]))
+        assert "https://b.info/rss/feed.xml" not in result["missing_internal_urls"]
+
+    def test_non_excluded_path_still_reported(self):
+        baseline = {"https://b.info/": self._page("https://b.info/")}
+        current = {
+            "https://b.info/": self._page("https://b.info/"),
+            "https://b.info/news/artikel": self._page("https://b.info/news/artikel"),
+        }
+        result = compare_snapshots(baseline, current, self._cfg(["/rss/"]))
+        assert "https://b.info/news/artikel" in result["new_internal_urls"]
+
+
 class TestScoreDiff:
     def test_green_for_no_changes(self):
         result = score_diff({