feat: exclude_paths — skip dynamic areas (RSS feeds, news archives) from scan

Adds crawl.exclude_paths config option (list of URL path prefixes).
Matching URLs are excluded at two levels:
- Crawler: not fetched at all (saves HTTP requests)
- Differ: filtered from both baseline and current before comparison,
  so legacy baseline entries under excluded paths don't appear as
  "missing URLs" after the option is added retroactively

Example config:
  crawl:
    exclude_paths: ["/rss/", "/feed/", "/news/archiv/"]

8 new tests (176 total).

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

tests/test_crawler.py

@@ -71,3 +71,25 @@ class TestShouldCrawl:
 
     def test_utm_query_param_rejected(self):
         assert not should_crawl("https://example.com/?utm_source=x", "example.com", SKIP_EXT)
+
+    def test_excluded_path_prefix_rejected(self):
+        assert not should_crawl(
+            "https://example.com/rss/feed.xml", "example.com", SKIP_EXT,
+            exclude_paths=["/rss/"],
+        )
+
+    def test_excluded_path_subpage_rejected(self):
+        assert not should_crawl(
+            "https://example.com/news/archiv/2024/artikel", "example.com", SKIP_EXT,
+            exclude_paths=["/news/archiv/"],
+        )
+
+    def test_similar_path_not_excluded(self):
+        """'/rss-link/' must not be excluded when only '/rss/' is configured."""
+        assert should_crawl(
+            "https://example.com/rss-link/", "example.com", SKIP_EXT,
+            exclude_paths=["/rss/"],
+        )
+
+    def test_no_exclude_paths_unchanged(self):
+        assert should_crawl("https://example.com/page/", "example.com", SKIP_EXT, exclude_paths=[])

tests/test_differ.py

@@ -279,6 +279,47 @@ class TestBrokenLinks:
         assert result["level"] == "green"
 
 
+class TestExcludePaths:
+    def _page(self, url, status=200):
+        return {
+            "url": url, "status": status, "text": "x",
+            "links": {"a": [], "script": [], "iframe": [], "link_rel": [], "form": [],
+                      "meta_refresh": [], "canonical": None},
+            "metadata": {"title": "T", "h1": [], "h2": []},
+            "hidden_content": [], "inline_scripts": [], "jsonld": [], "comment_links": [],
+        }
+
+    def _cfg(self, exclude):
+        return {**BASE_CFG, "crawl": {"exclude_paths": exclude}}
+
+    def test_new_url_under_excluded_path_not_reported(self):
+        baseline = {"https://b.info/": self._page("https://b.info/")}
+        current = {
+            "https://b.info/": self._page("https://b.info/"),
+            "https://b.info/rss/feed.xml": self._page("https://b.info/rss/feed.xml"),
+        }
+        result = compare_snapshots(baseline, current, self._cfg(["/rss/"]))
+        assert "https://b.info/rss/feed.xml" not in result["new_internal_urls"]
+
+    def test_missing_url_under_excluded_path_not_reported(self):
+        baseline = {
+            "https://b.info/": self._page("https://b.info/"),
+            "https://b.info/rss/feed.xml": self._page("https://b.info/rss/feed.xml"),
+        }
+        current = {"https://b.info/": self._page("https://b.info/")}
+        result = compare_snapshots(baseline, current, self._cfg(["/rss/"]))
+        assert "https://b.info/rss/feed.xml" not in result["missing_internal_urls"]
+
+    def test_non_excluded_path_still_reported(self):
+        baseline = {"https://b.info/": self._page("https://b.info/")}
+        current = {
+            "https://b.info/": self._page("https://b.info/"),
+            "https://b.info/news/artikel": self._page("https://b.info/news/artikel"),
+        }
+        result = compare_snapshots(baseline, current, self._cfg(["/rss/"]))
+        assert "https://b.info/news/artikel" in result["new_internal_urls"]
+
+
 class TestScoreDiff:
     def test_green_for_no_changes(self):
         result = score_diff({