feat: exclude_paths — skip dynamic areas (RSS feeds, news archives) from scan
Adds crawl.exclude_paths config option (list of URL path prefixes).
Matching URLs are excluded at two levels:
- Crawler: not fetched at all (saves HTTP requests)
- Differ: filtered from both baseline and current before comparison,
so legacy baseline entries under excluded paths don't appear as
"missing URLs" after the option is added retroactively
Example config:
crawl:
exclude_paths: ["/rss/", "/feed/", "/news/archiv/"]
8 new tests (176 total).
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
parent
b01344dcd8
commit
3a44cfb515
6 changed files with 93 additions and 2 deletions
|
|
@ -71,3 +71,25 @@ class TestShouldCrawl:
|
|||
|
||||
def test_utm_query_param_rejected(self):
|
||||
assert not should_crawl("https://example.com/?utm_source=x", "example.com", SKIP_EXT)
|
||||
|
||||
def test_excluded_path_prefix_rejected(self):
|
||||
assert not should_crawl(
|
||||
"https://example.com/rss/feed.xml", "example.com", SKIP_EXT,
|
||||
exclude_paths=["/rss/"],
|
||||
)
|
||||
|
||||
def test_excluded_path_subpage_rejected(self):
|
||||
assert not should_crawl(
|
||||
"https://example.com/news/archiv/2024/artikel", "example.com", SKIP_EXT,
|
||||
exclude_paths=["/news/archiv/"],
|
||||
)
|
||||
|
||||
def test_similar_path_not_excluded(self):
|
||||
"""'/rss-link/' must not be excluded when only '/rss/' is configured."""
|
||||
assert should_crawl(
|
||||
"https://example.com/rss-link/", "example.com", SKIP_EXT,
|
||||
exclude_paths=["/rss/"],
|
||||
)
|
||||
|
||||
def test_no_exclude_paths_unchanged(self):
|
||||
assert should_crawl("https://example.com/page/", "example.com", SKIP_EXT, exclude_paths=[])
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue