fix: M1 exclude_paths '/' jamulix, M2 seed-should_crawl filter, M3 alert dedup

- M1: Removed '/' from jamulix.de exclude_paths (blockaded ALL paths via startswith)
- M2: Seed URLs now pass through should_crawl() in crawler.py
- M3: Alert deduplication via last_alert.json (hash of reasons+level+score)
- Added 11 tests for alert deduplication (277 total, all green)
This commit is contained in:
Dieter Schlüter 2026-06-14 21:12:36 +02:00
commit 248788e282
4 changed files with 190 additions and 4 deletions

View file

@ -112,9 +112,18 @@ class Crawler:
queue: list[str] = [
normalize_url(self.target + p) or (self.target + p)
for p in self.include_paths
if should_crawl(
normalize_url(self.target + p) or (self.target + p),
self.base_netloc, self.skip_ext,
self.exclude_paths, self.include_paths,
)
]
else:
queue: list[str] = [normalize_url(self.target + "/") or self.target + "/"]
seed = normalize_url(self.target + "/") or self.target + "/"
queue: list[str] = [seed] if should_crawl(
seed, self.base_netloc, self.skip_ext,
self.exclude_paths, self.include_paths,
) else []
pages: list[dict] = []
errors: list[dict] = []