fix: M1 exclude_paths '/' jamulix, M2 seed-should_crawl filter, M3 alert dedup
- M1: Removed '/' from jamulix.de exclude_paths (blockaded ALL paths via startswith) - M2: Seed URLs now pass through should_crawl() in crawler.py - M3: Alert deduplication via last_alert.json (hash of reasons+level+score) - Added 11 tests for alert deduplication (277 total, all green)
This commit is contained in:
parent
2363c0dc9b
commit
248788e282
4 changed files with 190 additions and 4 deletions
|
|
@ -112,9 +112,18 @@ class Crawler:
|
|||
queue: list[str] = [
|
||||
normalize_url(self.target + p) or (self.target + p)
|
||||
for p in self.include_paths
|
||||
if should_crawl(
|
||||
normalize_url(self.target + p) or (self.target + p),
|
||||
self.base_netloc, self.skip_ext,
|
||||
self.exclude_paths, self.include_paths,
|
||||
)
|
||||
]
|
||||
else:
|
||||
queue: list[str] = [normalize_url(self.target + "/") or self.target + "/"]
|
||||
seed = normalize_url(self.target + "/") or self.target + "/"
|
||||
queue: list[str] = [seed] if should_crawl(
|
||||
seed, self.base_netloc, self.skip_ext,
|
||||
self.exclude_paths, self.include_paths,
|
||||
) else []
|
||||
pages: list[dict] = []
|
||||
errors: list[dict] = []
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue