diff --git a/config.yaml b/config.yaml index f8d9aef..72f2a40 100644 --- a/config.yaml +++ b/config.yaml @@ -14,6 +14,12 @@ crawl: max_pages: 200 delay_seconds: 1.0 respect_robots_txt: false + # Pfade, die vom Crawl und vom Vergleich komplett ausgeschlossen werden. + # Nützlich für Bereiche mit gewollt häufig wechselnden Inhalten (RSS-Feeds, News-Archive). + # Beispiel: ["/rss/", "/feed/", "/news/archiv/"] + # Alle URLs, deren Pfad mit einem dieser Einträge beginnt, werden ignoriert. + exclude_paths: [] + skip_extensions: - ".jpg" - ".jpeg" diff --git a/scanner/config.py b/scanner/config.py index c31cb3b..c45c340 100644 --- a/scanner/config.py +++ b/scanner/config.py @@ -19,6 +19,7 @@ DEFAULT_CONFIG: dict = { ".pdf", ".zip", ".woff", ".woff2", ".ttf", ".otf", ".eot", ".mp4", ".mp3", ".webm", ".avi", ".mov", ], + "exclude_paths": [], }, "scoring": { "new_external_domain": 50, diff --git a/scanner/crawler.py b/scanner/crawler.py index 813186b..1b77b20 100644 --- a/scanner/crawler.py +++ b/scanner/crawler.py @@ -52,7 +52,12 @@ def normalize_url(url: str) -> str | None: return None -def should_crawl(url: str, base_netloc: str, skip_extensions: list[str]) -> bool: +def should_crawl( + url: str, + base_netloc: str, + skip_extensions: list[str], + exclude_paths: tuple | list = (), +) -> bool: """Return True if url should be followed during a crawl.""" p = urlparse(url) if p.netloc != base_netloc: @@ -61,6 +66,8 @@ def should_crawl(url: str, base_netloc: str, skip_extensions: list[str]) -> bool for ext in skip_extensions: if path_lower.endswith(ext): return False + if any(p.path.startswith(ep) for ep in exclude_paths): + return False # Skip URLs whose query carries calendar/session/tracking noise if p.query and any(_is_noise_param(k) for k in parse_qs(p.query)): return False @@ -74,6 +81,7 @@ class Crawler: self.max_pages: int = cfg["crawl"]["max_pages"] self.delay: float = cfg["crawl"]["delay_seconds"] self.skip_ext: list[str] = cfg["crawl"]["skip_extensions"] + self.exclude_paths: list[str] = cfg["crawl"].get("exclude_paths", []) self.timeout: int = cfg["request_timeout"] self.headers = {"User-Agent": cfg["user_agent"]} self.session = requests.Session() @@ -164,7 +172,7 @@ class Crawler: for tag in soup.find_all("a", href=True): href = tag["href"].strip() abs_url = normalize_url(urljoin(base_url, href)) - if abs_url and should_crawl(abs_url, self.base_netloc, self.skip_ext): + if abs_url and should_crawl(abs_url, self.base_netloc, self.skip_ext, self.exclude_paths): result.append(abs_url) except Exception as exc: logger.debug("Link extraction failed on %s: %s", base_url, exc) diff --git a/scanner/differ.py b/scanner/differ.py index 1ac43e2..904db8a 100644 --- a/scanner/differ.py +++ b/scanner/differ.py @@ -12,6 +12,14 @@ from urllib.parse import urlparse # Normalisation # --------------------------------------------------------------------------- +def _is_excluded(url: str, exclude_paths: list[str]) -> bool: + """Return True if the URL's path starts with any of the configured exclude prefixes.""" + if not exclude_paths: + return False + path = urlparse(url).path + return any(path.startswith(ep) for ep in exclude_paths) + + def normalize_text(text: str, cfg: dict) -> str: """Apply noise-reduction before diffing.""" norm_cfg = cfg.get("normalization", {}) @@ -103,6 +111,11 @@ def compare_snapshots(baseline_pages: dict, current_pages: dict, cfg: dict) -> d page_diffs (list of per-page change records), new_pages_analysis (freshly seen pages checked for immediate issues) """ + excl = cfg.get("crawl", {}).get("exclude_paths", []) + if excl: + baseline_pages = {u: p for u, p in baseline_pages.items() if not _is_excluded(u, excl)} + current_pages = {u: p for u, p in current_pages.items() if not _is_excluded(u, excl)} + b_urls = set(baseline_pages) c_urls = set(current_pages) diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 56f13d4..a59720c 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -71,3 +71,25 @@ class TestShouldCrawl: def test_utm_query_param_rejected(self): assert not should_crawl("https://example.com/?utm_source=x", "example.com", SKIP_EXT) + + def test_excluded_path_prefix_rejected(self): + assert not should_crawl( + "https://example.com/rss/feed.xml", "example.com", SKIP_EXT, + exclude_paths=["/rss/"], + ) + + def test_excluded_path_subpage_rejected(self): + assert not should_crawl( + "https://example.com/news/archiv/2024/artikel", "example.com", SKIP_EXT, + exclude_paths=["/news/archiv/"], + ) + + def test_similar_path_not_excluded(self): + """'/rss-link/' must not be excluded when only '/rss/' is configured.""" + assert should_crawl( + "https://example.com/rss-link/", "example.com", SKIP_EXT, + exclude_paths=["/rss/"], + ) + + def test_no_exclude_paths_unchanged(self): + assert should_crawl("https://example.com/page/", "example.com", SKIP_EXT, exclude_paths=[]) diff --git a/tests/test_differ.py b/tests/test_differ.py index 63a949f..98f0155 100644 --- a/tests/test_differ.py +++ b/tests/test_differ.py @@ -279,6 +279,47 @@ class TestBrokenLinks: assert result["level"] == "green" +class TestExcludePaths: + def _page(self, url, status=200): + return { + "url": url, "status": status, "text": "x", + "links": {"a": [], "script": [], "iframe": [], "link_rel": [], "form": [], + "meta_refresh": [], "canonical": None}, + "metadata": {"title": "T", "h1": [], "h2": []}, + "hidden_content": [], "inline_scripts": [], "jsonld": [], "comment_links": [], + } + + def _cfg(self, exclude): + return {**BASE_CFG, "crawl": {"exclude_paths": exclude}} + + def test_new_url_under_excluded_path_not_reported(self): + baseline = {"https://b.info/": self._page("https://b.info/")} + current = { + "https://b.info/": self._page("https://b.info/"), + "https://b.info/rss/feed.xml": self._page("https://b.info/rss/feed.xml"), + } + result = compare_snapshots(baseline, current, self._cfg(["/rss/"])) + assert "https://b.info/rss/feed.xml" not in result["new_internal_urls"] + + def test_missing_url_under_excluded_path_not_reported(self): + baseline = { + "https://b.info/": self._page("https://b.info/"), + "https://b.info/rss/feed.xml": self._page("https://b.info/rss/feed.xml"), + } + current = {"https://b.info/": self._page("https://b.info/")} + result = compare_snapshots(baseline, current, self._cfg(["/rss/"])) + assert "https://b.info/rss/feed.xml" not in result["missing_internal_urls"] + + def test_non_excluded_path_still_reported(self): + baseline = {"https://b.info/": self._page("https://b.info/")} + current = { + "https://b.info/": self._page("https://b.info/"), + "https://b.info/news/artikel": self._page("https://b.info/news/artikel"), + } + result = compare_snapshots(baseline, current, self._cfg(["/rss/"])) + assert "https://b.info/news/artikel" in result["new_internal_urls"] + + class TestScoreDiff: def test_green_for_no_changes(self): result = score_diff({