integrity_scanner_fuer_statische_Webseiten/tests/test_crawler.py

"""Tests for crawler URL normalization and crawl filtering."""
import pytest
from scanner.crawler import normalize_url, should_crawl

SKIP_EXT = [".jpg", ".jpeg", ".png", ".pdf", ".zip", ".css", ".js"]


class TestNormalizeUrl:
    def test_strips_fragment(self):
        assert normalize_url("https://example.com/page#section") == "https://example.com/page"

    def test_lowercase_host(self):
        assert normalize_url("https://EXAMPLE.COM/path") == "https://example.com/path"

    def test_non_http_returns_none(self):
        assert normalize_url("javascript:void(0)") is None
        assert normalize_url("mailto:foo@bar.com") is None
        assert normalize_url("ftp://example.com") is None

    def test_sorts_query_params(self):
        a = normalize_url("https://example.com/?z=1&a=2")
        b = normalize_url("https://example.com/?a=2&z=1")
        assert a == b

    def test_strips_tracking_params(self):
        url = normalize_url("https://example.com/?utm_source=google&page=1")
        assert "utm_source" not in url
        assert "page=1" in url

    def test_strips_session_params(self):
        url = normalize_url("https://example.com/?sid=abc123&id=5")
        assert "sid" not in url
        assert "id=5" in url

    def test_strips_cache_buster_v(self):
        url = normalize_url("https://example.com/?v=1781223996&id=5")
        assert "v=1781223996" not in url
        assert "id=5" in url

    def test_keeps_non_noise_v_prefixed_param(self):
        # 'view' must NOT be dropped just because it starts with 'v'
        url = normalize_url("https://example.com/?view=gallery")
        assert "view=gallery" in url

    def test_adds_default_path(self):
        result = normalize_url("https://example.com")
        assert result.endswith("/") or "example.com" in result

    def test_empty_string_returns_none(self):
        assert normalize_url("") is None


class TestShouldCrawl:
    def test_same_host_plain_page(self):
        assert should_crawl("https://example.com/about", "example.com", SKIP_EXT)

    def test_different_host_rejected(self):
        assert not should_crawl("https://other.com/page", "example.com", SKIP_EXT)

    def test_image_rejected(self):
        assert not should_crawl("https://example.com/photo.jpg", "example.com", SKIP_EXT)

    def test_pdf_rejected(self):
        assert not should_crawl("https://example.com/doc.pdf", "example.com", SKIP_EXT)

    def test_html_accepted(self):
        assert should_crawl("https://example.com/page.html", "example.com", SKIP_EXT)

    def test_trailing_slash_accepted(self):
        assert should_crawl("https://example.com/section/", "example.com", SKIP_EXT)

    def test_utm_query_param_rejected(self):
        assert not should_crawl("https://example.com/?utm_source=x", "example.com", SKIP_EXT)

    def test_excluded_path_prefix_rejected(self):
        assert not should_crawl(
            "https://example.com/rss/feed.xml", "example.com", SKIP_EXT,
            exclude_paths=["/rss/"],
        )

    def test_excluded_path_subpage_rejected(self):
        assert not should_crawl(
            "https://example.com/news/archiv/2024/artikel", "example.com", SKIP_EXT,
            exclude_paths=["/news/archiv/"],
        )

    def test_similar_path_not_excluded(self):
        """'/rss-link/' must not be excluded when only '/rss/' is configured."""
        assert should_crawl(
            "https://example.com/rss-link/", "example.com", SKIP_EXT,
            exclude_paths=["/rss/"],
        )

    def test_no_exclude_paths_unchanged(self):
        assert should_crawl("https://example.com/page/", "example.com", SKIP_EXT, exclude_paths=[])