63 lines
2.3 KiB
Python
63 lines
2.3 KiB
Python
|
"""Tests for crawler URL normalization and crawl filtering."""
|
||
|
|
import pytest
|
||
|
|
from scanner.crawler import normalize_url, should_crawl
|
||
|
|
|
||
|
|
SKIP_EXT = [".jpg", ".jpeg", ".png", ".pdf", ".zip", ".css", ".js"]
|
||
|
|
|
||
|
|
|
||
|
|
class TestNormalizeUrl:
|
||
|
|
def test_strips_fragment(self):
|
||
|
|
assert normalize_url("https://example.com/page#section") == "https://example.com/page"
|
||
|
|
|
||
|
|
def test_lowercase_host(self):
|
||
|
|
assert normalize_url("https://EXAMPLE.COM/path") == "https://example.com/path"
|
||
|
|
|
||
|
|
def test_non_http_returns_none(self):
|
||
|
|
assert normalize_url("javascript:void(0)") is None
|
||
|
|
assert normalize_url("mailto:foo@bar.com") is None
|
||
|
|
assert normalize_url("ftp://example.com") is None
|
||
|
|
|
||
|
|
def test_sorts_query_params(self):
|
||
|
|
a = normalize_url("https://example.com/?z=1&a=2")
|
||
|
|
b = normalize_url("https://example.com/?a=2&z=1")
|
||
|
|
assert a == b
|
||
|
|
|
||
|
|
def test_strips_tracking_params(self):
|
||
|
|
url = normalize_url("https://example.com/?utm_source=google&page=1")
|
||
|
|
assert "utm_source" not in url
|
||
|
|
assert "page=1" in url
|
||
|
|
|
||
|
|
def test_strips_session_params(self):
|
||
|
|
url = normalize_url("https://example.com/?sid=abc123&id=5")
|
||
|
|
assert "sid" not in url
|
||
|
|
assert "id=5" in url
|
||
|
|
|
||
|
|
def test_adds_default_path(self):
|
||
|
|
result = normalize_url("https://example.com")
|
||
|
|
assert result.endswith("/") or "example.com" in result
|
||
|
|
|
||
|
|
def test_empty_string_returns_none(self):
|
||
|
|
assert normalize_url("") is None
|
||
|
|
|
||
|
|
|
||
|
|
class TestShouldCrawl:
|
||
|
|
def test_same_host_plain_page(self):
|
||
|
|
assert should_crawl("https://example.com/about", "example.com", SKIP_EXT)
|
||
|
|
|
||
|
|
def test_different_host_rejected(self):
|
||
|
|
assert not should_crawl("https://other.com/page", "example.com", SKIP_EXT)
|
||
|
|
|
||
|
|
def test_image_rejected(self):
|
||
|
|
assert not should_crawl("https://example.com/photo.jpg", "example.com", SKIP_EXT)
|
||
|
|
|
||
|
|
def test_pdf_rejected(self):
|
||
|
|
assert not should_crawl("https://example.com/doc.pdf", "example.com", SKIP_EXT)
|
||
|
|
|
||
|
|
def test_html_accepted(self):
|
||
|
|
assert should_crawl("https://example.com/page.html", "example.com", SKIP_EXT)
|
||
|
|
|
||
|
|
def test_trailing_slash_accepted(self):
|
||
|
|
assert should_crawl("https://example.com/section/", "example.com", SKIP_EXT)
|
||
|
|
|
||
|
|
def test_utm_query_param_rejected(self):
|
||
|
|
assert not should_crawl("https://example.com/?utm_source=x", "example.com", SKIP_EXT)
|