68243a97b9
Echte Bugs: - extractor: JSON-LD-Crash bei Nicht-Dict-Items (isinstance-Guard) - baseline/__main__: Crawl-Fehler ins Snapshot-Manifest -> erscheinen im Report - __main__: Whitelist-/Header-/Webshell-Checks nur noch auf status==200 - crawler: Noise-Param-Regex auf Voll-Key (view=/value= nicht mehr verworfen) - differ/__main__: unerwartetes JSON-LD via eigenem Kanal, auch auf unveraenderten Seiten erkannt, kein Re-Alert auf bereits genehmigte Typen Aufgeraeumt: - checker: check_internal_paths, find_broken_pages, canonical_is_hijacked, check_jsonld_types entfernt (nicht verdrahtet) - allowed_paths.yaml-Logik und tote Scoring-Keys entfernt - tote Imports entfernt Neuer aktiver Schutz: - Webshell-/Backdoor-Dateinamen-Check verdrahtet (suspicious_filename: 60). Regex wortgrenzen-verankert gegen Fehlalarm auf PSEMailerAntispam.js Effizienz/Struktur: - crawler nutzt requests.Session (keep-alive) - Cache-Buster-Normalisierung an einer Stelle (Extraktion) -> stabile Snapshots, differ wieder reiner Set-Diff Tests: 111 gruen (neu: test_checker.py + Regressionstests) CLAUDE.md aktualisiert Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
73 lines
2.7 KiB
Python
73 lines
2.7 KiB
Python
"""Tests for crawler URL normalization and crawl filtering."""
|
|
import pytest
|
|
from scanner.crawler import normalize_url, should_crawl
|
|
|
|
SKIP_EXT = [".jpg", ".jpeg", ".png", ".pdf", ".zip", ".css", ".js"]
|
|
|
|
|
|
class TestNormalizeUrl:
|
|
def test_strips_fragment(self):
|
|
assert normalize_url("https://example.com/page#section") == "https://example.com/page"
|
|
|
|
def test_lowercase_host(self):
|
|
assert normalize_url("https://EXAMPLE.COM/path") == "https://example.com/path"
|
|
|
|
def test_non_http_returns_none(self):
|
|
assert normalize_url("javascript:void(0)") is None
|
|
assert normalize_url("mailto:foo@bar.com") is None
|
|
assert normalize_url("ftp://example.com") is None
|
|
|
|
def test_sorts_query_params(self):
|
|
a = normalize_url("https://example.com/?z=1&a=2")
|
|
b = normalize_url("https://example.com/?a=2&z=1")
|
|
assert a == b
|
|
|
|
def test_strips_tracking_params(self):
|
|
url = normalize_url("https://example.com/?utm_source=google&page=1")
|
|
assert "utm_source" not in url
|
|
assert "page=1" in url
|
|
|
|
def test_strips_session_params(self):
|
|
url = normalize_url("https://example.com/?sid=abc123&id=5")
|
|
assert "sid" not in url
|
|
assert "id=5" in url
|
|
|
|
def test_strips_cache_buster_v(self):
|
|
url = normalize_url("https://example.com/?v=1781223996&id=5")
|
|
assert "v=1781223996" not in url
|
|
assert "id=5" in url
|
|
|
|
def test_keeps_non_noise_v_prefixed_param(self):
|
|
# 'view' must NOT be dropped just because it starts with 'v'
|
|
url = normalize_url("https://example.com/?view=gallery")
|
|
assert "view=gallery" in url
|
|
|
|
def test_adds_default_path(self):
|
|
result = normalize_url("https://example.com")
|
|
assert result.endswith("/") or "example.com" in result
|
|
|
|
def test_empty_string_returns_none(self):
|
|
assert normalize_url("") is None
|
|
|
|
|
|
class TestShouldCrawl:
|
|
def test_same_host_plain_page(self):
|
|
assert should_crawl("https://example.com/about", "example.com", SKIP_EXT)
|
|
|
|
def test_different_host_rejected(self):
|
|
assert not should_crawl("https://other.com/page", "example.com", SKIP_EXT)
|
|
|
|
def test_image_rejected(self):
|
|
assert not should_crawl("https://example.com/photo.jpg", "example.com", SKIP_EXT)
|
|
|
|
def test_pdf_rejected(self):
|
|
assert not should_crawl("https://example.com/doc.pdf", "example.com", SKIP_EXT)
|
|
|
|
def test_html_accepted(self):
|
|
assert should_crawl("https://example.com/page.html", "example.com", SKIP_EXT)
|
|
|
|
def test_trailing_slash_accepted(self):
|
|
assert should_crawl("https://example.com/section/", "example.com", SKIP_EXT)
|
|
|
|
def test_utm_query_param_rejected(self):
|
|
assert not should_crawl("https://example.com/?utm_source=x", "example.com", SKIP_EXT)
|