dschlueter/integrity_scanner_fuer_statische_Webseiten
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

feat: add local server fallback to AI analyzer, include_paths support, config_path in reports

Browse source 
- ai_analyzer: OpenRouter primary + localhost:8001/:8002 as dev fallback
- crawler: support include_paths for targeted crawling
- __main__: pass config_path through report pipeline
- tests: 266 tests passing
This commit is contained in:
Dieter Schlüter 2026-06-14 19:02:17 +02:00
commit 013cf794bc
13 changed files with 479 additions and 60 deletions
Download patch file Download diff file Expand all files Collapse all files

18
scanner/crawler.py
View file

@ -63,6 +63,7 @@ def should_crawl(
base_netloc: str,
skip_extensions: list[str],
exclude_paths: tuple | list = (),
include_paths: tuple | list = (),
) -> bool:
"""Return True if url should be followed during a crawl."""
p = urlparse(url)
@ -74,6 +75,8 @@ def should_crawl(
return False
if any(p.path.startswith(ep) for ep in exclude_paths):
return False
if include_paths and not any(p.path.startswith(ip) for ip in include_paths):
return False
# Skip URLs whose query carries calendar/session/tracking noise
if p.query and any(_is_noise_param(k) for k in parse_qs(p.query)):
return False
@ -88,6 +91,7 @@ class Crawler:
self.delay: float = cfg["crawl"]["delay_seconds"]
self.skip_ext: list[str] = cfg["crawl"]["skip_extensions"]
self.exclude_paths: list[str] = cfg["crawl"].get("exclude_paths", [])
self.include_paths: list[str] = cfg["crawl"].get("include_paths", [])
self.timeout: int = cfg["request_timeout"]
self.headers = {"User-Agent": cfg["user_agent"]}
self.session = requests.Session()
@ -104,7 +108,13 @@ class Crawler:
- errors: list of {url, error}
"""
seen: set[str] = set()
queue: list[str] = [normalize_url(self.target + "/") or self.target + "/"]
if self.include_paths:
queue: list[str] = [
normalize_url(self.target + p) or (self.target + p)
for p in self.include_paths
]
else:
queue: list[str] = [normalize_url(self.target + "/") or self.target + "/"]
pages: list[dict] = []
errors: list[dict] = []
@ -252,7 +262,8 @@ class Crawler:
raw = loc.text.strip() if loc.text else ""
url = normalize_url(raw)
if url and should_crawl(
url, self.base_netloc, self.skip_ext, self.exclude_paths
url, self.base_netloc, self.skip_ext, self.exclude_paths,
self.include_paths,
) and url not in seen and url not in queue:
queue.append(url)
added += 1
@ -267,7 +278,8 @@ class Crawler:
for tag in soup.find_all("a", href=True):
href = tag["href"].strip()
abs_url = normalize_url(urljoin(base_url, href))
if abs_url and should_crawl(abs_url, self.base_netloc, self.skip_ext, self.exclude_paths):
if abs_url and should_crawl(abs_url, self.base_netloc, self.skip_ext,
self.exclude_paths, self.include_paths):
result.append(abs_url)
except Exception as exc:
logger.debug("Link extraction failed on %s: %s", base_url, exc)