feat: externe Links auf Erreichbarkeit prüfen (check-ext-links)

Neuer Befehl `python -m scanner check-ext-links`:
- Liest alle externen <a>-Links aus dem letzten Snapshot
- Prüft jeden per HEAD-Request (Fallback auf GET bei HTTP 405)
- Meldet 4xx/5xx und Verbindungsfehler mit Quelladressen
- Report unter reports/<ts>_ext-links/
- Exit-Code 1 bei kaputten Links, 0 wenn alle erreichbar

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Your Name 2026-06-12 04:14:50 +02:00
commit ecd01abf98
3 changed files with 263 additions and 1 deletions

View file

@ -2,15 +2,20 @@
Additional integrity checks beyond baseline diff:
- link whitelist enforcement
- security-header audit
- canonical hijack detection
- suspicious URL filenames (from v4 heritage)
- external link reachability check
"""
import logging
import re
import time
from pathlib import Path
from urllib.parse import urlparse
import requests
import yaml
logger = logging.getLogger(__name__)
# Webshell / backdoor filename markers. Word boundaries are essential:
# without them "mailer" matches the legitimate "PSEMailerAntispam.js" and
# "gate" matches "navigate", producing false positives on every scan.
@ -105,6 +110,43 @@ def check_security_headers(response_headers: dict, required: list[str]) -> list[
# Suspicious filenames
# ---------------------------------------------------------------------------
# ---------------------------------------------------------------------------
# External link reachability
# ---------------------------------------------------------------------------
def check_external_links(
urls: list[str],
timeout: int = 10,
delay: float = 0.3,
) -> dict[str, dict]:
"""
Check a list of URLs for reachability via HEAD (fallback: GET).
Returns {url: {"status": int|None, "final_url": str|None, "error": str|None}}.
"""
results: dict[str, dict] = {}
session = requests.Session()
session.headers["User-Agent"] = "integrity-scanner/1.0 (external-link-check)"
try:
for i, url in enumerate(urls):
if i > 0 and delay > 0:
time.sleep(delay)
logger.debug("Checking external link: %s", url)
try:
resp = session.head(url, timeout=timeout, allow_redirects=True)
if resp.status_code == 405:
resp = session.get(url, timeout=timeout, allow_redirects=True, stream=True)
resp.close()
results[url] = {"status": resp.status_code, "final_url": resp.url, "error": None}
except requests.exceptions.RequestException as exc:
results[url] = {"status": None, "final_url": None, "error": str(exc)}
finally:
session.close()
return results
def is_suspicious_url(url: str) -> bool:
"""Flag URLs with names typical for webshells / backdoors."""
filename = urlparse(url).path.rsplit("/", 1)[-1]