68243a97b9
Echte Bugs: - extractor: JSON-LD-Crash bei Nicht-Dict-Items (isinstance-Guard) - baseline/__main__: Crawl-Fehler ins Snapshot-Manifest -> erscheinen im Report - __main__: Whitelist-/Header-/Webshell-Checks nur noch auf status==200 - crawler: Noise-Param-Regex auf Voll-Key (view=/value= nicht mehr verworfen) - differ/__main__: unerwartetes JSON-LD via eigenem Kanal, auch auf unveraenderten Seiten erkannt, kein Re-Alert auf bereits genehmigte Typen Aufgeraeumt: - checker: check_internal_paths, find_broken_pages, canonical_is_hijacked, check_jsonld_types entfernt (nicht verdrahtet) - allowed_paths.yaml-Logik und tote Scoring-Keys entfernt - tote Imports entfernt Neuer aktiver Schutz: - Webshell-/Backdoor-Dateinamen-Check verdrahtet (suspicious_filename: 60). Regex wortgrenzen-verankert gegen Fehlalarm auf PSEMailerAntispam.js Effizienz/Struktur: - crawler nutzt requests.Session (keep-alive) - Cache-Buster-Normalisierung an einer Stelle (Extraktion) -> stabile Snapshots, differ wieder reiner Set-Diff Tests: 111 gruen (neu: test_checker.py + Regressionstests) CLAUDE.md aktualisiert Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
111 lines
3.9 KiB
Python
111 lines
3.9 KiB
Python
"""
|
|
Additional integrity checks beyond baseline diff:
|
|
- link whitelist enforcement
|
|
- security-header audit
|
|
- canonical hijack detection
|
|
- suspicious URL filenames (from v4 heritage)
|
|
"""
|
|
import re
|
|
from pathlib import Path
|
|
from urllib.parse import urlparse
|
|
|
|
import yaml
|
|
|
|
# Webshell / backdoor filename markers. Word boundaries are essential:
|
|
# without them "mailer" matches the legitimate "PSEMailerAntispam.js" and
|
|
# "gate" matches "navigate", producing false positives on every scan.
|
|
_SUSPICIOUS_FILENAME_RE = re.compile(
|
|
r"\b(shell|backdoor|webshell|c99|r57|wso|b374k|cmd|reverse_shell|bindshell)\b",
|
|
re.I,
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Whitelist checks
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def load_whitelist(path: str | Path) -> set[str]:
|
|
"""Load a YAML list file into a set. Returns empty set if missing."""
|
|
p = Path(path)
|
|
if not p.exists():
|
|
return set()
|
|
data = yaml.safe_load(p.read_text(encoding="utf-8")) or []
|
|
if isinstance(data, list):
|
|
return {str(s).strip() for s in data if s}
|
|
return set()
|
|
|
|
|
|
def check_links_against_whitelist(
|
|
page: dict,
|
|
allowed_external_domains: set[str],
|
|
base_netloc: str,
|
|
) -> list[dict]:
|
|
"""
|
|
Return violations: external links to domains not in allowed_external_domains.
|
|
Also flags noscript links and comment links to external domains.
|
|
"""
|
|
violations: list[dict] = []
|
|
|
|
for link_type, links in page.get("links", {}).items():
|
|
if not isinstance(links, list):
|
|
continue
|
|
for link in links:
|
|
if not isinstance(link, dict):
|
|
continue
|
|
if link.get("class") != "external":
|
|
continue
|
|
domain = urlparse(link.get("url", "")).netloc
|
|
if domain and domain not in allowed_external_domains:
|
|
violations.append({
|
|
"type": "unlisted_external_domain",
|
|
"link_type": link_type,
|
|
"url": link.get("url", ""),
|
|
"domain": domain,
|
|
})
|
|
|
|
# Comment links
|
|
for href in page.get("comment_links", []):
|
|
domain = urlparse(href).netloc
|
|
if domain and domain != base_netloc and domain not in allowed_external_domains:
|
|
violations.append({
|
|
"type": "comment_link_to_unlisted_domain",
|
|
"link_type": "comment",
|
|
"url": href,
|
|
"domain": domain,
|
|
})
|
|
|
|
# noscript links
|
|
for hc in page.get("hidden_content", []):
|
|
if hc.get("type") == "noscript_links":
|
|
for href in hc.get("links", []):
|
|
domain = urlparse(href).netloc
|
|
if domain and domain != base_netloc and domain not in allowed_external_domains:
|
|
violations.append({
|
|
"type": "noscript_link_to_unlisted_domain",
|
|
"link_type": "noscript",
|
|
"url": href,
|
|
"domain": domain,
|
|
})
|
|
|
|
return violations
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Security headers
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def check_security_headers(response_headers: dict, required: list[str]) -> list[str]:
|
|
"""Return list of missing recommended security headers."""
|
|
# Header lookup is case-insensitive
|
|
present = {k.lower() for k in response_headers}
|
|
return [h for h in required if h.lower() not in present]
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Suspicious filenames
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def is_suspicious_url(url: str) -> bool:
|
|
"""Flag URLs with names typical for webshells / backdoors."""
|
|
filename = urlparse(url).path.rsplit("/", 1)[-1]
|
|
return bool(_SUSPICIOUS_FILENAME_RE.search(filename))
|