integrity_scanner_fuer_statische_Webseiten/scanner/checker.py

"""
Additional integrity checks beyond baseline diff:
- link whitelist enforcement
- security-header audit
- canonical hijack detection
- suspicious URL filenames (from v4 heritage)
"""
import re
from pathlib import Path
from urllib.parse import urlparse

import yaml

# Webshell / backdoor filename markers. Word boundaries are essential:
# without them "mailer" matches the legitimate "PSEMailerAntispam.js" and
# "gate" matches "navigate", producing false positives on every scan.
_SUSPICIOUS_FILENAME_RE = re.compile(
    r"\b(shell|backdoor|webshell|c99|r57|wso|b374k|cmd|reverse_shell|bindshell)\b",
    re.I,
)


# ---------------------------------------------------------------------------
# Whitelist checks
# ---------------------------------------------------------------------------

def load_whitelist(path: str | Path) -> set[str]:
    """Load a YAML list file into a set. Returns empty set if missing."""
    p = Path(path)
    if not p.exists():
        return set()
    data = yaml.safe_load(p.read_text(encoding="utf-8")) or []
    if isinstance(data, list):
        return {str(s).strip() for s in data if s}
    return set()


def check_links_against_whitelist(
    page: dict,
    allowed_external_domains: set[str],
    base_netloc: str,
) -> list[dict]:
    """
    Return violations: external links to domains not in allowed_external_domains.
    Also flags noscript links and comment links to external domains.
    """
    violations: list[dict] = []

    for link_type, links in page.get("links", {}).items():
        if not isinstance(links, list):
            continue
        for link in links:
            if not isinstance(link, dict):
                continue
            if link.get("class") != "external":
                continue
            domain = urlparse(link.get("url", "")).netloc
            if domain and domain not in allowed_external_domains:
                violations.append({
                    "type": "unlisted_external_domain",
                    "link_type": link_type,
                    "url": link.get("url", ""),
                    "domain": domain,
                })

    # Comment links
    for href in page.get("comment_links", []):
        domain = urlparse(href).netloc
        if domain and domain != base_netloc and domain not in allowed_external_domains:
            violations.append({
                "type": "comment_link_to_unlisted_domain",
                "link_type": "comment",
                "url": href,
                "domain": domain,
            })

    # noscript links
    for hc in page.get("hidden_content", []):
        if hc.get("type") == "noscript_links":
            for href in hc.get("links", []):
                domain = urlparse(href).netloc
                if domain and domain != base_netloc and domain not in allowed_external_domains:
                    violations.append({
                        "type": "noscript_link_to_unlisted_domain",
                        "link_type": "noscript",
                        "url": href,
                        "domain": domain,
                    })

    return violations


# ---------------------------------------------------------------------------
# Security headers
# ---------------------------------------------------------------------------

def check_security_headers(response_headers: dict, required: list[str]) -> list[str]:
    """Return list of missing recommended security headers."""
    # Header lookup is case-insensitive
    present = {k.lower() for k in response_headers}
    return [h for h in required if h.lower() not in present]


# ---------------------------------------------------------------------------
# Suspicious filenames
# ---------------------------------------------------------------------------

def is_suspicious_url(url: str) -> bool:
    """Flag URLs with names typical for webshells / backdoors."""
    filename = urlparse(url).path.rsplit("/", 1)[-1]
    return bool(_SUSPICIOUS_FILENAME_RE.search(filename))