integrity_scanner_fuer_statische_Webseiten/scanner/checker.py

"""
Additional integrity checks beyond baseline diff:
- link whitelist enforcement
- security-header audit
- suspicious URL filenames (from v4 heritage)
- external link reachability check
"""
import logging
import re
import time
from pathlib import Path
from urllib.parse import urlparse

import requests
import yaml

logger = logging.getLogger(__name__)

# Webshell / backdoor filename markers. Word boundaries are essential:
# without them "mailer" matches the legitimate "PSEMailerAntispam.js" and
# "gate" matches "navigate", producing false positives on every scan.
_SUSPICIOUS_FILENAME_RE = re.compile(
    r"\b(shell|backdoor|webshell|c99|r57|wso|b374k|cmd|reverse_shell|bindshell)\b",
    re.I,
)


# ---------------------------------------------------------------------------
# Whitelist checks
# ---------------------------------------------------------------------------

def load_whitelist(path: str | Path) -> set[str]:
    """Load a YAML list file into a set. Returns empty set if missing."""
    p = Path(path)
    if not p.exists():
        return set()
    data = yaml.safe_load(p.read_text(encoding="utf-8")) or []
    if isinstance(data, list):
        return {str(s).strip() for s in data if s}
    return set()


def check_links_against_whitelist(
    page: dict,
    allowed_external_domains: set[str],
    base_netloc: str,
) -> list[dict]:
    """
    Return violations: external links to domains not in allowed_external_domains.
    Also flags noscript links and comment links to external domains.
    """
    violations: list[dict] = []

    for link_type, links in page.get("links", {}).items():
        if not isinstance(links, list):
            continue
        for link in links:
            if not isinstance(link, dict):
                continue
            if link.get("class") != "external":
                continue
            domain = urlparse(link.get("url", "")).netloc
            if domain and domain not in allowed_external_domains:
                violations.append({
                    "type": "unlisted_external_domain",
                    "link_type": link_type,
                    "url": link.get("url", ""),
                    "domain": domain,
                })

    # Comment links
    for href in page.get("comment_links", []):
        domain = urlparse(href).netloc
        if domain and domain != base_netloc and domain not in allowed_external_domains:
            violations.append({
                "type": "comment_link_to_unlisted_domain",
                "link_type": "comment",
                "url": href,
                "domain": domain,
            })

    # noscript links
    for hc in page.get("hidden_content", []):
        if hc.get("type") == "noscript_links":
            for href in hc.get("links", []):
                domain = urlparse(href).netloc
                if domain and domain != base_netloc and domain not in allowed_external_domains:
                    violations.append({
                        "type": "noscript_link_to_unlisted_domain",
                        "link_type": "noscript",
                        "url": href,
                        "domain": domain,
                    })

    return violations


# ---------------------------------------------------------------------------
# Security headers
# ---------------------------------------------------------------------------

def check_security_headers(response_headers: dict, required: list[str]) -> list[str]:
    """Return list of missing recommended security headers."""
    # Header lookup is case-insensitive
    present = {k.lower() for k in response_headers}
    return [h for h in required if h.lower() not in present]


# ---------------------------------------------------------------------------
# Suspicious filenames
# ---------------------------------------------------------------------------

# ---------------------------------------------------------------------------
# External link reachability
# ---------------------------------------------------------------------------

def check_external_links(
    urls: list[str],
    timeout: int = 10,
    delay: float = 0.3,
) -> dict[str, dict]:
    """
    Check a list of URLs for reachability via HEAD (fallback: GET).

    Returns {url: {"status": int|None, "final_url": str|None, "error": str|None}}.
    """
    results: dict[str, dict] = {}
    session = requests.Session()
    session.headers["User-Agent"] = "integrity-scanner/1.0 (external-link-check)"

    try:
        for i, url in enumerate(urls):
            if i > 0 and delay > 0:
                time.sleep(delay)
            logger.debug("Checking external link: %s", url)
            try:
                resp = session.head(url, timeout=timeout, allow_redirects=True)
                if resp.status_code == 405:
                    resp = session.get(url, timeout=timeout, allow_redirects=True, stream=True)
                    resp.close()
                results[url] = {"status": resp.status_code, "final_url": resp.url, "error": None}
            except requests.exceptions.RequestException as exc:
                results[url] = {"status": None, "final_url": None, "error": str(exc)}
    finally:
        session.close()

    return results


# ---------------------------------------------------------------------------
# Asset hashing
# ---------------------------------------------------------------------------

def fetch_asset_hashes(
    urls: list[str],
    timeout: int = 15,
    delay: float = 0.2,
) -> dict[str, dict]:
    """
    Fetch binary assets and compute SHA-256 hashes via streaming GET.

    Returns {url: {"sha256": str|None, "size": int|None, "error": str|None}}.
    """
    import hashlib
    results: dict[str, dict] = {}
    session = requests.Session()
    session.headers["User-Agent"] = "integrity-scanner/1.0 (asset-check)"

    try:
        for i, url in enumerate(urls):
            if i > 0 and delay > 0:
                time.sleep(delay)
            logger.debug("Hashing asset: %s", url)
            try:
                resp = session.get(url, timeout=timeout, stream=True)
                if resp.status_code == 200:
                    h = hashlib.sha256()
                    size = 0
                    for chunk in resp.iter_content(chunk_size=65536):
                        h.update(chunk)
                        size += len(chunk)
                    results[url] = {"sha256": h.hexdigest(), "size": size, "error": None}
                else:
                    results[url] = {"sha256": None, "size": None, "error": f"HTTP {resp.status_code}"}
            except requests.exceptions.RequestException as exc:
                results[url] = {"sha256": None, "size": None, "error": str(exc)}
    finally:
        session.close()

    return results


def is_suspicious_url(url: str) -> bool:
    """Flag URLs with names typical for webshells / backdoors."""
    filename = urlparse(url).path.rsplit("/", 1)[-1]
    return bool(_SUSPICIOUS_FILENAME_RE.search(filename))