integrity_scanner_fuer_statische_Webseiten/scanner/checker.py

"""
Additional integrity checks beyond baseline diff:
- link whitelist enforcement
- security-header audit
- canonical hijack detection
- suspicious URL filenames (from v4 heritage)
"""
import re
from pathlib import Path
from urllib.parse import urlparse

import yaml

_SUSPICIOUS_FILENAME_RE = re.compile(
    r"(shell|backdoor|cmd|upload|gate|mailer|wso|c99|r57|b374k)",
    re.I,
)


# ---------------------------------------------------------------------------
# Whitelist checks
# ---------------------------------------------------------------------------

def load_whitelist(path: str | Path) -> set[str]:
    """Load a YAML list file into a set. Returns empty set if missing."""
    p = Path(path)
    if not p.exists():
        return set()
    data = yaml.safe_load(p.read_text(encoding="utf-8")) or []
    if isinstance(data, list):
        return {str(s).strip() for s in data if s}
    return set()


def check_links_against_whitelist(
    page: dict,
    allowed_external_domains: set[str],
    base_netloc: str,
) -> list[dict]:
    """
    Return violations: external links to domains not in allowed_external_domains.
    Also flags noscript links and comment links to external domains.
    """
    violations: list[dict] = []

    for link_type, links in page.get("links", {}).items():
        if not isinstance(links, list):
            continue
        for link in links:
            if not isinstance(link, dict):
                continue
            if link.get("class") != "external":
                continue
            domain = urlparse(link.get("url", "")).netloc
            if domain and domain not in allowed_external_domains:
                violations.append({
                    "type": "unlisted_external_domain",
                    "link_type": link_type,
                    "url": link.get("url", ""),
                    "domain": domain,
                })

    # Comment links
    for href in page.get("comment_links", []):
        domain = urlparse(href).netloc
        if domain and domain != base_netloc and domain not in allowed_external_domains:
            violations.append({
                "type": "comment_link_to_unlisted_domain",
                "link_type": "comment",
                "url": href,
                "domain": domain,
            })

    # noscript links
    for hc in page.get("hidden_content", []):
        if hc.get("type") == "noscript_links":
            for href in hc.get("links", []):
                domain = urlparse(href).netloc
                if domain and domain != base_netloc and domain not in allowed_external_domains:
                    violations.append({
                        "type": "noscript_link_to_unlisted_domain",
                        "link_type": "noscript",
                        "url": href,
                        "domain": domain,
                    })

    return violations


def check_internal_paths(
    page: dict,
    allowed_paths: set[str],
) -> list[dict]:
    """
    Flag internal links to paths not in the allowed_paths whitelist.
    If allowed_paths is empty, the check is skipped.
    """
    if not allowed_paths:
        return []
    violations: list[dict] = []
    for link in page.get("links", {}).get("a", []):
        if link.get("class") != "internal":
            continue
        path = urlparse(link.get("url", "")).path
        if path and path not in allowed_paths:
            violations.append({
                "type": "unlisted_internal_path",
                "url": link.get("url", ""),
                "path": path,
            })
    return violations


# ---------------------------------------------------------------------------
# Security headers
# ---------------------------------------------------------------------------

def check_security_headers(response_headers: dict, required: list[str]) -> list[str]:
    """Return list of missing recommended security headers."""
    # Header lookup is case-insensitive
    present = {k.lower() for k in response_headers}
    return [h for h in required if h.lower() not in present]


# ---------------------------------------------------------------------------
# Broken links
# ---------------------------------------------------------------------------

def find_broken_pages(pages: dict) -> list[dict]:
    """Return all pages whose HTTP status is >= 400 (broken / server error)."""
    return [
        {"url": p["url"], "status": p.get("status", 0)}
        for p in pages.values()
        if p.get("status", 0) >= 400
    ]


# ---------------------------------------------------------------------------
# Canonical hijack
# ---------------------------------------------------------------------------

def canonical_is_hijacked(page: dict, base_netloc: str) -> bool:
    """
    Return True if the canonical URL points to a different domain.
    Canonical appearing in both <link rel=canonical> and metadata is checked.
    """
    canonical = (
        page.get("links", {}).get("canonical")
        or page.get("metadata", {}).get("canonical")
    )
    if not canonical:
        return False
    netloc = urlparse(canonical).netloc
    return bool(netloc) and netloc != base_netloc


# ---------------------------------------------------------------------------
# Suspicious filenames
# ---------------------------------------------------------------------------

def is_suspicious_url(url: str) -> bool:
    """Flag URLs with names typical for webshells / backdoors."""
    filename = urlparse(url).path.rsplit("/", 1)[-1]
    return bool(_SUSPICIOUS_FILENAME_RE.search(filename))


# ---------------------------------------------------------------------------
# JSON-LD type check
# ---------------------------------------------------------------------------

def check_jsonld_types(page: dict, allowed_types: list[str]) -> list[dict]:
    """Return JSON-LD entries whose @type is not in the allowed list."""
    allowed = set(allowed_types)
    return [
        j for j in page.get("jsonld", [])
        if j.get("type") not in allowed
    ]