"""
HTML extraction with BeautifulSoup4+lxml.

Extracts: visible text, links (all types), metadata, hidden-content indicators,
inline script patterns, JSON-LD, and links buried in HTML comments.
"""
import json
import re
from urllib.parse import urlencode, urljoin, urlparse, parse_qsl, urlunparse

from bs4 import BeautifulSoup

# Query-param keys that are pure cache-busters on asset URLs (CSS/JS/img).
# They change on every request and must be stripped before storing/diffing.
CACHE_PARAM_KEYS = {"v", "ver", "version", "cache", "cb", "_", "_t"}


def strip_cache_params(url: str) -> str:
    """Remove cache-buster query params (e.g. style.css?v=12345) from a URL."""
    p = urlparse(url)
    if not p.query:
        return url
    kept = [(k, v) for k, v in parse_qsl(p.query) if k.lower() not in CACHE_PARAM_KEYS]
    return urlunparse(p._replace(query=urlencode(kept)))

# Inline-script patterns that warrant flagging
_SUSPICIOUS_INLINE = [
    (r"eval\s*\(", "eval()"),
    (r"atob\s*\(", "atob()"),
    (r"fromCharCode", "fromCharCode"),
    (r"document\.createElement\s*\(\s*[\"']script[\"']", "createElement(script)"),
    (r"\bfetch\s*\(", "fetch()"),
    (r"XMLHttpRequest", "XMLHttpRequest"),
    (r"base64_decode", "base64_decode"),
]

# CSS-based hiding patterns (applied to inline style attributes)
_HIDDEN_STYLE_CHECKS = [
    (r"display\s*:\s*none", "display:none"),
    (r"visibility\s*:\s*hidden", "visibility:hidden"),
    (r"opacity\s*:\s*0(?:\.0+)?(?!\d)", "opacity:0"),
    (r"font-size\s*:\s*0", "font-size:0"),
    (r"color\s*:\s*(?:white|#fff(?:fff)?|rgba?\(\s*255\s*,\s*255\s*,\s*255)", "white-text"),
    (r"width\s*:\s*0(?:px)?[;\s]", "width:0"),
    (r"height\s*:\s*0(?:px)?[;\s]", "height:0"),
    (r"position\s*:\s*absolute", "position:absolute"),  # combined with off-screen check below
    (r"(?:left|top)\s*:\s*-\d{3,}", "off-screen-position"),
    (r"clip\s*:\s*rect\s*\(\s*0\s*,?\s*0\s*,?\s*0\s*,?\s*0\s*\)", "clip:rect(0)"),
    (r"text-indent\s*:\s*-\d{3,}", "text-indent-offscreen"),
]


def extract_page(html: str, url: str, cfg: dict | None = None) -> dict:
    """
    Full extraction from page HTML.

    Returns a structured dict with all data needed for diffing and checking.
    """
    cfg = cfg or {}
    soup = BeautifulSoup(html, "lxml")

    # Remove ignored selectors before text extraction
    ignore_selectors = cfg.get("normalization", {}).get("ignore_selectors") or []
    for sel in ignore_selectors:
        for el in soup.select(sel):
            el.decompose()

    return {
        "url": url,
        "text": _visible_text(soup),
        "links": _extract_links(soup, url),
        "metadata": _extract_metadata(soup),
        "hidden_content": _find_hidden_content(soup),
        "inline_scripts": _check_inline_scripts(soup),
        "jsonld": _extract_jsonld(soup),
        "comment_links": _find_comment_links(html, url),
    }


# ---------------------------------------------------------------------------
# Text
# ---------------------------------------------------------------------------

def _visible_text(soup: BeautifulSoup) -> str:
    """Extract visible text — strips scripts, styles, noscript, head."""
    work = soup.__copy__()
    for tag in work(["script", "style", "noscript", "head", "meta", "link"]):
        tag.decompose()
    text = work.get_text(separator=" ")
    return re.sub(r"\s+", " ", text).strip()


# ---------------------------------------------------------------------------
# Links
# ---------------------------------------------------------------------------

def _extract_links(soup: BeautifulSoup, base_url: str) -> dict:
    """
    Extract all link categories:
      a, link, script[src], img[src], iframe[src], form[action],
      meta_refresh, canonical
    Each entry: {url, class: 'internal'|'external', ...extra}
    """
    base_netloc = urlparse(base_url).netloc

    def classify(u: str) -> str:
        return "internal" if urlparse(u).netloc == base_netloc else "external"

    def resolve(href: str | None) -> str | None:
        if not href:
            return None
        href = href.strip()
        if href.startswith(("javascript:", "mailto:", "tel:", "data:", "#")):
            return None
        abs_url = urljoin(base_url, href)
        p = urlparse(abs_url)
        if p.scheme not in ("http", "https"):
            return None
        # Drop fragment
        return abs_url.split("#")[0]

    result: dict = {
        "a": [],
        "link_rel": [],
        "script": [],
        "img": [],
        "audio": [],
        "video": [],
        "iframe": [],
        "form": [],
        "meta_refresh": [],
        "canonical": None,
    }

    for tag in soup.find_all("a", href=True):
        u = resolve(tag["href"])
        if u:
            result["a"].append({
                "url": u,
                "text": tag.get_text(" ", strip=True)[:200],
                "class": classify(u),
                "rel": " ".join(tag.get("rel", [])),
            })

    for tag in soup.find_all("link", href=True):
        u = resolve(tag["href"])
        if not u:
            continue
        rel = " ".join(tag.get("rel", []))
        if "canonical" in rel:
            result["canonical"] = u
        u = strip_cache_params(u)  # assets carry cache-busters → normalise away
        result["link_rel"].append({"url": u, "rel": rel, "class": classify(u)})

    for tag in soup.find_all("script", src=True):
        u = resolve(tag["src"])
        if u:
            u = strip_cache_params(u)
            result["script"].append({"url": u, "class": classify(u)})

    for tag in soup.find_all("img", src=True):
        u = resolve(tag["src"])
        if u:
            u = strip_cache_params(u)
            result["img"].append({"url": u, "class": classify(u)})

    for tag in soup.find_all(["audio", "video"]):
        media_type = tag.name  # "audio" or "video"
        if tag.get("src"):
            u = resolve(tag["src"])
            if u:
                u = strip_cache_params(u)
                result[media_type].append({"url": u, "class": classify(u)})
        for src_tag in tag.find_all("source", src=True):
            u = resolve(src_tag["src"])
            if u:
                u = strip_cache_params(u)
                result[media_type].append({"url": u, "class": classify(u)})

    for tag in soup.find_all("iframe", src=True):
        u = resolve(tag["src"])
        if u:
            result["iframe"].append({"url": u, "class": classify(u)})

    for tag in soup.find_all("form", action=True):
        u = resolve(tag["action"])
        if u:
            result["form"].append({"url": u, "class": classify(u)})

    for tag in soup.find_all("meta"):
        if tag.get("http-equiv", "").lower() == "refresh":
            content = tag.get("content", "")
            m = re.search(r"url\s*=\s*(.+)", content, re.I)
            if m:
                u = resolve(m.group(1).strip().strip("\"'"))
                if u:
                    result["meta_refresh"].append(u)

    return result


# ---------------------------------------------------------------------------
# Metadata
# ---------------------------------------------------------------------------

def _extract_metadata(soup: BeautifulSoup) -> dict:
    """Extract title, description, canonical, og-tags, robots, hreflang, H1/H2."""
    meta: dict = {}

    t = soup.find("title")
    meta["title"] = t.get_text(strip=True) if t else None

    canonical_tag = soup.find("link", rel="canonical")
    if canonical_tag:
        meta["canonical"] = canonical_tag.get("href", "").strip()

    for tag in soup.find_all("meta"):
        name = (tag.get("name") or "").lower().strip()
        prop = (tag.get("property") or "").lower().strip()
        content = tag.get("content", "")

        if name == "description":
            meta["description"] = content
        elif name == "robots":
            meta["robots"] = content
        elif name == "keywords":
            meta["keywords"] = content
        elif prop.startswith("og:"):
            meta[prop] = content
        elif prop.startswith("twitter:"):
            meta[prop] = content

    hreflang = []
    for tag in soup.find_all("link", rel="alternate"):
        hl = tag.get("hreflang")
        if hl:
            hreflang.append({"lang": hl, "href": tag.get("href", "")})
    meta["hreflang"] = hreflang

    meta["h1"] = [h.get_text(" ", strip=True) for h in soup.find_all("h1")]
    meta["h2"] = [h.get_text(" ", strip=True) for h in soup.find_all("h2")]

    return meta


# ---------------------------------------------------------------------------
# Hidden content
# ---------------------------------------------------------------------------

def _find_hidden_content(soup: BeautifulSoup) -> list[dict]:
    """
    Detect elements that are visually hidden but contain text or links.
    Catches common SEO-spam hiding techniques.
    """
    findings: list[dict] = []

    for tag in soup.find_all(style=True):
        style = tag.get("style", "")
        matched_indicators = []
        for pattern, label in _HIDDEN_STYLE_CHECKS:
            if re.search(pattern, style, re.I):
                matched_indicators.append(label)
        if not matched_indicators:
            continue

        text = tag.get_text(" ", strip=True)[:300]
        links = [a.get("href", "") for a in tag.find_all("a", href=True)]
        if text or links:
            findings.append({
                "type": "inline_style_hidden",
                "indicators": matched_indicators,
                "tag": tag.name,
                "text_preview": text,
                "links": links,
            })

    # <noscript> with links
    for tag in soup.find_all("noscript"):
        links = re.findall(r'href=["\']([^"\']+)["\']', str(tag), re.I)
        if links:
            findings.append({
                "type": "noscript_links",
                "links": links,
                "text_preview": tag.get_text(" ", strip=True)[:200],
            })

    return findings


# ---------------------------------------------------------------------------
# Inline scripts
# ---------------------------------------------------------------------------

def _check_inline_scripts(soup: BeautifulSoup) -> list[dict]:
    """Find inline <script> blocks that match suspicious patterns."""
    findings: list[dict] = []
    for i, tag in enumerate(soup.find_all("script"), 1):
        if tag.get("src"):
            continue
        code = tag.get_text() or ""
        hits = [label for pat, label in _SUSPICIOUS_INLINE if re.search(pat, code, re.I)]
        if hits:
            findings.append({
                "script_index": i,
                "patterns": hits,
                "preview": code[:400],
            })
    return findings


# ---------------------------------------------------------------------------
# JSON-LD
# ---------------------------------------------------------------------------

def _extract_jsonld(soup: BeautifulSoup) -> list[dict]:
    """Extract JSON-LD blocks and flag unexpected @types."""
    results: list[dict] = []
    for tag in soup.find_all("script", type="application/ld+json"):
        raw = tag.string or ""
        try:
            data = json.loads(raw)
        except (json.JSONDecodeError, ValueError):
            results.append({"type": "PARSE_ERROR", "unexpected": True, "preview": raw[:200]})
            continue
        items = data if isinstance(data, list) else [data]
        for item in items:
            if not isinstance(item, dict):
                continue  # JSON-LD must be an object; ignore stray scalars/arrays
            t = item.get("@type", "Unknown")
            results.append({"type": t, "preview": json.dumps(item, ensure_ascii=False)[:500]})
    return results


# ---------------------------------------------------------------------------
# Comment links
# ---------------------------------------------------------------------------

def _find_comment_links(html: str, base_url: str) -> list[str]:
    """
    Find external URLs hidden inside HTML comments.
    Internal links are excluded — CMS templates routinely comment out
    navigation fragments that link back to the same domain.
    """
    base_netloc = urlparse(base_url).netloc
    found: list[str] = []
    for comment in re.findall(r"<!--(.*?)-->", html, re.S):
        for href in re.findall(r'href=["\']([^"\']+)["\']', comment, re.I):
            if not href.startswith(("http://", "https://")):
                continue
            if urlparse(href).netloc == base_netloc:
                continue  # internal comment link — CMS artifact, not a threat
            found.append(href)
    return found