epub2md/epub2md/normalize.py

"""Document loading and conservative normalization of publisher-specific markup."""

from __future__ import annotations

import re
from collections.abc import Callable
from dataclasses import dataclass, field
from typing import Any

from lxml import etree

from .diagnostics import Diagnostics
from .utils import classes, epub_types, is_element, iter_elements, local, norm_ws, roles, text_content

HEAD_CLASS_RE = re.compile(r"^(?:head|heading|h|ueberschrift|uberschrift)[_-]?(\d)$")
TITLE_CLASS_RE = re.compile(r"^(?:title[_-]?document|chapter[_-]?title|document[_-]?number|chaptertitle)$")
TOC_TITLE_RE = re.compile(r"^\s*(inhalt|inhaltsverzeichnis|inhaltsübersicht|contents|table of contents|toc)\s*$", re.I)
MEDIA_TAGS = {"img", "svg", "math", "audio", "video", "object", "embed", "iframe", "image"}
PAGE_ATTR = "data-e2m-page"  # print page number of a page-break marker (rendered with --page-markers)
PAGE_ID_RE = re.compile(r"^(?:page|pg|seite)[_-]?(\d+|[ivxlcdm]+)$", re.I)  # id="page_47", <span id="page37">
PAGE_ID_SHORT_RE = re.compile(r"^p[_-]?(\d+)$", re.I)  # id="p17": only on empty <a>/<span>


@dataclass
class Doc:
    path: str
    root: Any
    body: Any
    title: str
    linear: bool = True
    types: list[str] = field(default_factory=list)
    skipped: str = ""  # reason, e.g. "html-toc"
    toc_ids: set[str] = field(default_factory=set)
    pagebreaks: int = 0


def has_content(el: Any) -> bool:
    """True if ``el`` contains visible text or media (anchors alone do not count)."""
    if not is_element(el):
        return bool(str(el).strip())
    if text_content(el).strip():
        return True
    return any(local(e.tag) in MEDIA_TAGS for e in iter_elements(el))


def remove_keep_tail(el: Any) -> None:
    """Remove ``el`` from its parent but keep its tail text in the flow."""
    parent = el.getparent()
    if parent is None:
        return
    tail = el.tail
    if tail:
        prev = el.getprevious()
        if prev is not None:
            prev.tail = (prev.tail or "") + tail
        else:
            parent.text = (parent.text or "") + tail
    parent.remove(el)


def _make(tag_like: Any, name: str) -> Any:
    ns = tag_like.tag[1:].split("}")[0] if tag_like.tag.startswith("{") else None
    return etree.Element(f"{{{ns}}}{name}" if ns else name)


def make_doc(path: str, root: Any, linear: bool) -> Doc:
    body = next((e for e in root.iter() if is_element(e) and local(e.tag) == "body"), None)
    if body is None:
        body = root
    head_title = next((e for e in root.iter() if is_element(e) and local(e.tag) == "title"), None)
    title = norm_ws(text_content(head_title)) if head_title is not None else ""
    types = epub_types(body) + [r.removeprefix("doc-") for r in roles(body)]
    return Doc(path=path, root=root, body=body, title=title, linear=linear, types=types)


def book_uses_title_tables(docs: list[Doc]) -> bool:
    for doc in docs:
        for el in iter_elements(doc.body):
            if local(el.tag) in ("table", "td") and any(TITLE_CLASS_RE.match(c) for c in classes(el)):
                return True
    return False


def normalize_doc(doc: Doc, diag: Diagnostics, title_tables: bool,
                  hidden: Callable[[Any], bool] | None = None) -> None:
    body = doc.body
    for el in list(iter_elements(body)):
        if not _attached(el, body):
            continue  # already removed or replaced together with an ancestor
        name = local(el.tag)
        if name in ("script", "style", "noscript", "template", "link", "meta"):
            remove_keep_tail(el)
            continue
        types = epub_types(el)
        if "pagebreak" in types or "doc-pagebreak" in roles(el):
            _replace_with_anchor(el, _pagebreak_label(el))
            doc.pagebreaks += 1
            continue
        if name in ("a", "span") and not el.get("href") and (page := _page_from_id(el)):
            el.set(PAGE_ATTR, page)
        if name == "table":
            _maybe_heading_table(el, title_tables)
        elif name in ("p", "div") and len(classes(el)) == 1 and (m := HEAD_CLASS_RE.match(classes(el)[0])):
            if not any(is_element(c) and local(c.tag) in ("p", "div", "table", "ul", "ol", "pre") for c in el):
                level = int(m.group(1)) + (1 if title_tables else 0)
                ns_prefix = el.tag.rsplit("}", 1)[0] + "}" if el.tag.startswith("{") else ""
                el.tag = f"{ns_prefix}h{max(1, min(6, level))}"
        if el is not body and ("toc" in types or "doc-toc" in roles(el)) and name not in ("a", "body"):
            doc.toc_ids.update(e.get("id") for e in iter_elements(el) if e.get("id"))
            remove_keep_tail(el)
            diag.info("html-toc-removed", "HTML-Inhaltsverzeichnis entfernt (ersetzt durch README.md).", doc.path)
    separated = _separate_field_spans(body, hidden or (lambda el: False))
    if separated:
        diag.info("inline-separator-added",
                  f"{separated} Leerzeichen zwischen aneinanderstoßenden Feld-Spans eingefügt.", doc.path)


def _separate_field_spans(body: Any, hidden: Callable[[Any], bool]) -> int:
    """Insert a space between adjacent class-styled <span> "fields" that touch without whitespace.

    Publishers sometimes lay out metadata as sibling spans (author | title | series) separated only by
    CSS. Narrow rule to keep drop caps ("D|as") and split words ("Haupt|bahnhof") intact: both spans
    carry different classes, no text between them, both have >= 2 characters and the right one starts
    with an uppercase letter or digit. Spans hidden by the book's CSS (display:none) get no separator.
    """
    count = 0
    for parent in iter_elements(body):
        children = [c for c in parent if is_element(c)]
        for left, right in zip(children, children[1:]):
            if local(left.tag) != "span" or local(right.tag) != "span" or left.getnext() is not right:
                continue
            if left.tail or not classes(left) or not classes(right) or classes(left) == classes(right):
                continue
            lt, rt = text_content(left), text_content(right)
            if len(lt.strip()) < 2 or len(rt.strip()) < 2 or lt[-1].isspace() or rt[0].isspace():
                continue
            if (rt[0].isupper() or rt[0].isdigit()) and not hidden(left) and not hidden(right):
                left.tail = " "
                count += 1
    return count


def _attached(el: Any, body: Any) -> bool:
    node = el
    while node is not None:
        if node is body:
            return True
        node = node.getparent()
    return False


def _pagebreak_label(el: Any) -> str:
    """Page number of an epub:type="pagebreak" element (aria-label, title, text or id)."""
    for value in (el.get("aria-label"), el.get("title"), text_content(el)):
        value = norm_ws(value or "")
        if value and len(value) <= 12:
            return re.sub(r"^(?:seite|page|s\.|p\.)\s*", "", value, flags=re.I)
    m = PAGE_ID_RE.match(el.get("id") or "") or PAGE_ID_SHORT_RE.match(el.get("id") or "")
    return m.group(1) if m else ""


def _page_from_id(el: Any) -> str:
    el_id = el.get("id") or ""
    if m := PAGE_ID_RE.match(el_id):
        return m.group(1)
    if (m := PAGE_ID_SHORT_RE.match(el_id)) and not norm_ws(text_content(el)) and len(el) == 0:
        return m.group(1)
    return ""


def _replace_with_anchor(el: Any, page: str = "") -> None:
    ids = [e.get("id") for e in iter_elements(el) if e.get("id")]
    if not ids and not page:
        remove_keep_tail(el)
        return
    anchor = _make(el, "a")
    if ids:
        anchor.set("id", ids[0])
    if page:
        anchor.set(PAGE_ATTR, page)
    anchor.tail = el.tail
    el.getparent().replace(el, anchor)


def _maybe_heading_table(table: Any, title_tables: bool) -> None:
    """Layout tables used as headings (e.g. <table class="Head1">) become <hN>."""
    rows = [e for e in iter_elements(table) if local(e.tag) == "tr"]
    if len(rows) != 1:
        return
    if any(local(e.tag) in ("table", "img", "svg", "math", "pre", "ul", "ol") for e in iter_elements(table) if e is not table):
        return
    cls = classes(table)
    cells = [e for e in rows[0] if is_element(e) and local(e.tag) in ("td", "th")]
    cell_classes = [c for cell in cells for c in classes(cell)]
    level = None
    if any(TITLE_CLASS_RE.match(c) for c in cls + cell_classes):
        level = 1
    else:
        for c in cls:
            m = HEAD_CLASS_RE.match(c)
            if m:
                level = int(m.group(1)) + (1 if title_tables else 0)
                break
    if level is None:
        return
    level = max(1, min(6, level))
    texts = []
    ids = [e.get("id") for e in iter_elements(table) if e.get("id")]
    pages = {e.get("id"): page for e in iter_elements(table) if local(e.tag) in ("a", "span") and (page := _page_from_id(e))}
    for cell in cells:
        parts = []
        for node in cell.iter():
            if is_element(node) and local(node.tag) == "br":
                parts.append(" ")
            if is_element(node) and node.text:
                parts.append(node.text)
            if node is not cell and node.tail:
                parts.append(node.tail)
        t = norm_ws("".join(parts))
        if t:
            texts.append(t)
    heading = _make(table, f"h{level}")
    for anchor_id in ids:
        a = _make(table, "a")
        a.set("id", anchor_id)
        if anchor_id in pages:
            a.set(PAGE_ATTR, pages[anchor_id])
        heading.append(a)
    if len(heading):
        heading[-1].tail = " ".join(texts)
    else:
        heading.text = " ".join(texts)
    heading.tail = table.tail
    table.getparent().replace(table, heading)


def is_html_toc_doc(doc: Doc, nav_path: str, landmark_toc_docs: set[str]) -> bool:
    if doc.path == nav_path or doc.path in landmark_toc_docs or "toc" in doc.types:
        return True
    heading = next((e for e in iter_elements(doc.body) if local(e.tag) in ("h1", "h2", "h3")), None)
    title = norm_ws(text_content(heading)) if heading is not None else doc.title
    if not TOC_TITLE_RE.match(title or "") and not TOC_TITLE_RE.match(doc.title or ""):
        return False
    total = len(norm_ws(text_content(doc.body))) - (len(norm_ws(text_content(heading))) if heading is not None else 0)
    links = [a for a in iter_elements(doc.body) if local(a.tag) == "a" and a.get("href") and "://" not in a.get("href")]
    link_text = sum(len(norm_ws(text_content(a))) for a in links)
    return len(links) >= 3 and total > 0 and link_text / total >= 0.6