dschlueter
5f58589c2e
Zwei <span>-Geschwister mit unterschiedlichen Klassen, ohne Text dazwischen und mit Großbuchstabe oder Ziffer rechts bekommen ein trennendes Leerzeichen. Initialen (D|as) und Wortteile (Haupt|bahnhof) bleiben unberührt. Im Mathebuch werden so 49 Stellen korrigiert, die anderen drei Beispielbücher bleiben unverändert. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
215 lines
8.6 KiB
Python
215 lines
8.6 KiB
Python
"""Document loading and conservative normalization of publisher-specific markup."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import re
|
|
from dataclasses import dataclass, field
|
|
from typing import Any
|
|
|
|
from lxml import etree
|
|
|
|
from .diagnostics import Diagnostics
|
|
from .utils import classes, epub_types, is_element, iter_elements, local, norm_ws, roles, text_content
|
|
|
|
HEAD_CLASS_RE = re.compile(r"^(?:head|heading|h|ueberschrift|uberschrift)[_-]?(\d)$")
|
|
TITLE_CLASS_RE = re.compile(r"^(?:title[_-]?document|chapter[_-]?title|document[_-]?number|chaptertitle)$")
|
|
TOC_TITLE_RE = re.compile(r"^\s*(inhalt|inhaltsverzeichnis|inhaltsübersicht|contents|table of contents|toc)\s*$", re.I)
|
|
MEDIA_TAGS = {"img", "svg", "math", "audio", "video", "object", "embed", "iframe", "image"}
|
|
|
|
|
|
@dataclass
|
|
class Doc:
|
|
path: str
|
|
root: Any
|
|
body: Any
|
|
title: str
|
|
linear: bool = True
|
|
types: list[str] = field(default_factory=list)
|
|
skipped: str = "" # reason, e.g. "html-toc"
|
|
toc_ids: set[str] = field(default_factory=set)
|
|
pagebreaks: int = 0
|
|
|
|
|
|
def has_content(el: Any) -> bool:
|
|
"""True if ``el`` contains visible text or media (anchors alone do not count)."""
|
|
if not is_element(el):
|
|
return bool(str(el).strip())
|
|
if text_content(el).strip():
|
|
return True
|
|
return any(local(e.tag) in MEDIA_TAGS for e in iter_elements(el))
|
|
|
|
|
|
def remove_keep_tail(el: Any) -> None:
|
|
"""Remove ``el`` from its parent but keep its tail text in the flow."""
|
|
parent = el.getparent()
|
|
if parent is None:
|
|
return
|
|
tail = el.tail
|
|
if tail:
|
|
prev = el.getprevious()
|
|
if prev is not None:
|
|
prev.tail = (prev.tail or "") + tail
|
|
else:
|
|
parent.text = (parent.text or "") + tail
|
|
parent.remove(el)
|
|
|
|
|
|
def _make(tag_like: Any, name: str) -> Any:
|
|
ns = tag_like.tag[1:].split("}")[0] if tag_like.tag.startswith("{") else None
|
|
return etree.Element(f"{{{ns}}}{name}" if ns else name)
|
|
|
|
|
|
def make_doc(path: str, root: Any, linear: bool) -> Doc:
|
|
body = next((e for e in root.iter() if is_element(e) and local(e.tag) == "body"), None)
|
|
if body is None:
|
|
body = root
|
|
head_title = next((e for e in root.iter() if is_element(e) and local(e.tag) == "title"), None)
|
|
title = norm_ws(text_content(head_title)) if head_title is not None else ""
|
|
types = epub_types(body) + [r.removeprefix("doc-") for r in roles(body)]
|
|
return Doc(path=path, root=root, body=body, title=title, linear=linear, types=types)
|
|
|
|
|
|
def book_uses_title_tables(docs: list[Doc]) -> bool:
|
|
for doc in docs:
|
|
for el in iter_elements(doc.body):
|
|
if local(el.tag) in ("table", "td") and any(TITLE_CLASS_RE.match(c) for c in classes(el)):
|
|
return True
|
|
return False
|
|
|
|
|
|
def normalize_doc(doc: Doc, diag: Diagnostics, title_tables: bool) -> None:
|
|
body = doc.body
|
|
for el in list(iter_elements(body)):
|
|
if not _attached(el, body):
|
|
continue # already removed or replaced together with an ancestor
|
|
name = local(el.tag)
|
|
if name in ("script", "style", "noscript", "template", "link", "meta"):
|
|
remove_keep_tail(el)
|
|
continue
|
|
types = epub_types(el)
|
|
if "pagebreak" in types or "doc-pagebreak" in roles(el):
|
|
_replace_with_anchor(el)
|
|
doc.pagebreaks += 1
|
|
continue
|
|
if name == "table":
|
|
_maybe_heading_table(el, title_tables)
|
|
elif name in ("p", "div") and len(classes(el)) == 1 and (m := HEAD_CLASS_RE.match(classes(el)[0])):
|
|
if not any(is_element(c) and local(c.tag) in ("p", "div", "table", "ul", "ol", "pre") for c in el):
|
|
level = int(m.group(1)) + (1 if title_tables else 0)
|
|
ns_prefix = el.tag.rsplit("}", 1)[0] + "}" if el.tag.startswith("{") else ""
|
|
el.tag = f"{ns_prefix}h{max(1, min(6, level))}"
|
|
if el is not body and ("toc" in types or "doc-toc" in roles(el)) and name not in ("a", "body"):
|
|
doc.toc_ids.update(e.get("id") for e in iter_elements(el) if e.get("id"))
|
|
remove_keep_tail(el)
|
|
diag.info("html-toc-removed", "HTML-Inhaltsverzeichnis entfernt (ersetzt durch README.md).", doc.path)
|
|
separated = _separate_field_spans(body)
|
|
if separated:
|
|
diag.info("inline-separator-added",
|
|
f"{separated} Leerzeichen zwischen aneinanderstoßenden Feld-Spans eingefügt.", doc.path)
|
|
|
|
|
|
def _separate_field_spans(body: Any) -> int:
|
|
"""Insert a space between adjacent class-styled <span> "fields" that touch without whitespace.
|
|
|
|
Publishers sometimes lay out metadata as sibling spans (author | title | series) separated only by
|
|
CSS. Narrow rule to keep drop caps ("D|as") and split words ("Haupt|bahnhof") intact: both spans
|
|
carry different classes, no text between them, both have >= 2 characters and the right one starts
|
|
with an uppercase letter or digit.
|
|
"""
|
|
count = 0
|
|
for parent in iter_elements(body):
|
|
children = [c for c in parent if is_element(c)]
|
|
for left, right in zip(children, children[1:]):
|
|
if local(left.tag) != "span" or local(right.tag) != "span" or left.getnext() is not right:
|
|
continue
|
|
if left.tail or not classes(left) or not classes(right) or classes(left) == classes(right):
|
|
continue
|
|
lt, rt = text_content(left), text_content(right)
|
|
if len(lt.strip()) < 2 or len(rt.strip()) < 2 or lt[-1].isspace() or rt[0].isspace():
|
|
continue
|
|
if rt[0].isupper() or rt[0].isdigit():
|
|
left.tail = " "
|
|
count += 1
|
|
return count
|
|
|
|
|
|
def _attached(el: Any, body: Any) -> bool:
|
|
node = el
|
|
while node is not None:
|
|
if node is body:
|
|
return True
|
|
node = node.getparent()
|
|
return False
|
|
|
|
|
|
def _replace_with_anchor(el: Any) -> None:
|
|
ids = [e.get("id") for e in iter_elements(el) if e.get("id")]
|
|
if not ids:
|
|
remove_keep_tail(el)
|
|
return
|
|
anchor = _make(el, "a")
|
|
anchor.set("id", ids[0])
|
|
anchor.tail = el.tail
|
|
el.getparent().replace(el, anchor)
|
|
|
|
|
|
def _maybe_heading_table(table: Any, title_tables: bool) -> None:
|
|
"""Layout tables used as headings (e.g. <table class="Head1">) become <hN>."""
|
|
rows = [e for e in iter_elements(table) if local(e.tag) == "tr"]
|
|
if len(rows) != 1:
|
|
return
|
|
if any(local(e.tag) in ("table", "img", "svg", "math", "pre", "ul", "ol") for e in iter_elements(table) if e is not table):
|
|
return
|
|
cls = classes(table)
|
|
cells = [e for e in rows[0] if is_element(e) and local(e.tag) in ("td", "th")]
|
|
cell_classes = [c for cell in cells for c in classes(cell)]
|
|
level = None
|
|
if any(TITLE_CLASS_RE.match(c) for c in cls + cell_classes):
|
|
level = 1
|
|
else:
|
|
for c in cls:
|
|
m = HEAD_CLASS_RE.match(c)
|
|
if m:
|
|
level = int(m.group(1)) + (1 if title_tables else 0)
|
|
break
|
|
if level is None:
|
|
return
|
|
level = max(1, min(6, level))
|
|
texts = []
|
|
ids = [e.get("id") for e in iter_elements(table) if e.get("id")]
|
|
for cell in cells:
|
|
parts = []
|
|
for node in cell.iter():
|
|
if is_element(node) and local(node.tag) == "br":
|
|
parts.append(" ")
|
|
if is_element(node) and node.text:
|
|
parts.append(node.text)
|
|
if node is not cell and node.tail:
|
|
parts.append(node.tail)
|
|
t = norm_ws("".join(parts))
|
|
if t:
|
|
texts.append(t)
|
|
heading = _make(table, f"h{level}")
|
|
for anchor_id in ids:
|
|
a = _make(table, "a")
|
|
a.set("id", anchor_id)
|
|
heading.append(a)
|
|
if len(heading):
|
|
heading[-1].tail = " ".join(texts)
|
|
else:
|
|
heading.text = " ".join(texts)
|
|
heading.tail = table.tail
|
|
table.getparent().replace(table, heading)
|
|
|
|
|
|
def is_html_toc_doc(doc: Doc, nav_path: str, landmark_toc_docs: set[str]) -> bool:
|
|
if doc.path == nav_path or doc.path in landmark_toc_docs or "toc" in doc.types:
|
|
return True
|
|
heading = next((e for e in iter_elements(doc.body) if local(e.tag) in ("h1", "h2", "h3")), None)
|
|
title = norm_ws(text_content(heading)) if heading is not None else doc.title
|
|
if not TOC_TITLE_RE.match(title or "") and not TOC_TITLE_RE.match(doc.title or ""):
|
|
return False
|
|
total = len(norm_ws(text_content(doc.body))) - (len(norm_ws(text_content(heading))) if heading is not None else 0)
|
|
links = [a for a in iter_elements(doc.body) if local(a.tag) == "a" and a.get("href") and "://" not in a.get("href")]
|
|
link_text = sum(len(norm_ws(text_content(a))) for a in links)
|
|
return len(links) >= 3 and total > 0 and link_text / total >= 0.6
|