dschlueter
86aa93c12c
- Kapitelerkennung: erstes Kapitel nur aus einem Bild = Cover ("[Titelinfo]",
"Deckblatt"), Titel "Titel" = Titelseite
- epub2md_merge: Vorspann reicht bis vor den Hauptteil (Teil, Vorwort,
Einleitung, nummerierter Titel, > 1000 Wörter), kurze Zwischenseiten
(Auflage, Verlagsbrief, "Über dieses Buch") bleiben davor
- book.lua: Cover am Inhalt erkannt; LaTeX-Weg (Filter in beiden Schritten):
\tableofcontents hinter dem Vorspann, \clearpage je Kapitel, kein \maketitle
vor dem Cover, A4, seitengroßes Cover, DejaVu-Schriften und Mathe-Symbole
für fehlende Glyphen (Mathebuch: >10.000 fehlende Zeichen → 0)
- Buch-README: bei Formeln LaTeX-Weg empfohlen, sonst weasyprint
- Alle sechs Beispielbücher geprüft: Seite 1 Cover, Verzeichnis nach dem Vorspann
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
392 lines
16 KiB
Python
392 lines
16 KiB
Python
"""Merge an epub2md output directory into one complete Markdown file.
|
||
|
||
The chapter order comes from manifest.json; the file name is the original EPUB name with ".md".
|
||
Links between chapters become links to anchors inside the merged file, colliding anchors are
|
||
renamed per chapter, media paths are made relative to the merged file and duplicated footnote
|
||
definitions are dropped. Code (fenced blocks, inline code) is never modified.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import json
|
||
import os
|
||
import posixpath
|
||
import re
|
||
import sys
|
||
from dataclasses import dataclass, field
|
||
from pathlib import Path
|
||
from typing import NoReturn
|
||
from urllib.parse import quote, unquote
|
||
|
||
from . import __version__
|
||
from .errors import EXIT_INPUT, EXIT_INTERNAL
|
||
from .linkcheck import check_links
|
||
from .utils import to_yaml
|
||
|
||
FENCE_OPEN = re.compile(r"^(\s*)(`{3,}|~{3,})")
|
||
# order matters: inline code is matched first and returned unchanged
|
||
TOKEN = re.compile(
|
||
r"(?P<code>(?<![\\`])(?P<ticks>`+)(?!`)(?:[^`\n]|`(?!(?P=ticks)))+?(?P=ticks)(?!`))"
|
||
r"|(?P<mdlink>(?P<lpre>(?<!\\)!?\[(?:[^\[\]\\\n]|\\.|\[(?:[^\[\]\\\n]|\\.)*\]|\n(?![ \t]*\n))*\]\()(?P<ltarget><[^>\n]*>|[^)\s]+)(?P<lpost>(?:\s+\"[^\"]*\")?\)))"
|
||
r"|(?P<attr>\b(?P<aname>href|src|id)=\"(?P<avalue>[^\"]*)\")"
|
||
)
|
||
FN_DEF = re.compile(r"^\[\^([^\]]+)\]:")
|
||
|
||
|
||
class MergeError(Exception):
|
||
pass
|
||
|
||
|
||
@dataclass
|
||
class ChapterFile:
|
||
number: int
|
||
title: str
|
||
rel: str # e.g. chapters/001-x.md (relative to the output directory)
|
||
body: str = ""
|
||
start_anchor: str = ""
|
||
anchor_map: dict[str, str] = field(default_factory=dict)
|
||
types: list[str] = field(default_factory=list)
|
||
words: int = 0
|
||
|
||
|
||
@dataclass
|
||
class MergeResult:
|
||
target: Path
|
||
chapters: int
|
||
renamed_anchors: int
|
||
dropped_footnote_defs: int
|
||
unresolved: list[str]
|
||
broken: list[dict[str, str]]
|
||
|
||
|
||
def strip_frontmatter(text: str) -> str:
|
||
if text.startswith("---\n"):
|
||
end = text.find("\n---\n", 4)
|
||
if end >= 0:
|
||
return text[end + 5 :].lstrip("\n")
|
||
return text
|
||
|
||
|
||
def load_book(out_dir: Path) -> tuple[dict, list[ChapterFile]]:
|
||
manifest_path = out_dir / "manifest.json"
|
||
report_path = out_dir / "conversion-report.json"
|
||
if not out_dir.is_dir():
|
||
raise MergeError(f"Verzeichnis nicht gefunden: {out_dir}")
|
||
report = json.loads(report_path.read_text(encoding="utf-8")) if report_path.is_file() else {}
|
||
if manifest_path.is_file():
|
||
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
|
||
entries = [(c["number"], c["title"], c["file"], c.get("types") or [], c.get("word_count", 0))
|
||
for c in manifest.get("chapters", [])]
|
||
else:
|
||
files = sorted((out_dir / "chapters").glob("*.md"))
|
||
entries = [(i + 1, p.stem, f"chapters/{p.name}", [], 0) for i, p in enumerate(files)]
|
||
chapters = [ChapterFile(n, t, rel, types=list(ty), words=w) for n, t, rel, ty, w in entries if (out_dir / rel).is_file()]
|
||
missing = [rel for _, _, rel, _, _ in entries if not (out_dir / rel).is_file()]
|
||
if missing:
|
||
raise MergeError(f"Kapiteldateien fehlen: {', '.join(missing[:5])}")
|
||
if not chapters:
|
||
raise MergeError(f"Keine Kapiteldateien in {out_dir} – ist das eine epub2md-Ausgabe?")
|
||
for ch in chapters:
|
||
ch.body = strip_frontmatter((out_dir / ch.rel).read_text(encoding="utf-8"))
|
||
return report, chapters
|
||
|
||
|
||
# Front matter that precedes the table of contents in a printed book (cover, title page, imprint …).
|
||
FRONT_TYPES = {"cover", "titlepage", "halftitlepage", "colophon", "copyright-page", "imprint", "dedication",
|
||
"epigraph", "frontispiece", "seriespage", "imprimatur"}
|
||
TOC_PLACEHOLDER = '<div id="epub2md-toc"></div>' # pandoc/book.lua puts the table of contents here
|
||
TITLE_PLACEHOLDER = '<div id="epub2md-titlepage"></div>' # … and Pandoc's title block here (no own title page)
|
||
|
||
|
||
# Where the main matter starts: a part, preface, introduction …, a numbered title or a longer text.
|
||
MAIN_TYPES = {"part", "preface", "foreword", "introduction", "prologue", "preamble", "appendix", "bodymatter", "toc"}
|
||
NUMBERED_TITLE = re.compile(r"^\s*(\d+|[IVXLC]+\.\s|teil\b|kapitel\b|part\b|chapter\b|buch\b|book\b)", re.I)
|
||
FRONT_MAX_WORDS = 1000
|
||
FRONT_MAX_CHAPTERS = 12
|
||
|
||
|
||
def front_matter_count(chapters: list[ChapterFile]) -> int:
|
||
"""Number of leading chapters before the table of contents: cover, title page, imprint and the short
|
||
pages between them (series page, edition note, publisher's letter …) up to the start of the main matter."""
|
||
k = 0
|
||
while k < min(len(chapters), FRONT_MAX_CHAPTERS):
|
||
ch = chapters[k]
|
||
if MAIN_TYPES & set(ch.types) or NUMBERED_TITLE.match(ch.title) or ch.words > FRONT_MAX_WORDS:
|
||
break
|
||
k += 1
|
||
if k < len(chapters) and any(FRONT_TYPES & set(c.types) for c in chapters[:k]):
|
||
return k
|
||
n = 0 # no clear start of the main matter: only the unbroken run of typed front matter
|
||
while n < len(chapters) and FRONT_TYPES & set(chapters[n].types):
|
||
n += 1
|
||
return n
|
||
|
||
|
||
ISO_DATE = re.compile(r"^(\d{4})-(\d{2})-(\d{2})(?:[T ][\d:.]+(?:Z|[+-]\d{2}:?\d{2})?)?$")
|
||
|
||
|
||
def display_date(value: str, language: str) -> str:
|
||
"""'2025-01-13T08:00:00+00:00' → '13.01.2025' (German) or '2025-01-13'; other formats unchanged."""
|
||
m = ISO_DATE.match(value.strip())
|
||
if not m:
|
||
return value
|
||
year, month, day = m.groups()
|
||
return f"{day}.{month}.{year}" if language.lower().startswith("de") else f"{year}-{month}-{day}"
|
||
|
||
|
||
def default_target(out_dir: Path, report: dict) -> Path:
|
||
name = (report.get("input") or {}).get("name") or (out_dir.name + ".epub")
|
||
stem = name[:-5] if name.lower().endswith(".epub") else name
|
||
return out_dir / f"{stem}.md"
|
||
|
||
|
||
def _non_code_segments(text: str) -> list[tuple[bool, str]]:
|
||
"""Split into (is_code, text) segments by fenced code blocks (also indented ones)."""
|
||
segments: list[tuple[bool, str]] = []
|
||
buf: list[str] = []
|
||
fence: str | None = None
|
||
for line in text.split("\n"):
|
||
if fence is None:
|
||
m = FENCE_OPEN.match(line)
|
||
if m:
|
||
if buf:
|
||
segments.append((False, "\n".join(buf)))
|
||
buf = [line]
|
||
fence = m.group(2)
|
||
continue
|
||
buf.append(line)
|
||
else:
|
||
buf.append(line)
|
||
stripped = line.strip()
|
||
if stripped and set(stripped) == {fence[0]} and len(stripped) >= len(fence):
|
||
segments.append((True, "\n".join(buf)))
|
||
buf = []
|
||
fence = None
|
||
if buf:
|
||
segments.append((fence is not None, "\n".join(buf)))
|
||
return segments
|
||
|
||
|
||
def _join(segments: list[tuple[bool, str]]) -> str:
|
||
return "\n".join(s for _, s in segments)
|
||
|
||
|
||
def assign_anchors(chapters: list[ChapterFile]) -> int:
|
||
"""Map every anchor id to a book-wide unique name; returns the number of renamed anchors."""
|
||
used: set[str] = set()
|
||
renamed = 0
|
||
stems = {}
|
||
for ch in chapters:
|
||
stem = posixpath.splitext(posixpath.basename(ch.rel))[0]
|
||
stems[ch.rel] = stem
|
||
for ch in chapters:
|
||
ids: list[str] = []
|
||
for is_code, seg in _non_code_segments(ch.body):
|
||
if is_code:
|
||
continue
|
||
for m in TOKEN.finditer(seg):
|
||
if m.group("attr") and m.group("aname") == "id" and m.group("avalue") not in ids:
|
||
ids.append(m.group("avalue"))
|
||
start = stems[ch.rel]
|
||
while start in used or start in ids:
|
||
start = "kapitel-" + start
|
||
ch.start_anchor = start
|
||
used.add(start)
|
||
for raw in ids:
|
||
name = raw
|
||
if name in used:
|
||
name = f"c{ch.number:03d}-{raw}"
|
||
n = 2
|
||
while name in used:
|
||
name = f"c{ch.number:03d}-{raw}-{n}"
|
||
n += 1
|
||
renamed += 1
|
||
used.add(name)
|
||
ch.anchor_map[raw] = name
|
||
return renamed
|
||
|
||
|
||
def merge(out_dir: Path, target: Path | None = None, *, frontmatter: bool = True, toc: bool = False,
|
||
force: bool = False) -> MergeResult:
|
||
out_dir = out_dir.resolve()
|
||
report, chapters = load_book(out_dir)
|
||
target = (target or default_target(out_dir, report)).resolve()
|
||
if target.exists() and not force:
|
||
raise MergeError(f"Zieldatei existiert bereits: {target} (mit --force überschreiben)")
|
||
if target.parent.exists() is False:
|
||
raise MergeError(f"Zielverzeichnis existiert nicht: {target.parent}")
|
||
renamed = assign_anchors(chapters)
|
||
by_rel = {ch.rel: ch for ch in chapters}
|
||
target_dir = target.parent
|
||
unresolved: list[str] = []
|
||
|
||
def rewrite_target(link: str, ch: ChapterFile) -> str:
|
||
bracket = link.startswith("<") and link.endswith(">")
|
||
raw = link[1:-1] if bracket else link
|
||
if not raw or re.match(r"^[A-Za-z][A-Za-z0-9+.-]*:", raw):
|
||
return link # external URL, mailto:, …
|
||
path, _, frag = raw.partition("#")
|
||
frag = unquote(frag)
|
||
if path:
|
||
resolved = posixpath.normpath(posixpath.join(posixpath.dirname(ch.rel), unquote(path)))
|
||
else:
|
||
resolved = ch.rel
|
||
dest = by_rel.get(resolved)
|
||
if dest is not None:
|
||
if frag and frag in dest.anchor_map:
|
||
new = "#" + dest.anchor_map[frag]
|
||
else:
|
||
if frag:
|
||
unresolved.append(f"{ch.rel}: {raw}")
|
||
new = "#" + dest.start_anchor
|
||
else:
|
||
absolute = out_dir / resolved
|
||
rel = os.path.relpath(absolute, target_dir).replace(os.sep, "/")
|
||
new = quote(rel, safe="/-._~") + (("#" + quote(frag, safe="-._~:")) if frag else "")
|
||
return f"<{new}>" if bracket else new
|
||
|
||
def rewrite_segment(seg: str, ch: ChapterFile) -> str:
|
||
def repl(m: re.Match[str]) -> str:
|
||
if m.group("code"):
|
||
return m.group(0)
|
||
if m.group("mdlink"):
|
||
pre = m.group("lpre")
|
||
inner = TOKEN.sub(repl, pre[pre.index("[") + 1 : -2]) # images/links inside the link text
|
||
return pre[: pre.index("[") + 1] + inner + "](" + rewrite_target(m.group("ltarget"), ch) + m.group("lpost")
|
||
name, value = m.group("aname"), m.group("avalue")
|
||
if name == "id":
|
||
return f'id="{ch.anchor_map.get(value, value)}"'
|
||
return f'{name}="{rewrite_target(value, ch)}"'
|
||
|
||
return TOKEN.sub(repl, seg)
|
||
|
||
seen_defs: set[str] = set()
|
||
dropped = 0
|
||
parts: list[str] = []
|
||
if frontmatter:
|
||
md = report.get("metadata") or {}
|
||
fm: dict = {}
|
||
for key, src in (("title", "title"), ("subtitle", "subtitle"), ("author", "authors"), ("lang", "language"),
|
||
("publisher", "publisher"), ("date", "published"), ("identifier", "identifier"),
|
||
("rights", "rights"), ("description", "description")):
|
||
if md.get(src):
|
||
fm[key] = display_date(md[src], md.get("language", "")) if key == "date" else md[src]
|
||
if fm:
|
||
parts.append("---\n" + to_yaml(fm) + "---")
|
||
front = front_matter_count(chapters)
|
||
has_title_page = any({"titlepage", "halftitlepage"} & set(ch.types) for ch in chapters[:front])
|
||
cover_first = bool(chapters) and "cover" in chapters[0].types
|
||
if toc:
|
||
lines = ["## Inhalt" if (report.get("metadata") or {}).get("language", "de").startswith("de") else "## Contents", ""]
|
||
lines += [f"{ch.number}. [{_escape_link_text(ch.title)}](#{ch.start_anchor})" for ch in chapters]
|
||
toc_block = "\n".join(lines)
|
||
else:
|
||
toc_block = TOC_PLACEHOLDER
|
||
if not has_title_page and not cover_first:
|
||
parts.append(TITLE_PLACEHOLDER)
|
||
if front == 0:
|
||
parts.append(toc_block)
|
||
for i, ch in enumerate(chapters):
|
||
segments = [(is_code, seg if is_code else rewrite_segment(seg, ch)) for is_code, seg in _non_code_segments(ch.body)]
|
||
body, n = _drop_duplicate_definitions(_join(segments), seen_defs)
|
||
dropped += n
|
||
parts.append(f'<a id="{ch.start_anchor}"></a>\n\n{body.strip()}')
|
||
if i == 0 and cover_first and not has_title_page:
|
||
parts.append(TITLE_PLACEHOLDER)
|
||
if i + 1 == front:
|
||
parts.append(toc_block)
|
||
text = "\n\n".join(parts).rstrip() + "\n"
|
||
|
||
existing = {p.resolve().as_posix() for p in out_dir.rglob("*") if p.is_file()}
|
||
broken = check_links({target.as_posix(): text}, existing)
|
||
target.write_text(text, encoding="utf-8")
|
||
return MergeResult(target, len(chapters), renamed, dropped, unresolved, broken)
|
||
|
||
|
||
def _escape_link_text(text: str) -> str:
|
||
return text.replace("\\", "\\\\").replace("[", "\\[").replace("]", "\\]")
|
||
|
||
|
||
def _drop_duplicate_definitions(body: str, seen: set[str]) -> tuple[str, int]:
|
||
"""Remove footnote definitions whose label was already defined in an earlier chapter."""
|
||
out: list[str] = []
|
||
dropped = 0
|
||
skipping = False
|
||
for is_code, seg in _non_code_segments(body):
|
||
if is_code:
|
||
if not skipping:
|
||
out.append(seg)
|
||
continue
|
||
kept: list[str] = []
|
||
for line in seg.split("\n"):
|
||
m = FN_DEF.match(line)
|
||
if m:
|
||
label = m.group(1)
|
||
skipping = label in seen
|
||
if skipping:
|
||
dropped += 1
|
||
continue
|
||
seen.add(label)
|
||
elif skipping and (line.startswith(" ") or not line.strip()):
|
||
continue
|
||
else:
|
||
skipping = False
|
||
kept.append(line)
|
||
out.append("\n".join(kept))
|
||
return "\n".join(out), dropped
|
||
|
||
|
||
class _Parser(argparse.ArgumentParser):
|
||
def error(self, message: str) -> NoReturn:
|
||
self.print_usage(sys.stderr)
|
||
print(f"{self.prog}: Fehler: {message}", file=sys.stderr)
|
||
sys.exit(EXIT_INPUT)
|
||
|
||
|
||
def build_parser() -> argparse.ArgumentParser:
|
||
p = _Parser(
|
||
prog="epub2md_merge.py",
|
||
description="Fügt die Kapiteldateien einer epub2md-Ausgabe in Lesereihenfolge zu einer "
|
||
"vollständigen Markdown-Datei zusammen (Name = Original-EPUB mit Endung .md).",
|
||
epilog="Rückgabecodes: 0 Erfolg, 1 ungültige Eingabe/Zieldatei existiert, 4 interner Fehler.\n"
|
||
"Beispiel: python3 epub2md_merge.py output/programmierhandbuch",
|
||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||
)
|
||
p.add_argument("output_dir", metavar="AUSGABE_VERZEICHNIS", help="von epub2md.py erzeugtes Verzeichnis")
|
||
p.add_argument("-o", "--output", metavar="DATEI.md",
|
||
help="Zieldatei (Standard: AUSGABE_VERZEICHNIS/<Original-Name>.md; Medienpfade werden angepasst)")
|
||
p.add_argument("--frontmatter", action=argparse.BooleanOptionalAction, default=True,
|
||
help="YAML-Frontmatter mit Buchmetadaten an den Anfang setzen (Standard: an)")
|
||
p.add_argument("--toc", action="store_true", help="Inhaltsverzeichnis mit Links auf die Kapitel einfügen")
|
||
p.add_argument("--force", action="store_true", help="bestehende Zieldatei überschreiben")
|
||
p.add_argument("--version", action="version", version=f"%(prog)s {__version__}")
|
||
return p
|
||
|
||
|
||
def main(argv: list[str] | None = None) -> int:
|
||
try:
|
||
args = build_parser().parse_args(argv)
|
||
except SystemExit as exc:
|
||
return exc.code if isinstance(exc.code, int) else EXIT_INPUT
|
||
try:
|
||
result = merge(Path(args.output_dir), Path(args.output) if args.output else None,
|
||
frontmatter=args.frontmatter, toc=args.toc, force=args.force)
|
||
except (MergeError, json.JSONDecodeError, OSError) as exc:
|
||
print(f"epub2md_merge: Fehler: {exc}", file=sys.stderr)
|
||
return EXIT_INPUT
|
||
except Exception as exc: # noqa: BLE001 – last line of defense
|
||
print(f"epub2md_merge: Interner Fehler: {type(exc).__name__}: {exc}", file=sys.stderr)
|
||
return EXIT_INTERNAL
|
||
for item in result.unresolved[:10]:
|
||
print(f"Warnung: Anker nicht gefunden, Link zeigt auf Kapitelanfang: {item}", file=sys.stderr)
|
||
for b in result.broken[:10]:
|
||
print(f"Warnung: ungültiger Link {b['link']} ({b['reason']})", file=sys.stderr)
|
||
print(f"epub2md_merge: {result.chapters} Kapitel zusammengefügt, {result.renamed_anchors} Anker umbenannt, "
|
||
f"{result.dropped_footnote_defs} doppelte Fußnotendefinitionen entfernt, {len(result.broken)} ungültige Links")
|
||
print(f"Ausgabe: {result.target}")
|
||
return 0
|
||
|
||
|
||
if __name__ == "__main__":
|
||
sys.exit(main())
|