diff --git a/README.md b/README.md index dc68fb8..6be2eac 100644 --- a/README.md +++ b/README.md @@ -209,6 +209,38 @@ weil CommonMark keine `[^…]`-Syntax kennt. - **Register:** Buchstabengruppen werden zu `##`-Überschriften, Untereinträge erhalten `– `. Interne Registerlinks werden auf Kapiteldatei + Anker umgeschrieben. Seitenzahlen bleiben als Text sichtbar. +## Gesamtdatei erzeugen: `epub2md_merge.py` + +Fügt die Kapiteldateien einer epub2md-Ausgabe in Lesereihenfolge (aus `manifest.json`) zu **einer** +Markdown-Datei zusammen. Die Datei heißt wie das Original-EPUB, aber mit der Endung `.md`. + +```bash +python3 epub2md_merge.py output/programmierhandbuch +# → output/programmierhandbuch/.md + +python3 epub2md_merge.py output/programmierhandbuch -o buecher/handbuch.md --toc --force +``` + +| Option | Wirkung | +| --- | --- | +| `-o`, `--output DATEI.md` | anderer Zielort; Medienpfade werden relativ zum Ziel umgeschrieben | +| `--frontmatter` / `--no-frontmatter` | ein YAML-Kopf mit Buchmetadaten (`title`, `author`, `lang`, …) für Pandoc (Standard: an) | +| `--toc` | Inhaltsverzeichnis mit Sprunglinks auf die Kapitel einfügen | +| `--force` | bestehende Zieldatei überschreiben (sonst Abbruch mit Exit 1) | + +Beim Zusammenfügen passiert Folgendes: + +- Die Frontmatter der einzelnen Kapitel wird entfernt. +- Jedes Kapitel erhält einen Startanker (``). +- Links zwischen Kapiteln (`003-x.md#anker`) werden zu Sprunglinks in der Gesamtdatei. +- Anker, die in mehreren Kapiteln vorkommen (z. B. `Sec1` in jedem Kapitel eines Springer-Buchs), werden + ab dem zweiten Vorkommen in `c012-Sec1` umbenannt, samt aller Links darauf. +- Medienpfade (`../media/…`) werden auf den Ort der Gesamtdatei umgeschrieben. +- Fußnotendefinitionen, die in mehreren Kapiteln stehen, erscheinen nur einmal. +- Codeblöcke und Inline-Code werden nie verändert. + +Am Ende werden alle Links der Gesamtdatei geprüft. + ## Weiterverarbeitung mit Pandoc ```bash @@ -260,6 +292,7 @@ Verlustfreiheit behauptet (`lossless: false`). ```text epub2md.py Einstiegspunkt +epub2md_merge.py Einstiegspunkt: Gesamtdatei aus einer Ausgabe erzeugen (epub2md/merge.py) epub2md/ ├── cli.py argparse, Exit-Codes ├── pipeline.py Ablaufsteuerung, Schreiben (atomar), Bericht/Manifest diff --git a/epub2md/merge.py b/epub2md/merge.py new file mode 100644 index 0000000..cba21ff --- /dev/null +++ b/epub2md/merge.py @@ -0,0 +1,333 @@ +"""Merge an epub2md output directory into one complete Markdown file. + +The chapter order comes from manifest.json; the file name is the original EPUB name with ".md". +Links between chapters become links to anchors inside the merged file, colliding anchors are +renamed per chapter, media paths are made relative to the merged file and duplicated footnote +definitions are dropped. Code (fenced blocks, inline code) is never modified. +""" + +from __future__ import annotations + +import argparse +import json +import os +import posixpath +import re +import sys +from dataclasses import dataclass, field +from pathlib import Path +from typing import NoReturn +from urllib.parse import quote, unquote + +from . import __version__ +from .errors import EXIT_INPUT, EXIT_INTERNAL +from .linkcheck import check_links +from .utils import to_yaml + +FENCE_OPEN = re.compile(r"^(\s*)(`{3,}|~{3,})") +# order matters: inline code is matched first and returned unchanged +TOKEN = re.compile( + r"(?P(?`+)(?!`)(?:[^`\n]|`(?!(?P=ticks)))+?(?P=ticks)(?!`))" + r"|(?P(?P(?<[^>\n]*>|[^)\s]+)(?P(?:\s+\"[^\"]*\")?\)))" + r"|(?P\b(?Phref|src|id)=\"(?P[^\"]*)\")" +) +FN_DEF = re.compile(r"^\[\^([^\]]+)\]:") + + +class MergeError(Exception): + pass + + +@dataclass +class ChapterFile: + number: int + title: str + rel: str # e.g. chapters/001-x.md (relative to the output directory) + body: str = "" + start_anchor: str = "" + anchor_map: dict[str, str] = field(default_factory=dict) + + +@dataclass +class MergeResult: + target: Path + chapters: int + renamed_anchors: int + dropped_footnote_defs: int + unresolved: list[str] + broken: list[dict[str, str]] + + +def strip_frontmatter(text: str) -> str: + if text.startswith("---\n"): + end = text.find("\n---\n", 4) + if end >= 0: + return text[end + 5 :].lstrip("\n") + return text + + +def load_book(out_dir: Path) -> tuple[dict, list[ChapterFile]]: + manifest_path = out_dir / "manifest.json" + report_path = out_dir / "conversion-report.json" + if not out_dir.is_dir(): + raise MergeError(f"Verzeichnis nicht gefunden: {out_dir}") + report = json.loads(report_path.read_text(encoding="utf-8")) if report_path.is_file() else {} + if manifest_path.is_file(): + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + entries = [(c["number"], c["title"], c["file"]) for c in manifest.get("chapters", [])] + else: + files = sorted((out_dir / "chapters").glob("*.md")) + entries = [(i + 1, p.stem, f"chapters/{p.name}") for i, p in enumerate(files)] + chapters = [ChapterFile(n, t, rel) for n, t, rel in entries if (out_dir / rel).is_file()] + missing = [rel for _, _, rel in entries if not (out_dir / rel).is_file()] + if missing: + raise MergeError(f"Kapiteldateien fehlen: {', '.join(missing[:5])}") + if not chapters: + raise MergeError(f"Keine Kapiteldateien in {out_dir} – ist das eine epub2md-Ausgabe?") + for ch in chapters: + ch.body = strip_frontmatter((out_dir / ch.rel).read_text(encoding="utf-8")) + return report, chapters + + +def default_target(out_dir: Path, report: dict) -> Path: + name = (report.get("input") or {}).get("name") or (out_dir.name + ".epub") + stem = name[:-5] if name.lower().endswith(".epub") else name + return out_dir / f"{stem}.md" + + +def _non_code_segments(text: str) -> list[tuple[bool, str]]: + """Split into (is_code, text) segments by fenced code blocks (also indented ones).""" + segments: list[tuple[bool, str]] = [] + buf: list[str] = [] + fence: str | None = None + for line in text.split("\n"): + if fence is None: + m = FENCE_OPEN.match(line) + if m: + if buf: + segments.append((False, "\n".join(buf))) + buf = [line] + fence = m.group(2) + continue + buf.append(line) + else: + buf.append(line) + stripped = line.strip() + if stripped and set(stripped) == {fence[0]} and len(stripped) >= len(fence): + segments.append((True, "\n".join(buf))) + buf = [] + fence = None + if buf: + segments.append((fence is not None, "\n".join(buf))) + return segments + + +def _join(segments: list[tuple[bool, str]]) -> str: + return "\n".join(s for _, s in segments) + + +def assign_anchors(chapters: list[ChapterFile]) -> int: + """Map every anchor id to a book-wide unique name; returns the number of renamed anchors.""" + used: set[str] = set() + renamed = 0 + stems = {} + for ch in chapters: + stem = posixpath.splitext(posixpath.basename(ch.rel))[0] + stems[ch.rel] = stem + for ch in chapters: + ids: list[str] = [] + for is_code, seg in _non_code_segments(ch.body): + if is_code: + continue + for m in TOKEN.finditer(seg): + if m.group("attr") and m.group("aname") == "id" and m.group("avalue") not in ids: + ids.append(m.group("avalue")) + start = stems[ch.rel] + while start in used or start in ids: + start = "kapitel-" + start + ch.start_anchor = start + used.add(start) + for raw in ids: + name = raw + if name in used: + name = f"c{ch.number:03d}-{raw}" + n = 2 + while name in used: + name = f"c{ch.number:03d}-{raw}-{n}" + n += 1 + renamed += 1 + used.add(name) + ch.anchor_map[raw] = name + return renamed + + +def merge(out_dir: Path, target: Path | None = None, *, frontmatter: bool = True, toc: bool = False, + force: bool = False) -> MergeResult: + out_dir = out_dir.resolve() + report, chapters = load_book(out_dir) + target = (target or default_target(out_dir, report)).resolve() + if target.exists() and not force: + raise MergeError(f"Zieldatei existiert bereits: {target} (mit --force überschreiben)") + if target.parent.exists() is False: + raise MergeError(f"Zielverzeichnis existiert nicht: {target.parent}") + renamed = assign_anchors(chapters) + by_rel = {ch.rel: ch for ch in chapters} + target_dir = target.parent + unresolved: list[str] = [] + + def rewrite_target(link: str, ch: ChapterFile) -> str: + bracket = link.startswith("<") and link.endswith(">") + raw = link[1:-1] if bracket else link + if not raw or re.match(r"^[A-Za-z][A-Za-z0-9+.-]*:", raw): + return link # external URL, mailto:, … + path, _, frag = raw.partition("#") + frag = unquote(frag) + if path: + resolved = posixpath.normpath(posixpath.join(posixpath.dirname(ch.rel), unquote(path))) + else: + resolved = ch.rel + dest = by_rel.get(resolved) + if dest is not None: + if frag and frag in dest.anchor_map: + new = "#" + dest.anchor_map[frag] + else: + if frag: + unresolved.append(f"{ch.rel}: {raw}") + new = "#" + dest.start_anchor + else: + absolute = out_dir / resolved + rel = os.path.relpath(absolute, target_dir).replace(os.sep, "/") + new = quote(rel, safe="/-._~") + (("#" + quote(frag, safe="-._~:")) if frag else "") + return f"<{new}>" if bracket else new + + def rewrite_segment(seg: str, ch: ChapterFile) -> str: + def repl(m: re.Match[str]) -> str: + if m.group("code"): + return m.group(0) + if m.group("mdlink"): + pre = m.group("lpre") + inner = TOKEN.sub(repl, pre[pre.index("[") + 1 : -2]) # images/links inside the link text + return pre[: pre.index("[") + 1] + inner + "](" + rewrite_target(m.group("ltarget"), ch) + m.group("lpost") + name, value = m.group("aname"), m.group("avalue") + if name == "id": + return f'id="{ch.anchor_map.get(value, value)}"' + return f'{name}="{rewrite_target(value, ch)}"' + + return TOKEN.sub(repl, seg) + + seen_defs: set[str] = set() + dropped = 0 + parts: list[str] = [] + if frontmatter: + md = report.get("metadata") or {} + fm: dict = {} + for key, src in (("title", "title"), ("subtitle", "subtitle"), ("author", "authors"), ("lang", "language"), + ("publisher", "publisher"), ("date", "published"), ("identifier", "identifier"), + ("rights", "rights"), ("description", "description")): + if md.get(src): + fm[key] = md[src] + if fm: + parts.append("---\n" + to_yaml(fm) + "---") + if toc: + lines = ["## Inhalt" if (report.get("metadata") or {}).get("language", "de").startswith("de") else "## Contents", ""] + lines += [f"{ch.number}. [{_escape_link_text(ch.title)}](#{ch.start_anchor})" for ch in chapters] + parts.append("\n".join(lines)) + for ch in chapters: + segments = [(is_code, seg if is_code else rewrite_segment(seg, ch)) for is_code, seg in _non_code_segments(ch.body)] + body, n = _drop_duplicate_definitions(_join(segments), seen_defs) + dropped += n + parts.append(f'\n\n{body.strip()}') + text = "\n\n".join(parts).rstrip() + "\n" + + existing = {p.resolve().as_posix() for p in out_dir.rglob("*") if p.is_file()} + broken = check_links({target.as_posix(): text}, existing) + target.write_text(text, encoding="utf-8") + return MergeResult(target, len(chapters), renamed, dropped, unresolved, broken) + + +def _escape_link_text(text: str) -> str: + return text.replace("\\", "\\\\").replace("[", "\\[").replace("]", "\\]") + + +def _drop_duplicate_definitions(body: str, seen: set[str]) -> tuple[str, int]: + """Remove footnote definitions whose label was already defined in an earlier chapter.""" + out: list[str] = [] + dropped = 0 + skipping = False + for is_code, seg in _non_code_segments(body): + if is_code: + if not skipping: + out.append(seg) + continue + kept: list[str] = [] + for line in seg.split("\n"): + m = FN_DEF.match(line) + if m: + label = m.group(1) + skipping = label in seen + if skipping: + dropped += 1 + continue + seen.add(label) + elif skipping and (line.startswith(" ") or not line.strip()): + continue + else: + skipping = False + kept.append(line) + out.append("\n".join(kept)) + return "\n".join(out), dropped + + +class _Parser(argparse.ArgumentParser): + def error(self, message: str) -> NoReturn: + self.print_usage(sys.stderr) + print(f"{self.prog}: Fehler: {message}", file=sys.stderr) + sys.exit(EXIT_INPUT) + + +def build_parser() -> argparse.ArgumentParser: + p = _Parser( + prog="epub2md_merge.py", + description="Fügt die Kapiteldateien einer epub2md-Ausgabe in Lesereihenfolge zu einer " + "vollständigen Markdown-Datei zusammen (Name = Original-EPUB mit Endung .md).", + epilog="Rückgabecodes: 0 Erfolg, 1 ungültige Eingabe/Zieldatei existiert, 4 interner Fehler.\n" + "Beispiel: python3 epub2md_merge.py output/programmierhandbuch", + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + p.add_argument("output_dir", metavar="AUSGABE_VERZEICHNIS", help="von epub2md.py erzeugtes Verzeichnis") + p.add_argument("-o", "--output", metavar="DATEI.md", + help="Zieldatei (Standard: AUSGABE_VERZEICHNIS/.md; Medienpfade werden angepasst)") + p.add_argument("--frontmatter", action=argparse.BooleanOptionalAction, default=True, + help="YAML-Frontmatter mit Buchmetadaten an den Anfang setzen (Standard: an)") + p.add_argument("--toc", action="store_true", help="Inhaltsverzeichnis mit Links auf die Kapitel einfügen") + p.add_argument("--force", action="store_true", help="bestehende Zieldatei überschreiben") + p.add_argument("--version", action="version", version=f"%(prog)s {__version__}") + return p + + +def main(argv: list[str] | None = None) -> int: + try: + args = build_parser().parse_args(argv) + except SystemExit as exc: + return exc.code if isinstance(exc.code, int) else EXIT_INPUT + try: + result = merge(Path(args.output_dir), Path(args.output) if args.output else None, + frontmatter=args.frontmatter, toc=args.toc, force=args.force) + except (MergeError, json.JSONDecodeError, OSError) as exc: + print(f"epub2md_merge: Fehler: {exc}", file=sys.stderr) + return EXIT_INPUT + except Exception as exc: # noqa: BLE001 – last line of defense + print(f"epub2md_merge: Interner Fehler: {type(exc).__name__}: {exc}", file=sys.stderr) + return EXIT_INTERNAL + for item in result.unresolved[:10]: + print(f"Warnung: Anker nicht gefunden, Link zeigt auf Kapitelanfang: {item}", file=sys.stderr) + for b in result.broken[:10]: + print(f"Warnung: ungültiger Link {b['link']} ({b['reason']})", file=sys.stderr) + print(f"epub2md_merge: {result.chapters} Kapitel zusammengefügt, {result.renamed_anchors} Anker umbenannt, " + f"{result.dropped_footnote_defs} doppelte Fußnotendefinitionen entfernt, {len(result.broken)} ungültige Links") + print(f"Ausgabe: {result.target}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/epub2md_merge.py b/epub2md_merge.py new file mode 100755 index 0000000..442fa84 --- /dev/null +++ b/epub2md_merge.py @@ -0,0 +1,9 @@ +#!/usr/bin/env python3 +"""Entry point: python3 epub2md_merge.py AUSGABE_VERZEICHNIS [-o DATEI.md] [--toc] [--force]""" + +import sys + +from epub2md.merge import main + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/test_merge.py b/tests/test_merge.py new file mode 100644 index 0000000..91898ae --- /dev/null +++ b/tests/test_merge.py @@ -0,0 +1,73 @@ +import re + +import yaml + +from conftest import PNG, Book, Doc +from epub2md.linkcheck import anchors_in, check_links +from epub2md.merge import main as merge_main + +A = ('

Kapitel A

Siehe B, Kapitel B, ' + 'hier und Anmerkung1.

' + '

Bild

' + '

Bildlink

' + '
[kein](link.md) id="sec" `code`
') +B = ('

Kapitel B

Zurück zu A, nochmal Anmerkung' + '1.

' + '') + + +def book() -> Book: + return Book(title="Merge-Test", docs=[Doc("a.xhtml", A), Doc("b.xhtml", B)], media={"img/a.png": PNG}) + + +def test_merge_complete_book(convert, tmp_path): + run = convert(book()) + assert merge_main([str(run.out)]) == 0 + target = run.out / f"{run.report['input']['name'][:-5]}.md" + text = target.read_text(encoding="utf-8") + # one book-level frontmatter, no chapter frontmatter + fm = yaml.safe_load(text.split("---\n")[1]) + assert fm["title"] == "Merge-Test" and fm["author"] == ["Erika Musterfrau"] and fm["lang"] == "de" + assert text.count("chapter_number") == 0 + # reading order + assert text.index("# Kapitel A") < text.index("# Kapitel B") + # links between chapters point into the merged file; colliding id "sec" renamed in chapter B + assert '' in text and '' in text + assert "[B](#c002-sec)" in text and "[A](#sec)" in text and "[hier](#sec)" in text + assert "[Kapitel B](#002-kapitel-b)" in text and '' in text + # media paths relative to the merged file, also inside image links + assert "![Bild](media/images/0001-bild.png)" in text + assert "[![Bildlink](media/images/0001-bild.png)](#002-kapitel-b)" in text + # code untouched + assert '[kein](link.md) id="sec" `code`' in text + # footnote defined in both chapters -> only once in the merged file + assert text.count("[^fn-001]:") == 1 + assert check_links({target.as_posix(): text}, {p.as_posix() for p in run.out.rglob("*") if p.is_file()}) == [] + prose = re.sub(r"^```.*?^```$", "", text, flags=re.M | re.S) + ids = re.findall(r'id="([^"]+)"', prose) + assert len(ids) == len(set(ids)) + + +def test_merge_refuses_overwrite_without_force(convert): + run = convert(book()) + assert merge_main([str(run.out)]) == 0 + assert merge_main([str(run.out)]) == 1 + assert merge_main([str(run.out), "--force"]) == 0 + + +def test_merge_to_other_location_rewrites_media_paths(convert, tmp_path): + run = convert(book()) + (tmp_path / "ziel").mkdir() + target = tmp_path / "ziel" / "buch.md" + assert merge_main([str(run.out), "-o", str(target), "--toc", "--no-frontmatter"]) == 0 + text = target.read_text(encoding="utf-8") + assert not text.startswith("---") + assert f"![Bild](../{run.out.name}/media/images/0001-bild.png)" in text + assert "1. [Kapitel A](#001-kapitel-a)" in text + assert "001-kapitel-a" in anchors_in(text) + + +def test_merge_invalid_directory(tmp_path): + assert merge_main([str(tmp_path / "gibt-es-nicht")]) == 1 + (tmp_path / "leer").mkdir() + assert merge_main([str(tmp_path / "leer")]) == 1