diff --git a/README.md b/README.md index 6be2eac..dc68fb8 100644 --- a/README.md +++ b/README.md @@ -209,38 +209,6 @@ weil CommonMark keine `[^…]`-Syntax kennt. - **Register:** Buchstabengruppen werden zu `##`-Überschriften, Untereinträge erhalten `– `. Interne Registerlinks werden auf Kapiteldatei + Anker umgeschrieben. Seitenzahlen bleiben als Text sichtbar. -## Gesamtdatei erzeugen: `epub2md_merge.py` - -Fügt die Kapiteldateien einer epub2md-Ausgabe in Lesereihenfolge (aus `manifest.json`) zu **einer** -Markdown-Datei zusammen. Die Datei heißt wie das Original-EPUB, aber mit der Endung `.md`. - -```bash -python3 epub2md_merge.py output/programmierhandbuch -# → output/programmierhandbuch/.md - -python3 epub2md_merge.py output/programmierhandbuch -o buecher/handbuch.md --toc --force -``` - -| Option | Wirkung | -| --- | --- | -| `-o`, `--output DATEI.md` | anderer Zielort; Medienpfade werden relativ zum Ziel umgeschrieben | -| `--frontmatter` / `--no-frontmatter` | ein YAML-Kopf mit Buchmetadaten (`title`, `author`, `lang`, …) für Pandoc (Standard: an) | -| `--toc` | Inhaltsverzeichnis mit Sprunglinks auf die Kapitel einfügen | -| `--force` | bestehende Zieldatei überschreiben (sonst Abbruch mit Exit 1) | - -Beim Zusammenfügen passiert Folgendes: - -- Die Frontmatter der einzelnen Kapitel wird entfernt. -- Jedes Kapitel erhält einen Startanker (``). -- Links zwischen Kapiteln (`003-x.md#anker`) werden zu Sprunglinks in der Gesamtdatei. -- Anker, die in mehreren Kapiteln vorkommen (z. B. `Sec1` in jedem Kapitel eines Springer-Buchs), werden - ab dem zweiten Vorkommen in `c012-Sec1` umbenannt, samt aller Links darauf. -- Medienpfade (`../media/…`) werden auf den Ort der Gesamtdatei umgeschrieben. -- Fußnotendefinitionen, die in mehreren Kapiteln stehen, erscheinen nur einmal. -- Codeblöcke und Inline-Code werden nie verändert. - -Am Ende werden alle Links der Gesamtdatei geprüft. - ## Weiterverarbeitung mit Pandoc ```bash @@ -292,7 +260,6 @@ Verlustfreiheit behauptet (`lossless: false`). ```text epub2md.py Einstiegspunkt -epub2md_merge.py Einstiegspunkt: Gesamtdatei aus einer Ausgabe erzeugen (epub2md/merge.py) epub2md/ ├── cli.py argparse, Exit-Codes ├── pipeline.py Ablaufsteuerung, Schreiben (atomar), Bericht/Manifest diff --git a/epub2md/html_converter.py b/epub2md/html_converter.py index 7490a17..f3aeeb8 100644 --- a/epub2md/html_converter.py +++ b/epub2md/html_converter.py @@ -616,7 +616,7 @@ class Renderer: if self.ch: self.ch.stats.tables_html += 1 self.diag.warn("table-html", f"Tabelle enthält {reason}; als HTML erhalten.", self._source(el), self.file) - return [self.clean_html(el)] # clean_html keeps the id on itself + return ([anchor] if anchor else []) + [self.clean_html(el)] out = [anchor] if anchor else [] if caption_el is not None: cap = join_inline(self.inline_children(caption_el), hard_break=" ") diff --git a/epub2md/linkcheck.py b/epub2md/linkcheck.py index 1503f0b..78241ff 100644 --- a/epub2md/linkcheck.py +++ b/epub2md/linkcheck.py @@ -10,9 +10,7 @@ from urllib.parse import unquote FENCE = re.compile(r"^(`{3,}|~{3,}).*?$.*?^\1\s*$", re.M | re.S) # inline code spans never cross a blank line and never start at an escaped backtick INLINE_CODE = re.compile(r"(?]*>|[^)\s]+)(?:\s+\"[^\"]*\")?\)") +MD_LINK = re.compile(r"!?\[(?:[^\]\\]|\\.)*\]\((<[^>]*>|[^)\s]+)(?:\s+\"[^\"]*\")?\)") HTML_ATTR = re.compile(r"""\b(?:href|src)="([^"]*)\"""") ANCHOR = re.compile(r"""\bid="([^"]+)\"""") HEADING = re.compile(r"^#{1,6}\s+(.*?)\s*#*\s*$", re.M) @@ -40,14 +38,7 @@ def anchors_in(text: str) -> set[str]: def extract_links(text: str) -> list[str]: body = strip_code(text) - links: list[str] = [] - - def collect(text: str) -> None: - for m in MD_LINK.finditer(text): - links.append(m.group(2)) - collect(m.group(1)) # links/images inside the link text - - collect(body) + links = [m.group(1) for m in MD_LINK.finditer(body)] links += HTML_ATTR.findall(body) return [link[1:-1] if link.startswith("<") and link.endswith(">") else link for link in links] diff --git a/epub2md/merge.py b/epub2md/merge.py deleted file mode 100644 index cba21ff..0000000 --- a/epub2md/merge.py +++ /dev/null @@ -1,333 +0,0 @@ -"""Merge an epub2md output directory into one complete Markdown file. - -The chapter order comes from manifest.json; the file name is the original EPUB name with ".md". -Links between chapters become links to anchors inside the merged file, colliding anchors are -renamed per chapter, media paths are made relative to the merged file and duplicated footnote -definitions are dropped. Code (fenced blocks, inline code) is never modified. -""" - -from __future__ import annotations - -import argparse -import json -import os -import posixpath -import re -import sys -from dataclasses import dataclass, field -from pathlib import Path -from typing import NoReturn -from urllib.parse import quote, unquote - -from . import __version__ -from .errors import EXIT_INPUT, EXIT_INTERNAL -from .linkcheck import check_links -from .utils import to_yaml - -FENCE_OPEN = re.compile(r"^(\s*)(`{3,}|~{3,})") -# order matters: inline code is matched first and returned unchanged -TOKEN = re.compile( - r"(?P(?`+)(?!`)(?:[^`\n]|`(?!(?P=ticks)))+?(?P=ticks)(?!`))" - r"|(?P(?P(?<[^>\n]*>|[^)\s]+)(?P(?:\s+\"[^\"]*\")?\)))" - r"|(?P\b(?Phref|src|id)=\"(?P[^\"]*)\")" -) -FN_DEF = re.compile(r"^\[\^([^\]]+)\]:") - - -class MergeError(Exception): - pass - - -@dataclass -class ChapterFile: - number: int - title: str - rel: str # e.g. chapters/001-x.md (relative to the output directory) - body: str = "" - start_anchor: str = "" - anchor_map: dict[str, str] = field(default_factory=dict) - - -@dataclass -class MergeResult: - target: Path - chapters: int - renamed_anchors: int - dropped_footnote_defs: int - unresolved: list[str] - broken: list[dict[str, str]] - - -def strip_frontmatter(text: str) -> str: - if text.startswith("---\n"): - end = text.find("\n---\n", 4) - if end >= 0: - return text[end + 5 :].lstrip("\n") - return text - - -def load_book(out_dir: Path) -> tuple[dict, list[ChapterFile]]: - manifest_path = out_dir / "manifest.json" - report_path = out_dir / "conversion-report.json" - if not out_dir.is_dir(): - raise MergeError(f"Verzeichnis nicht gefunden: {out_dir}") - report = json.loads(report_path.read_text(encoding="utf-8")) if report_path.is_file() else {} - if manifest_path.is_file(): - manifest = json.loads(manifest_path.read_text(encoding="utf-8")) - entries = [(c["number"], c["title"], c["file"]) for c in manifest.get("chapters", [])] - else: - files = sorted((out_dir / "chapters").glob("*.md")) - entries = [(i + 1, p.stem, f"chapters/{p.name}") for i, p in enumerate(files)] - chapters = [ChapterFile(n, t, rel) for n, t, rel in entries if (out_dir / rel).is_file()] - missing = [rel for _, _, rel in entries if not (out_dir / rel).is_file()] - if missing: - raise MergeError(f"Kapiteldateien fehlen: {', '.join(missing[:5])}") - if not chapters: - raise MergeError(f"Keine Kapiteldateien in {out_dir} – ist das eine epub2md-Ausgabe?") - for ch in chapters: - ch.body = strip_frontmatter((out_dir / ch.rel).read_text(encoding="utf-8")) - return report, chapters - - -def default_target(out_dir: Path, report: dict) -> Path: - name = (report.get("input") or {}).get("name") or (out_dir.name + ".epub") - stem = name[:-5] if name.lower().endswith(".epub") else name - return out_dir / f"{stem}.md" - - -def _non_code_segments(text: str) -> list[tuple[bool, str]]: - """Split into (is_code, text) segments by fenced code blocks (also indented ones).""" - segments: list[tuple[bool, str]] = [] - buf: list[str] = [] - fence: str | None = None - for line in text.split("\n"): - if fence is None: - m = FENCE_OPEN.match(line) - if m: - if buf: - segments.append((False, "\n".join(buf))) - buf = [line] - fence = m.group(2) - continue - buf.append(line) - else: - buf.append(line) - stripped = line.strip() - if stripped and set(stripped) == {fence[0]} and len(stripped) >= len(fence): - segments.append((True, "\n".join(buf))) - buf = [] - fence = None - if buf: - segments.append((fence is not None, "\n".join(buf))) - return segments - - -def _join(segments: list[tuple[bool, str]]) -> str: - return "\n".join(s for _, s in segments) - - -def assign_anchors(chapters: list[ChapterFile]) -> int: - """Map every anchor id to a book-wide unique name; returns the number of renamed anchors.""" - used: set[str] = set() - renamed = 0 - stems = {} - for ch in chapters: - stem = posixpath.splitext(posixpath.basename(ch.rel))[0] - stems[ch.rel] = stem - for ch in chapters: - ids: list[str] = [] - for is_code, seg in _non_code_segments(ch.body): - if is_code: - continue - for m in TOKEN.finditer(seg): - if m.group("attr") and m.group("aname") == "id" and m.group("avalue") not in ids: - ids.append(m.group("avalue")) - start = stems[ch.rel] - while start in used or start in ids: - start = "kapitel-" + start - ch.start_anchor = start - used.add(start) - for raw in ids: - name = raw - if name in used: - name = f"c{ch.number:03d}-{raw}" - n = 2 - while name in used: - name = f"c{ch.number:03d}-{raw}-{n}" - n += 1 - renamed += 1 - used.add(name) - ch.anchor_map[raw] = name - return renamed - - -def merge(out_dir: Path, target: Path | None = None, *, frontmatter: bool = True, toc: bool = False, - force: bool = False) -> MergeResult: - out_dir = out_dir.resolve() - report, chapters = load_book(out_dir) - target = (target or default_target(out_dir, report)).resolve() - if target.exists() and not force: - raise MergeError(f"Zieldatei existiert bereits: {target} (mit --force überschreiben)") - if target.parent.exists() is False: - raise MergeError(f"Zielverzeichnis existiert nicht: {target.parent}") - renamed = assign_anchors(chapters) - by_rel = {ch.rel: ch for ch in chapters} - target_dir = target.parent - unresolved: list[str] = [] - - def rewrite_target(link: str, ch: ChapterFile) -> str: - bracket = link.startswith("<") and link.endswith(">") - raw = link[1:-1] if bracket else link - if not raw or re.match(r"^[A-Za-z][A-Za-z0-9+.-]*:", raw): - return link # external URL, mailto:, … - path, _, frag = raw.partition("#") - frag = unquote(frag) - if path: - resolved = posixpath.normpath(posixpath.join(posixpath.dirname(ch.rel), unquote(path))) - else: - resolved = ch.rel - dest = by_rel.get(resolved) - if dest is not None: - if frag and frag in dest.anchor_map: - new = "#" + dest.anchor_map[frag] - else: - if frag: - unresolved.append(f"{ch.rel}: {raw}") - new = "#" + dest.start_anchor - else: - absolute = out_dir / resolved - rel = os.path.relpath(absolute, target_dir).replace(os.sep, "/") - new = quote(rel, safe="/-._~") + (("#" + quote(frag, safe="-._~:")) if frag else "") - return f"<{new}>" if bracket else new - - def rewrite_segment(seg: str, ch: ChapterFile) -> str: - def repl(m: re.Match[str]) -> str: - if m.group("code"): - return m.group(0) - if m.group("mdlink"): - pre = m.group("lpre") - inner = TOKEN.sub(repl, pre[pre.index("[") + 1 : -2]) # images/links inside the link text - return pre[: pre.index("[") + 1] + inner + "](" + rewrite_target(m.group("ltarget"), ch) + m.group("lpost") - name, value = m.group("aname"), m.group("avalue") - if name == "id": - return f'id="{ch.anchor_map.get(value, value)}"' - return f'{name}="{rewrite_target(value, ch)}"' - - return TOKEN.sub(repl, seg) - - seen_defs: set[str] = set() - dropped = 0 - parts: list[str] = [] - if frontmatter: - md = report.get("metadata") or {} - fm: dict = {} - for key, src in (("title", "title"), ("subtitle", "subtitle"), ("author", "authors"), ("lang", "language"), - ("publisher", "publisher"), ("date", "published"), ("identifier", "identifier"), - ("rights", "rights"), ("description", "description")): - if md.get(src): - fm[key] = md[src] - if fm: - parts.append("---\n" + to_yaml(fm) + "---") - if toc: - lines = ["## Inhalt" if (report.get("metadata") or {}).get("language", "de").startswith("de") else "## Contents", ""] - lines += [f"{ch.number}. [{_escape_link_text(ch.title)}](#{ch.start_anchor})" for ch in chapters] - parts.append("\n".join(lines)) - for ch in chapters: - segments = [(is_code, seg if is_code else rewrite_segment(seg, ch)) for is_code, seg in _non_code_segments(ch.body)] - body, n = _drop_duplicate_definitions(_join(segments), seen_defs) - dropped += n - parts.append(f'\n\n{body.strip()}') - text = "\n\n".join(parts).rstrip() + "\n" - - existing = {p.resolve().as_posix() for p in out_dir.rglob("*") if p.is_file()} - broken = check_links({target.as_posix(): text}, existing) - target.write_text(text, encoding="utf-8") - return MergeResult(target, len(chapters), renamed, dropped, unresolved, broken) - - -def _escape_link_text(text: str) -> str: - return text.replace("\\", "\\\\").replace("[", "\\[").replace("]", "\\]") - - -def _drop_duplicate_definitions(body: str, seen: set[str]) -> tuple[str, int]: - """Remove footnote definitions whose label was already defined in an earlier chapter.""" - out: list[str] = [] - dropped = 0 - skipping = False - for is_code, seg in _non_code_segments(body): - if is_code: - if not skipping: - out.append(seg) - continue - kept: list[str] = [] - for line in seg.split("\n"): - m = FN_DEF.match(line) - if m: - label = m.group(1) - skipping = label in seen - if skipping: - dropped += 1 - continue - seen.add(label) - elif skipping and (line.startswith(" ") or not line.strip()): - continue - else: - skipping = False - kept.append(line) - out.append("\n".join(kept)) - return "\n".join(out), dropped - - -class _Parser(argparse.ArgumentParser): - def error(self, message: str) -> NoReturn: - self.print_usage(sys.stderr) - print(f"{self.prog}: Fehler: {message}", file=sys.stderr) - sys.exit(EXIT_INPUT) - - -def build_parser() -> argparse.ArgumentParser: - p = _Parser( - prog="epub2md_merge.py", - description="Fügt die Kapiteldateien einer epub2md-Ausgabe in Lesereihenfolge zu einer " - "vollständigen Markdown-Datei zusammen (Name = Original-EPUB mit Endung .md).", - epilog="Rückgabecodes: 0 Erfolg, 1 ungültige Eingabe/Zieldatei existiert, 4 interner Fehler.\n" - "Beispiel: python3 epub2md_merge.py output/programmierhandbuch", - formatter_class=argparse.RawDescriptionHelpFormatter, - ) - p.add_argument("output_dir", metavar="AUSGABE_VERZEICHNIS", help="von epub2md.py erzeugtes Verzeichnis") - p.add_argument("-o", "--output", metavar="DATEI.md", - help="Zieldatei (Standard: AUSGABE_VERZEICHNIS/.md; Medienpfade werden angepasst)") - p.add_argument("--frontmatter", action=argparse.BooleanOptionalAction, default=True, - help="YAML-Frontmatter mit Buchmetadaten an den Anfang setzen (Standard: an)") - p.add_argument("--toc", action="store_true", help="Inhaltsverzeichnis mit Links auf die Kapitel einfügen") - p.add_argument("--force", action="store_true", help="bestehende Zieldatei überschreiben") - p.add_argument("--version", action="version", version=f"%(prog)s {__version__}") - return p - - -def main(argv: list[str] | None = None) -> int: - try: - args = build_parser().parse_args(argv) - except SystemExit as exc: - return exc.code if isinstance(exc.code, int) else EXIT_INPUT - try: - result = merge(Path(args.output_dir), Path(args.output) if args.output else None, - frontmatter=args.frontmatter, toc=args.toc, force=args.force) - except (MergeError, json.JSONDecodeError, OSError) as exc: - print(f"epub2md_merge: Fehler: {exc}", file=sys.stderr) - return EXIT_INPUT - except Exception as exc: # noqa: BLE001 – last line of defense - print(f"epub2md_merge: Interner Fehler: {type(exc).__name__}: {exc}", file=sys.stderr) - return EXIT_INTERNAL - for item in result.unresolved[:10]: - print(f"Warnung: Anker nicht gefunden, Link zeigt auf Kapitelanfang: {item}", file=sys.stderr) - for b in result.broken[:10]: - print(f"Warnung: ungültiger Link {b['link']} ({b['reason']})", file=sys.stderr) - print(f"epub2md_merge: {result.chapters} Kapitel zusammengefügt, {result.renamed_anchors} Anker umbenannt, " - f"{result.dropped_footnote_defs} doppelte Fußnotendefinitionen entfernt, {len(result.broken)} ungültige Links") - print(f"Ausgabe: {result.target}") - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/epub2md_merge.py b/epub2md_merge.py deleted file mode 100755 index 442fa84..0000000 --- a/epub2md_merge.py +++ /dev/null @@ -1,9 +0,0 @@ -#!/usr/bin/env python3 -"""Entry point: python3 epub2md_merge.py AUSGABE_VERZEICHNIS [-o DATEI.md] [--toc] [--force]""" - -import sys - -from epub2md.merge import main - -if __name__ == "__main__": - sys.exit(main()) diff --git a/tests/test_links.py b/tests/test_links.py index 6a4bf8f..6b22d8e 100644 --- a/tests/test_links.py +++ b/tests/test_links.py @@ -66,18 +66,3 @@ def test_link_checker_detects_problems(): } broken = check_links(files, {"media/images/a.png"}) assert {b["link"] for b in broken} == {"002-b.md#nein", "003.md"} - - -def test_link_checker_ignores_escaped_brackets(): - """An escaped \\[ must not start a link that swallows the following paragraphs.""" - files = { - "a.md": '\\[»\\] Hinweis\n\nText [kaputt](fehlt.md) und \\[1\\]\n\n[ok](#x)\n', - } - broken = check_links(files, set()) - assert [b["link"] for b in broken] == ["fehlt.md"] - - -def test_link_checker_handles_image_links(): - files = {"a.md": "[![Abb.](bild.png)](fehlt.md) [![ok](bild.png)](#x) \n"} - broken = check_links(files, {"bild.png"}) - assert [b["link"] for b in broken] == ["fehlt.md"] diff --git a/tests/test_merge.py b/tests/test_merge.py deleted file mode 100644 index 91898ae..0000000 --- a/tests/test_merge.py +++ /dev/null @@ -1,73 +0,0 @@ -import re - -import yaml - -from conftest import PNG, Book, Doc -from epub2md.linkcheck import anchors_in, check_links -from epub2md.merge import main as merge_main - -A = ('

Kapitel A

Siehe B, Kapitel B, ' - 'hier und Anmerkung1.

' - '

Bild

' - '

Bildlink

' - '
[kein](link.md) id="sec" `code`
') -B = ('

Kapitel B

Zurück zu A, nochmal Anmerkung' - '1.

' - '') - - -def book() -> Book: - return Book(title="Merge-Test", docs=[Doc("a.xhtml", A), Doc("b.xhtml", B)], media={"img/a.png": PNG}) - - -def test_merge_complete_book(convert, tmp_path): - run = convert(book()) - assert merge_main([str(run.out)]) == 0 - target = run.out / f"{run.report['input']['name'][:-5]}.md" - text = target.read_text(encoding="utf-8") - # one book-level frontmatter, no chapter frontmatter - fm = yaml.safe_load(text.split("---\n")[1]) - assert fm["title"] == "Merge-Test" and fm["author"] == ["Erika Musterfrau"] and fm["lang"] == "de" - assert text.count("chapter_number") == 0 - # reading order - assert text.index("# Kapitel A") < text.index("# Kapitel B") - # links between chapters point into the merged file; colliding id "sec" renamed in chapter B - assert '' in text and '' in text - assert "[B](#c002-sec)" in text and "[A](#sec)" in text and "[hier](#sec)" in text - assert "[Kapitel B](#002-kapitel-b)" in text and '' in text - # media paths relative to the merged file, also inside image links - assert "![Bild](media/images/0001-bild.png)" in text - assert "[![Bildlink](media/images/0001-bild.png)](#002-kapitel-b)" in text - # code untouched - assert '[kein](link.md) id="sec" `code`' in text - # footnote defined in both chapters -> only once in the merged file - assert text.count("[^fn-001]:") == 1 - assert check_links({target.as_posix(): text}, {p.as_posix() for p in run.out.rglob("*") if p.is_file()}) == [] - prose = re.sub(r"^```.*?^```$", "", text, flags=re.M | re.S) - ids = re.findall(r'id="([^"]+)"', prose) - assert len(ids) == len(set(ids)) - - -def test_merge_refuses_overwrite_without_force(convert): - run = convert(book()) - assert merge_main([str(run.out)]) == 0 - assert merge_main([str(run.out)]) == 1 - assert merge_main([str(run.out), "--force"]) == 0 - - -def test_merge_to_other_location_rewrites_media_paths(convert, tmp_path): - run = convert(book()) - (tmp_path / "ziel").mkdir() - target = tmp_path / "ziel" / "buch.md" - assert merge_main([str(run.out), "-o", str(target), "--toc", "--no-frontmatter"]) == 0 - text = target.read_text(encoding="utf-8") - assert not text.startswith("---") - assert f"![Bild](../{run.out.name}/media/images/0001-bild.png)" in text - assert "1. [Kapitel A](#001-kapitel-a)" in text - assert "001-kapitel-a" in anchors_in(text) - - -def test_merge_invalid_directory(tmp_path): - assert merge_main([str(tmp_path / "gibt-es-nicht")]) == 1 - (tmp_path / "leer").mkdir() - assert merge_main([str(tmp_path / "leer")]) == 1 diff --git a/tests/test_tables_lists.py b/tests/test_tables_lists.py index 8817fb6..3fab505 100644 --- a/tests/test_tables_lists.py +++ b/tests/test_tables_lists.py @@ -63,10 +63,3 @@ def test_unknown_elements_are_reported_and_keep_html(convert): kept = convert(Book(docs=[Doc("c.xhtml", body)]), "--keep-html") assert 'Inhalt bleibt' in kept.chapter("001") assert "evil" not in kept.chapter("001") - - -def test_complex_table_anchor_is_emitted_once(convert): - body = ('

T

Tabelle 1

' - "
a
bc
") - text = convert(Book(docs=[Doc("c.xhtml", body)])).chapter("001") - assert text.count('id="t1"') == 1