Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 557bbd3b36 | |||
| cc36372329 | |||
| 8afa1cc6d4 |
9 changed files with 71 additions and 65 deletions
6
.gitignore
vendored
6
.gitignore
vendored
|
|
@ -18,12 +18,8 @@ dist/
|
|||
.qa-fixtures/
|
||||
.qa-reports/
|
||||
|
||||
# Urheberrechtlich geschützte Beispiel-Bücher und Konvertierungsausgaben
|
||||
# Urheberrechtlich geschützte Beispiel-Bücher
|
||||
Beispiele/
|
||||
buch.epub
|
||||
buch.md
|
||||
buch-md/
|
||||
medien/
|
||||
|
||||
# Editor/OS
|
||||
.idea/
|
||||
|
|
|
|||
|
|
@ -78,6 +78,7 @@ python3 epub2md.py buch.epub output/buch --force --report berichte/buch.json
|
|||
| `--math {auto,tex,mathml,image}` | `auto` | siehe [Mathematik](#math-modi) |
|
||||
| `--images {relative,flat,original-path}` | `relative` | `relative`: `media/images/0001-name.ext`; `flat`: `media/name.ext`; `original-path`: `media/<Pfad im EPUB>` |
|
||||
| `--slug-language {de,en,none}` | `de` | Dateinamen: `de` ä→ae, ß→ss; `en` ä→a, ß→ss; `none` Unicode bleibt |
|
||||
| `--code-language SPRACHE` | – | Sprachlabel für Codeblöcke ohne erkannte Sprache (z. B. `python`, `c`; Aliase wie `py` erlaubt). Erkannte Labels haben Vorrang; im Bericht `detected_by: "option"` |
|
||||
| `--keep-html` | aus | Unbekannte/nicht sicher konvertierbare Elemente als bereinigtes HTML erhalten (sonst: Text übernommen + Eintrag in `diagnostics/unsupported-elements.md`) |
|
||||
| `--strict` | aus | Abbruch bei kritischen Fehlern (fehlende/unlesbare Spine-Dokumente, unsichere ZIP-Pfade, ungültige Spine-Verweise) |
|
||||
| `--force` | aus | Bestehendes Ausgabeverzeichnis ersetzen – siehe [Sicherheit](#sicherheit-und-fehlerbehandlung) |
|
||||
|
|
@ -150,6 +151,8 @@ Literaturlisten mit eigenen Labels (`[BryL06]`) werden nicht doppelt nummeriert.
|
|||
|
||||
**Tipp:** Hat ein Buch Teile als `h1` und Kapitel als `h2` (typisch bei Romanen), erzeugt
|
||||
`--chapter-level 2` eine Datei pro Kapitel. Mit dem Standard `1` entsteht eine Datei pro Teil.
|
||||
Ist eine Kapiteldatei größer als 30 000 Wörter und hat sie mindestens zwei `##`-Unterabschnitte, meldet
|
||||
epub2md eine Warnung `chapter-large` mit der passenden `--chapter-level`-Empfehlung.
|
||||
|
||||
## Math-Modi
|
||||
|
||||
|
|
@ -198,7 +201,9 @@ weil CommonMark keine `[^…]`-Syntax kennt.
|
|||
- **Syntax-Highlighting:** Farb-Spans, `<strong>`/`<em>` in Listings und Zeilennummern werden entfernt.
|
||||
Einrückung, Tabs, `<`, `>`, `&`, `$`, `|`, Backslashes und Backticks bleiben exakt erhalten. Die Sprache
|
||||
kommt aus Klassen/Attributen (`language-python`, `data-lang`, …) oder starken Inhaltssignalen
|
||||
(`#include`, `def …:`, Shebang, …). Ist sie unsicher, bleibt das Label leer.
|
||||
(`#include`, `def …:`, Shebang, …). Ist sie unsicher, bleibt das Label leer. Bei einsprachigen
|
||||
Büchern kann `--code-language python` (o. Ä.) alle übrigen Blöcke kennzeichnen. Achtung: Das betrifft
|
||||
auch Programmausgaben und Shell-Befehle im Buch.
|
||||
- **Gescannte Texte:** nur Bilder, kein Text (keine OCR).
|
||||
- **DRM:** wird erkannt und abgelehnt.
|
||||
- **Register:** Buchstabengruppen werden zu `##`-Überschriften, Untereinträge erhalten `– `. Interne
|
||||
|
|
|
|||
|
|
@ -1,11 +0,0 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
pandoc buch.epub \
|
||||
--from=epub \
|
||||
--to=markdown+fenced_code_blocks+backtick_code_blocks+tex_math_dollars+footnotes+pipe_tables \
|
||||
--wrap=none \
|
||||
--extract-media=medien \
|
||||
--output=buch.md
|
||||
|
||||
./split-md-chapters.py buch.md buch-md
|
||||
|
||||
|
|
@ -9,6 +9,7 @@ from pathlib import Path
|
|||
from typing import NoReturn
|
||||
|
||||
from . import __version__
|
||||
from .codeblocks import LANG_ALIASES
|
||||
from .errors import EXIT_INPUT, EXIT_INTERNAL, Epub2MdError
|
||||
from .pipeline import Options, convert, print_summary
|
||||
|
||||
|
|
@ -37,6 +38,14 @@ class _Parser(argparse.ArgumentParser):
|
|||
sys.exit(EXIT_INPUT)
|
||||
|
||||
|
||||
def code_language(value: str) -> str:
|
||||
lang = LANG_ALIASES.get(value.strip().lower())
|
||||
if lang is None:
|
||||
known = ", ".join(sorted(set(LANG_ALIASES.values())))
|
||||
raise argparse.ArgumentTypeError(f"unbekannte Sprache {value!r} (bekannt: {known})")
|
||||
return lang
|
||||
|
||||
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
p = _Parser(
|
||||
prog="epub2md.py",
|
||||
|
|
@ -70,6 +79,9 @@ def build_parser() -> argparse.ArgumentParser:
|
|||
help="Dateinamen-Regeln: de (ä→ae), en (ä→a), none (Unicode behalten) (Standard: de)")
|
||||
p.add_argument("--keep-html", action="store_true",
|
||||
help="Nicht zuverlässig konvertierbare Elemente als bereinigtes HTML erhalten")
|
||||
p.add_argument("--code-language", type=code_language, default=None, metavar="SPRACHE",
|
||||
help="Sprachlabel für Codeblöcke ohne erkannte Sprache, z. B. python oder c "
|
||||
"(erkannte Labels haben Vorrang; Standard: kein Label)")
|
||||
p.add_argument("--strict", action="store_true",
|
||||
help="Bei kritischen Fehlern (fehlende Spine-Dateien, unsichere Pfade, kaputtes XHTML) abbrechen")
|
||||
p.add_argument("--force", action="store_true",
|
||||
|
|
@ -101,6 +113,7 @@ def main(argv: list[str] | None = None) -> int:
|
|||
images=args.images,
|
||||
slug_language=args.slug_language,
|
||||
keep_html=args.keep_html,
|
||||
code_language=args.code_language or "",
|
||||
strict=args.strict,
|
||||
verbose=args.verbose,
|
||||
dry_run=args.dry_run,
|
||||
|
|
|
|||
|
|
@ -89,6 +89,7 @@ class RenderOptions:
|
|||
extract_media: bool = True
|
||||
keep_html: bool = False
|
||||
language: str = "de"
|
||||
code_language: str = "" # --code-language: label for blocks without a detected language
|
||||
|
||||
|
||||
def md_escape(text: str, table: bool = False) -> str:
|
||||
|
|
@ -421,6 +422,8 @@ class Renderer:
|
|||
if not lang:
|
||||
lang = detect_language_from_content(code)
|
||||
how = "content" if lang else ""
|
||||
if not lang and self.opt.code_language:
|
||||
lang, how = self.opt.code_language, "option"
|
||||
if self.ch:
|
||||
self.ch.stats.code_blocks += 1
|
||||
if not lang:
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ from __future__ import annotations
|
|||
|
||||
import os
|
||||
import posixpath
|
||||
import re
|
||||
import shutil
|
||||
import sys
|
||||
import tempfile
|
||||
|
|
@ -28,6 +29,7 @@ from .utils import is_inside, iter_elements, sha256_file, slugify
|
|||
from . import writer
|
||||
|
||||
MARKER_FILES = ("conversion-report.json", "manifest.json")
|
||||
LARGE_CHAPTER_WORDS = 30000 # above this a deeper --chapter-level is suggested (RAG/readability)
|
||||
|
||||
|
||||
@dataclass
|
||||
|
|
@ -43,6 +45,7 @@ class Options:
|
|||
images: str = "relative"
|
||||
slug_language: str = "de"
|
||||
keep_html: bool = False
|
||||
code_language: str = ""
|
||||
strict: bool = False
|
||||
verbose: bool = False
|
||||
dry_run: bool = False
|
||||
|
|
@ -178,7 +181,9 @@ def _convert(reader: EpubReader, input_path: Path, output_dir: Path, opts: Optio
|
|||
cover_out = media.register(md.cover_path, role="cover", hint="cover").output
|
||||
language = (md.language or "de").lower()
|
||||
renderer = Renderer(
|
||||
RenderOptions(opts.format, opts.footnotes, opts.math, opts.extract_media, opts.keep_html, language),
|
||||
RenderOptions(format=opts.format, footnotes=opts.footnotes, math=opts.math,
|
||||
extract_media=opts.extract_media, keep_html=opts.keep_html, language=language,
|
||||
code_language=opts.code_language),
|
||||
resolver, media, notes, diag, set(reader.names),
|
||||
)
|
||||
width = max(3, len(str(len(chapters) + 1)))
|
||||
|
|
@ -189,6 +194,7 @@ def _convert(reader: EpubReader, input_path: Path, output_dir: Path, opts: Optio
|
|||
diag.step(f"Rendere {ch.filename}")
|
||||
ch.markdown = renderer.render_chapter(ch)
|
||||
ch.stats.word_count = _word_count(ch.markdown)
|
||||
_check_chapter_size(ch, opts.chapter_level, diag)
|
||||
endnotes: Chapter | None = None
|
||||
endnotes_md = ""
|
||||
if opts.footnotes == "end-of-book" and renderer.book_notes:
|
||||
|
|
@ -288,9 +294,22 @@ def _assign_filenames(chapters: list[Chapter], slug_language: str) -> None:
|
|||
ch.filename = f"chapters/{candidate}.md"
|
||||
|
||||
|
||||
def _word_count(markdown: str) -> int:
|
||||
import re
|
||||
def _check_chapter_size(ch: Chapter, chapter_level: int, diag: Diagnostics) -> None:
|
||||
"""Suggest a deeper --chapter-level for very large chapters with enough sub-headings."""
|
||||
if ch.stats.word_count <= LARGE_CHAPTER_WORDS or chapter_level >= 6:
|
||||
return
|
||||
sections = len(re.findall(r"^## ", ch.markdown, re.M))
|
||||
if sections >= 2:
|
||||
diag.warn(
|
||||
"chapter-large",
|
||||
f"Kapitel „{ch.title}“ hat {ch.stats.word_count} Wörter und {sections} Unterabschnitte; "
|
||||
f"--chapter-level {chapter_level + 1} würde sie als eigene Dateien ausgeben.",
|
||||
ch.docs[0] if ch.docs else "",
|
||||
ch.filename,
|
||||
)
|
||||
|
||||
|
||||
def _word_count(markdown: str) -> int:
|
||||
text = re.sub(r"^(`{3,}).*?^\1\s*$", " ", markdown, flags=re.M | re.S)
|
||||
text = re.sub(r"<[^>]+>|\]\([^)]*\)", " ", text)
|
||||
return len(re.findall(r"\w+", text))
|
||||
|
|
|
|||
|
|
@ -1,45 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
if len(sys.argv) != 3:
|
||||
raise SystemExit(f"Aufruf: {sys.argv[0]} EINGABE.md AUSGABE_ORDNER")
|
||||
|
||||
src = Path(sys.argv[1])
|
||||
outdir = Path(sys.argv[2])
|
||||
outdir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
text = src.read_text(encoding="utf-8")
|
||||
parts = re.split(r"(?m)^(?=#\s+)", text)
|
||||
|
||||
def slugify(title: str) -> str:
|
||||
title = title.lower()
|
||||
title = re.sub(r"[^\w\s-]", "", title, flags=re.UNICODE)
|
||||
title = re.sub(r"[\s_-]+", "-", title).strip("-")
|
||||
return title[:80] or "kapitel"
|
||||
|
||||
index = []
|
||||
chapter_no = 0
|
||||
|
||||
for part in parts:
|
||||
if not part.strip():
|
||||
continue
|
||||
|
||||
m = re.match(r"^#\s+(.+?)\s*$", part, re.M)
|
||||
if m:
|
||||
chapter_no += 1
|
||||
title = m.group(1).strip()
|
||||
filename = f"{chapter_no:03d}-{slugify(title)}.md"
|
||||
else:
|
||||
filename = "000-vorspann.md"
|
||||
title = "Vorspann"
|
||||
|
||||
(outdir / filename).write_text(part.rstrip() + "\n", encoding="utf-8")
|
||||
index.append(f"- [{title}]({filename})")
|
||||
|
||||
(outdir / "README.md").write_text(
|
||||
"# Inhaltsverzeichnis\n\n" + "\n".join(index) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
|
|
@ -122,3 +122,14 @@ def test_glossary_anchor_is_placed_before_bold_term(convert):
|
|||
'<dl><dt id="g">API</dt><dd>Schnittstelle.</dd></dl></section>')
|
||||
text = convert(Book(docs=[Doc("c.xhtml", body)])).chapter("002")
|
||||
assert '<a id="g"></a>**API:** Schnittstelle.' in text
|
||||
|
||||
|
||||
def test_large_chapter_with_subheadings_suggests_deeper_chapter_level(convert):
|
||||
words = " ".join(["Wort"] * 11000)
|
||||
body = "<h1>Teil 1</h1>" + "".join(f"<h2>Kapitel {i}</h2><p>{words}</p>" for i in range(3))
|
||||
run = convert(Book(docs=[Doc("t.xhtml", body)]))
|
||||
hints = [w for w in run.report["warnings"] if w["code"] == "chapter-large"]
|
||||
assert len(hints) == 1
|
||||
assert "--chapter-level 2" in hints[0]["message"] and "3 Unterabschnitte" in hints[0]["message"]
|
||||
small = convert(Book(docs=[Doc("t.xhtml", "<h1>Teil</h1><h2>A</h2><p>x</p><h2>B</h2><p>y</p>")]))
|
||||
assert not any(w["code"] == "chapter-large" for w in small.report["warnings"])
|
||||
|
|
|
|||
|
|
@ -86,3 +86,18 @@ def test_prose_is_not_forced_into_code(convert):
|
|||
def test_unlabeled_code_is_reported(convert):
|
||||
run = convert(Book(docs=[Doc("c.xhtml", "<h1>U</h1><pre>foo bar</pre>")]))
|
||||
assert run.report["code_blocks"]["languages"] == {"(keine)": 1}
|
||||
|
||||
|
||||
def test_code_language_option_fills_only_unlabeled_blocks(convert):
|
||||
body = "<h1>C</h1><pre>x = compute()</pre><pre>#include <stdio.h>\nint main(void) { return 0; }</pre>"
|
||||
run = convert(Book(docs=[Doc("c.xhtml", body)]), "--code-language", "py")
|
||||
langs = [lang for _, lang, _ in fenced_blocks(run.chapter("001"))]
|
||||
assert langs == ["python", "c"]
|
||||
by = {item["language"]: item["detected_by"] for item in run.report["code_blocks"]["items"]}
|
||||
assert by == {"python": "option", "c": "content"}
|
||||
|
||||
|
||||
def test_code_language_option_rejects_unknown_values(convert):
|
||||
run = convert(Book(docs=[Doc("c.xhtml", "<h1>C</h1><pre>x</pre>")]), "--code-language", "quatsch")
|
||||
assert run.rc == 1
|
||||
assert "quatsch" in run.stderr
|
||||
|
|
|
|||
Loading…
Reference in a new issue