diff --git a/BEDIENUNGSANLEITUNG.md b/BEDIENUNGSANLEITUNG.md index afb00f6..b7a473a 100644 --- a/BEDIENUNGSANLEITUNG.md +++ b/BEDIENUNGSANLEITUNG.md @@ -434,6 +434,7 @@ python3 epub2md_merge.py --help | `--chapter-level N` | ab welcher Überschriftenebene eine neue Datei beginnt (Standard 1) | | `--code-language SPRACHE` | Sprache für Codeblöcke ohne erkannte Sprache | | `--keep-wrap-markers` | Druck-Umbruchzeichen `↪` im Code stehen lassen | +| `--keep-invisible-chars` | unsichtbare Zeichen (Nullbreiten-Leerzeichen, weiche Trennstriche) behalten | | `--math auto/tex/mathml/image` | Formeldarstellung | | `--footnotes end-of-chapter/end-of-book/inline/keep-links` | Fußnoten-Platzierung | | `--format gfm/commonmark` | Markdown-Variante; `commonmark` für sehr strikte Programme | diff --git a/README.md b/README.md index f48bfdd..caf194f 100644 --- a/README.md +++ b/README.md @@ -97,6 +97,7 @@ python3 epub2md.py buch.epub output/buch --force --report berichte/buch.json | `--slug-language {de,en,none}` | `de` | Dateinamen: `de` ä→ae, ß→ss; `en` ä→a, ß→ss; `none` Unicode bleibt | | `--code-language SPRACHE` | – | Sprachlabel für Codeblöcke ohne erkannte Sprache (z. B. `python`, `c`; Aliase wie `py` erlaubt). Erkannte Labels haben Vorrang; im Bericht `detected_by: "option"` | | `--keep-wrap-markers` | aus | Druck-Umbruchzeichen `↪`/`➥` in Codeblöcken stehen lassen. Standard: Fortsetzungszeilen werden wieder zur ursprünglichen Codezeile zusammengefügt (Anzahl im Bericht: `wrap_lines_joined`) | +| `--keep-invisible-chars` | aus | Nullbreiten-Zeichen und weiche Trennstriche im Fließtext behalten. Standard: entfernen, weil sie Suche und RAG stören (Anzahl im Bericht: `text_cleanup`). Code bleibt immer unverändert | | `--keep-html` | aus | Unbekannte/nicht sicher konvertierbare Elemente als bereinigtes HTML erhalten (sonst: Text übernommen + Eintrag in `diagnostics/unsupported-elements.md`) | | `--strict` | aus | Abbruch bei kritischen Fehlern (fehlende/unlesbare Spine-Dokumente, unsichere ZIP-Pfade, ungültige Spine-Verweise) | | `--force` | aus | Bestehendes Ausgabeverzeichnis ersetzen – siehe [Sicherheit](#sicherheit-und-fehlerbehandlung) | diff --git a/epub2md/chapters.py b/epub2md/chapters.py index 5845852..0b16de3 100644 --- a/epub2md/chapters.py +++ b/epub2md/chapters.py @@ -9,7 +9,17 @@ from typing import Any from .diagnostics import Diagnostics from .models import Chapter, NavEntry from .normalize import Doc, has_content -from .utils import classes, epub_types, is_element, iter_elements, local, norm_ws, roles, text_content +from .utils import ( + classes, + epub_types, + is_element, + iter_elements, + local, + norm_ws, + remove_invisible, + roles, + text_content, +) HEADINGS = {"h1": 1, "h2": 2, "h3": 3, "h4": 4, "h5": 5, "h6": 6} CONTAINERS = {"body", "div", "section", "article", "aside", "main", "header", "footer", "nav", "span", "hgroup"} @@ -270,6 +280,7 @@ class ChapterBuilder: if not doc.linear and all(not d or d == doc.path for d in ch.docs[:1]): types.append("auxiliary") ch.types = list(dict.fromkeys(types)) + ch.title = remove_invisible(ch.title)[0] # titles feed file names, frontmatter and README return ch def _title_from_content(self, items: list[tuple[str, Any]], number: int) -> tuple[str, str]: diff --git a/epub2md/cli.py b/epub2md/cli.py index 4679493..4f40c58 100644 --- a/epub2md/cli.py +++ b/epub2md/cli.py @@ -85,6 +85,9 @@ def build_parser() -> argparse.ArgumentParser: p.add_argument("--keep-wrap-markers", action="store_true", help="Druck-Umbruchzeichen (↪/➥) in Codeblöcken nicht entfernen; Standard: fortgesetzte " "Zeilen werden wieder zur ursprünglichen Codezeile zusammengefügt") + p.add_argument("--keep-invisible-chars", action="store_true", + help="Nullbreiten-Zeichen und weiche Trennstriche (U+00AD) im Fließtext behalten; Standard: entfernen " + "(stören Suche und Weiterverarbeitung). Code bleibt immer unverändert") p.add_argument("--strict", action="store_true", help="Bei kritischen Fehlern abbrechen: fehlende/unlesbare Spine-Dokumente, nicht dekodierbares " "XHTML, unsichere ZIP-Pfade, übersprungene (verdächtige) Medien") @@ -119,6 +122,7 @@ def main(argv: list[str] | None = None) -> int: keep_html=args.keep_html, code_language=args.code_language or "", keep_wrap_markers=args.keep_wrap_markers, + keep_invisible_chars=args.keep_invisible_chars, strict=args.strict, verbose=args.verbose, dry_run=args.dry_run, diff --git a/epub2md/html_converter.py b/epub2md/html_converter.py index 68c0c4d..d842b90 100644 --- a/epub2md/html_converter.py +++ b/epub2md/html_converter.py @@ -45,6 +45,7 @@ from .utils import ( local, norm_ws, rel_link, + remove_invisible, resolve_href, text_content, url_path, @@ -93,6 +94,7 @@ class RenderOptions: language: str = "de" code_language: str = "" # --code-language: label for blocks without a detected language keep_wrap_markers: bool = False # --keep-wrap-markers: do not join ↪-continued code lines + keep_invisible_chars: bool = False # --keep-invisible-chars: keep zero-width chars and soft hyphens def md_escape(text: str, table: bool = False) -> str: @@ -231,6 +233,7 @@ class Renderer: notes: NoteIndex, diag: Diagnostics, zip_names: set[str], entry_problem: Callable[[str], str | None] | None = None) -> None: self.entry_problem = entry_problem or (lambda path: None) + self.invisible_removed = 0 self.opt = options self.zip_names = zip_names self.in_table_html = False @@ -293,7 +296,7 @@ class Renderer: if doc: self.doc = doc if isinstance(item, str): - buf.append(("t", md_escape(item, self.in_table))) + buf.append(("t", self._text(item))) elif isinstance(item, IdCarrier): anchor = self.resolver.anchor_for(self.doc, item.element) if anchor: @@ -755,12 +758,12 @@ class Renderer: def inline_children(self, el: Any) -> list[Piece]: pieces: list[Piece] = [] if el.text: - pieces.append(("t", md_escape(el.text, self.in_table))) + pieces.append(("t", self._text(el.text))) for child in el: if is_element(child): pieces.extend(self.inline(child)) if child.tail: - pieces.append(("t", md_escape(child.tail, self.in_table))) + pieces.append(("t", self._text(child.tail))) return pieces def inline(self, el: Any) -> list[Piece]: @@ -1046,7 +1049,16 @@ class Renderer: title_part = f' "{title.replace(chr(34), chr(39))}"' if title else "" return [("r", f"![{self._alt(alt)}]({url_path(rel_link(self.file, mf.output))}{title_part})")] + def _text(self, text: str) -> str: + """Escape prose text; zero-width characters and soft hyphens are dropped (P6).""" + if not self.opt.keep_invisible_chars: + text, n = remove_invisible(text) + self.invisible_removed += n + return md_escape(text, self.in_table) + def _alt(self, alt: str) -> str: + if not self.opt.keep_invisible_chars: + alt = remove_invisible(alt)[0] return WS.sub(" ", alt).replace("\\", "\\\\").replace("[", "\\[").replace("]", "\\]").strip() def _is_formula_image(self, el: Any, alt: str, path: str) -> bool: diff --git a/epub2md/pipeline.py b/epub2md/pipeline.py index e5dd9dc..7c683cb 100644 --- a/epub2md/pipeline.py +++ b/epub2md/pipeline.py @@ -47,6 +47,7 @@ class Options: keep_html: bool = False code_language: str = "" keep_wrap_markers: bool = False + keep_invisible_chars: bool = False strict: bool = False verbose: bool = False dry_run: bool = False @@ -235,7 +236,8 @@ def _render(reader: EpubReader, package: Any, chapters: list[Chapter], resolver: renderer = Renderer( RenderOptions(format=opts.format, footnotes=opts.footnotes, math=opts.math, extract_media=opts.extract_media, keep_html=opts.keep_html, language=language, - code_language=opts.code_language, keep_wrap_markers=opts.keep_wrap_markers), + code_language=opts.code_language, keep_wrap_markers=opts.keep_wrap_markers, + keep_invisible_chars=opts.keep_invisible_chars), resolver, media, notes, diag, set(reader.names), reader.entry_problem, ) width = max(3, len(str(len(chapters) + 1))) @@ -495,6 +497,7 @@ def _report(input_path: Path, sha: str, generated_at: str, package: Any, opts: O "referenced": sum(1 for n in notes.notes.values() if n.ref_count), "rendered": len(renderer.book_notes)}, "special_sections": special, + "text_cleanup": {"invisible_chars_removed": renderer.invisible_removed}, "warnings": warnings, "errors": errors, "infos": [d.as_dict() for d in diag.items if d.level == "info"], diff --git a/epub2md/utils.py b/epub2md/utils.py index 5d88b3e..d9115c0 100644 --- a/epub2md/utils.py +++ b/epub2md/utils.py @@ -185,6 +185,15 @@ def text_content(el: Any) -> str: return "".join(parts) +INVISIBLE = re.compile("[\u200b\u200c\u200d\u2060\ufeff\u00ad]") # zero-width chars, BOM, soft hyphen + + +def remove_invisible(text: str) -> tuple[str, int]: + """Remove zero-width characters and soft hyphens; returns (text, number removed).""" + cleaned, n = INVISIBLE.subn("", text) + return cleaned, n + + def norm_ws(text: str) -> str: return re.sub(r"[ \t\r\n\f]+", " ", text or "").strip() diff --git a/tests/test_styles.py b/tests/test_styles.py new file mode 100644 index 0000000..8ce166f --- /dev/null +++ b/tests/test_styles.py @@ -0,0 +1,18 @@ +"""Typography and layout: invisible characters, CSS emphasis, scene breaks, boxes, styles, page markers.""" + +from conftest import Book, Doc + +ZW, SHY = "​", "­" + + +def test_p6_invisible_characters_removed_outside_code(convert): + body = (f"

Kapitel 1.{ZW}1

Lese{SHY}rinnen und 1.{ZW}1 sowie{ZW}Wort.

" + f"
a{ZW}b

x{SHY}y

") + run = convert(Book(docs=[Doc("c.xhtml", body)])) + text = run.chapter("001") + assert "Leserinnen und 1.1 sowieWort." in text + assert "# Kapitel 1.1" in text and 'chapter_title: "Kapitel 1.1"' in text + assert f"a{ZW}b" in text and f"x{SHY}y" in text # code stays verbatim + assert run.report["text_cleanup"]["invisible_chars_removed"] == 4 + kept = convert(Book(docs=[Doc("c.xhtml", body)]), "--keep-invisible-chars").chapter("001") + assert f"Lese{SHY}rinnen" in kept