feat: unsichtbare Zeichen im Fließtext entfernen (P6)
Nullbreiten-Zeichen (U+200B–U+200D, U+2060, U+FEFF) und weiche Trennstriche (U+00AD) werden aus Fließtext, Alternativtexten und Kapiteltiteln entfernt; Code bleibt unverändert. Im Mathebuch sind das 198 Zeichen (z. B. '1.1'). Abschaltbar mit --keep-invisible-chars; die Anzahl steht im Bericht unter text_cleanup. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
2125555e2a
commit
a54fdda2fa
8 changed files with 64 additions and 5 deletions
|
|
@ -434,6 +434,7 @@ python3 epub2md_merge.py --help
|
|||
| `--chapter-level N` | ab welcher Überschriftenebene eine neue Datei beginnt (Standard 1) |
|
||||
| `--code-language SPRACHE` | Sprache für Codeblöcke ohne erkannte Sprache |
|
||||
| `--keep-wrap-markers` | Druck-Umbruchzeichen `↪` im Code stehen lassen |
|
||||
| `--keep-invisible-chars` | unsichtbare Zeichen (Nullbreiten-Leerzeichen, weiche Trennstriche) behalten |
|
||||
| `--math auto/tex/mathml/image` | Formeldarstellung |
|
||||
| `--footnotes end-of-chapter/end-of-book/inline/keep-links` | Fußnoten-Platzierung |
|
||||
| `--format gfm/commonmark` | Markdown-Variante; `commonmark` für sehr strikte Programme |
|
||||
|
|
|
|||
|
|
@ -97,6 +97,7 @@ python3 epub2md.py buch.epub output/buch --force --report berichte/buch.json
|
|||
| `--slug-language {de,en,none}` | `de` | Dateinamen: `de` ä→ae, ß→ss; `en` ä→a, ß→ss; `none` Unicode bleibt |
|
||||
| `--code-language SPRACHE` | – | Sprachlabel für Codeblöcke ohne erkannte Sprache (z. B. `python`, `c`; Aliase wie `py` erlaubt). Erkannte Labels haben Vorrang; im Bericht `detected_by: "option"` |
|
||||
| `--keep-wrap-markers` | aus | Druck-Umbruchzeichen `↪`/`➥` in Codeblöcken stehen lassen. Standard: Fortsetzungszeilen werden wieder zur ursprünglichen Codezeile zusammengefügt (Anzahl im Bericht: `wrap_lines_joined`) |
|
||||
| `--keep-invisible-chars` | aus | Nullbreiten-Zeichen und weiche Trennstriche im Fließtext behalten. Standard: entfernen, weil sie Suche und RAG stören (Anzahl im Bericht: `text_cleanup`). Code bleibt immer unverändert |
|
||||
| `--keep-html` | aus | Unbekannte/nicht sicher konvertierbare Elemente als bereinigtes HTML erhalten (sonst: Text übernommen + Eintrag in `diagnostics/unsupported-elements.md`) |
|
||||
| `--strict` | aus | Abbruch bei kritischen Fehlern (fehlende/unlesbare Spine-Dokumente, unsichere ZIP-Pfade, ungültige Spine-Verweise) |
|
||||
| `--force` | aus | Bestehendes Ausgabeverzeichnis ersetzen – siehe [Sicherheit](#sicherheit-und-fehlerbehandlung) |
|
||||
|
|
|
|||
|
|
@ -9,7 +9,17 @@ from typing import Any
|
|||
from .diagnostics import Diagnostics
|
||||
from .models import Chapter, NavEntry
|
||||
from .normalize import Doc, has_content
|
||||
from .utils import classes, epub_types, is_element, iter_elements, local, norm_ws, roles, text_content
|
||||
from .utils import (
|
||||
classes,
|
||||
epub_types,
|
||||
is_element,
|
||||
iter_elements,
|
||||
local,
|
||||
norm_ws,
|
||||
remove_invisible,
|
||||
roles,
|
||||
text_content,
|
||||
)
|
||||
|
||||
HEADINGS = {"h1": 1, "h2": 2, "h3": 3, "h4": 4, "h5": 5, "h6": 6}
|
||||
CONTAINERS = {"body", "div", "section", "article", "aside", "main", "header", "footer", "nav", "span", "hgroup"}
|
||||
|
|
@ -270,6 +280,7 @@ class ChapterBuilder:
|
|||
if not doc.linear and all(not d or d == doc.path for d in ch.docs[:1]):
|
||||
types.append("auxiliary")
|
||||
ch.types = list(dict.fromkeys(types))
|
||||
ch.title = remove_invisible(ch.title)[0] # titles feed file names, frontmatter and README
|
||||
return ch
|
||||
|
||||
def _title_from_content(self, items: list[tuple[str, Any]], number: int) -> tuple[str, str]:
|
||||
|
|
|
|||
|
|
@ -85,6 +85,9 @@ def build_parser() -> argparse.ArgumentParser:
|
|||
p.add_argument("--keep-wrap-markers", action="store_true",
|
||||
help="Druck-Umbruchzeichen (↪/➥) in Codeblöcken nicht entfernen; Standard: fortgesetzte "
|
||||
"Zeilen werden wieder zur ursprünglichen Codezeile zusammengefügt")
|
||||
p.add_argument("--keep-invisible-chars", action="store_true",
|
||||
help="Nullbreiten-Zeichen und weiche Trennstriche (U+00AD) im Fließtext behalten; Standard: entfernen "
|
||||
"(stören Suche und Weiterverarbeitung). Code bleibt immer unverändert")
|
||||
p.add_argument("--strict", action="store_true",
|
||||
help="Bei kritischen Fehlern abbrechen: fehlende/unlesbare Spine-Dokumente, nicht dekodierbares "
|
||||
"XHTML, unsichere ZIP-Pfade, übersprungene (verdächtige) Medien")
|
||||
|
|
@ -119,6 +122,7 @@ def main(argv: list[str] | None = None) -> int:
|
|||
keep_html=args.keep_html,
|
||||
code_language=args.code_language or "",
|
||||
keep_wrap_markers=args.keep_wrap_markers,
|
||||
keep_invisible_chars=args.keep_invisible_chars,
|
||||
strict=args.strict,
|
||||
verbose=args.verbose,
|
||||
dry_run=args.dry_run,
|
||||
|
|
|
|||
|
|
@ -45,6 +45,7 @@ from .utils import (
|
|||
local,
|
||||
norm_ws,
|
||||
rel_link,
|
||||
remove_invisible,
|
||||
resolve_href,
|
||||
text_content,
|
||||
url_path,
|
||||
|
|
@ -93,6 +94,7 @@ class RenderOptions:
|
|||
language: str = "de"
|
||||
code_language: str = "" # --code-language: label for blocks without a detected language
|
||||
keep_wrap_markers: bool = False # --keep-wrap-markers: do not join ↪-continued code lines
|
||||
keep_invisible_chars: bool = False # --keep-invisible-chars: keep zero-width chars and soft hyphens
|
||||
|
||||
|
||||
def md_escape(text: str, table: bool = False) -> str:
|
||||
|
|
@ -231,6 +233,7 @@ class Renderer:
|
|||
notes: NoteIndex, diag: Diagnostics, zip_names: set[str],
|
||||
entry_problem: Callable[[str], str | None] | None = None) -> None:
|
||||
self.entry_problem = entry_problem or (lambda path: None)
|
||||
self.invisible_removed = 0
|
||||
self.opt = options
|
||||
self.zip_names = zip_names
|
||||
self.in_table_html = False
|
||||
|
|
@ -293,7 +296,7 @@ class Renderer:
|
|||
if doc:
|
||||
self.doc = doc
|
||||
if isinstance(item, str):
|
||||
buf.append(("t", md_escape(item, self.in_table)))
|
||||
buf.append(("t", self._text(item)))
|
||||
elif isinstance(item, IdCarrier):
|
||||
anchor = self.resolver.anchor_for(self.doc, item.element)
|
||||
if anchor:
|
||||
|
|
@ -755,12 +758,12 @@ class Renderer:
|
|||
def inline_children(self, el: Any) -> list[Piece]:
|
||||
pieces: list[Piece] = []
|
||||
if el.text:
|
||||
pieces.append(("t", md_escape(el.text, self.in_table)))
|
||||
pieces.append(("t", self._text(el.text)))
|
||||
for child in el:
|
||||
if is_element(child):
|
||||
pieces.extend(self.inline(child))
|
||||
if child.tail:
|
||||
pieces.append(("t", md_escape(child.tail, self.in_table)))
|
||||
pieces.append(("t", self._text(child.tail)))
|
||||
return pieces
|
||||
|
||||
def inline(self, el: Any) -> list[Piece]:
|
||||
|
|
@ -1046,7 +1049,16 @@ class Renderer:
|
|||
title_part = f' "{title.replace(chr(34), chr(39))}"' if title else ""
|
||||
return [("r", f")}{title_part})")]
|
||||
|
||||
def _text(self, text: str) -> str:
|
||||
"""Escape prose text; zero-width characters and soft hyphens are dropped (P6)."""
|
||||
if not self.opt.keep_invisible_chars:
|
||||
text, n = remove_invisible(text)
|
||||
self.invisible_removed += n
|
||||
return md_escape(text, self.in_table)
|
||||
|
||||
def _alt(self, alt: str) -> str:
|
||||
if not self.opt.keep_invisible_chars:
|
||||
alt = remove_invisible(alt)[0]
|
||||
return WS.sub(" ", alt).replace("\\", "\\\\").replace("[", "\\[").replace("]", "\\]").strip()
|
||||
|
||||
def _is_formula_image(self, el: Any, alt: str, path: str) -> bool:
|
||||
|
|
|
|||
|
|
@ -47,6 +47,7 @@ class Options:
|
|||
keep_html: bool = False
|
||||
code_language: str = ""
|
||||
keep_wrap_markers: bool = False
|
||||
keep_invisible_chars: bool = False
|
||||
strict: bool = False
|
||||
verbose: bool = False
|
||||
dry_run: bool = False
|
||||
|
|
@ -235,7 +236,8 @@ def _render(reader: EpubReader, package: Any, chapters: list[Chapter], resolver:
|
|||
renderer = Renderer(
|
||||
RenderOptions(format=opts.format, footnotes=opts.footnotes, math=opts.math,
|
||||
extract_media=opts.extract_media, keep_html=opts.keep_html, language=language,
|
||||
code_language=opts.code_language, keep_wrap_markers=opts.keep_wrap_markers),
|
||||
code_language=opts.code_language, keep_wrap_markers=opts.keep_wrap_markers,
|
||||
keep_invisible_chars=opts.keep_invisible_chars),
|
||||
resolver, media, notes, diag, set(reader.names), reader.entry_problem,
|
||||
)
|
||||
width = max(3, len(str(len(chapters) + 1)))
|
||||
|
|
@ -495,6 +497,7 @@ def _report(input_path: Path, sha: str, generated_at: str, package: Any, opts: O
|
|||
"referenced": sum(1 for n in notes.notes.values() if n.ref_count),
|
||||
"rendered": len(renderer.book_notes)},
|
||||
"special_sections": special,
|
||||
"text_cleanup": {"invisible_chars_removed": renderer.invisible_removed},
|
||||
"warnings": warnings,
|
||||
"errors": errors,
|
||||
"infos": [d.as_dict() for d in diag.items if d.level == "info"],
|
||||
|
|
|
|||
|
|
@ -185,6 +185,15 @@ def text_content(el: Any) -> str:
|
|||
return "".join(parts)
|
||||
|
||||
|
||||
INVISIBLE = re.compile("[\u200b\u200c\u200d\u2060\ufeff\u00ad]") # zero-width chars, BOM, soft hyphen
|
||||
|
||||
|
||||
def remove_invisible(text: str) -> tuple[str, int]:
|
||||
"""Remove zero-width characters and soft hyphens; returns (text, number removed)."""
|
||||
cleaned, n = INVISIBLE.subn("", text)
|
||||
return cleaned, n
|
||||
|
||||
|
||||
def norm_ws(text: str) -> str:
|
||||
return re.sub(r"[ \t\r\n\f]+", " ", text or "").strip()
|
||||
|
||||
|
|
|
|||
18
tests/test_styles.py
Normal file
18
tests/test_styles.py
Normal file
|
|
@ -0,0 +1,18 @@
|
|||
"""Typography and layout: invisible characters, CSS emphasis, scene breaks, boxes, styles, page markers."""
|
||||
|
||||
from conftest import Book, Doc
|
||||
|
||||
ZW, SHY = "", ""
|
||||
|
||||
|
||||
def test_p6_invisible_characters_removed_outside_code(convert):
|
||||
body = (f"<h1>Kapitel 1.{ZW}1</h1><p>Lese{SHY}rinnen und 1.{ZW}1 sowie{ZW}Wort.</p>"
|
||||
f"<pre>a{ZW}b</pre><p><code>x{SHY}y</code></p>")
|
||||
run = convert(Book(docs=[Doc("c.xhtml", body)]))
|
||||
text = run.chapter("001")
|
||||
assert "Leserinnen und 1.1 sowieWort." in text
|
||||
assert "# Kapitel 1.1" in text and 'chapter_title: "Kapitel 1.1"' in text
|
||||
assert f"a{ZW}b" in text and f"x{SHY}y" in text # code stays verbatim
|
||||
assert run.report["text_cleanup"]["invisible_chars_removed"] == 4
|
||||
kept = convert(Book(docs=[Doc("c.xhtml", body)]), "--keep-invisible-chars").chapter("001")
|
||||
assert f"Lese{SHY}rinnen" in kept
|
||||
Loading…
Reference in a new issue