Markdown-Sonderzeichen maskieren, Verszeilen mit Satzende erhalten
- md_escape im Span-Patch: \ ` * [ ] < > $ ~ ^ und "_" an Wortgrenzen werden maskiert (__add__ nicht mehr fett, <p> nicht mehr HTML); Code bleibt wörtlich - Folgeschritte maskierungsfest: norm_title, clean_inline, link_urls (Positionstabelle inkl. Backslash), link_anchors, --split-Titel - Kurze Zeile mit Satzende ist nur bei eingerückter Folgezeile ein Absatzende, sonst eine Verszeile; Einrückung gegen den Spaltenrand der Seite gemessen (trennt auch verklebte einzeilige Dialogabsätze) - Tests, README, Bedienungsanleitung, TODO Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Dieter Schlüter
parent
3d418e3987
commit
d6a8c11b0a
1 changed files with 148 additions and 28 deletions
108
pdf2md.py
108
pdf2md.py
|
|
@ -148,19 +148,34 @@ NUMBERED_TITLE = re.compile(
|
|||
# Hilfsfunktionen für Text
|
||||
# ===========================================================================
|
||||
|
||||
# Markdown-Sonderzeichen im PDF-Text maskieren, damit z. B. "__add__" nicht fett
|
||||
# wird. "_" mitten im Wort ("snake_case") bleibt unmaskiert.
|
||||
MD_SPECIAL = "\\`*_[]<>$~^"
|
||||
MD_ESCAPE = re.compile(r"[\\`*\[\]<>$~^]|(?<![^\W_])_|_(?![^\W_])")
|
||||
MD_UNESCAPE = re.compile(r"\\([\\`*_\[\]<>$~^])")
|
||||
|
||||
|
||||
def md_escape(text: str) -> str:
|
||||
return MD_ESCAPE.sub(lambda m: "\\" + m.group(0), text)
|
||||
|
||||
|
||||
def md_unescape(text: str) -> str:
|
||||
return MD_UNESCAPE.sub(r"\1", text)
|
||||
|
||||
|
||||
def ends_sentence(text: str) -> bool:
|
||||
return bool(SENTENCE_END.search(text.rstrip()))
|
||||
|
||||
|
||||
def clean_inline(text: str) -> str:
|
||||
"""Markdown-Hervorhebungen entfernen und Leerraum vereinheitlichen."""
|
||||
text = re.sub(r"\*\*|__|(?<!\w)_|_(?!\w)", "", text)
|
||||
text = re.sub(r"(?<!\\)\*\*|(?<!\\)__|(?<![\w\\])_|(?<!\\)_(?!\w)", "", text)
|
||||
return re.sub(r"\s+", " ", text).strip()
|
||||
|
||||
|
||||
def norm_title(text: str) -> str:
|
||||
"""Vergleichsschlüssel für Überschriften und Gliederungseinträge."""
|
||||
return clean_inline(text).translate(DASHES).lower()
|
||||
return md_unescape(clean_inline(text)).translate(DASHES).lower()
|
||||
|
||||
|
||||
def is_other_structure(line: str) -> bool:
|
||||
|
|
@ -572,7 +587,7 @@ def continues_block(following: list[str], free: int, running_heads: set[str]) ->
|
|||
return False
|
||||
elif LIST_ITEM.match(text) and not ORDERED_ITEM.match(text):
|
||||
return False # "14. Jahrhundert" ist dagegen oft keine Liste
|
||||
word = clean_inline(text).replace(NBSP, " ").split()
|
||||
word = md_unescape(clean_inline(text)).replace(NBSP, " ").split()
|
||||
# Breite aus der vorigen Zeile geschätzt: ein Zeichen Spielraum.
|
||||
return bool(word) and len(word[0]) + 2 > free
|
||||
return False
|
||||
|
|
@ -715,16 +730,32 @@ def _match_printed(tokens: list[str]) -> tuple[str, int]:
|
|||
return url, consumed - (len(joined) - len(url))
|
||||
|
||||
|
||||
def _unescape_map(text: str) -> tuple[str, list[int]]:
|
||||
"""Text ohne Markdown-Maskierung und Position jedes Zeichens im Original."""
|
||||
plain, index, i = [], [], 0
|
||||
while i < len(text):
|
||||
index.append(i) # Position samt Backslash, damit er beim Kopieren bleibt
|
||||
if text[i] == "\\" and i + 1 < len(text) and text[i + 1] in MD_SPECIAL:
|
||||
i += 1
|
||||
plain.append(text[i])
|
||||
i += 1
|
||||
return "".join(plain), index + [len(text)]
|
||||
|
||||
|
||||
def _link_line(line: str, known: dict[str, str]) -> str:
|
||||
# URLs auf dem unmaskierten Text suchen ("Leitlinien\_" → "Leitlinien_"),
|
||||
# den übrigen Text unverändert (maskiert) übernehmen.
|
||||
plain, index = _unescape_map(line)
|
||||
out = []
|
||||
pos = 0
|
||||
while match := URL_START.search(line, pos):
|
||||
out.append(line[pos:match.start()])
|
||||
tokens = re.findall(r"\S+\s*", line[match.start():])
|
||||
while match := URL_START.search(plain, pos):
|
||||
out.append(line[index[pos]:index[match.start()]])
|
||||
tokens = re.findall(r"\S+\s*", plain[match.start():])
|
||||
url, consumed = _match_known(tokens, known) or _match_printed(tokens)
|
||||
out.append(_autolink(url) if "." in url else line[match.start():][:consumed])
|
||||
pos = match.start() + consumed
|
||||
out.append(line[pos:])
|
||||
end = match.start() + consumed
|
||||
out.append(_autolink(url) if "." in url else line[index[match.start()]:index[end]])
|
||||
pos = end
|
||||
out.append(line[index[pos]:])
|
||||
return "".join(out)
|
||||
|
||||
|
||||
|
|
@ -780,12 +811,19 @@ def load_link_anchors(doc: pymupdf.Document) -> list[tuple[str, list[str], list[
|
|||
return anchors
|
||||
|
||||
|
||||
def _md_literal(text: str) -> str:
|
||||
"""Regex für Text, der im Markdown maskiert sein kann ("\\_", "\\*" …)."""
|
||||
return "".join(
|
||||
(r"\\?" if char in MD_SPECIAL else "") + re.escape(char) for char in text
|
||||
)
|
||||
|
||||
|
||||
def _words_pattern(words: list[str]) -> str:
|
||||
"""Wörter mit beliebigem Leerraum, Hervorhebung oder Trennung dazwischen."""
|
||||
parts = []
|
||||
for word in words:
|
||||
core = word.rstrip("-\u00ad") # Silbentrennung am Zeilenende
|
||||
parts.append(re.escape(core) + (r"-?" if core != word else ""))
|
||||
parts.append(_md_literal(core) + (r"-?" if core != word else ""))
|
||||
return r"[\s*_]*".join(parts)
|
||||
|
||||
|
||||
|
|
@ -816,8 +854,7 @@ def link_anchors(markdown: str, anchors: list[tuple[str, list[str], list[str]]])
|
|||
)
|
||||
if match:
|
||||
start, end = match.span("anchor")
|
||||
label = markdown[start:end].replace("[", "\\[").replace("]", "\\]")
|
||||
markdown = f"{markdown[:start]}[{label}]({uri}){markdown[end:]}"
|
||||
markdown = f"{markdown[:start]}[{markdown[start:end]}]({uri}){markdown[end:]}"
|
||||
pos = start
|
||||
break
|
||||
return markdown
|
||||
|
|
@ -827,7 +864,11 @@ def _inside_link(markdown: str, index: int) -> bool:
|
|||
"""Liegt die Stelle schon in einem Link (<…>, [..](…))?"""
|
||||
line_start = markdown.rfind("\n", 0, index) + 1
|
||||
before = markdown[line_start:index]
|
||||
return before.count("<") > before.count(">") or before.count("[") > before.count("]") or (
|
||||
|
||||
def count(char: str) -> int: # maskierte Zeichen ("\\[") zählen nicht
|
||||
return len(re.findall(r"(?<!\\)" + re.escape(char), before))
|
||||
|
||||
return count("<") > count(">") or count("[") > count("]") or (
|
||||
before.rfind("](") > before.rfind(")")
|
||||
)
|
||||
|
||||
|
|
@ -859,6 +900,8 @@ def _mark_line_end_hyphens(spans: list[dict]) -> list[dict]:
|
|||
nxt = spans[i + 1] if i + 1 < len(spans) else None
|
||||
if not span.get("char_flags", VISIBLE_TEXT) & VISIBLE_TEXT:
|
||||
span = {**span, "flags": span.get("flags", 0) & ~MONOSPACE_FLAG}
|
||||
if not (span.get("flags", 0) & MONOSPACE_FLAG): # Code bleibt wörtlich
|
||||
span = {**span, "text": md_escape(span["text"])}
|
||||
text = span["text"].rstrip()
|
||||
line_end = nxt is not None and (span.get("block"), span.get("line")) != (
|
||||
nxt.get("block"), nxt.get("line")
|
||||
|
|
@ -911,8 +954,12 @@ def _line_text(line: dict) -> str:
|
|||
return "".join(span["text"] for span in line["spans"]).strip()
|
||||
|
||||
|
||||
def split_by_indent(textlines: list[dict]) -> list[list[dict]]:
|
||||
"""Zeilen eines Textblocks in Absätze aufteilen."""
|
||||
def split_by_indent(textlines: list[dict], column_left: float | None = None) -> list[list[dict]]:
|
||||
"""Zeilen eines Textblocks in Absätze aufteilen.
|
||||
|
||||
column_left: linker Rand der Textspalte, falls bekannt – sonst der Blockrand
|
||||
(dann bleiben gleich eingerückte einzeilige Absätze unerkannt).
|
||||
"""
|
||||
if len(textlines) < 2:
|
||||
return [textlines]
|
||||
|
||||
|
|
@ -920,6 +967,8 @@ def split_by_indent(textlines: list[dict]) -> list[list[dict]]:
|
|||
em = sizes[len(sizes) // 2] if sizes else 10.0
|
||||
lefts = sorted(line["bbox"][0] for line in textlines)
|
||||
left = lefts[len(lefts) // 4] # Blockrand, robust gegen Einzüge
|
||||
if column_left is not None:
|
||||
left = min(left, column_left)
|
||||
widths = [line["bbox"][2] - line["bbox"][0] for line in textlines]
|
||||
longest = max(widths)
|
||||
|
||||
|
|
@ -1028,6 +1077,14 @@ def _fit_tolerance(lines: list[dict], right: float, em: float) -> float:
|
|||
return max(FIT_TOLERANCE_EM * em, FIT_TOLERANCE * (right - left))
|
||||
|
||||
|
||||
def _column_left(lines: list[dict], em: float) -> float:
|
||||
"""Linker Rand der Textspalte (Einzüge einzeiliger Absätze erkennen)."""
|
||||
bx0 = min(line["bbox"][0] for line in lines)
|
||||
bx1 = max(line["bbox"][2] for line in lines)
|
||||
edges = [x0 for x0, x1 in _page_lines if x0 < bx1 and x1 > bx0 and x0 >= bx0 - 10 * em]
|
||||
return min(edges + [bx0])
|
||||
|
||||
|
||||
def _is_soft_break(prev: dict, line: dict, right: float, em: float, tol: float) -> bool:
|
||||
"""Hätte das erste Wort der Zeile nicht mehr in die vorige gepasst?"""
|
||||
prev_text = _raw_text(prev).rstrip()
|
||||
|
|
@ -1119,11 +1176,20 @@ def lines_to_md(textlines: list[dict], original, ignore_code: bool = False) -> s
|
|||
tol = _fit_tolerance(lines, right, em)
|
||||
end = _block_end(lines, right, em, tol)
|
||||
|
||||
# Harte Umbrüche ohne Satzende; nach Satzende ist es ein Absatzende.
|
||||
# Harte Umbrüche zählen; ein Satzende mit eingerückter Folgezeile ist dagegen
|
||||
# ein Absatzende (Roman), ohne Einrückung eine Verszeile (Gedicht).
|
||||
soft = [_is_soft_break(a, b, right, em, tol) for a, b in zip(lines, lines[1:])]
|
||||
left = _column_left(lines, em)
|
||||
|
||||
def new_paragraph(prev: dict, line: dict) -> bool:
|
||||
indent = line["bbox"][0] - left
|
||||
return ends_sentence(_raw_text(prev).strip()) and (
|
||||
INDENT_MIN_EM * em <= indent <= INDENT_MAX_EM * em
|
||||
)
|
||||
|
||||
hard = sum(
|
||||
1 for is_soft, line in zip(soft, lines)
|
||||
if not is_soft and not ends_sentence(_raw_text(line).strip())
|
||||
1 for is_soft, a, b in zip(soft, lines, lines[1:])
|
||||
if not is_soft and not new_paragraph(a, b)
|
||||
)
|
||||
|
||||
verbatim = bool(soft) and hard / len(soft) >= HARD_BLOCK_RATIO
|
||||
|
|
@ -1131,7 +1197,7 @@ def lines_to_md(textlines: list[dict], original, ignore_code: bool = False) -> s
|
|||
if end and len(lines) == 1 and _is_code(lines, ignore_code):
|
||||
return _code_block([lines], [0])
|
||||
# Fließtext: Absätze am Einzug trennen, wie bisher ausgeben.
|
||||
groups = split_by_indent(textlines)
|
||||
groups = split_by_indent(textlines, left)
|
||||
md = "".join(original(group, ignore_code) for group in groups)
|
||||
return md.rstrip("\n").rstrip() + end + "\n\n" if end else md
|
||||
|
||||
|
|
@ -1726,7 +1792,7 @@ def split_markdown(markdown: str, mode: str) -> list[dict]:
|
|||
part = ""
|
||||
for n, (index, level, text) in enumerate(cuts):
|
||||
end = cuts[n + 1][0] if n + 1 < len(cuts) else len(lines)
|
||||
title = clean_inline(text)
|
||||
title = md_unescape(clean_inline(text))
|
||||
if level == part_level:
|
||||
part = title
|
||||
number = re.match(r"(\d+)\b", title) if level == chapter_level != part_level else None
|
||||
|
|
@ -1803,7 +1869,7 @@ def write_split(folder: Path, sections: list[dict], meta: dict) -> list[Path]:
|
|||
files.append(folder / name)
|
||||
|
||||
entries = "\n".join(
|
||||
f"- [{s['title']}]({f.name})" for s, f in zip(sections, files)
|
||||
f"- [{md_escape(s['title'])}]({f.name})" for s, f in zip(sections, files)
|
||||
)
|
||||
index = (
|
||||
make_frontmatter(meta)
|
||||
|
|
|
|||
Loading…
Reference in a new issue