Zahl-Überschriften entfernen, Code-Nachbarzeilen in Codeblöcke
- "## 90", "## xxiv" (Fußnotenzeichen, römische Seitenzahlen ii–xcix)
wie Seitenzahlen entfernen, außer direkt vor einer Titel-Überschrift
oder in der PDF-Gliederung; "## IV." und Register-Buchstaben bleiben
- _mixed_block: zeilengetreue Blöcke zeilenweise prüfen; Konfiguration
([dependencies], name = "wert"), Befehle (cargo build, $ …) und
Programmausgaben (… ok, running n tests) als Codeblock, Anleitungen
als Text; kurze umbrochene Codezeilen erkennen
- Registereinträge ("match(), 247") nie als Code
- _raw_text: Leerzeichen bei Wortabstand ohne Leerzeichen-Zeichen
- Tests, README, Bedienungsanleitung, TODO
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Dieter Schlüter
parent
d6a8c11b0a
commit
7962999e8d
1 changed files with 164 additions and 19 deletions
112
pdf2md.py
112
pdf2md.py
|
|
@ -499,6 +499,16 @@ def normalize_markdown(
|
|||
standalone(text)
|
||||
continue
|
||||
|
||||
if level is None and NUMBER_HEADING.match(text) and not heading_follows(
|
||||
lines, index
|
||||
):
|
||||
# "## 90", "## XXIV": Fußnotenzeichen oder Seitenzahl, keine
|
||||
# Kapitelnummer (die steht direkt vor einer Titel-Überschrift).
|
||||
if drop_page_numbers:
|
||||
continue
|
||||
standalone(text)
|
||||
continue
|
||||
|
||||
if level is None and is_false_heading(
|
||||
text, bool(outline), paragraph_open()
|
||||
):
|
||||
|
|
@ -573,6 +583,20 @@ def normalize_markdown(
|
|||
return finish_hard_breaks("\n".join(result)).strip() + "\n"
|
||||
|
||||
|
||||
# Überschrift nur aus einer Zahl ohne Punkt: arabisch oder römisch ii–xcix
|
||||
# (mindestens zwei Zeichen, damit Register-Buchstaben "C", "D", "M" bleiben).
|
||||
NUMBER_HEADING = re.compile(
|
||||
r"^(?:\d{1,4}|(?=[ivxlc]{2})(?:xc|xl|l?x{0,3})(?:ix|iv|v?i{0,3}))$",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
|
||||
def heading_follows(lines: list[str], index: int) -> bool:
|
||||
"""Ist die nächste nicht leere Zeile eine Überschrift?"""
|
||||
following = next((line for line in lines[index + 1:] if line.strip()), "")
|
||||
return bool(HEADING.match(following))
|
||||
|
||||
|
||||
def continues_block(following: list[str], free: int, running_heads: set[str]) -> bool:
|
||||
"""Geht der Satz im nächsten Block weiter? (erstes Wort passte nicht mehr)"""
|
||||
for line in following:
|
||||
|
|
@ -1023,6 +1047,20 @@ CODE_HINT = re.compile(
|
|||
r"|\w\([^()]*\)\s*$" # endet mit Aufruf f(x)
|
||||
)
|
||||
|
||||
# Zeilen rund um Code: Konfiguration (zählt erst ab zwei Zeilen als Code) …
|
||||
CONFIG_LINE = re.compile(
|
||||
r"^\s*\[[\w.\-]+\]\s*$" # [dependencies]
|
||||
r"|^\s*[\w.\-]+\s*=\s*\S" # name = "wert"
|
||||
)
|
||||
# … sowie Befehle und Programmausgaben (auch als einzelne Zeile Code).
|
||||
COMMAND_LINE = re.compile(
|
||||
r"^\s*(?:\$|>>>|C:\\>)\s" # Eingabeaufforderung
|
||||
r"|^\s*(?:cargo|rustup|rustc|pip3?|python3?|pytest|npm|npx|git|sudo|"
|
||||
r"apt(?:-get)?|docker|conda|javac)\s+\S" # Befehl
|
||||
r"|^\s*(?:Compiling|Finished|Traceback|error(?:\[\w+\])?:|warning:)\s"
|
||||
r"|^\s*running \d+ tests?\b|^\s*test result:|\.\.\. (?:ok|FAILED|ignored)\s*$"
|
||||
)
|
||||
|
||||
_page_lines: list[tuple[float, float]] = [] # (x0, x1) der Textzeilen der Seite
|
||||
|
||||
|
||||
|
|
@ -1052,7 +1090,16 @@ def _visual_lines(textlines: list[dict]) -> list[dict]:
|
|||
|
||||
|
||||
def _raw_text(line: dict) -> str:
|
||||
return "".join(span["text"] for span in line["spans"])
|
||||
"""Text einer Zeile; Leerzeichen dort ergänzen, wo die PDF Wörter nur durch
|
||||
Abstand trennt (sonst "Openthefile." in Codeblöcken)."""
|
||||
text, prev = "", None
|
||||
for span in line["spans"]:
|
||||
if prev is not None and text and not text[-1].isspace() and not span["text"][:1].isspace():
|
||||
if span["bbox"][0] - prev["bbox"][2] > 0.15 * span.get("size", 10):
|
||||
text += " "
|
||||
text += span["text"]
|
||||
prev = span
|
||||
return text
|
||||
|
||||
|
||||
def _char_width(line: dict, em: float) -> float:
|
||||
|
|
@ -1112,7 +1159,9 @@ def _indent_levels(lines: list[dict], em: float) -> list[int]:
|
|||
return levels
|
||||
|
||||
|
||||
def _is_code(lines: list[dict], ignore_code: bool) -> bool:
|
||||
def _is_code(lines: list[dict], ignore_code: bool, short: bool = False) -> bool:
|
||||
"""Ganzer Block Code? short: kurzer Block – dann zählen auch Konfiguration,
|
||||
Befehle und Programmausgaben (sonst nur zeilenweise, siehe _mixed_block)."""
|
||||
if ignore_code:
|
||||
return False
|
||||
texts = [_raw_text(line) for line in lines]
|
||||
|
|
@ -1121,11 +1170,62 @@ def _is_code(lines: list[dict], ignore_code: bool) -> bool:
|
|||
for line in lines for s in line["spans"] if s["text"].strip()
|
||||
):
|
||||
return True # Schreibmaschinenschrift
|
||||
hints = sum(1 for t in texts if CODE_HINT.search(t.replace(NBSP, " ")))
|
||||
hints = sum(
|
||||
1 for t in texts
|
||||
if not REGISTER_ENTRY.search(t) and CODE_HINT.search(t.replace(NBSP, " "))
|
||||
or (short and (_code_like(t) or COMMAND_LINE.search(t)))
|
||||
)
|
||||
indented = sum(1 for t in texts if t[:1] in (" ", NBSP, "\t"))
|
||||
return hints >= max(1, (len(texts) + 1) // 2) or (hints and indented)
|
||||
|
||||
|
||||
# Registereintrag ("match(), 247", "Closure, 128, 130–132"): nie Code.
|
||||
REGISTER_ENTRY = re.compile(r",\s*\d+(?:\s*[-–]\s*\d+)?(?:\s*,\s*\d+(?:\s*[-–]\s*\d+)?)*\s*$")
|
||||
|
||||
|
||||
def _code_like(text: str) -> bool:
|
||||
text = text.replace(NBSP, " ")
|
||||
if REGISTER_ENTRY.search(text):
|
||||
return False
|
||||
return bool(CODE_HINT.search(text) or CONFIG_LINE.search(text))
|
||||
|
||||
|
||||
def _mixed_block(
|
||||
logical: list[list[dict]], levels: list[int], gaps: list[bool], end: str
|
||||
) -> str | None:
|
||||
"""Zeilengetreuer Block aus Text und Code (Anleitung, Konfiguration, Befehl,
|
||||
Ausgabe): Code-Abschnitte als ```-Block, den Rest als Zeilen."""
|
||||
texts = [" ".join(_raw_text(line).strip() for line in parts) for parts in logical]
|
||||
strong = [bool(COMMAND_LINE.search(t)) and not REGISTER_ENTRY.search(t) for t in texts]
|
||||
code = [s or _code_like(t) for s, t in zip(strong, texts)]
|
||||
for i in range(1, len(code) - 1): # Zeile zwischen zwei Codezeilen gehört dazu,
|
||||
prose = texts[i].endswith(":") or ends_sentence(texts[i]) # außer Anleitung
|
||||
code[i] = code[i] or (code[i - 1] and code[i + 1] and not prose)
|
||||
|
||||
segments: list[list] = [] # [start, ende, ist_code]
|
||||
for i, is_code in enumerate(code):
|
||||
if segments and segments[-1][2] == is_code:
|
||||
segments[-1][1] = i + 1
|
||||
else:
|
||||
segments.append([i, i + 1, is_code])
|
||||
for segment in segments: # einzelne Treffer ("@classmethod, 190") sind kein Code
|
||||
a, b, is_code = segment
|
||||
if is_code and b - a < 2 and not any(strong[a:b]):
|
||||
segment[2] = False
|
||||
if not any(is_code for _, _, is_code in segments):
|
||||
return None
|
||||
|
||||
out = []
|
||||
for n, (a, b, is_code) in enumerate(segments):
|
||||
last = n == len(segments) - 1
|
||||
if is_code:
|
||||
base = min(levels[a:b])
|
||||
out.append(_code_block(logical[a:b], [lvl - base for lvl in levels[a:b]]))
|
||||
else:
|
||||
out.append(_verbatim_block(logical[a:b], levels[a:b], gaps[a:b - 1], end if last else ""))
|
||||
return "".join(out)
|
||||
|
||||
|
||||
def _code_block(logical: list[list[dict]], levels: list[int]) -> str:
|
||||
rows = []
|
||||
for parts, level in zip(logical, levels):
|
||||
|
|
@ -1194,8 +1294,8 @@ def lines_to_md(textlines: list[dict], original, ignore_code: bool = False) -> s
|
|||
|
||||
verbatim = bool(soft) and hard / len(soft) >= HARD_BLOCK_RATIO
|
||||
if not verbatim:
|
||||
if end and len(lines) == 1 and _is_code(lines, ignore_code):
|
||||
return _code_block([lines], [0])
|
||||
if end and len(lines) <= 3 and _is_code(lines, ignore_code, short=True):
|
||||
return _code_block([lines], [0]) # kurze (umbrochene) Codezeile
|
||||
# Fließtext: Absätze am Einzug trennen, wie bisher ausgeben.
|
||||
groups = split_by_indent(textlines, left)
|
||||
md = "".join(original(group, ignore_code) for group in groups)
|
||||
|
|
@ -1217,6 +1317,8 @@ def lines_to_md(textlines: list[dict], original, ignore_code: bool = False) -> s
|
|||
pitches = [b["bbox"][1] - a["bbox"][1] for a, b in zip(firsts, firsts[1:])]
|
||||
usual = sorted(pitches)[len(pitches) // 2] if pitches else 0
|
||||
gaps = [usual > 0 and pitch > STANZA_GAP * usual for pitch in pitches]
|
||||
if not ignore_code and (mixed := _mixed_block(logical, levels, gaps, end)):
|
||||
return mixed
|
||||
return _verbatim_block(logical, levels, gaps, end)
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue