--keep-lines für Gedichtbände, OCR-Neuerkennung repariert, OCR-Rauschen entfernen
- --ocr always / schiefe OCR-Ebenen: alte unsichtbare Textebene vor der
OCR entfernen (nur in der Kopie im Speicher); vorher blieb der alte
Text erhalten bzw. stand doppelt da ("bliht" statt "blüht")
- --keep-lines: jede PDF-Zeile bleibt eine Zeile, Strophen als Absätze,
Einrückungen bleiben, nichts wird zusammengefügt (auch keine Trennung)
- OCR-Text selbst erkennen (unsichtbar, wie pymupdf4llm 1.28 intern);
Buchkante "|" entfernen, Rauschzeilen und Fetzen neben dem Textbereich
verwerfen, Zeilen mit verschiedenen Grundlinien trennen, Überschriften
ohne Buchstaben/Ziffern verwerfen
- Fraktur: frk+deu dokumentiert, Grenzen in TODO
- Tests: Scan mit falscher OCR-Ebene, Gedicht in schmaler Spalte,
OCR-Rauschen; README, Bedienungsanleitung, TODO
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Dieter Schlüter
parent
1ba5b5005f
commit
3e5195efc2
1 changed files with 265 additions and 13 deletions
133
pdf2md.py
133
pdf2md.py
|
|
@ -32,7 +32,7 @@ from pathlib import Path
|
|||
import pymupdf
|
||||
import pymupdf4llm
|
||||
from pymupdf4llm.helpers import document_layout
|
||||
from pymupdf4llm.ocr import tesseract_api
|
||||
from pymupdf4llm.ocr import OCRMode, tesseract_api
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
|
|
@ -499,6 +499,9 @@ def normalize_markdown(
|
|||
standalone(text)
|
||||
continue
|
||||
|
||||
if not HAS_ALNUM.search(text):
|
||||
continue # Überschrift nur aus Zeichen ("# ||", OCR-Rauschen)
|
||||
|
||||
if level is None and NUMBER_HEADING.match(text) and not heading_follows(
|
||||
lines, index
|
||||
):
|
||||
|
|
@ -1023,6 +1026,8 @@ def _mark_line_end_hyphens(spans: list[dict]) -> list[dict]:
|
|||
nxt = spans[i + 1] if i + 1 < len(spans) else None
|
||||
if not span.get("char_flags", VISIBLE_TEXT) & VISIBLE_TEXT:
|
||||
span = {**span, "flags": span.get("flags", 0) & ~MONOSPACE_FLAG}
|
||||
if not span.get("char_flags", VISIBLE_TEXT) & VISIBLE_TEXT: # OCR-Text
|
||||
span = {**span, "text": OCR_NOISE.sub("", span["text"])} # Scanrand-"|"
|
||||
if not (span.get("flags", 0) & MONOSPACE_FLAG): # Code bleibt wörtlich
|
||||
span = {**span, "text": md_escape(span["text"])}
|
||||
text = span["text"].rstrip()
|
||||
|
|
@ -1161,14 +1166,22 @@ COMMAND_LINE = re.compile(
|
|||
)
|
||||
|
||||
_page_lines: list[tuple[float, float]] = [] # (x0, x1) der Textzeilen der Seite
|
||||
_page_text: list[tuple[float, float] | None] = [None] # Textbereich aus echten Zeilen
|
||||
_keep_lines = [False] # --keep-lines: jede PDF-Zeile bleibt eine Zeile
|
||||
REAL_WORD = re.compile(r"[^\W\d_]{3,}")
|
||||
|
||||
|
||||
def _remember_page_lines(boxes) -> None:
|
||||
_page_lines.clear()
|
||||
real: list[tuple[float, float]] = []
|
||||
for box in boxes:
|
||||
if getattr(box, "boxclass", None) in TEXT_BOXES:
|
||||
for line in getattr(box, "textlines", None) or []:
|
||||
_page_lines.append((line["bbox"][0], line["bbox"][2]))
|
||||
text = "".join(span["text"] for span in line["spans"])
|
||||
if len(REAL_WORD.findall(text)) >= 2: # keine Seitenzahl, kein Scan-Rauschen
|
||||
real.append((line["bbox"][0], line["bbox"][2]))
|
||||
_page_text[0] = (min(x0 for x0, _ in real), max(x1 for _, x1 in real)) if real else None
|
||||
|
||||
|
||||
def _visual_lines(textlines: list[dict]) -> list[dict]:
|
||||
|
|
@ -1258,16 +1271,20 @@ def _indent_levels(lines: list[dict], em: float) -> list[int]:
|
|||
return levels
|
||||
|
||||
|
||||
def _is_monospace(lines: list[dict]) -> bool:
|
||||
return all(
|
||||
s["flags"] & 8 and s.get("char_flags", VISIBLE_TEXT) & VISIBLE_TEXT
|
||||
for line in lines for s in line["spans"] if s["text"].strip()
|
||||
)
|
||||
|
||||
|
||||
def _is_code(lines: list[dict], ignore_code: bool, short: bool = False) -> bool:
|
||||
"""Ganzer Block Code? short: kurzer Block – dann zählen auch Konfiguration,
|
||||
Befehle und Programmausgaben (sonst nur zeilenweise, siehe _mixed_block)."""
|
||||
if ignore_code:
|
||||
return False
|
||||
texts = [_raw_text(line) for line in lines]
|
||||
if all(
|
||||
s["flags"] & 8 and s.get("char_flags", VISIBLE_TEXT) & VISIBLE_TEXT
|
||||
for line in lines for s in line["spans"] if s["text"].strip()
|
||||
):
|
||||
if _is_monospace(lines):
|
||||
return True # Schreibmaschinenschrift
|
||||
hints = sum(
|
||||
1 for t in texts
|
||||
|
|
@ -1366,11 +1383,68 @@ def _block_end(lines: list[dict], right: float, em: float, tol: float) -> str:
|
|||
return f"{BLOCK_END}{max(0, int(free))}{paragraph}{BLOCK_END}"
|
||||
|
||||
|
||||
OCR_NOISE = re.compile(r"[|¦]") # Buchkante/Seitenrand als "|" gelesen
|
||||
HAS_ALNUM = re.compile(r"[^\W_]")
|
||||
|
||||
|
||||
def _clean_ocr_lines(textlines: list[dict]) -> list[dict]:
|
||||
"""OCR-Seiten: Zeilen mit Wörtern auf verschiedenen Grundlinien trennen
|
||||
(schiefer Scan: "UndDon tauchteweißen") und Rauschen entfernen."""
|
||||
result = []
|
||||
for line in textlines:
|
||||
groups: list[list[dict]] = []
|
||||
for span in sorted(line["spans"], key=lambda s: s["origin"][1]):
|
||||
text = OCR_NOISE.sub("", span["text"])
|
||||
if not text.strip():
|
||||
continue
|
||||
span = {**span, "text": text}
|
||||
if groups and abs(span["origin"][1] - groups[-1][0]["origin"][1]) <= 0.4 * span["size"]:
|
||||
groups[-1].append(span)
|
||||
else:
|
||||
groups.append([span])
|
||||
for spans in groups:
|
||||
spans.sort(key=lambda s: s["bbox"][0])
|
||||
if not HAS_ALNUM.search("".join(s["text"] for s in spans)):
|
||||
continue # nur Rauschen ("„+", "%*")
|
||||
bbox = (
|
||||
min(s["bbox"][0] for s in spans), min(s["bbox"][1] for s in spans),
|
||||
max(s["bbox"][2] for s in spans), max(s["bbox"][3] for s in spans),
|
||||
)
|
||||
result.append({**line, "bbox": bbox, "spans": spans})
|
||||
result.sort(key=lambda line: (line["bbox"][1], line["bbox"][0]))
|
||||
return result
|
||||
|
||||
|
||||
def _is_ocr_text(textlines: list[dict]) -> bool:
|
||||
"""OCR-Text ist unsichtbar (Render-Modus 3) – wie utils.is_ocr_text in pymupdf4llm."""
|
||||
spans = [s for line in textlines for s in line["spans"] if s["text"].strip()]
|
||||
return bool(spans) and not any(s.get("char_flags", VISIBLE_TEXT) & VISIBLE_TEXT for s in spans)
|
||||
|
||||
|
||||
def _outside_text_area(lines: list[dict], em: float) -> bool:
|
||||
"""Block ganz links/rechts neben dem Textbereich (OCR-Rauschen vom Scanrand)."""
|
||||
if _page_text[0] is None:
|
||||
return False
|
||||
left, right = _page_text[0]
|
||||
bx0 = min(line["bbox"][0] for line in lines)
|
||||
bx1 = max(line["bbox"][2] for line in lines)
|
||||
return bx1 < left - 0.5 * em or bx0 > right + 0.5 * em
|
||||
|
||||
|
||||
def lines_to_md(textlines: list[dict], original, ignore_code: bool = False) -> str:
|
||||
"""Textblock ausgeben: Fließtext wie bisher, sonst zeilengetreu."""
|
||||
ocr = _is_ocr_text(textlines)
|
||||
if ocr:
|
||||
textlines = _clean_ocr_lines(textlines)
|
||||
if not textlines:
|
||||
return ""
|
||||
lines = _visual_lines(textlines)
|
||||
sizes = sorted(s["size"] for line in lines for s in line["spans"])
|
||||
em = sizes[len(sizes) // 2] if sizes else 10.0
|
||||
if ocr and _outside_text_area(lines, em):
|
||||
return "" # OCR-Seite: Rauschen am Rand verwerfen
|
||||
if _keep_lines[0]:
|
||||
return _keep_lines_md(lines, em, ignore_code)
|
||||
right = _column_right(lines, em)
|
||||
tol = _fit_tolerance(lines, right, em)
|
||||
end = _block_end(lines, right, em, tol)
|
||||
|
|
@ -1421,6 +1495,24 @@ def lines_to_md(textlines: list[dict], original, ignore_code: bool = False) -> s
|
|||
return _verbatim_block(logical, levels, gaps, end)
|
||||
|
||||
|
||||
def _keep_lines_md(lines: list[dict], em: float, ignore_code: bool) -> str:
|
||||
"""--keep-lines: jede Zeile bleibt eine Zeile, nichts wird zusammengefügt.
|
||||
|
||||
Strophen-/Absatzabstände werden Leerzeilen, Einrückungen bleiben. Ein Block
|
||||
mit mehreren Zeilen endet mit Absatzende (Strophe), eine Einzelzeile mit
|
||||
hartem Umbruch (sie kann zur nächsten Zeile gehören).
|
||||
"""
|
||||
logical = [[line] for line in lines]
|
||||
levels = _indent_levels(lines, em)
|
||||
if len(lines) > 1 and not ignore_code and _is_monospace(lines):
|
||||
return _code_block(logical, levels)
|
||||
pitches = [b["bbox"][1] - a["bbox"][1] for a, b in zip(lines, lines[1:])]
|
||||
usual = sorted(pitches)[len(pitches) // 2] if pitches else 0
|
||||
gaps = [usual > 0 and pitch > STANZA_GAP * usual for pitch in pitches]
|
||||
end = PARAGRAPH_END if len(lines) > 1 else HARD_BREAK
|
||||
return _verbatim_block(logical, levels, gaps, end)
|
||||
|
||||
|
||||
def patch_pymupdf4llm_paragraphs() -> None:
|
||||
original = getattr(document_layout, "text_to_md", None)
|
||||
if original is None or getattr(original, "pdf2md_patched", False):
|
||||
|
|
@ -1686,6 +1778,7 @@ def keep_existing_ocr(page, **kwargs) -> None:
|
|||
|
||||
def convert_pages(source: str, run: str, pages: list[int], options: dict) -> str:
|
||||
_jpeg_quality[0] = options.get("jpeg_quality", DEFAULT_JPEG_QUALITY) # auch im Worker
|
||||
_keep_lines[0] = options.get("keep_lines", False)
|
||||
kwargs = dict(
|
||||
pages=pages,
|
||||
header=options["keep_headers"],
|
||||
|
|
@ -1695,8 +1788,10 @@ def convert_pages(source: str, run: str, pages: list[int], options: dict) -> str
|
|||
image_format=options["image_format"],
|
||||
)
|
||||
if run == RUN_OCR:
|
||||
# FORCE_DROP_OLD: vorhandene (schlechte) OCR-Ebene verwerfen und neu
|
||||
# erkennen; force_ocr=True hieße FORCE_KEEP_OLD und behielte den alten Text.
|
||||
kwargs.update(
|
||||
force_ocr=True,
|
||||
use_ocr=OCRMode.FORCE_DROP_OLD,
|
||||
ocr_function=tesseract_api.exec_ocr,
|
||||
ocr_language=options["ocr_language"],
|
||||
ocr_dpi=options["ocr_dpi"],
|
||||
|
|
@ -1708,7 +1803,24 @@ def convert_pages(source: str, run: str, pages: list[int], options: dict) -> str
|
|||
|
||||
# pymupdf4llm schreibt Statusmeldungen auf stdout; wir melden selbst.
|
||||
with contextlib.redirect_stdout(io.StringIO()):
|
||||
return pymupdf4llm.to_markdown(source, **kwargs)
|
||||
if run != RUN_OCR:
|
||||
return pymupdf4llm.to_markdown(source, **kwargs)
|
||||
# Fremde OCR-Ebenen entfernt pymupdf4llm nicht (nur eigene); sonst stünde
|
||||
# alter und neuer Text doppelt da. Nur die geöffnete Kopie wird geändert.
|
||||
with pymupdf.open(source) as doc:
|
||||
for pno in pages:
|
||||
drop_invisible_text(doc[pno])
|
||||
return pymupdf4llm.to_markdown(doc, **kwargs)
|
||||
|
||||
|
||||
def drop_invisible_text(page: pymupdf.Page) -> None:
|
||||
"""Unsichtbaren Text (vorhandene OCR-Ebene) entfernen; Bild und Grafik bleiben."""
|
||||
page.add_redact_annot(page.rect, fill=False)
|
||||
page.apply_redactions(
|
||||
images=pymupdf.PDF_REDACT_IMAGE_NONE,
|
||||
graphics=pymupdf.PDF_REDACT_LINE_ART_NONE,
|
||||
text=pymupdf.PDF_REDACT_TEXT_REMOVE_INVISIBLE,
|
||||
)
|
||||
|
||||
|
||||
def _init_worker() -> None:
|
||||
|
|
@ -2232,6 +2344,12 @@ def main() -> int:
|
|||
help="Sprache im EPUB, z. B. en (Standard: aus --ocr-language, "
|
||||
"deu -> de, eng -> en)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--keep-lines",
|
||||
action="store_true",
|
||||
help="jede Zeile der PDF bleibt eine Zeile (Gedichte, Liedtexte); kein "
|
||||
"Zusammenfügen, auch keine Silbentrennung",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--keep-headers",
|
||||
action="store_true",
|
||||
|
|
@ -2319,6 +2437,7 @@ def main() -> int:
|
|||
ocr_language=args.ocr_language,
|
||||
ocr_dpi=args.ocr_dpi,
|
||||
jpeg_quality=args.jpeg_quality,
|
||||
keep_lines=args.keep_lines,
|
||||
)
|
||||
markdown = extract_markdown(source, plan, options, args.jobs)
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue