Gedicht-Scans: Zeilen direkt aus Tesseract, Fehler aus pymupdf4llm 1.28
- --keep-lines auf Scanseiten: Zeilen direkt aus Tesseract statt über die
Layout-Analyse (die verschränkte eng stehende Verse und gab Zeilen
überlappender Blöcke doppelt aus); Stücke auf gleicher Höhe verbinden,
Rauschen außerhalb der Textspalte verwerfen, Strophen über den
Grundlinienabstand, Einrückung aus der x-Position, Gedichttitel aus der
PDF-Gliederung (richtige Schreibweise statt OCR-Lesart)
- Bildtext im neuen Format von pymupdf4llm 1.28 ("<!-- Start of picture
text -->") wieder entfernen
- Zeilen überlappender Layout-Blöcke nur einmal ausgeben
- OCR-Seitenzahlen ("= 58 —", "— VIII —") erkennen, auch am Strophenende
- Fraktur: deu_latf aus tessdata_best empfohlen (besser als frk)
- Tests, README, Bedienungsanleitung (Gedicht-/Fraktur-Abschnitt), TODO
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Dieter Schlüter
parent
3e5195efc2
commit
180d98ae04
1 changed files with 225 additions and 19 deletions
151
pdf2md.py
151
pdf2md.py
|
|
@ -14,6 +14,7 @@ Ablauf:
|
|||
|
||||
import argparse
|
||||
import contextlib
|
||||
import difflib
|
||||
import io
|
||||
import json
|
||||
import multiprocessing
|
||||
|
|
@ -75,9 +76,10 @@ OMITTED_IMAGE = re.compile(
|
|||
)
|
||||
|
||||
# Text, den pymupdf4llm innerhalb von Bildern findet (Fotos, OCR-Reste).
|
||||
PICTURE_TEXT = re.compile(
|
||||
r"\*\*----- Start of picture text -----\*\*<br>.*?"
|
||||
r"\*\*----- End of picture text -----\*\*<br>\n?",
|
||||
PICTURE_TEXT = re.compile( # Format bis pymupdf4llm 1.27 bzw. ab 1.28
|
||||
r"(?:\*\*----- Start of picture text -----\*\*<br>|<!-- Start of picture text -->)"
|
||||
r".*?"
|
||||
r"(?:\*\*----- End of picture text -----\*\*<br>|<!-- End of picture text -->)\n?",
|
||||
re.DOTALL,
|
||||
)
|
||||
|
||||
|
|
@ -103,7 +105,8 @@ HTML_BLOCK = re.compile(
|
|||
|
||||
# Seitenzahl allein in einer Zeile: "12", "- 12 -", "Seite 12"
|
||||
PAGE_NUMBER = re.compile(
|
||||
r"^(?:[-–—]\s*)?(?:(?:seite|page|s\.)\s*)?\d{1,4}(?:\s*[-–—])?$",
|
||||
r"^(?:[-–—=]\s*)?(?:(?:seite|page|s\.)\s*)?\d{1,4}(?:\s*[-–—=])?$"
|
||||
r"|^[-–—=]\s*[ivxlc]{1,7}\s*[-–—=]$", # "— VIII —" (OCR: "= 58 —")
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
|
|
@ -409,6 +412,11 @@ def normalize_markdown(
|
|||
standalone(clean_inline(line.rstrip(HARD_BREAK + PARAGRAPH_END)))
|
||||
continue
|
||||
|
||||
if line.endswith(PARAGRAPH_END) and drop_page_numbers and PAGE_NUMBER.match(
|
||||
clean_inline(line.removesuffix(PARAGRAPH_END))
|
||||
):
|
||||
continue # Seitenzahl am Strophen-/Seitenende
|
||||
|
||||
if line.endswith(PARAGRAPH_END):
|
||||
content = line.removesuffix(PARAGRAPH_END)
|
||||
if not paragraph and result and result[-1].endswith(HARD_BREAK):
|
||||
|
|
@ -1171,7 +1179,26 @@ _keep_lines = [False] # --keep-lines: jede PDF-Zeile bleibt eine Zeile
|
|||
REAL_WORD = re.compile(r"[^\W\d_]{3,}")
|
||||
|
||||
|
||||
def _dedupe_box_lines(boxes) -> None:
|
||||
"""Überlappende Layout-Blöcke enthalten dieselben Zeilen doppelt
|
||||
(Gedichtseiten in Scans) – jede Zeile nur im ersten Block behalten."""
|
||||
seen: set = set()
|
||||
for box in boxes:
|
||||
lines = getattr(box, "textlines", None)
|
||||
if not lines or getattr(box, "boxclass", None) not in TEXT_BOXES:
|
||||
continue
|
||||
kept = []
|
||||
for line in lines:
|
||||
key = (tuple(round(v) for v in line["bbox"]),
|
||||
"".join(span["text"] for span in line["spans"]))
|
||||
if key not in seen:
|
||||
seen.add(key)
|
||||
kept.append(line)
|
||||
box.textlines = kept
|
||||
|
||||
|
||||
def _remember_page_lines(boxes) -> None:
|
||||
_dedupe_box_lines(boxes)
|
||||
_page_lines.clear()
|
||||
real: list[tuple[float, float]] = []
|
||||
for box in boxes:
|
||||
|
|
@ -1805,6 +1832,15 @@ def convert_pages(source: str, run: str, pages: list[int], options: dict) -> str
|
|||
with contextlib.redirect_stdout(io.StringIO()):
|
||||
if run != RUN_OCR:
|
||||
return pymupdf4llm.to_markdown(source, **kwargs)
|
||||
if options.get("keep_lines"):
|
||||
with pymupdf.open(source) as doc:
|
||||
return "".join(
|
||||
ocr_lines_md(
|
||||
doc[pno], options["ocr_language"], options["ocr_dpi"],
|
||||
options.get("titles", {}).get(pno, []),
|
||||
)
|
||||
for pno in pages
|
||||
)
|
||||
# Fremde OCR-Ebenen entfernt pymupdf4llm nicht (nur eigene); sonst stünde
|
||||
# alter und neuer Text doppelt da. Nur die geöffnete Kopie wird geändert.
|
||||
with pymupdf.open(source) as doc:
|
||||
|
|
@ -1813,6 +1849,108 @@ def convert_pages(source: str, run: str, pages: list[int], options: dict) -> str
|
|||
return pymupdf4llm.to_markdown(doc, **kwargs)
|
||||
|
||||
|
||||
# --keep-lines auf Scanseiten: Zeilen direkt aus Tesseract, ohne Layout-Modell.
|
||||
# Dieses legt bei Gedicht-Scans eng stehende Verse zusammen ("UndDon tauchte…")
|
||||
# und gibt Zeilen sich überlappender Blöcke doppelt aus.
|
||||
|
||||
OCR_TITLE_LINES = 8 # Titel suchen wir unter den ersten Zeilen der Seite
|
||||
OCR_STANZA_GAP = 1.4 # Grundlinienabstand (× üblichem) für eine Strophengrenze
|
||||
PAGE_LABEL = re.compile(r"^seite\s+\d+$", re.IGNORECASE) # Gliederungseintrag "Seite 4"
|
||||
|
||||
|
||||
def _ocr_segments(page: pymupdf.Page, language: str, dpi: int) -> list[list]:
|
||||
"""Textzeilen-Stücke [x0, y0, x1, y1, text] aus Tesseract, ohne Rauschen."""
|
||||
pix = page.get_pixmap(dpi=dpi) # RGB: MuPDFs OCR liest Graustufen nicht
|
||||
segments = []
|
||||
with pymupdf.open("pdf", pix.pdfocr_tobytes(language=language)) as ocr:
|
||||
for block in ocr[0].get_text("dict")["blocks"]:
|
||||
for line in block.get("lines", []):
|
||||
text = OCR_NOISE.sub("", "".join(s["text"] for s in line["spans"]))
|
||||
text = re.sub(r"\s+", " ", text).strip()
|
||||
if HAS_ALNUM.search(text): # Grundlinie für Zeilen- und Strophenabstand
|
||||
segments.append([*line["bbox"], text, line["spans"][0]["origin"][1]])
|
||||
real = [seg for seg in segments if len(REAL_WORD.findall(seg[4])) >= 2]
|
||||
if real: # Spalte aus echten Zeilen; Fetzen am Scanrand liegen außerhalb
|
||||
left = min(seg[0] for seg in real)
|
||||
right = max(seg[2] for seg in real)
|
||||
height = sorted(seg[3] - seg[1] for seg in real)[len(real) // 2]
|
||||
segments = [
|
||||
seg for seg in segments
|
||||
if seg[2] > left - height and seg[0] < right + height
|
||||
and (len(REAL_WORD.findall(seg[4])) >= 1 or PAGE_NUMBER.match(seg[4]))
|
||||
]
|
||||
return segments
|
||||
|
||||
|
||||
def _ocr_rows(segments: list[list]) -> list[list]:
|
||||
"""Stücke auf gleicher Höhe zu Zeilen verbinden: [x0, y0, x1, y1, text].
|
||||
|
||||
Verglichen wird mit dem Mittelpunkt des ersten Stücks einer Zeile – ein
|
||||
wachsender Bereich würde sonst die Nachbarzeilen mit einsammeln."""
|
||||
if not segments:
|
||||
return []
|
||||
heights = sorted(seg[3] - seg[1] for seg in segments)
|
||||
height = heights[len(heights) // 2]
|
||||
rows: list[list] = [] # [x0, y0, x1, y1, stücke, mittelpunkt]
|
||||
for seg in sorted(segments, key=lambda s: (s[1] + s[3]) / 2):
|
||||
center = (seg[1] + seg[3]) / 2
|
||||
row = next((r for r in rows if abs(r[5] - center) < 0.5 * height
|
||||
and all(seg[0] >= o[2] or seg[2] <= o[0] for o in r[4])), None)
|
||||
if row is None:
|
||||
rows.append([*seg[:4], [seg], center])
|
||||
else:
|
||||
row[0], row[1] = min(row[0], seg[0]), min(row[1], seg[1])
|
||||
row[2], row[3] = max(row[2], seg[2]), max(row[3], seg[3])
|
||||
row[4].append(seg)
|
||||
rows.sort(key=lambda r: r[5])
|
||||
return [
|
||||
[*r[:4], " ".join(seg[4] for seg in sorted(r[4], key=lambda s: s[0])),
|
||||
max(seg[5] for seg in r[4])]
|
||||
for r in rows
|
||||
]
|
||||
|
||||
|
||||
def _title_key(text: str) -> str:
|
||||
return re.sub(r"[\W\d_]+", "", text.lower())
|
||||
|
||||
|
||||
def ocr_lines_md(page: pymupdf.Page, language: str, dpi: int, titles: list) -> str:
|
||||
"""Scanseite zeilengetreu: Verse, Strophen (Abstand), Einrückung, Titel aus
|
||||
der PDF-Gliederung (richtige Schreibweise statt OCR-Lesart)."""
|
||||
rows = _ocr_rows(_ocr_segments(page, language, dpi))
|
||||
if not rows:
|
||||
return ""
|
||||
heights = sorted(r[3] - r[1] for r in rows)
|
||||
height = heights[len(heights) // 2]
|
||||
left = min(r[0] for r in rows if not PAGE_NUMBER.match(r[4]))
|
||||
pitches = sorted(b[5] - a[5] for a, b in zip(rows, rows[1:])) or [height]
|
||||
pitch = pitches[len(pitches) // 2] # Abstand der Grundlinien
|
||||
|
||||
headings: dict[int, str] = {}
|
||||
for level, title in titles: # Gliederungstitel dieser Seite wiederfinden
|
||||
key = _title_key(title)
|
||||
scored = [
|
||||
(difflib.SequenceMatcher(None, key, _title_key(r[4])).ratio(), i)
|
||||
for i, r in enumerate(rows[:OCR_TITLE_LINES]) if i not in headings
|
||||
]
|
||||
if scored and (best := max(scored))[0] >= 0.6:
|
||||
headings[best[1]] = "#" * min(level, 6) + " " + md_escape(title)
|
||||
|
||||
out: list[str] = []
|
||||
for i, row in enumerate(rows):
|
||||
if i in headings:
|
||||
out += ["", headings[i], ""]
|
||||
continue
|
||||
indent = int(max(0.0, row[0] - left) / (1.2 * height))
|
||||
text = TEXT_INDENT * indent + md_escape(row[4])
|
||||
gap = i + 1 < len(rows) and rows[i + 1][5] - row[5] > OCR_STANZA_GAP * pitch
|
||||
last = i + 1 == len(rows) or (i + 1) in headings
|
||||
out.append(text + (PARAGRAPH_END if gap or (last and i + 1 < len(rows)) else HARD_BREAK))
|
||||
if gap:
|
||||
out.append("")
|
||||
return "\n".join(out).strip("\n") + "\n\n"
|
||||
|
||||
|
||||
def drop_invisible_text(page: pymupdf.Page) -> None:
|
||||
"""Unsichtbaren Text (vorhandene OCR-Ebene) entfernen; Bild und Grafik bleiben."""
|
||||
page.add_redact_annot(page.rect, fill=False)
|
||||
|
|
@ -2392,6 +2530,10 @@ def main() -> int:
|
|||
with pymupdf.open(source) as doc:
|
||||
kinds, plan = plan_pages(doc, args.ocr, args.ocr_language, args.ocr_dpi)
|
||||
outline = load_outline(doc)
|
||||
titles: dict[int, list[tuple[int, str]]] = {} # Seite → Gliederungstitel
|
||||
for level, title, page_no in doc.get_toc():
|
||||
if page_no > 0 and title.strip() and not PAGE_LABEL.match(title.strip()):
|
||||
titles.setdefault(page_no - 1, []).append((level, title.strip()))
|
||||
uris = load_link_uris(doc)
|
||||
anchors = load_link_anchors(doc)
|
||||
page_count = doc.page_count
|
||||
|
|
@ -2438,6 +2580,7 @@ def main() -> int:
|
|||
ocr_dpi=args.ocr_dpi,
|
||||
jpeg_quality=args.jpeg_quality,
|
||||
keep_lines=args.keep_lines,
|
||||
titles=titles,
|
||||
)
|
||||
markdown = extract_markdown(source, plan, options, args.jobs)
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue