Seitenzahlen bei --split, Teilen nach Länge, Links mit eigenem Text
- add_source_pages: source_pages [von, bis] im Frontmatter; Startseite aus
der PDF-Gliederung, sonst Textsuche ohne Verzeichnisseiten; Ende davor,
wenn der nächste Abschnitt oben auf der Seite beginnt
- split_by_length: ohne verwertbare Gliederung an Absatzgrenzen nach Länge
(ca. 30 000 Zeichen), Codeblöcke bleiben ganz
- load_link_anchors/link_anchors: Links mit eigenem Text ("hier") über
Linktext und Wörter davor wiederfinden und als [Text](Ziel) setzen
(Zensur-Report: 75 von 76)
- Tests, README, Bedienungsanleitung, TODO
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Dieter Schlüter
parent
b8ad0310d0
commit
3d418e3987
1 changed files with 273 additions and 17 deletions
172
pdf2md.py
172
pdf2md.py
|
|
@ -744,6 +744,94 @@ def link_urls(markdown: str, uris: list[str] | None = None) -> str:
|
|||
return "\n".join(lines)
|
||||
|
||||
|
||||
# Links mit eigenem Text ("hier", "Bericht"): Linktext und die Wörter davor aus
|
||||
# der PDF lesen und die Stelle im Markdown als [Text](Ziel) markieren.
|
||||
|
||||
ANCHOR_CONTEXT_WORDS = 3
|
||||
URL_LIKE_WORD = re.compile(r"://|^www\.|/|\w\.(?:[a-z]{2,4})\.?$", re.IGNORECASE)
|
||||
|
||||
|
||||
def load_link_anchors(doc: pymupdf.Document) -> list[tuple[str, list[str], list[str]]]:
|
||||
"""(Ziel, Wörter davor, Linkwörter) für Links, deren Text keine URL ist."""
|
||||
anchors: list[tuple[str, list[str], list[str]]] = []
|
||||
for page in doc:
|
||||
words = page.get_text("words", sort=True)
|
||||
links = [link for link in page.get_links() if (link.get("uri") or "").startswith("http")]
|
||||
merged: list[tuple[str, list[int]]] = [] # (Ziel, Indizes der Wörter)
|
||||
for link in links:
|
||||
rect = link["from"]
|
||||
hits = [
|
||||
i for i, w in enumerate(words)
|
||||
if rect.contains(pymupdf.Point((w[0] + w[2]) / 2, (w[1] + w[3]) / 2))
|
||||
]
|
||||
if not hits:
|
||||
continue
|
||||
if merged and merged[-1][0] == link["uri"] and hits[0] <= merged[-1][1][-1] + 1:
|
||||
merged[-1][1].extend(i for i in hits if i not in merged[-1][1])
|
||||
else:
|
||||
merged.append((link["uri"], hits))
|
||||
for uri, hits in merged:
|
||||
text = [words[i][4] for i in sorted(hits)]
|
||||
if any(URL_LIKE_WORD.search(word) for word in text):
|
||||
continue # URL als Linktext: erledigt link_urls
|
||||
first = min(hits)
|
||||
before = [w[4] for w in words[max(0, first - ANCHOR_CONTEXT_WORDS):first]]
|
||||
anchors.append((clean_uri(uri), before, text))
|
||||
return anchors
|
||||
|
||||
|
||||
def _words_pattern(words: list[str]) -> str:
|
||||
"""Wörter mit beliebigem Leerraum, Hervorhebung oder Trennung dazwischen."""
|
||||
parts = []
|
||||
for word in words:
|
||||
core = word.rstrip("-\u00ad") # Silbentrennung am Zeilenende
|
||||
parts.append(re.escape(core) + (r"-?" if core != word else ""))
|
||||
return r"[\s*_]*".join(parts)
|
||||
|
||||
|
||||
# Zwischen Kontextwörtern: Satzzeichen, Fußnotenziffern, Linkziele "](…)".
|
||||
CONTEXT_GAP = r"(?:[\W_\d]|\]\([^)]*\))*"
|
||||
|
||||
|
||||
def _context_pattern(words: list[str]) -> str:
|
||||
cores = [core for word in words for core in re.findall(r"[^\W\d_]+", word)]
|
||||
return CONTEXT_GAP.join(re.escape(core) for core in cores)
|
||||
|
||||
|
||||
def link_anchors(markdown: str, anchors: list[tuple[str, list[str], list[str]]]) -> str:
|
||||
"""Linktexte im Markdown als [Text](Ziel) setzen, in Lesereihenfolge."""
|
||||
pos = 0
|
||||
for uri, before, text in anchors:
|
||||
anchor = r"(?P<anchor>" + _words_pattern(text) + r")"
|
||||
patterns = []
|
||||
if context := _context_pattern(before):
|
||||
patterns.append(re.compile(context + CONTEXT_GAP + anchor))
|
||||
if len(" ".join(text)) >= 6:
|
||||
patterns.append(re.compile(r"(?<![\w\[])" + anchor)) # ohne Kontext
|
||||
for pattern in patterns:
|
||||
match = next(
|
||||
(m for m in pattern.finditer(markdown, pos)
|
||||
if not _inside_link(markdown, m.start("anchor"))),
|
||||
None,
|
||||
)
|
||||
if match:
|
||||
start, end = match.span("anchor")
|
||||
label = markdown[start:end].replace("[", "\\[").replace("]", "\\]")
|
||||
markdown = f"{markdown[:start]}[{label}]({uri}){markdown[end:]}"
|
||||
pos = start
|
||||
break
|
||||
return markdown
|
||||
|
||||
|
||||
def _inside_link(markdown: str, index: int) -> bool:
|
||||
"""Liegt die Stelle schon in einem Link (<…>, [..](…))?"""
|
||||
line_start = markdown.rfind("\n", 0, index) + 1
|
||||
before = markdown[line_start:index]
|
||||
return before.count("<") > before.count(">") or before.count("[") > before.count("]") or (
|
||||
before.rfind("](") > before.rfind(")")
|
||||
)
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# Korrektur für pymupdf4llm: Trennstriche am Zeilenende, OCR-Ebenen
|
||||
# ===========================================================================
|
||||
|
|
@ -1580,6 +1668,34 @@ def iter_headings(lines: list[str]):
|
|||
yield index, len(match.group(1)), match.group(2).strip()
|
||||
|
||||
|
||||
SPLIT_CHARS = 30_000 # ohne Gliederung: Abschnitte von etwa dieser Länge
|
||||
|
||||
|
||||
def split_by_length(markdown: str, size: int = SPLIT_CHARS) -> list[dict]:
|
||||
"""Ohne verwertbare Überschriften: an Absatzgrenzen in etwa gleich lange Stücke."""
|
||||
blocks, fence = [], False
|
||||
for part in re.split(r"(\n\n+)", markdown):
|
||||
if blocks and fence:
|
||||
blocks[-1] += part # Codeblöcke nicht zerteilen
|
||||
elif part.strip() or not blocks:
|
||||
blocks.append(part)
|
||||
else:
|
||||
blocks[-1] += part
|
||||
fence ^= sum(1 for line in part.split("\n") if FENCE.match(line)) % 2 == 1
|
||||
sections, current = [], ""
|
||||
for block in blocks:
|
||||
if current and len(current) + len(block) > size:
|
||||
sections.append(current)
|
||||
current = ""
|
||||
current += block
|
||||
if current.strip():
|
||||
sections.append(current)
|
||||
return [
|
||||
{"title": f"Abschnitt {n}", "body": body.strip() + "\n"}
|
||||
for n, body in enumerate(sections, start=1)
|
||||
]
|
||||
|
||||
|
||||
def split_markdown(markdown: str, mode: str) -> list[dict]:
|
||||
"""Buch in Abschnitte teilen: "teile" (oberste Ebene) oder "kapitel".
|
||||
|
||||
|
|
@ -1590,7 +1706,7 @@ def split_markdown(markdown: str, mode: str) -> list[dict]:
|
|||
counts = Counter(level for _, level, _ in headings)
|
||||
levels = sorted(level for level, n in counts.items() if n >= 2 and level <= 3)
|
||||
if not levels:
|
||||
return [{"title": "Vorspann", "body": markdown}]
|
||||
return split_by_length(markdown)
|
||||
part_level, chapter_level = levels[0], levels[-1]
|
||||
cut_levels = {part_level} if mode == "teile" else {part_level, chapter_level}
|
||||
|
||||
|
|
@ -1623,6 +1739,50 @@ def split_markdown(markdown: str, mode: str) -> list[dict]:
|
|||
return sections
|
||||
|
||||
|
||||
def _page_key(text: str) -> str:
|
||||
return re.sub(r"[\W_]+", "", clean_inline(text).lower())
|
||||
|
||||
|
||||
def add_source_pages(doc: pymupdf.Document, sections: list[dict]) -> None:
|
||||
"""Seiten der PDF (1-basiert) je Abschnitt.
|
||||
|
||||
Startseite aus der PDF-Gliederung, sonst die erste Seite (nach der vorigen),
|
||||
auf der die Überschrift steht – Verzeichnisseiten ausgenommen.
|
||||
"""
|
||||
outline = {norm_title(title): page - 1 for _, title, page in doc.get_toc() if page > 0}
|
||||
pages = [_page_key(page.get_text()) for page in doc]
|
||||
keys = [_page_key(section["title"])[:40] for section in sections]
|
||||
listing = { # Verzeichnisse: viele Abschnittstitel auf einer Seite
|
||||
i for i, text in enumerate(pages) if sum(1 for k in keys if k and k in text) >= 3
|
||||
}
|
||||
starts, page = [], 0
|
||||
for section, key in zip(sections, keys):
|
||||
found = outline.get(norm_title(section["title"]))
|
||||
if found is None and section["title"] != "Vorspann" and key:
|
||||
found = next(
|
||||
(i for i in range(page, len(pages)) if i not in listing and key in pages[i]),
|
||||
None,
|
||||
)
|
||||
page = max(page, found) if found is not None else page
|
||||
starts.append(page)
|
||||
for n, section in enumerate(sections):
|
||||
end = starts[n + 1] if n + 1 < len(sections) else len(pages) - 1
|
||||
# Beginnt der nächste Abschnitt oben auf einer Seite, endet dieser davor.
|
||||
if n + 1 < len(sections) and end > starts[n] and _starts_page(doc[end], keys[n + 1]):
|
||||
end -= 1
|
||||
section["source_pages"] = [starts[n] + 1, max(starts[n], end) + 1]
|
||||
|
||||
|
||||
def _starts_page(page: pymupdf.Page, key: str) -> bool:
|
||||
"""Steht über der Überschrift nur Kolumnentitel/Seitenzahl (≤ 3 Wörter je Block)?"""
|
||||
blocks = [b for b in page.get_text("blocks") if b[6] == 0 and b[4].strip()]
|
||||
heading = next((b for b in blocks if key and key in _page_key(b[4])), None)
|
||||
if heading is None or not _page_key(heading[4]).startswith(key):
|
||||
return False # nicht gefunden oder Text davor im selben Block
|
||||
above = [b for b in blocks if b[3] <= heading[1] + 1 and b is not heading]
|
||||
return all(len(b[4].split()) <= 3 for b in above)
|
||||
|
||||
|
||||
def write_split(folder: Path, sections: list[dict], meta: dict) -> list[Path]:
|
||||
"""Abschnitte als nummerierte Dateien plus index.md (Reihenfolge) schreiben."""
|
||||
folder.mkdir(parents=True, exist_ok=True)
|
||||
|
|
@ -1635,6 +1795,7 @@ def write_split(folder: Path, sections: list[dict], meta: dict) -> list[Path]:
|
|||
"part": section.get("part"),
|
||||
"chapter": section.get("chapter"),
|
||||
"order": order,
|
||||
"source_pages": section.get("source_pages"),
|
||||
"book": meta.get("title"),
|
||||
"author": meta.get("author"),
|
||||
}
|
||||
|
|
@ -1847,6 +2008,7 @@ def main() -> int:
|
|||
kinds, plan = plan_pages(doc, args.ocr, args.ocr_language, args.ocr_dpi)
|
||||
outline = load_outline(doc)
|
||||
uris = load_link_uris(doc)
|
||||
anchors = load_link_anchors(doc)
|
||||
page_count = doc.page_count
|
||||
file_title, file_author = title_author_from_filename(source.stem)
|
||||
title = (
|
||||
|
|
@ -1904,6 +2066,7 @@ def main() -> int:
|
|||
running_head_min=max(3, page_count // 10),
|
||||
)
|
||||
markdown = link_urls(markdown, uris)
|
||||
markdown = link_anchors(markdown, anchors)
|
||||
|
||||
base = folder or target.parent
|
||||
meta = {
|
||||
|
|
@ -1913,7 +2076,12 @@ def main() -> int:
|
|||
"cover": cover.relative_to(base).as_posix() if cover else None,
|
||||
}
|
||||
if folder:
|
||||
files = write_split(folder, split_markdown(markdown, args.split), meta)
|
||||
sections = split_markdown(markdown, args.split)
|
||||
if sections and sections[0]["title"] == "Abschnitt 1":
|
||||
print("Hinweis: keine Gliederung erkannt – nach Länge geteilt.")
|
||||
with pymupdf.open(source) as doc:
|
||||
add_source_pages(doc, sections)
|
||||
files = write_split(folder, sections, meta)
|
||||
else:
|
||||
target.write_text(make_frontmatter(meta) + markdown, encoding="utf-8")
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue