3d418e3987
- add_source_pages: source_pages [von, bis] im Frontmatter; Startseite aus
der PDF-Gliederung, sonst Textsuche ohne Verzeichnisseiten; Ende davor,
wenn der nächste Abschnitt oben auf der Seite beginnt
- split_by_length: ohne verwertbare Gliederung an Absatzgrenzen nach Länge
(ca. 30 000 Zeichen), Codeblöcke bleiben ganz
- load_link_anchors/link_anchors: Links mit eigenem Text ("hier") über
Linktext und Wörter davor wiederfinden und als [Text](Ziel) setzen
(Zensur-Report: 75 von 76)
- Tests, README, Bedienungsanleitung, TODO
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
234 lines
9.1 KiB
Python
234 lines
9.1 KiB
Python
"""--split, --from-md, YAML-Kopf und --jpeg-quality."""
|
||
|
||
import zipfile
|
||
|
||
import pymupdf
|
||
|
||
import pytest
|
||
|
||
import pdf2md as p
|
||
from conftest import needs_pandoc
|
||
from test_e2e import run
|
||
|
||
BOOK = """# Das Buch
|
||
|
||
## Inhaltsverzeichnis
|
||
|
||
Teil I
|
||
|
||
1 Anfang
|
||
|
||
## Teil I: Der Anfang
|
||
|
||
Einleitung zum Teil.
|
||
|
||
### 1 Anfang
|
||
|
||
Text eins.
|
||
|
||
```
|
||
# kein Kapitel, nur ein Kommentar
|
||
```
|
||
|
||
### 2 Weiter
|
||
|
||
Text zwei.
|
||
|
||
## Impressum
|
||
|
||
Alle Rechte vorbehalten.
|
||
"""
|
||
|
||
|
||
def test_frontmatter_round_trip():
|
||
meta = {"title": "Ein \"Buch\": Teil 1", "author": "Max Muster", "chapter": 3, "leer": ""}
|
||
text = p.make_frontmatter(meta) + "# Text\n"
|
||
parsed, body = p.split_frontmatter(text)
|
||
assert parsed == {"title": "Ein \"Buch\": Teil 1", "author": "Max Muster", "chapter": 3}
|
||
assert body == "# Text\n"
|
||
|
||
|
||
def test_text_without_frontmatter_is_unchanged():
|
||
assert p.split_frontmatter("---\n\nText\n") == ({}, "---\n\nText\n")
|
||
|
||
|
||
def test_slugify():
|
||
assert p.slugify("2 Ein Brief – eine Liste – ein Vermächtnis") == "2-ein-brief-eine-liste-ein-vermaechtnis"
|
||
assert p.slugify("Größe & Maß") == "groesse-mass"
|
||
assert p.slugify("!!!") == "abschnitt"
|
||
|
||
|
||
def test_split_by_chapters():
|
||
sections = p.split_markdown(BOOK, "kapitel")
|
||
assert [s["title"] for s in sections] == [
|
||
"Vorspann", "Teil I: Der Anfang", "1 Anfang", "2 Weiter", "Impressum",
|
||
]
|
||
assert sections[0]["body"].startswith("# Das Buch") and "## Inhaltsverzeichnis" in sections[0]["body"]
|
||
assert sections[2]["part"] == "Teil I: Der Anfang" and sections[2]["chapter"] == 1
|
||
assert "# kein Kapitel" in sections[2]["body"] # Codeblock wird nicht geschnitten
|
||
assert "".join(s["body"] for s in sections).replace("\n", "") == BOOK.replace("\n", "")
|
||
|
||
|
||
def test_split_by_parts():
|
||
sections = p.split_markdown(BOOK, "teile")
|
||
assert [s["title"] for s in sections] == ["Vorspann", "Teil I: Der Anfang", "Impressum"]
|
||
assert "### 2 Weiter" in sections[1]["body"]
|
||
|
||
|
||
def test_write_split_and_read_back(tmp_path):
|
||
sections = p.split_markdown(BOOK, "kapitel")
|
||
files = p.write_split(tmp_path / "buch", sections, {"title": "Das Buch", "author": "Max"})
|
||
assert [f.name for f in files][:3] == ["00_vorspann.md", "01_teil-i-der-anfang.md", "02_1-anfang.md"]
|
||
meta, body, base = p.read_markdown_source(tmp_path / "buch")
|
||
assert meta["title"] == "Das Buch" and base == tmp_path / "buch"
|
||
assert body.replace("\n", "") == BOOK.replace("\n", "")
|
||
head, _ = p.split_frontmatter(files[2].read_text())
|
||
assert head == {"title": "1 Anfang", "part": "Teil I: Der Anfang", "chapter": 1,
|
||
"order": 2, "book": "Das Buch", "author": "Max"}
|
||
|
||
|
||
def test_missing_index_entry_is_reported(tmp_path):
|
||
folder = tmp_path / "buch"
|
||
p.write_split(folder, p.split_markdown(BOOK, "teile"), {"title": "X"})
|
||
(folder / "01_teil-i-der-anfang.md").unlink()
|
||
with pytest.raises(RuntimeError, match="nicht vorhanden"):
|
||
p.read_markdown_source(folder)
|
||
|
||
|
||
# --- Ende zu Ende ------------------------------------------------------------------
|
||
|
||
def test_cli_split_writes_folder(pdfs, tmp_path):
|
||
run(pdfs["digital"], tmp_path / "buch.md", "--split", "kapitel", "--images")
|
||
folder = tmp_path / "buch"
|
||
names = sorted(f.name for f in folder.glob("*.md"))
|
||
assert names[0] == "00_vorspann.md" and "index.md" in names
|
||
assert any(n.endswith("_1-der-notar.md") for n in names)
|
||
assert (folder / "images" / "cover.jpg").is_file()
|
||
text = "".join(f.read_text() for f in folder.glob("*.md"))
|
||
assert "](images/" in text and "_images/" not in text
|
||
|
||
|
||
@needs_pandoc
|
||
def test_from_md_folder_follows_index_order(pdfs, tmp_path):
|
||
run(pdfs["digital"], tmp_path / "buch.md", "--split", "kapitel", "--epub")
|
||
index = tmp_path / "buch" / "index.md"
|
||
lines = index.read_text().split("\n")
|
||
i = next(n for n, line in enumerate(lines) if "1-der-notar" in line)
|
||
j = next(n for n, line in enumerate(lines) if "2-ein-brief" in line)
|
||
lines[i], lines[j] = lines[j], lines[i]
|
||
index.write_text("\n".join(lines))
|
||
|
||
run("--from-md", tmp_path / "buch", "--epub-file", tmp_path / "neu.epub")
|
||
with zipfile.ZipFile(tmp_path / "neu.epub") as z:
|
||
nav = z.read("EPUB/nav.xhtml").decode()
|
||
opf = z.read("EPUB/content.opf").decode()
|
||
assert nav.index("2 Ein Brief") < nav.index("1 Der Notar")
|
||
assert "Erika Muster" in opf and "cover" in opf
|
||
|
||
|
||
@needs_pandoc
|
||
def test_from_md_file_uses_frontmatter_and_overrides(pdfs, tmp_path):
|
||
run(pdfs["digital"], tmp_path / "buch.md", "--images")
|
||
md = tmp_path / "buch.md"
|
||
md.write_text(md.read_text().replace("Der Regen", "Der starke Regen"))
|
||
run("--from-md", md, "--title", "Neuer Titel")
|
||
with zipfile.ZipFile(tmp_path / "buch.epub") as z:
|
||
opf = z.read("EPUB/content.opf").decode()
|
||
text = "".join(z.read(n).decode() for n in z.namelist() if n.endswith(".xhtml"))
|
||
assert ">Neuer Titel<" in opf and "Erika Muster" in opf and "cover" in opf
|
||
assert "Der starke Regen" in text and "title:" not in text
|
||
|
||
|
||
def test_jpeg_quality_changes_image_size(pdfs, tmp_path):
|
||
def size(quality):
|
||
out = tmp_path / f"q{quality}" / "buch.md"
|
||
run(pdfs["digital"], out, "--images", "--jpeg-quality", str(quality))
|
||
return sum(f.stat().st_size for f in (out.parent / "buch_images").glob("*.jpg"))
|
||
|
||
assert size(30) < size(95)
|
||
|
||
|
||
# --- Seitenzahlen, Teilen nach Länge -----------------------------------------------
|
||
|
||
def _book_pdf(outline: bool):
|
||
doc = pymupdf.open()
|
||
texts = [
|
||
"Inhalt\n1 Anfang\n2 Weiter\n3 Schluss",
|
||
"1 Anfang\nText eins.",
|
||
"Noch mehr Text\nzum Anfang,\nüber mehrere\nZeilen.\n2 Weiter\nText zwei.",
|
||
"3 Schluss\nText drei.",
|
||
]
|
||
for text in texts:
|
||
doc.new_page().insert_text((72, 72), text, fontsize=11)
|
||
if outline:
|
||
doc.set_toc([[1, "1 Anfang", 2], [1, "2 Weiter", 3], [1, "3 Schluss", 4]])
|
||
return doc
|
||
|
||
|
||
@pytest.mark.parametrize("outline", [True, False])
|
||
def test_source_pages(outline):
|
||
sections = [{"title": t} for t in ("Vorspann", "1 Anfang", "2 Weiter", "3 Schluss")]
|
||
p.add_source_pages(_book_pdf(outline), sections)
|
||
assert [s["source_pages"] for s in sections] == [[1, 1], [2, 3], [3, 3], [4, 4]]
|
||
|
||
|
||
def test_split_without_headings_by_length():
|
||
paragraphs = [f"Absatz {n} " + " ".join(["Text"] * 40) for n in range(30)]
|
||
code = "```\n" + "\n\n".join(f"zeile_{n} = {n}" for n in range(20)) + "\n```"
|
||
markdown = "\n\n".join(paragraphs[:15] + [code] + paragraphs[15:]) + "\n"
|
||
assert p.split_markdown(markdown, "kapitel")[0]["title"] == "Abschnitt 1"
|
||
sections = p.split_by_length(markdown, size=1500)
|
||
assert len(sections) > 3
|
||
assert [s["title"] for s in sections[:2]] == ["Abschnitt 1", "Abschnitt 2"]
|
||
assert all(s["body"].count("```") in (0, 2) for s in sections) # Code bleibt ganz
|
||
assert "".join(s["body"] for s in sections).replace("\n", "") == markdown.replace("\n", "")
|
||
|
||
|
||
def test_cli_split_writes_source_pages(pdfs, tmp_path):
|
||
run(pdfs["digital"], tmp_path / "buch.md", "--split", "kapitel")
|
||
files = {f.name: p.split_frontmatter(f.read_text())[0] for f in (tmp_path / "buch").glob("0*.md")}
|
||
notar = next(meta for name, meta in files.items() if name.endswith("_1-der-notar.md"))
|
||
assert notar["source_pages"] == [2, 3]
|
||
|
||
|
||
# --- Links mit eigenem Text --------------------------------------------------------
|
||
|
||
def _link_pdf(path):
|
||
doc = pymupdf.open()
|
||
page = doc.new_page()
|
||
page.insert_text((72, 72), "Den ganzen Bericht finden Sie hier im Netz.", fontsize=11)
|
||
page.insert_text((72, 90), "Siehe auch www.example.org/info zum Thema.", fontsize=11)
|
||
for word, uri in (("hier", "https://beispiel.de/bericht?utm_source=x"),
|
||
("www.example.org/info", "https://www.example.org/info")):
|
||
rect = page.search_for(word)[0]
|
||
page.insert_link({"kind": pymupdf.LINK_URI, "from": rect, "uri": uri})
|
||
doc.save(path)
|
||
return path
|
||
|
||
|
||
def test_load_link_anchors_skips_url_texts(tmp_path):
|
||
doc = pymupdf.open(_link_pdf(tmp_path / "links.pdf"))
|
||
assert p.load_link_anchors(doc) == [
|
||
("https://beispiel.de/bericht", ["Bericht", "finden", "Sie"], ["hier"])
|
||
]
|
||
|
||
|
||
def test_link_anchors_uses_context_and_skips_existing_links():
|
||
anchors = [("https://a.de/x", ["finden", "Sie"], ["hier"])]
|
||
md = "Hier steht hier. Den Bericht finden Sie hier im Netz.\n"
|
||
assert p.link_anchors(md, anchors) == (
|
||
"Hier steht hier. Den Bericht finden Sie [hier](https://a.de/x) im Netz.\n"
|
||
)
|
||
# Fußnotenziffer und Trennung zwischen Kontext und Linktext stören nicht
|
||
anchors = [("https://b.de/", ["überprüfen.81"], ["Landes-", "medienanstalten"])]
|
||
md = "Die Landesmedienanstalten.\n\nzu überprüfen.[81 ] Die Landesmedienanstalten prüfen.\n"
|
||
anchors = [("https://b.de/", ["überprüfen.81", "Die"], ["Landes-", "medienanstalten"])]
|
||
assert "Die [Landesmedienanstalten](https://b.de/) prüfen" in p.link_anchors(md, anchors)
|
||
|
||
|
||
def test_cli_sets_anchor_links(tmp_path):
|
||
out = tmp_path / "links.md"
|
||
run(_link_pdf(tmp_path / "links.pdf"), out)
|
||
md = out.read_text()
|
||
assert "[hier](https://beispiel.de/bericht)" in md
|
||
assert "<https://www.example.org/info>" in md or "[www.example.org/info]" in md
|