147 lines
6.4 KiB
Python
147 lines
6.4 KiB
Python
|
"""Links: Test-PDF mit typischen Varianten gebrochener URLs und Linktexte.
|
|||
|
|
|
|||
|
|
Die PDF lässt sich auch zum Ansehen erzeugen:
|
|||
|
|
python3 -c "import sys; sys.path[:0]=['.', 'tests']; import test_links; test_links.make_link_pdf('links.pdf')"
|
|||
|
|
"""
|
|||
|
|
|
|||
|
|
import pymupdf
|
|||
|
|
|
|||
|
|
import pdf2md as p
|
|||
|
|
from test_e2e import run
|
|||
|
|
|
|||
|
|
FONT, SIZE, LEFT, WIDTH = "tiro", 11, 50, 320
|
|||
|
|
FILLER = "Wie schon im letzten Bericht dargestellt wurde die Lage danach genauer untersucht".split()
|
|||
|
|
|
|||
|
|
# (Zeilen, Linkziel, Text des Links in der PDF, Abstand vor dem Absatz)
|
|||
|
|
PARAGRAPHS = [
|
|||
|
|
# 1. URL innerhalb des Absatzes am Bindestrich umbrochen, Tracking-Parameter
|
|||
|
|
(["Quelle: https://example.org/artikel/insa-umfrage-deutsche-",
|
|||
|
|
"sehen-meinungsfreiheit/. Danach geht der Satz ganz normal weiter."],
|
|||
|
|
"https://example.org/artikel/insa-umfrage-deutsche-sehen-meinungsfreiheit/?utm_source=chatgpt.com",
|
|||
|
|
None, 0),
|
|||
|
|
# 2. URL am "/" umbrochen (mit Leerzeichen danach), Endung in der nächsten Zeile
|
|||
|
|
(["Mehr unter https://example.org/docs/",
|
|||
|
|
"handbuch-2024.html und im Anhang des Berichts."],
|
|||
|
|
"https://example.org/docs/handbuch-2024.html", None, 14),
|
|||
|
|
# 3. URL geht in einem neuen Block weiter (großer Abstand)
|
|||
|
|
(["Siehe https://example.org/presse/safer-internet-day-"],
|
|||
|
|
None, None, 14),
|
|||
|
|
(["juuuport-scouts-seminar/. Ende des Satzes."],
|
|||
|
|
None, None, 40),
|
|||
|
|
# 4. ohne Linkziel: "und/oder" gehört nicht zur URL
|
|||
|
|
(["Wir prüfen https://kein-ziel.de/ und/oder andere Seiten im Netz."], None, None, 14),
|
|||
|
|
# 5. Link mit eigenem Text "hier"
|
|||
|
|
(["Den ganzen Bericht finden Sie hier im Netz der Organisation."],
|
|||
|
|
"https://example.org/bericht", ["hier"], 14),
|
|||
|
|
# 6. Linktext über zwei Zeilen mit Trennung mitten im Wort
|
|||
|
|
(["Zu den Empfängern gehört RUBIN: News-Poly-",
|
|||
|
|
"graph: Multimodal Orchestration for Media Verification. Danach folgt Text."],
|
|||
|
|
"https://example.org/rubin", ["RUBIN:", "News-Poly-", "graph:", "Multimodal",
|
|||
|
|
"Orchestration", "for", "Media", "Verification."], 14),
|
|||
|
|
]
|
|||
|
|
CROSS_BLOCK_URI = "https://example.org/presse/safer-internet-day-juuuport-scouts-seminar/"
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _fill(line: str) -> str:
|
|||
|
|
"""Zeile vorn mit Füllwörtern bis an den rechten Rand bringen (Blocksatz-ähnlich)."""
|
|||
|
|
for n in range(len(FILLER), 0, -1):
|
|||
|
|
filled = " ".join(FILLER[:n]) + " " + line
|
|||
|
|
if pymupdf.get_text_length(filled, FONT, SIZE) <= WIDTH:
|
|||
|
|
return filled
|
|||
|
|
return line
|
|||
|
|
|
|||
|
|
|
|||
|
|
def make_link_pdf(path) -> str:
|
|||
|
|
doc = pymupdf.open()
|
|||
|
|
page = doc.new_page(width=420, height=595)
|
|||
|
|
y = 60
|
|||
|
|
cross_rects = []
|
|||
|
|
for lines, uri, anchor, gap in PARAGRAPHS:
|
|||
|
|
y += gap
|
|||
|
|
rects = []
|
|||
|
|
for n, line in enumerate(lines):
|
|||
|
|
text = _fill(line) if n < len(lines) - 1 or len(lines) == 1 and line.endswith("-") else line
|
|||
|
|
page.insert_text((LEFT, y), text, fontname=FONT, fontsize=SIZE)
|
|||
|
|
words = [w for w in page.get_text("words") if abs(w[3] - y) < 4]
|
|||
|
|
if uri and anchor is None: # URL-Teile dieser Zeile
|
|||
|
|
rects += [pymupdf.Rect(w[:4]) for w in words if "/" in w[4] or "html" in w[4]
|
|||
|
|
or w[4].startswith(("sehen", "handbuch"))]
|
|||
|
|
elif uri:
|
|||
|
|
rects += [pymupdf.Rect(w[:4]) for w in words if w[4] in anchor]
|
|||
|
|
if "safer-internet" in line or "juuuport" in line:
|
|||
|
|
cross_rects += [pymupdf.Rect(w[:4]) for w in words
|
|||
|
|
if "safer" in w[4] or "juuuport" in w[4]]
|
|||
|
|
y += 14
|
|||
|
|
for rect in rects:
|
|||
|
|
page.insert_link({"kind": pymupdf.LINK_URI, "from": rect, "uri": uri})
|
|||
|
|
for rect in cross_rects:
|
|||
|
|
page.insert_link({"kind": pymupdf.LINK_URI, "from": rect, "uri": CROSS_BLOCK_URI})
|
|||
|
|
doc.save(str(path))
|
|||
|
|
return str(path)
|
|||
|
|
|
|||
|
|
|
|||
|
|
def convert(tmp_path) -> str:
|
|||
|
|
out = tmp_path / "links.md"
|
|||
|
|
run(make_link_pdf(tmp_path / "links.pdf"), out)
|
|||
|
|
return out.read_text(encoding="utf-8")
|
|||
|
|
|
|||
|
|
|
|||
|
|
# --- Ende zu Ende ------------------------------------------------------------------
|
|||
|
|
|
|||
|
|
def test_url_broken_at_hyphen_is_repaired_without_tracking(tmp_path):
|
|||
|
|
md = convert(tmp_path)
|
|||
|
|
assert "<https://example.org/artikel/insa-umfrage-deutsche-sehen-meinungsfreiheit/>." in md
|
|||
|
|
assert "utm_source" not in md
|
|||
|
|
|
|||
|
|
|
|||
|
|
def test_url_broken_after_slash_is_joined(tmp_path):
|
|||
|
|
assert "<https://example.org/docs/handbuch-2024.html> und im Anhang" in convert(tmp_path)
|
|||
|
|
|
|||
|
|
|
|||
|
|
def test_url_continued_in_next_block_is_joined(tmp_path):
|
|||
|
|
md = convert(tmp_path)
|
|||
|
|
assert f"<{CROSS_BLOCK_URI}>. Ende des Satzes." in md
|
|||
|
|
assert "\njuuuport" not in md
|
|||
|
|
|
|||
|
|
|
|||
|
|
def test_word_pair_after_url_is_not_part_of_it(tmp_path):
|
|||
|
|
assert "<https://kein-ziel.de/> und/oder andere Seiten" in convert(tmp_path)
|
|||
|
|
|
|||
|
|
|
|||
|
|
def test_anchor_links(tmp_path):
|
|||
|
|
md = convert(tmp_path)
|
|||
|
|
assert "finden Sie [hier](https://example.org/bericht) im Netz" in md
|
|||
|
|
assert ("[RUBIN: News-Polygraph: Multimodal Orchestration for Media Verification.]"
|
|||
|
|
"(https://example.org/rubin)") in md
|
|||
|
|
|
|||
|
|
|
|||
|
|
# --- Einzelfälle auf Markdown-Ebene -------------------------------------------------
|
|||
|
|
|
|||
|
|
def test_url_continued_as_list_item_is_joined():
|
|||
|
|
uri = "https://www.nlm.de/aktuell/safer-internet-day-juuuport-scouts-seminar-vor"
|
|||
|
|
md = "Siehe https://www.nlm.de/aktuell/safer-internet-day-\n\n- juuuport-scouts-\n\nseminar-vor/. Weiter.\n"
|
|||
|
|
assert p.link_urls(md, [uri]) == f"Siehe <{uri}>. Weiter.\n"
|
|||
|
|
assert p.link_urls(md) == f"Siehe <{uri}/>. Weiter.\n" # ohne Linkziel: gedruckte Form
|
|||
|
|
|
|||
|
|
|
|||
|
|
def test_url_at_line_end_does_not_swallow_next_paragraph():
|
|||
|
|
md = "Normal https://a.de/x-\n\nDer nächste Absatz.\n\nSiehe https://b.de/\n\n- erster Punkt\n"
|
|||
|
|
assert p.link_urls(md) == (
|
|||
|
|
"Normal <https://a.de/x->\n\nDer nächste Absatz.\n\nSiehe <https://b.de/>\n\n- erster Punkt\n"
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
|
|||
|
|
def test_extension_only_after_dot():
|
|||
|
|
assert p.link_urls("A https://b.de/a. html. B https://c.de/ html.\n") == (
|
|||
|
|
"A <https://b.de/a.html>. B <https://c.de/> html.\n"
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
|
|||
|
|
def test_fuzzy_anchor_with_lost_ligature():
|
|||
|
|
anchors = [("https://x.de/rubin", ["haben,", "gehört"],
|
|||
|
|
["RUBIN", "–", "News-Poly-", "graph:", "Media-Content", "Verification", "(MOVERA)."])]
|
|||
|
|
md = "erhalten haben, gehört RUBIN – News-Polygraph: Media-Content Verifcation (MOVERA). In diesem Programm\n"
|
|||
|
|
assert p.link_anchors(md, anchors) == (
|
|||
|
|
"erhalten haben, gehört [RUBIN – News-Polygraph: Media-Content Verifcation (MOVERA).]"
|
|||
|
|
"(https://x.de/rubin) In diesem Programm\n"
|
|||
|
|
)
|