feat: Formeln im Fließtext (pdf2md --formulas), etwa dreimal schneller

- pdf2md-Formelliste lesen: jede Formel mit Seite/Rechteck/PDF-Text, 300-dpi-Bild
- eingebettete Formeln: Satzzeichen am Ende abgetrennt, als Text dahinter (final_suffix)
- Normalform: \underset-Grenzwerte, Gleichungsnummer als \tag, \textbf, \cfrac, wörtliches \n
- Tempo: Verweis- und Ausschnitt-Cache, pix2text gebündelt nach Pixelbudget, Lesen und
  Prüfen überlappend (forkserver), Qwen 8 Plätze, Render-Vorfilter, kein neuer Scan,
  je Paket nur dessen Datensätze, Zeiten je Stufe
- Bildprüfung: max. 1600 px Breite, 2,5 MP, Proportionen-Abbruch
- gemessen und abgeschaltet: PDF-Textebene als Bestätigung (pdftext=0)
- Corral 10 %: 79 % automatisch, Pilot 869 Formeln 70 → ~24 min

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Dieter Schlüter 2026-10-10 03:31:57 +02:00
commit 6373db73a4

View file

@ -23,9 +23,9 @@ from PIL import Image
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from formula_img2latex.pipeline import _ref_for, latest_by_engine, open_all # noqa: E402
from formula_img2latex.pipeline import latest_by_engine, open_all, original_bytes # noqa: E402
from formula_img2latex.render import render_many # noqa: E402
from formula_img2latex.tex import strip_delimiters # noqa: E402
from formula_img2latex.tex import split_tag, strip_delimiters # noqa: E402
LABEL = {"local": "Qwen", "p2t": "pix2text", "local2": "Gemma", "cloud": "GLM"}
STATUS = {"auto_accepted": ("automatisch ersetzt", "good"), "accepted_by_human": ("von Hand übernommen", "good"),
@ -75,12 +75,21 @@ def main() -> int:
p = latest_by_engine(f).get("p2t")
tex = p.get("tex") if p else None
shown.append(strip_delimiters(tex) if tex else None)
rendered = render_many([(t or "\\text{–}", f.display) for t, f in zip(shown, formulas)], preset)
# already set during the image check (same LaTeX): reuse that picture instead of running pdflatex again
def checked(f, tex: str | None) -> Path | None:
best = (f.verification or {}).get("best") or {}
rigid = ws.review_dir / f"{f.id}-rigid.png"
return rigid if tex and best.get("tex") == split_tag(tex)[0] and rigid.exists() else None
reuse = [checked(f, t) for f, t in zip(formulas, shown)]
todo = [k for k, r in enumerate(reuse) if r is None]
fresh = dict(zip(todo, render_many([(shown[k] or "\\text{–}", formulas[k].display) for k in todo], preset)))
counts: dict[str, int] = {}
secs: dict[str, list[float]] = {}
cards = []
for n, (f, tex, rr) in enumerate(zip(formulas, shown, rendered), 1):
for n, (f, tex) in enumerate(zip(formulas, shown), 1):
rr = fresh.get(n - 1)
counts[f.status] = counts.get(f.status, 0) + 1
latest = latest_by_engine(f)
for e, r in latest.items():
@ -89,11 +98,14 @@ def main() -> int:
try:
o = out / "img" / f"{f.id}-o.png"
if not o.exists(): # the original never changes; cutting it from the PDF is the slow part
save_png(original_png(source.original_bytes(_ref_for(source, f))), o)
save_png(original_png(original_bytes(source, ws, f)), o)
orig = f"<img loading=lazy src='img/{f.id}-o.png'>"
except Exception: # noqa: BLE001
orig = "<i>Original nicht lesbar</i>"
if tex and rr.ok:
if tex and reuse[n - 1] is not None:
save_png(original_png(reuse[n - 1].read_bytes()), out / "img" / f"{f.id}-l.png")
latex = f"<img loading=lazy src='img/{f.id}-l.png'>"
elif tex and rr is not None and rr.ok:
g = (rr.rasterize(1.6) * 255).astype(np.uint8)
save_png(Image.fromarray(255 - g), out / "img" / f"{f.id}-l.png")
latex = f"<img loading=lazy src='img/{f.id}-l.png'>"