feat: Formeln im Fließtext (pdf2md --formulas), etwa dreimal schneller
- pdf2md-Formelliste lesen: jede Formel mit Seite/Rechteck/PDF-Text, 300-dpi-Bild - eingebettete Formeln: Satzzeichen am Ende abgetrennt, als Text dahinter (final_suffix) - Normalform: \underset-Grenzwerte, Gleichungsnummer als \tag, \textbf, \cfrac, wörtliches \n - Tempo: Verweis- und Ausschnitt-Cache, pix2text gebündelt nach Pixelbudget, Lesen und Prüfen überlappend (forkserver), Qwen 8 Plätze, Render-Vorfilter, kein neuer Scan, je Paket nur dessen Datensätze, Zeiten je Stufe - Bildprüfung: max. 1600 px Breite, 2,5 MP, Proportionen-Abbruch - gemessen und abgeschaltet: PDF-Textebene als Bestätigung (pdftext=0) - Corral 10 %: 79 % automatisch, Pilot 869 Formeln 70 → ~24 min Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
2478bc5cb9
commit
6373db73a4
1 changed files with 479 additions and 112 deletions
|
|
@ -23,9 +23,9 @@ from PIL import Image
|
|||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
|
||||
from formula_img2latex.pipeline import _ref_for, latest_by_engine, open_all # noqa: E402
|
||||
from formula_img2latex.pipeline import latest_by_engine, open_all, original_bytes # noqa: E402
|
||||
from formula_img2latex.render import render_many # noqa: E402
|
||||
from formula_img2latex.tex import strip_delimiters # noqa: E402
|
||||
from formula_img2latex.tex import split_tag, strip_delimiters # noqa: E402
|
||||
|
||||
LABEL = {"local": "Qwen", "p2t": "pix2text", "local2": "Gemma", "cloud": "GLM"}
|
||||
STATUS = {"auto_accepted": ("automatisch ersetzt", "good"), "accepted_by_human": ("von Hand übernommen", "good"),
|
||||
|
|
@ -75,12 +75,21 @@ def main() -> int:
|
|||
p = latest_by_engine(f).get("p2t")
|
||||
tex = p.get("tex") if p else None
|
||||
shown.append(strip_delimiters(tex) if tex else None)
|
||||
rendered = render_many([(t or "\\text{–}", f.display) for t, f in zip(shown, formulas)], preset)
|
||||
# already set during the image check (same LaTeX): reuse that picture instead of running pdflatex again
|
||||
def checked(f, tex: str | None) -> Path | None:
|
||||
best = (f.verification or {}).get("best") or {}
|
||||
rigid = ws.review_dir / f"{f.id}-rigid.png"
|
||||
return rigid if tex and best.get("tex") == split_tag(tex)[0] and rigid.exists() else None
|
||||
|
||||
reuse = [checked(f, t) for f, t in zip(formulas, shown)]
|
||||
todo = [k for k, r in enumerate(reuse) if r is None]
|
||||
fresh = dict(zip(todo, render_many([(shown[k] or "\\text{–}", formulas[k].display) for k in todo], preset)))
|
||||
|
||||
counts: dict[str, int] = {}
|
||||
secs: dict[str, list[float]] = {}
|
||||
cards = []
|
||||
for n, (f, tex, rr) in enumerate(zip(formulas, shown, rendered), 1):
|
||||
for n, (f, tex) in enumerate(zip(formulas, shown), 1):
|
||||
rr = fresh.get(n - 1)
|
||||
counts[f.status] = counts.get(f.status, 0) + 1
|
||||
latest = latest_by_engine(f)
|
||||
for e, r in latest.items():
|
||||
|
|
@ -89,11 +98,14 @@ def main() -> int:
|
|||
try:
|
||||
o = out / "img" / f"{f.id}-o.png"
|
||||
if not o.exists(): # the original never changes; cutting it from the PDF is the slow part
|
||||
save_png(original_png(source.original_bytes(_ref_for(source, f))), o)
|
||||
save_png(original_png(original_bytes(source, ws, f)), o)
|
||||
orig = f"<img loading=lazy src='img/{f.id}-o.png'>"
|
||||
except Exception: # noqa: BLE001
|
||||
orig = "<i>Original nicht lesbar</i>"
|
||||
if tex and rr.ok:
|
||||
if tex and reuse[n - 1] is not None:
|
||||
save_png(original_png(reuse[n - 1].read_bytes()), out / "img" / f"{f.id}-l.png")
|
||||
latex = f"<img loading=lazy src='img/{f.id}-l.png'>"
|
||||
elif tex and rr is not None and rr.ok:
|
||||
g = (rr.rasterize(1.6) * 255).astype(np.uint8)
|
||||
save_png(Image.fromarray(255 - g), out / "img" / f"{f.id}-l.png")
|
||||
latex = f"<img loading=lazy src='img/{f.id}-l.png'>"
|
||||
|
|
|
|||
Loading…
Reference in a new issue