From 6373db73a45d26163369fc381f777814ff7592f0 Mon Sep 17 00:00:00 2001 From: dschlueter Date: Sat, 10 Oct 2026 03:31:57 +0200 Subject: [PATCH] =?UTF-8?q?feat:=20Formeln=20im=20Flie=C3=9Ftext=20(pdf2md?= =?UTF-8?q?=20--formulas),=20etwa=20dreimal=20schneller?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - pdf2md-Formelliste lesen: jede Formel mit Seite/Rechteck/PDF-Text, 300-dpi-Bild - eingebettete Formeln: Satzzeichen am Ende abgetrennt, als Text dahinter (final_suffix) - Normalform: \underset-Grenzwerte, Gleichungsnummer als \tag, \textbf, \cfrac, wörtliches \n - Tempo: Verweis- und Ausschnitt-Cache, pix2text gebündelt nach Pixelbudget, Lesen und Prüfen überlappend (forkserver), Qwen 8 Plätze, Render-Vorfilter, kein neuer Scan, je Paket nur dessen Datensätze, Zeiten je Stufe - Bildprüfung: max. 1600 px Breite, 2,5 MP, Proportionen-Abbruch - gemessen und abgeschaltet: PDF-Textebene als Bestätigung (pdftext=0) - Corral 10 %: 79 % automatisch, Pilot 869 Formeln 70 → ~24 min Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 31 +++ formula_img2latex/cli.py | 4 +- formula_img2latex/compare.py | 5 + formula_img2latex/consensus.py | 30 ++- formula_img2latex/decide.py | 4 +- formula_img2latex/engines/p2t.py | 63 +++++- formula_img2latex/pipeline.py | 287 ++++++++++++++++++++-------- formula_img2latex/review/server.py | 6 +- formula_img2latex/sources/pdf2md.py | 35 +++- formula_img2latex/store.py | 1 + formula_img2latex/tex.py | 46 +++++ formula_img2latex/verify.py | 18 +- tests/test_pipeline.py | 31 ++- tools/buch_bericht.py | 24 ++- tools/vergleich.py | 4 +- 15 files changed, 478 insertions(+), 111 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 190863f..5b87e85 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,36 @@ # Changelog +## Unveröffentlicht + +Alle Formeln eines PDF-Buchs, auch im Fließtext, und etwa dreimal schneller. + +- **pdf2md --formulas** (Nachbarprojekt): Die Formelerkennung von pix2text (MFD) findet jede Formel auf der + Seite, eingebettet oder abgesetzt, auch in Merkkästen. Sie wird zeichengenau aus dem Text geschnitten + (300 dpi) und mit Seite, Rechteck und PDF-Text in `formulas-*.json` vermerkt. "Formel"-Boxen mit Text + werden zu Text, solche mit Kurven zu Grafiken. formula_img2latex liest die Liste + (`sources/pdf2md.listed`). Corral, 10 % der 9239 Formeln: **79 % automatisch** (im Text 83 %). +- **Eingebettete Formeln:** Ein Satzzeichen am Ende wird vor dem Vergleich abgetrennt und als Text hinter + die Formel gesetzt (`final_suffix`). +- **Normalform:** `\underset{…}{\lim}` = `\lim_{…}`, Gleichungsnummer am Ende = `\tag`, `\textbf` als + Fettdruck, `\cfrac` = `\frac` (pix2text schreibt `\dfrac`), wörtliches `\n` aus Modellantworten entfällt. +- **Tempo** (Pilot 869 Formeln: 70 → ~24 min): + - Markdown-Verweise und Ausschnitte nur einmal lesen (`crops/`); + - pix2text gebündelt nach Pixelbudget, auf der freiesten GPU, einmal geladen; + - Lesen und Prüfen überlappen, die Prüfprozesse starten per `forkserver`; + - Qwen mit 8 parallelen Anfragen (`llama-server --parallel 8 -c 32768`); + - pdflatex beim Entscheiden nur, wenn gleiche Bilder möglich sind; + - kein neuer Scan beim Fortsetzen; + - je Paket nur dessen Datensätze laden; + - Zeiten je Stufe in der Statuszeile. +- **Bildprüfung:** Originale höchstens 1600 px breit, über 2,5 MP nicht, Lesarten mit mehr als 2,5-fach + falschen Proportionen sofort abgelehnt. Vorher hielten einzelne Formeln den Lauf über 10 Minuten auf. +- **Gemessen und verworfen:** + - PDF-Textebene als Bestätigung: 12 von 319 widersprachen Qwen, fehlende Punkte und Indizes; Schalter + `pdftext` bleibt aus. + - pix2texts eigene Sicherheit: Fehler auch bei 1,0. + - Qwen3-VL-8B: nicht schneller als das MoE-30B-Modell (3B aktiv), etwas schlechter. +- **Offen:** Formeln in Tabellen. + ## 0.4.0 – 2026-10-09 Bessere Normalform: Lesarten, die sich nur in der Schreibweise unterscheiden, gelten als gleich. diff --git a/formula_img2latex/cli.py b/formula_img2latex/cli.py index d22ee95..ddfb14b 100644 --- a/formula_img2latex/cli.py +++ b/formula_img2latex/cli.py @@ -185,11 +185,11 @@ def _dispatch(args: argparse.Namespace) -> int: source, ws = pipeline.open_all(args.source, args.workspace, create=create, pdf=getattr(args, "pdf", None)) q = args.quiet if args.command == "scan": - pipeline.scan(source, ws, quiet=q) + pipeline.scan(source, ws, quiet=q, force=True) # explicit scan: always re-read elif args.command == "recognize": pipeline.recognize(source, ws, args.engine, args.model, args.url, args.only_unresolved, args.include_figures, args.limit, - args.workers or {"local": 4, "p2t": 1}.get(args.engine, 8), q) + args.workers or {"p2t": 1}.get(args.engine, 8), q) elif args.command == "verify": pipeline.verify_all(source, ws, None if args.preset == "auto" else args.preset, args.include_figures, args.workers, args.redo, q) diff --git a/formula_img2latex/compare.py b/formula_img2latex/compare.py index 1bdebae..e22d303 100644 --- a/formula_img2latex/compare.py +++ b/formula_img2latex/compare.py @@ -31,6 +31,7 @@ from .imageio import trim from .render import Rendered BIN_THRESHOLD = 0.4 +MAX_ASPECT_FACTOR = 2.5 # accepted readings: 0.6 to 1.54 (Corral); beyond this no comparison at all MAX_ASPECT_DEVIATION = 0.25 TARGET_STROKE = 2.0 # strokes thicker than this (px) are scaled down before comparing OVERSAMPLE = 4 @@ -220,6 +221,10 @@ def compare(orig_ink: np.ndarray, rendered: Rendered, tolerance: float | None = sy, sx = h / hi.shape[0], w / hi.shape[1] aspect = sx / sy metrics: dict[str, float] = {"aspect": aspect, "stroke": sw, "norm": norm} + if not 1 / MAX_ASPECT_FACTOR <= aspect <= MAX_ASPECT_FACTOR: + # another layout altogether (a three-line derivation read as one line): it cannot pass, and its + # canvas would be huge (one comparison took over 10 minutes) + return Comparison(False, f"Proportionen weichen stark ab (Faktor {max(aspect, 1 / aspect):.1f})", metrics) if abs(aspect - 1.0) > MAX_ASPECT_DEVIATION: # Different layout (e.g. another typesetting engine); still compare at the height scale so a # human gets an overlay, but the metrics will keep it out of automatic acceptance. diff --git a/formula_img2latex/consensus.py b/formula_img2latex/consensus.py index 9b0cf81..2333796 100644 --- a/formula_img2latex/consensus.py +++ b/formula_img2latex/consensus.py @@ -15,6 +15,7 @@ from __future__ import annotations import os import re +from collections import Counter import numpy as np @@ -22,7 +23,7 @@ from .compare import _register from .confusables import tokenize from .imageio import trim from .render import render_many -from .tex import fix_literal_newlines, prepare_for_render, split_tag, strip_delimiters +from .tex import fix_literal_newlines, prepare_for_render, split_tag, strip_delimiters, trailing_number_to_tag # tokens that carry no symbol _DROP = {"\\,", "\\;", "\\:", "\\!", "\\ ", "~", "\\quad", "\\qquad", "\\left", "\\right", "\\big", "\\Big", "\\bigg", @@ -35,7 +36,7 @@ _SYNONYM = { "\\overrightarrow": "\\vec", "\\overline": "\\bar", "\\widehat": "\\hat", "\\widetilde": "\\tilde", "\\rightarrow": "\\to", "\\longrightarrow": "\\to", "\\dfrac": "\\frac", "\\tfrac": "\\frac", "\\cfrac": "\\frac", "\\leq": "\\le", "\\geq": "\\ge", "\\neq": "\\ne", "\\ldots": "\\dots", "\\prime": "'", - "\\mathbf": "\\boldsymbol", "\\bm": "\\boldsymbol", "\\lbrace": "\\{", "\\rbrace": "\\}", "\\vert": "|", + "\\mathbf": "\\boldsymbol", "\\bm": "\\boldsymbol", "\\textbf": "\\boldsymbol", "\\lbrace": "\\{", "\\rbrace": "\\}", "\\vert": "|", "\\lvert": "|", "\\rvert": "|", "\\Vert": "\\|", "\\lVert": "\\|", "\\rVert": "\\|", "\\gets": "\\leftarrow", "\\land": "\\wedge", "\\lor": "\\vee", "\\lnot": "\\neg", "\\leqslant": "\\le", "\\geqslant": "\\ge", "\\varnothing": "\\emptyset", "\\colon": ":", "\\cdotp": "\\cdot", "\\ast": "*", @@ -57,6 +58,10 @@ _OPERATORS = {"sin", "cos", "tan", "cot", "sec", "csc", "arcsin", "arccos", "arc # upright text groups; one level of nested braces (\mathrm{f \ddot{u} r}) _TEXT_GROUP = re.compile(r"\\(operatorname\*?|mathrm|text|textrm|mbox|textnormal|textup)\s*\{((?:[^{}]|\{[^{}]*\})*)\}") _BOLD = {"\\boldsymbol", "\\mathbf", "\\bm", "\\pmb"} +# a limit written below/above an operator with \underset/\overset is the same as its sub-/superscript in +# display style (pix2text: \underset{x\to0}{\operatorname*{lim}}); for other symbols the position differs +_UNDERSET = re.compile(r"\\(under|over)set\s*\{((?:[^{}]|\{(?:[^{}]|\{[^{}]*\})*\})*)\}" # 2 brace levels + r"\s*\{\s*(\\(?:lim|liminf|limsup|sum|prod|coprod|max|min|sup|inf|bigcup|bigcap|det|gcd|Pr))\s*\}") # number of arguments of a command (after synonyms); every other command takes none, so a group after # \Delta or after another group is free-standing (\Delta{x_{n}} = \Delta x_n, t^{2}\,{\sqrt{2}} = t^2\sqrt{2}) _ARITY = {c: 2 for c in ("\\frac", "\\cfrac", "\\binom", "\\dbinom", "\\tbinom", "\\overset", "\\underset", @@ -78,9 +83,10 @@ def canonical(tex: str, bold: bool = False) -> str: *bold*: whether bold vs. normal counts (False: \\boldsymbol{v} and v are the same reading). """ - body, tag = split_tag(strip_delimiters(fix_literal_newlines(tex))) + body, tag = split_tag(trailing_number_to_tag(strip_delimiters(fix_literal_newlines(tex)))) body = prepare_for_render(body) body = _TEXT_GROUP.sub(_text_group, body) + body = _UNDERSET.sub(lambda m: f"{m.group(3)}{'_' if m.group(1) == 'under' else '^'}{{{m.group(2)}}}", body) body = re.sub(r'\{?\\"\{?([aouAOU])\}?\}?', lambda m: _UMLAUT[m.group(1)], body) # f{\"u}r -> für body = re.sub(r"\^\s*\{\s*((?:\\prime\s*)+)\}", lambda m: "'" * m.group(1).count("\\prime"), body) # f^{\prime} body = re.sub(r"\^\s*\\prime(?![A-Za-z])", "'", body) @@ -215,6 +221,19 @@ def _flatten_groups(toks: list[str]) -> list[str]: return _emit(_parse(toks)[0]) +def _letters(canon: str) -> Counter: + """Latin letters and digits a canonical form prints (operator names spelled out, \\sin -> s i n). Two + readings whose counts differ cannot render to the same picture.""" + out: Counter = Counter() + for tok in canon.split("#tag(")[0].split(" "): + if tok.startswith("\\"): + if tok[1:] in _OPERATORS: + out.update(tok[1:]) + elif len(tok) == 1 and tok.isascii() and tok.isalnum(): + out[tok] += 1 + return out + + def render_equal(a: str, b: str, display: bool, preset: str = "cm", zoom: float = 4.0) -> bool: """True if both readings render to the same picture (render vs. render).""" ra, rb = render_many([(a, display), (b, display)], preset) @@ -241,7 +260,8 @@ def same_reading(a: str, b: str, display: bool, preset: str = "cm", bold: bool = """ if not a or not b: return None - if canonical(a, bold) == canonical(b, bold): + ca, cb = canonical(a, bold), canonical(b, bold) + if ca == cb: return "normalform" if structural is None: structural = os.environ.get("FORMULA_STRUCTURAL", "0") == "1" @@ -249,6 +269,8 @@ def same_reading(a: str, b: str, display: bool, preset: str = "cm", bold: bool = from .structure import structural_equal if structural_equal(a, b, bold): return "strukturell" + if _letters(ca) != _letters(cb): # different letters or digits: pdflatex need not even try + return None if render_equal(strip_delimiters(a), strip_delimiters(b), display, preset): return "render" return None diff --git a/formula_img2latex/decide.py b/formula_img2latex/decide.py index 078e4ae..c9931c1 100644 --- a/formula_img2latex/decide.py +++ b/formula_img2latex/decide.py @@ -77,6 +77,7 @@ CONSENSUS_DEFAULTS: dict[str, float] = { "require_third": 1, # a third reading (cloud, other model family) must agree as well "arbiter": 0, # 1: Qwen = Gemma against pix2text is accepted if the image clearly shows their "arbiter_margin": 5.0, # reading: evidence (compare.contrast) at least this large (rule p2t) + "pdftext": 0, # 1: the PDF text layer of a flat formula may confirm pix2text instead of Qwen "structural": 0, # 1: readings with the same unevaluated sympy tree agree (structure.py). Off: # after the multi-line normal form it found nothing more (Göllmann, 0.4.0) } @@ -200,7 +201,8 @@ def decide_pair_tiebreak(readings: list[dict[str, Any] | None], agree: list[tupl return "not_formula", "Qwen" + (" und Gemma" if g else "") + ": keine Formel", None first = 1 if p.get("tex") else None # reading shown for review: pix2text's if frozenset((0, 1)) in pairs: - how, who, confirm = pairs[frozenset((0, 1))], "pix2text und Qwen einig", q + who = "pix2text und PDF-Text einig" if q.get("engine") == "pdftext" else "pix2text und Qwen einig" + how, confirm = pairs[frozenset((0, 1))], q elif g is None: return "needs_review", f"pix2text und Qwen uneinig, {TIEBREAK_MISSING}", first elif frozenset((1, 2)) in pairs: diff --git a/formula_img2latex/engines/p2t.py b/formula_img2latex/engines/p2t.py index 7ed8e70..fbf2d11 100644 --- a/formula_img2latex/engines/p2t.py +++ b/formula_img2latex/engines/p2t.py @@ -15,6 +15,7 @@ import time from PIL import Image from ..errors import EXIT_TOOL, FormulaError +from ..tex import trailing_number_to_tag from .vlm import Recognition, prepare_image DEFAULT_MODEL = "mfr-1.5" @@ -38,15 +39,22 @@ def tidy_p2t(tex: str) -> str: tex = _BOLDMATH.sub(lambda m: "\\boldsymbol{" + re.sub(r"\s+", "", m.group(1)) + "}", tex) tex = _BF.sub(lambda m: "\\mathbf{" + re.sub(r"\s+", "", m.group(2)) + "}", tex) tex = re.sub(r"\\cfrac(?![A-Za-z])", r"\\dfrac", tex) - return post_process(tex).strip() + return trailing_number_to_tag(post_process(tex).strip()) def resolve_device(device: str) -> str: - """Device "auto": the first visible GPU if onnxruntime has CUDA, else the CPU.""" + """Device "auto": the visible GPU with the most free memory (not the one the VLM server fills), else the CPU.""" if device != "auto": return device import onnxruntime - return "cuda" if "CUDAExecutionProvider" in onnxruntime.get_available_providers() else "cpu" + if "CUDAExecutionProvider" not in onnxruntime.get_available_providers(): + return "cpu" + try: + import torch + free = [torch.cuda.mem_get_info(i)[0] for i in range(torch.cuda.device_count())] + except Exception: # no torch or no driver access: the first GPU + return "cuda" + return f"cuda:{max(range(len(free)), key=free.__getitem__)}" if free else "cuda" def ort_provider(device: str) -> tuple[str, dict]: @@ -61,6 +69,10 @@ def ort_provider(device: str) -> tuple[str, dict]: raise ValueError(f"unbekanntes Gerät {device!r} (cpu, cuda, cuda:N)") +BATCH_MAX = 32 # images per bundle at most +BATCH_PIXELS = 4_000_000 # largest image x count per bundle (the GPU is shared with the VLM servers) + + class Pix2TextEngine: def __init__(self, model: str = DEFAULT_MODEL, device: str = "auto") -> None: self.name = "p2t" @@ -115,3 +127,48 @@ class Pix2TextEngine: tex = tidy_p2t(raw) return Recognition(self.name, self.model, tex or None, None, raw[:2000], None, time.time() - t0, p_mean=score) + + def recognize_many(self, items: list[tuple[bytes, str]], batch: int = BATCH_MAX) -> list[Recognition]: + """Read many formula images (data, name), one model call per bundle: on the GPU many times faster than + one call per image. Bundles are cut by a pixel budget (largest image x count; a bundle is padded to its + largest image), so big images go alone and the GPU, shared with the VLM servers, does not run out of + memory. Falls back to single images if a bundle fails.""" + try: + self.check() + except Exception as e: + return [Recognition(self.name, self.model, None, None, "", str(e)[:300], 0.0) for _ in items] + out: list[Recognition | None] = [None] * len(items) + ready = [] + for i, (data, name) in enumerate(items): + try: + ready.append((i, Image.open(io.BytesIO(prepare_image(data, name))).convert("RGB"))) + except Exception as e: # unreadable image: an error reading, the others go on + out[i] = Recognition(self.name, self.model, None, None, "", str(e)[:300], 0.0) + ready.sort(key=lambda t: t[1].size[0] * t[1].size[1]) + bundles: list[list] = [] + cur: list = [] + width = height = 0 + for i, img in ready: + w, h = max(width, img.size[0]), max(height, img.size[1]) + if cur and (len(cur) >= batch or w * h * (len(cur) + 1) > BATCH_PIXELS): + bundles.append(cur) + cur, w, h = [], img.size[0], img.size[1] + cur.append((i, img)) + width, height = w, h + if cur: + bundles.append(cur) + for bundle in bundles: + t0 = time.time() + try: + with self._lock: + results = self._ocr._one_batch([img for _, img in bundle], None) + except Exception: # a model error (e.g. GPU memory): read this bundle one by one + for i, _ in bundle: + out[i] = self.recognize(*items[i]) + continue + each = (time.time() - t0) / len(bundle) + for (i, _), res in zip(bundle, results): + raw = self._ocr.post_process(res["text"]).strip() + out[i] = Recognition(self.name, self.model, tidy_p2t(raw) or None, None, raw[:2000], None, each, + p_mean=float(res.get("score") or 0.0) or None) + return [r for r in out if r is not None] diff --git a/formula_img2latex/pipeline.py b/formula_img2latex/pipeline.py index 1bb9691..96b09c5 100644 --- a/formula_img2latex/pipeline.py +++ b/formula_img2latex/pipeline.py @@ -3,12 +3,15 @@ from __future__ import annotations import hashlib +import json +import multiprocessing import io import os import re import shutil import statistics import threading +import time import sys from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor, as_completed from pathlib import Path @@ -30,10 +33,11 @@ from .render import PRESETS from .sources import open_source from .sources.base import ImageRef, Source from .store import HUMAN_STATUSES, REPLACE_STATUSES, Formula, Workspace, now_iso, sha256_bytes -from .tex import normalize_for_compare, split_tag, strip_delimiters +from .tex import normalize_for_compare, split_tag, strip_delimiters, text_to_tex from .verify import verify PRESET_SAMPLE = 12 +RECOGNIZE_BATCH = 32 # images per pix2text call _SAVE_LOCK = threading.Lock() @@ -44,7 +48,21 @@ def log(msg: str, quiet: bool = False) -> None: # --------------------------------------------------------------------------------------- scan -def scan(source: Source, ws: Workspace, quiet: bool = False) -> dict[str, int]: +def _markdown_fingerprint(source: Source) -> str: + """Changes whenever a Markdown file of the source changes (size, time) or the source description does.""" + parts = [json.dumps(source.describe(), sort_keys=True)] + for md in source.markdown_files(): + st = md.stat() + parts.append(f"{md}:{st.st_size}:{st.st_mtime_ns}") + return hashlib.sha256("\n".join(parts).encode()).hexdigest()[:16] + + +def scan(source: Source, ws: Workspace, quiet: bool = False, force: bool = False) -> dict[str, int]: + fingerprint = _markdown_fingerprint(source) + last = ws.meta.get("last_scan") or {} + if not force and last.get("fingerprint") == fingerprint: # resuming: nothing changed, skip the slow part + log(f"scan: unverändert seit {last.get('at', '?')}, {last.get('formulas', '?')} Bilder", quiet) + return {k: last.get(k, 0) for k in ("formula", "maybe", "figure")} refs = source.image_refs() groups: dict[str, tuple[Formula, list[ImageRef]]] = {} for ref in refs: @@ -67,19 +85,41 @@ def scan(source: Source, ws: Workspace, quiet: bool = False) -> dict[str, int]: f.alt = ref.alt f.occurrences = [{"file": r.file.relative_to(source.root).as_posix(), "line": r.line, "md": r.md, "context": r.context} for r in rs] - data = source.original_bytes(ref) + _add_pdf_text_reading(source, f, ref) + data = original_bytes(source, ws, f, ref) f.width, f.height = _image_size(data) f.classify = classify(data, f.asset, f.known_formula, inline=not f.display) counts[f.classify["label"]] += 1 ws.save(f) ws.meta["source"] = source.describe() - ws.meta["last_scan"] = {"at": now_iso(), "images": len(refs), "formulas": len(groups), **counts} + ws.meta["last_scan"] = {"at": now_iso(), "images": len(refs), "formulas": len(groups), **counts, + "fingerprint": fingerprint} ws.save_meta() log(f"scan: {len(refs)} Bildverweise, {len(groups)} verschiedene Bilder: {counts['formula']} Formeln, " f"{counts['maybe']} unklar, {counts['figure']} Abbildungen", quiet) return counts +def _add_pdf_text_reading(source: Source, f: Formula, ref: ImageRef) -> None: + """`pdf2md --formulas` lists the PDF text of each formula: a "flat" one (one baseline, one size: no lost + super-/subscript, fraction or limit) becomes the reading "pdftext", which may confirm pix2text instead of Qwen.""" + entry = source.listed(ref) if hasattr(source, "listed") else None + if not entry or not entry.get("flat") or any(c.get("engine") == "pdftext" for c in f.candidates): + return + tex = text_to_tex(entry.get("text") or "") + if tex: + f.candidates.append({"engine": "pdftext", "model": "PDF-Textebene", "tex": tex, "is_formula": True, + "error": None, "seconds": 0.0, "created_at": now_iso()}) + + +def pdftext_confirms(f: Formula, bold: bool = False) -> bool: + """The PDF text layer reads the same formula as pix2text (same canonical form).""" + from .consensus import canonical + latest = latest_by_engine(f) + p, t = latest.get("p2t"), latest.get("pdftext") + return bool(p and t and p.get("tex") and canonical(p["tex"], bold) == canonical(t["tex"], bold)) + + def _image_size(data: bytes) -> tuple[int, int]: try: with Image.open(io.BytesIO(data)) as im: @@ -95,17 +135,45 @@ def _wanted(f: Formula, include_figures: bool) -> bool: return include_figures or label != "figure" +def _records(ws: Workspace, only_ids: set[str] | None) -> list[Formula]: + """The records of *only_ids* (one part of a run), or all: a big book has 10 000 of them, loading all of them + for every part of 50 cost seconds each time.""" + if only_ids is None: + return ws.all() + return [f for f in (ws.load(i) for i in sorted(only_ids)) if f is not None] + + +_REFS: dict[tuple[str, int], tuple[dict, dict]] = {} # (file, mtime) -> ({(line, md): ref}, {md: ref}) + + def _ref_for(source: Source, f: Formula) -> ImageRef: + """The image reference of a formula's first occurrence (each Markdown file is parsed once while unchanged).""" occ = f.occurrences[0] md = source.root / occ["file"] - from .sources.base import scan_markdown - for r in scan_markdown(md): - if r.line == occ["line"] and r.md == occ["md"]: - return r - for r in scan_markdown(md): - if r.md == occ["md"]: - return r - raise FileNotFoundError(f"Bildverweis nicht mehr gefunden: {occ['file']}:{occ['line']}") + key = (str(md), md.stat().st_mtime_ns) + if key not in _REFS: + from .sources.base import scan_markdown + refs = scan_markdown(md) + _REFS[key] = ({(r.line, r.md): r for r in refs}, {r.md: r for r in reversed(refs)}) + by_line, by_md = _REFS[key] + ref = by_line.get((occ["line"], occ["md"])) or by_md.get(occ["md"]) + if ref is None: + raise FileNotFoundError(f"Bildverweis nicht mehr gefunden: {occ['file']}:{occ['line']}") + return ref + + +def original_bytes(source: Source, ws: Workspace, f: Formula, ref: ImageRef | None = None) -> bytes: + """The original image of a formula. Crops from a PDF are expensive (600 dpi, or template matching): + they are cut once and kept in /crops/.""" + if getattr(source, "pdf", None) is None: + return source.original_bytes(ref or _ref_for(source, f)) + path = ws.root / "crops" / f"{f.id}.png" + if path.exists(): + return path.read_bytes() + data = source.original_bytes(ref or _ref_for(source, f)) + path.parent.mkdir(exist_ok=True) + path.write_bytes(data) + return data # --------------------------------------------------------------------------------- recognize @@ -120,15 +188,26 @@ def needs_tiebreak(f: Formula) -> bool: return f.status == "needs_review" and TIEBREAK_MISSING in (f.reason or "") +_ENGINES: dict[tuple, Any] = {} + + +def _engine(name: str, model: str | None, url: str | None) -> Any: + """One engine per name/model/url for the whole run: pix2text would otherwise load its model for every part.""" + key = (name, model, url) + if key not in _ENGINES: + _ENGINES[key] = make_engine(name, model, url) + return _ENGINES[key] + + def recognize(source: Source, ws: Workspace, engine_name: str, model: str | None = None, url: str | None = None, only_unresolved: bool = False, include_figures: bool = False, limit: int = 0, - workers: int = 4, quiet: bool = False, only_ids: set[str] | None = None, + workers: int = 8, quiet: bool = False, only_ids: set[str] | None = None, only_needing_third: bool = False, only_needing_tiebreak: bool = False) -> dict[str, int]: - engine = make_engine(engine_name, model, url) + engine = _engine(engine_name, model, url) engine.check() todo = [] - for f in ws.all(): - if not _wanted(f, include_figures) or (only_ids is not None and f.id not in only_ids): + for f in _records(ws, only_ids): + if not _wanted(f, include_figures): continue if only_unresolved and f.status in ("auto_accepted", "not_formula"): continue @@ -146,33 +225,42 @@ def recognize(source: Source, ws: Workspace, engine_name: str, model: str | None stats = {"asked": 0, "latex": 0, "not_formula": 0, "error": 0, "tokens_in": 0, "tokens_out": 0} def job(f: Formula) -> tuple[Formula, Any]: - ref = _ref_for(source, f) ctx = f.occurrences[0].get("context", "") - return f, engine.recognize(source.original_bytes(ref), f.asset, ctx, f.display) + return f, engine.recognize(original_bytes(source, ws, f), f.asset, ctx, f.display) - with ThreadPoolExecutor(max_workers=max(1, workers)) as pool: - futures = [pool.submit(job, f) for f in todo] - for i, fut in enumerate(as_completed(futures), 1): - f, rec = fut.result() - stats["asked"] += 1 - stats["tokens_in"] += rec.tokens_in - stats["tokens_out"] += rec.tokens_out - cand = {"engine": engine.name, "model": engine.model, "tex": rec.tex, "is_formula": rec.is_formula, - "p_min": rec.p_min, "p_mean": rec.p_mean, "self_conf": rec.self_conf, "prompt": PROMPT_VERSION, - "thinking": engine.thinking, - "error": rec.error, "seconds": round(rec.seconds, 2), "created_at": now_iso()} - with _SAVE_LOCK: # another engine may run in parallel: append to the current record - f = ws.load(f.id) or f - f.candidates.append(cand) - ws.save(f) - if rec.error: - stats["error"] += 1 - elif rec.tex: - stats["latex"] += 1 - else: - stats["not_formula"] += 1 - if not quiet and (i % 25 == 0 or i == len(todo)): - log(f" {i}/{len(todo)}") + def store(i: int, f: Formula, rec: Any) -> None: + stats["asked"] += 1 + stats["tokens_in"] += rec.tokens_in + stats["tokens_out"] += rec.tokens_out + cand = {"engine": engine.name, "model": engine.model, "tex": rec.tex, "is_formula": rec.is_formula, + "p_min": rec.p_min, "p_mean": rec.p_mean, "self_conf": rec.self_conf, "prompt": PROMPT_VERSION, + "thinking": engine.thinking, + "error": rec.error, "seconds": round(rec.seconds, 2), "created_at": now_iso()} + with _SAVE_LOCK: # another engine may run in parallel: append to the current record + f = ws.load(f.id) or f + f.candidates.append(cand) + ws.save(f) + if rec.error: + stats["error"] += 1 + elif rec.tex: + stats["latex"] += 1 + else: + stats["not_formula"] += 1 + if not quiet and (i % 25 == 0 or i == len(todo)): + log(f" {i}/{len(todo)}") + + if hasattr(engine, "recognize_many"): # pix2text: batches on the GPU instead of one call per image + i = 0 + for start in range(0, len(todo), RECOGNIZE_BATCH): + part = todo[start:start + RECOGNIZE_BATCH] + for f, rec in zip(part, engine.recognize_many([(original_bytes(source, ws, f), f.asset) for f in part])): + i += 1 + store(i, f, rec) + else: + with ThreadPoolExecutor(max_workers=max(1, workers)) as pool: + futures = [pool.submit(job, f) for f in todo] + for i, fut in enumerate(as_completed(futures), 1): + store(i, *fut.result()) log(f"recognize: {stats['latex']} LaTeX-Kandidaten, {stats['not_formula']} „keine Formel“, " f"{stats['error']} Fehler; Tokens {stats['tokens_in']} ein / {stats['tokens_out']} aus", quiet) return stats @@ -237,7 +325,7 @@ def choose_preset(source: Source, ws: Workspace, formulas: list[Formula], quiet: sample = [f for f in formulas if _distinct_candidates(f)][:PRESET_SAMPLE] if not sample: return "cm" - inks = [load_ink(source.original_bytes(_ref_for(source, f)), f.asset) for f in sample] + inks = [load_ink(original_bytes(source, ws, f), f.asset) for f in sample] scores: dict[str, float] = {} for preset in PRESETS: rs = render_many([(_distinct_candidates(f)[0], f.display) for f in sample], preset) @@ -252,8 +340,7 @@ def choose_preset(source: Source, ws: Workspace, formulas: list[Formula], quiet: def verify_all(source: Source, ws: Workspace, preset: str | None = None, include_figures: bool = False, workers: int = 0, redo: bool = False, quiet: bool = False, progress: Callable[[int, int], None] | None = None, only_ids: set[str] | None = None) -> dict[str, int]: - formulas = [f for f in ws.all() if _wanted(f, include_figures) and _distinct_candidates(f) - and (only_ids is None or f.id in only_ids)] + formulas = [f for f in _records(ws, only_ids) if _wanted(f, include_figures) and _distinct_candidates(f)] mode = decide_mode(ws) if mode == "consensus" and consensus_rule(ws) == "vlm3": mode = "consensus-vlm3" @@ -266,15 +353,16 @@ def verify_all(source: Source, ws: Workspace, preset: str | None = None, include ws.save_meta() jobs = [] for f in formulas: - ref = _ref_for(source, f) - jobs.append((f.id, source.original_bytes(ref), f.asset, _verify_texes(f, mode), f.display, preset, + jobs.append((f.id, original_bytes(source, ws, f), f.asset, _verify_texes(f, mode), f.display, preset, mode == "pixel")) log(f"verify: {len(jobs)} Formeln, Schrift {preset}", quiet) by_id = {f.id: f for f in formulas} ws.review_dir.mkdir(parents=True, exist_ok=True) workers = workers or max(1, (os.cpu_count() or 2) - 2) done = 0 - with ProcessPoolExecutor(max_workers=workers) as pool: + # forkserver, not fork: the reading thread (run_consensus) may be inside pix2text/CUDA at that moment, and a + # forked copy of a process mid-way in another thread inherits half-held locks (workers died, the run hung) + with ProcessPoolExecutor(max_workers=workers, mp_context=multiprocessing.get_context("forkserver")) as pool: for fut in as_completed([pool.submit(_verify_job, j) for j in jobs]): fid, results, pngs = fut.result() f = by_id[fid] @@ -385,6 +473,14 @@ def decide_all(ws: Workspace, quiet: bool = False, preset: str | None = None, return counts +def _split_final_punct(tex: str) -> tuple[str, str]: + """(formula without a final period, comma or semicolon, that punctuation or "").""" + m = re.search(r"(?:\\[,;!]|\\quad|\\qquad|\s)*([.,;])\s*$", tex) + if not m or not tex[:m.start()].strip(): + return tex, "" + return tex[:m.start()].rstrip(), m.group(1) + + def _strip_final_punct(tex: str) -> str: return re.sub(r"(\\[,;!]|\\quad|\\qquad|\s)*[.,;]\s*$", "", strip_delimiters(tex or "")).strip() @@ -399,14 +495,15 @@ def decide_all_consensus(ws: Workspace, quiet: bool, preset: str, only_ids: set[ th = consensus_thresholds(ws.settings.get("consensus_thresholds")) rule = consensus_rule(ws) decide_fn = decide_pair_tiebreak if rule == "p2t" else decide_consensus - formulas = [f for f in ws.all() if f.candidates and f.status not in HUMAN_STATUSES - and (only_ids is None or f.id in only_ids)] + formulas = [f for f in _records(ws, only_ids) if f.candidates and f.status not in HUMAN_STATUSES] plan = [] for f in formulas: bold = bool(th.get("bold_relevant")) if rule == "p2t": latest = latest_by_engine(f) rs = [latest.get("local"), latest.get("p2t"), latest.get("local2")] + if th.get("pdftext") and pdftext_confirms(f, bold): # the PDF text layer confirms pix2text + rs[0] = latest["pdftext"] pairs = [(0, 1), (1, 2), (0, 2)] else: rs = readings_of(f) @@ -416,9 +513,10 @@ def decide_all_consensus(ws: Workspace, quiet: bool, preset: str, only_ids: set[ if rs[i] is None or rs[j] is None: continue a, b = rs[i].get("tex") or "", rs[j].get("tex") or "" - # rule vlm3: a confirming reading (j >= 2) may lack the final period or comma. Rule p2t: strict, - # pix2text's reading is the one used, so its final punctuation must be confirmed too - if rule != "p2t" and j >= 2: + # rule vlm3: a confirming reading (j >= 2) may lack the final period or comma. Rule p2t: strict for + # displayed formulas; an inline one is cut out of a sentence, its final comma or period belongs to the + # sentence (the readers see it or not): compared without, written after the formula (final_suffix) + if (rule != "p2t" and j >= 2) or (rule == "p2t" and not f.display): a, b = _strip_final_punct(a), _strip_final_punct(b) how = same_reading(a, b, f.display, preset, bold, bool(th.get("structural"))) if how: @@ -445,6 +543,9 @@ def decide_all_consensus(ws: Workspace, quiet: bool, preset: str, only_ids: set[ status, reason, use = decide_fn(rs, agree, veto, pok, book_ppt, th) f.status, f.reason = status, reason f.final_tex = _with_tag(f, tex) if status == "auto_accepted" and tex else None + f.final_suffix = "" + if f.final_tex and rule == "p2t" and not f.display: + f.final_tex, f.final_suffix = _split_final_punct(f.final_tex) f.decided_at = now_iso() ws.save(f) counts[status] = counts.get(status, 0) + 1 @@ -476,7 +577,7 @@ def apply(source: Source, ws: Workspace, dry_run: bool = False, quiet: bool = Fa by_occ: dict[tuple[str, int, str], str] = {} for f in ws.all(): if f.status in REPLACE_STATUSES and f.final_tex: - math = source.format_math(f.final_tex, f.display) + math = source.format_math(f.final_tex, f.display) + f.final_suffix for occ in f.occurrences: by_occ[(occ["file"], occ["line"], occ["md"])] = math state = ws.meta.setdefault("applied", {}) @@ -578,38 +679,62 @@ def run_consensus(source: Source, ws: Workspace, chunk: int = 50, preset: str | tiebreak_ok = True totals: dict[str, int] = {} done: list[str] = [] - for start in range(0, len(todo), chunk): - part = set(todo[start:start + chunk]) - if rule == "p2t": # pix2text first (fast), then Qwen - for eng in ("p2t", "local"): - recognize(source, ws, eng, only_ids=part, include_figures=include_figures, quiet=True) + parts = [set(todo[start:start + chunk]) for start in range(0, len(todo), chunk)] + use_pdftext = bool(consensus_thresholds(ws.settings.get("consensus_thresholds")).get("pdftext")) + secs: list[dict[str, float]] = [{} for _ in parts] + + def timed(n: int, label: str, fn: Callable, *args: Any, **kwargs: Any) -> Any: + t0 = time.time() + try: + return fn(*args, **kwargs) + finally: + secs[n][label] = secs[n].get(label, 0.0) + time.time() - t0 + + def read(n: int) -> None: + """GPU stage of part n: runs while the CPU checks part n-1 against the images.""" + if rule == "p2t": # pix2text first (fast), then Qwen – unless the PDF text layer already confirms it + timed(n, "pix2text", recognize, source, ws, "p2t", only_ids=parts[n], include_figures=include_figures, + quiet=True) + ask = parts[n] + if use_pdftext: + ask = {fid for fid in parts[n] if not ((f := ws.load(fid)) and pdftext_confirms(f))} + timed(n, "Qwen", recognize, source, ws, "local", only_ids=ask, include_figures=include_figures, quiet=True) else: # both VLMs run on their own GPU: ask them at the same time with ThreadPoolExecutor(max_workers=2) as both: - for fut in [both.submit(recognize, source, ws, eng, only_ids=part, include_figures=include_figures, + for fut in [both.submit(recognize, source, ws, eng, only_ids=parts[n], include_figures=include_figures, quiet=True) for eng in ("local", "local2")]: fut.result() - verify_all(source, ws, preset, include_figures, quiet=True, only_ids=part) - if rule == "p2t" and tiebreak_ok and any(needs_tiebreak(f) for f in map(ws.load, part) if f): - try: - recognize(source, ws, "local2", only_ids=part, only_needing_tiebreak=True, + + with ThreadPoolExecutor(max_workers=1) as ahead: + reading = ahead.submit(read, 0) if parts else None + for n, part in enumerate(parts): + t_part = time.time() + reading.result() + if n + 1 < len(parts): + reading = ahead.submit(read, n + 1) + timed(n, "Prüfung", verify_all, source, ws, preset, include_figures, quiet=True, only_ids=part) + if rule == "p2t" and tiebreak_ok and any(needs_tiebreak(f) for f in map(ws.load, part) if f): + try: # Gemma runs on the other GPU, alongside Qwen reading the next part + timed(n, "Gemma", recognize, source, ws, "local2", only_ids=part, only_needing_tiebreak=True, include_figures=include_figures, quiet=True) + timed(n, "Prüfung", verify_all, source, ws, preset, include_figures, quiet=True, only_ids=part) + except FormulaError as e: # Gemma not running: the disputed formulas stay in review + tiebreak_ok = False + say(f"Stichwahl (Gemma) nicht möglich, strittige Formeln bleiben im Review: {e}") + if rule != "p2t" and use_cloud: + recognize(source, ws, "cloud", cloud_model, only_ids=part, only_needing_third=True, + include_figures=include_figures, workers=8, quiet=True) verify_all(source, ws, preset, include_figures, quiet=True, only_ids=part) - except FormulaError as e: # Gemma not running: the disputed formulas stay in review - tiebreak_ok = False - say(f"Stichwahl (Gemma) nicht möglich, strittige Formeln bleiben im Review: {e}") - if rule != "p2t" and use_cloud: - recognize(source, ws, "cloud", cloud_model, only_ids=part, only_needing_third=True, - include_figures=include_figures, workers=8, quiet=True) - verify_all(source, ws, preset, include_figures, quiet=True, only_ids=part) - done.extend(sorted(part)) - totals = {} - for fid in done: - f = ws.load(fid) - if f is not None: - totals[f.status] = totals.get(f.status, 0) + 1 - auto = totals.get("auto_accepted", 0) - image = sum(totals.get(k, 0) for k in ("rejected_keep_image", "not_formula", "no_candidate", "render_failed")) - review = totals.get("needs_review", 0) - say(f"{len(done):5d}/{len(todo)} verarbeitet: automatisch ersetzt {auto} ({auto / len(done):.0%}), " - f"Review {review}, bleibt Bild/keine Formel {image}") + done.extend(sorted(part)) + totals = {} + for fid in done: + f = ws.load(fid) + if f is not None: + totals[f.status] = totals.get(f.status, 0) + 1 + auto = totals.get("auto_accepted", 0) + image = sum(totals.get(k, 0) for k in ("rejected_keep_image", "not_formula", "no_candidate", "render_failed")) + review = totals.get("needs_review", 0) + say(f"{len(done):5d}/{len(todo)} verarbeitet: automatisch ersetzt {auto} ({auto / len(done):.0%}), " + f"Review {review}, bleibt Bild/keine Formel {image} [Paket {time.time() - t_part:.0f} s; " + + ", ".join(f"{k} {v:.0f} s" for k, v in secs[n].items()) + "]") return totals diff --git a/formula_img2latex/review/server.py b/formula_img2latex/review/server.py index 9c9c35d..27c3de9 100644 --- a/formula_img2latex/review/server.py +++ b/formula_img2latex/review/server.py @@ -18,7 +18,7 @@ from urllib.parse import unquote, urlparse from ..checks import pandoc_mathml_ok from ..imageio import load_ink -from ..pipeline import _ref_for, comparison_pngs, readings_of +from ..pipeline import comparison_pngs, original_bytes, readings_of from ..sources.base import Source from ..store import Formula, Workspace, now_iso from ..tex import strip_delimiters, unsafe_reason @@ -114,7 +114,7 @@ def serve(source: Source, ws: Workspace, port: int = 8765, show_all: bool = Fals reason = unsafe_reason(tex) if reason: return {"ok": False, "error": reason} - orig = load_ink(source.original_bytes(_ref_for(source, f)), f.asset) + orig = load_ink(original_bytes(source, ws, f), f.asset) v = verify(orig, tex, f.display, preset, max_climb=0) pngs = comparison_pngs(v.comparison) pandoc = pandoc_mathml_ok([(tex, f.display)])[0] @@ -132,7 +132,7 @@ def serve(source: Source, ws: Workspace, port: int = 8765, show_all: bool = Fals return {"error": "Pandoc kann diese Formel nicht nach MathML umsetzen"} best = ((f.verification or {}).get("best") or {}).get("tex", "") f.status = "accepted_by_human" if tex == best else "corrected_by_human" - f.final_tex = tex + f.final_tex, f.final_suffix = tex, "" # the human's LaTeX is used as it is elif action == "keep": f.status, f.final_tex = "kept_by_human", None elif action == "figure": diff --git a/formula_img2latex/sources/pdf2md.py b/formula_img2latex/sources/pdf2md.py index d8d3cda..79f6b86 100644 --- a/formula_img2latex/sources/pdf2md.py +++ b/formula_img2latex/sources/pdf2md.py @@ -1,19 +1,22 @@ """pdf2md output: a single .md with _images/, or a --split folder with index.md and images/. pymupdf-layout classifies displayed formulas as boxes of class "formula"; pymupdf4llm writes them -as images named "--." at low resolution. If the PDF is -available, the formula is cut out of the vector page again at high resolution for comparison -(see pdfcrop.py); otherwise the stored image is used. +as images named "--." at low resolution. With `pdf2md --formulas` +every formula found on the page (also inside text and boxes) is an image "--f.png" +(300 dpi from the vector page), listed with page, rectangle and PDF text in /formulas-*.json; it is +used as it is. Box images (150 dpi) are cut out of the vector page again at 600 dpi if the PDF is +available (see pdfcrop.py), by locating the stored image; otherwise the stored image is used. """ from __future__ import annotations +import json import re from pathlib import Path from .base import ImageRef, Source -PDF_IMAGE_RE = re.compile(r"^(?P.+)-(?P\d{4})-(?P\d{2})\.(?:png|jpe?g)$", re.I) +PDF_IMAGE_RE = re.compile(r"^(?P.+)-(?P\d{4})-(?P\d{2}|f\d{3})\.(?:png|jpe?g)$", re.I) class Pdf2MdSource(Source): @@ -23,6 +26,7 @@ class Pdf2MdSource(Source): super().__init__(path) self.pdf = pdf self._cropper = None + self._listed: dict[str, dict] | None = None @classmethod def detect(cls, path: Path) -> bool: @@ -40,13 +44,30 @@ class Pdf2MdSource(Source): def page_box(self, ref: ImageRef) -> tuple[int, int] | None: m = PDF_IMAGE_RE.match(Path(ref.target).name) - return (int(m.group("page")), int(m.group("box"))) if m else None + if not m or m.group("box").startswith("f"): + return None + return int(m.group("page")), int(m.group("box")) + + def listed(self, ref: ImageRef) -> dict | None: + """Entry of a formula found by `pdf2md --formulas`: {"page", "rect", "type", "score"}.""" + if self._listed is None: + self._listed = {} + for f in sorted(ref.asset_path().parent.glob("formulas-*.json")): + for key, entry in json.loads(f.read_text(encoding="utf-8")).items(): + self._listed[Path(key).name] = entry # early pdf2md versions wrote the full path + return self._listed.get(Path(ref.target).name) def source_name(self, ref: ImageRef) -> str: + entry = self.listed(ref) + if entry: + kind = "abgesetzt" if entry["type"] == "isolated" else "im Text" + return f"Seite {entry['page']}, Formel {Path(ref.target).stem.rsplit('-f', 1)[-1]} ({kind})" pb = self.page_box(ref) return f"Seite {pb[0]}, Box {pb[1]}" if pb else ref.target def known_formula(self, ref: ImageRef) -> bool: + if self.listed(ref): + return True # found by the formula detector on the page if self.pdf is None or self.page_box(ref) is None: return False crop = self._crop(ref) @@ -58,6 +79,10 @@ class Pdf2MdSource(Source): if self._cropper is None: from ..pdfcrop import PdfCropper self._cropper = PdfCropper(self.pdf) + if self.listed(ref): + # pdf2md --formulas saved it at 300 dpi from the vector page: sharp enough (measured on the Corral + # pilot: 78 % automatic), a 600-dpi crop only makes the models and the image check slower + return ref.asset_path().read_bytes(), "formula" pb = self.page_box(ref) return self._cropper.crop(pb[0], pb[1], ref.asset_path()) if pb else None diff --git a/formula_img2latex/store.py b/formula_img2latex/store.py index 036d892..6c177b8 100644 --- a/formula_img2latex/store.py +++ b/formula_img2latex/store.py @@ -87,6 +87,7 @@ class Formula: status: str = "pending" reason: str = "" final_tex: str | None = None + final_suffix: str = "" # text after the formula: the sentence's punctuation cut out with an inline formula reviewer: str | None = None decided_at: str | None = None diff --git a/formula_img2latex/tex.py b/formula_img2latex/tex.py index 9c3f7ff..794966e 100644 --- a/formula_img2latex/tex.py +++ b/formula_img2latex/tex.py @@ -115,6 +115,52 @@ def fix_literal_newlines(tex: str) -> str: _TAG_RE = re.compile(r"\\tag\*?\s*\{([^{}]*)\}") +# PDF text layer -> LaTeX (only symbols whose spelling is unambiguous); anything else makes it unusable +_UNICODE_TEX = { + "\u2212": "-", "\u2264": "\\le ", "\u2265": "\\ge ", "\u2260": "\\ne ", "\u221e": "\\infty ", "\u2192": "\\to ", + "\u00b7": "\\cdot ", "\u22c5": "\\cdot ", "\u00d7": "\\times ", "\u2208": "\\in ", "\u00b1": "\\pm ", "\u2032": "'", + "\u2026": "\\dots ", "\u2248": "\\approx ", "\u2202": "\\partial ", "\u2207": "\\nabla ", "\u2205": "\\emptyset ", + "\u2229": "\\cap ", "\u222a": "\\cup ", "\u2282": "\\subset ", "\u2286": "\\subseteq ", "\u21d2": "\\Rightarrow ", + "\u21d4": "\\Leftrightarrow ", "\u2200": "\\forall ", "\u2203": "\\exists ", "\u2218": "\\circ ", "\u00b0": "^{\\circ}", +} +_GREEK_NAMES = ("alpha beta gamma delta epsilon zeta eta theta iota kappa lambda mu nu xi omicron pi rho varsigma sigma tau " + "upsilon phi chi psi omega").split() +for _i, _n in enumerate(_GREEK_NAMES): + _UNICODE_TEX[chr(0x3B1 + _i)] = "\\" + _n + " " if _n != "omicron" else "o" + if _n not in ("varsigma", "omicron"): + _UNICODE_TEX[chr(0x391 + _i)] = ("\\" + _n.capitalize() + " ") if _n in ( + "gamma", "delta", "theta", "lambda", "xi", "pi", "sigma", "upsilon", "phi", "psi", "omega") else None +_UNICODE_TEX = {k: v for k, v in _UNICODE_TEX.items() if v is not None} + + +def text_to_tex(text: str) -> str | None: + """A formula of the PDF text layer ("R < 1", "0 \u2264 a") as LaTeX, or None if it contains a symbol without + a safe spelling (font-specific glyphs such as "P" for a sum sign cannot be told apart and simply won't match).""" + out = [] + for c in text: + if c.isascii() and (c.isalnum() or c in " +-=<>()[],.;:/|'!*"): + out.append(c) + elif c in _UNICODE_TEX: + out.append(_UNICODE_TEX[c]) + else: + return None + tex = "".join(out).strip() + return tex or None + + +# an equation number written as text after a wide gap: "...\\qquad\\qquad(7.16)" (pix2text) +_TRAILING_NUMBER_RE = re.compile( + r"(?:\\qquad|\\quad|(?:\\[;,: ]|~){3,})(?:\\q?quad|\\[;,:! ]|~|\s)*\(\s*(\d+(?:\.\d+)*[a-z]?)\s*\)\s*$") + + +def trailing_number_to_tag(tex: str) -> str: + """`a=b\\qquad(7.16)` -> `a=b\\tag{7.16}`: the same equation number as a model's \\tag. Only after a wide + gap (\\quad, \\qquad or three small spaces), so f(2) stays a formula.""" + if _TAG_RE.search(tex): + return tex + return _TRAILING_NUMBER_RE.sub(lambda m: f"\\tag{{{m.group(1)}}}", tex.rstrip()) + + def split_tag(tex: str) -> tuple[str, str | None]: """Remove a \\tag{...}; return (formula, tag text or None).""" m = _TAG_RE.search(tex) diff --git a/formula_img2latex/verify.py b/formula_img2latex/verify.py index 455cdb3..2ae5781 100644 --- a/formula_img2latex/verify.py +++ b/formula_img2latex/verify.py @@ -10,6 +10,7 @@ from __future__ import annotations from dataclasses import dataclass, field +import cv2 import numpy as np from .compare import Comparison, compare, contrast @@ -19,8 +20,10 @@ from .render import render_many SWITCH_MARGIN = 2.0 # evidence (see compare.contrast) by which a variant must win to replace the candidate MAX_CLIMB = 3 MIN_NCC_FOR_VARIANTS = 0.6 # hopeless candidates are not worth the variant test -MAX_ORIG_PIXELS = 4_000_000 # larger crops are figures or text boxes (largest accepted formula: 1.7 MP); - # the elastic comparison of a full-page figure killed its worker process +MAX_COMPARE_WIDTH = 1600 # wider originals are shrunk first: the elastic comparison grows with the square of the + # width (a 3500-px derivation kept one process busy for minutes); a rough check needs no more +MAX_ORIG_PIXELS = 2_500_000 # larger crops are figures or text boxes (largest accepted formula: 1.7 MP); a full-page + # figure killed its worker process, a 4-MP box kept one busy for over 10 minutes @dataclass @@ -75,6 +78,15 @@ def _check_variants(orig: np.ndarray, cand: Comparison, tex: str, display: bool, return min_c, worst, worst_kind, n, eq, rival +def _shrink(orig: np.ndarray) -> np.ndarray: + """The original at most MAX_COMPARE_WIDTH wide (ink image, values 0..1).""" + h, w = orig.shape[:2] + if w <= MAX_COMPARE_WIDTH: + return orig + f = MAX_COMPARE_WIDTH / w + return cv2.resize(orig, (MAX_COMPARE_WIDTH, max(1, round(h * f))), interpolation=cv2.INTER_AREA) + + def image_arbiter(orig: np.ndarray, a: str, b: str, display: bool, preset: str) -> float | None: """Evidence that the original shows reading *a* rather than *b* (compare.contrast: positive = a). @@ -83,6 +95,7 @@ def image_arbiter(orig: np.ndarray, a: str, b: str, display: bool, preset: str) """ if orig.size > MAX_ORIG_PIXELS: return None + orig = _shrink(orig) ra, rb = render_many([(a, display), (b, display)], preset) ca = compare(orig, ra) if not ca.ok: @@ -96,6 +109,7 @@ def verify(orig: np.ndarray, tex: str, display: bool, preset: str, max_climb: in """Verify *tex* against the trimmed ink image *orig*.""" if orig.size > MAX_ORIG_PIXELS: return Verification(tex, False, f"Ausschnitt zu groß für den Bildvergleich ({orig.size / 1e6:.1f} MP)") + orig = _shrink(orig) history: list[str] = [] current = tex while True: diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index e76371f..9c3b2a4 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -315,11 +315,20 @@ def test_canonical_form() -> None: (r"\begin{aligned}\displaystyle a&\displaystyle=0\,.\\ \displaystyle\end{aligned}", r"a=0."), # pix2text's \cfrac; model readings with literal \n line breaks (JSON escape gone wrong) (r"\cfrac{df}{dx}=\cfrac{df}{du}\cdot\cfrac{du}{dx}", r"\frac{df}{dx}=\frac{df}{du}\cdot\frac{du}{dx}"), - (r"\begin{aligned}\n\lim_{x\to0} f &= 1 \\\nk_2 &= 2\n\end{aligned}", r"\lim_{x\to 0}f=1\\k_2=2")] + (r"\begin{aligned}\n\lim_{x\to0} f &= 1 \\\nk_2 &= 2\n\end{aligned}", r"\lim_{x\to 0}f=1\\k_2=2"), + # pix2text (Corral): limits with \underset, equation number as text, \textbf + (r"\underset{x\to-\infty}{\operatorname*{lim}}\ f(x)\,=\,L", r"\lim_{x \to -\infty} f(x) = L"), + (r"\underset{\underset{m/n~\mathrm{rational}}{m/n\to\sqrt{2}}}{\operatorname*{lim}}3^{m/n}", + r"\lim_{\underset{m/n~\mathrm{rational}}{m/n\to\sqrt{2}}}3^{m/n}"), + (r"y\ =\ x^{\prime}\sin\theta\qquad\qquad(7.12)", r"y = x' \sin \theta \tag{7.12}"), + (r"A=1\;\;\;\;\;\;(8.1)", r"A=1\tag{8.1}"), + (r"A_{v}~=~\frac{A_{o}}{\textbf{1}-\textbf{T}}", r"A_v = \frac{A_o}{1 - T}")] differ += [(r"\begin{pmatrix}1&2\end{pmatrix}", r"\begin{pmatrix}12\end{pmatrix}"), (r"a\\b\\c", r"a\\bc"), (r"f(x)=\begin{cases}1\\0\end{cases}", r"f(x)=\begin{aligned}1\\0\end{aligned}"), (r"x^{2}y", r"x^{2y}"), (r"\frac{a}{b}c", r"\frac{a}{bc}"), (r"\cdot\cdot", r"\dots"), - (r"\nu", "u"), (r"a\neq b", "a b"), (r"a\\\nu", r"a\\u")] # real \n commands stay + (r"\nu", "u"), (r"a\neq b", "a b"), (r"a\\\nu", r"a\\u"), # real \n commands stay + (r"y=f(2)", r"y=f\tag{2}"), (r"y=x\qquad(7.12)", r"y=x"), # number only after a wide gap + (r"\underset{x}{a}", "a_x"), (r"\overset{n}{\sum}", r"\sum_n")] assert [p for p in same if canonical(p[0]) != canonical(p[1])] == [] assert [p for p in differ if canonical(p[0]) == canonical(p[1])] == [] @@ -366,6 +375,7 @@ def test_tidy_p2t() -> None: r"{\mathrm{{\bf~ r o t}}} \, v": r"{\mathbf{rot}}\,v", r"\mathrm{\bf~ r o t} \, E": r"\mathbf{rot}\,E", r"\cfrac{d f}{d x}": r"\dfrac{d f}{d x}", # Pandoc has no \cfrac + r"y=x\qquad\qquad(7.12)": r"y=x\tag{7.12}", # equation number as text } for raw, tidy in cases.items(): assert tidy_p2t(raw) == tidy, (raw, tidy_p2t(raw)) @@ -423,3 +433,20 @@ def test_verify_skips_formulas_without_current_reading(epub2md_dir: Path) -> Non {"engine": "local", "model": "A", "tex": None, "is_formula": False, "prompt": 2}] # current: none ws.save(f) pipeline.verify_all(source, ws, preset="cm", workers=1, quiet=True) # must not crash + + +def test_inline_final_punctuation() -> None: + from formula_img2latex.pipeline import _split_final_punct + assert _split_final_punct("a^{x},") == ("a^{x}", ",") + assert _split_final_punct(r"u=\pi-2x\,.") == (r"u=\pi-2x", ".") + assert _split_final_punct("x") == ("x", "") + assert _split_final_punct(".") == (".", "") # nothing but punctuation stays as it is + + +def test_pdf_text_to_tex() -> None: + from formula_img2latex.consensus import canonical + from formula_img2latex.tex import text_to_tex + assert canonical(text_to_tex("R < 1")) == canonical("R<1") + assert canonical(text_to_tex("0 ≤ a")) == canonical(r"0\le a") + assert canonical(text_to_tex("[1,∞)")) == canonical(r"[1,\infty)") + assert text_to_tex("ℝ") is None # no safe spelling: not usable diff --git a/tools/buch_bericht.py b/tools/buch_bericht.py index bc3a01c..bb2ef53 100644 --- a/tools/buch_bericht.py +++ b/tools/buch_bericht.py @@ -23,9 +23,9 @@ from PIL import Image sys.path.insert(0, str(Path(__file__).resolve().parents[1])) -from formula_img2latex.pipeline import _ref_for, latest_by_engine, open_all # noqa: E402 +from formula_img2latex.pipeline import latest_by_engine, open_all, original_bytes # noqa: E402 from formula_img2latex.render import render_many # noqa: E402 -from formula_img2latex.tex import strip_delimiters # noqa: E402 +from formula_img2latex.tex import split_tag, strip_delimiters # noqa: E402 LABEL = {"local": "Qwen", "p2t": "pix2text", "local2": "Gemma", "cloud": "GLM"} STATUS = {"auto_accepted": ("automatisch ersetzt", "good"), "accepted_by_human": ("von Hand übernommen", "good"), @@ -75,12 +75,21 @@ def main() -> int: p = latest_by_engine(f).get("p2t") tex = p.get("tex") if p else None shown.append(strip_delimiters(tex) if tex else None) - rendered = render_many([(t or "\\text{–}", f.display) for t, f in zip(shown, formulas)], preset) + # already set during the image check (same LaTeX): reuse that picture instead of running pdflatex again + def checked(f, tex: str | None) -> Path | None: + best = (f.verification or {}).get("best") or {} + rigid = ws.review_dir / f"{f.id}-rigid.png" + return rigid if tex and best.get("tex") == split_tag(tex)[0] and rigid.exists() else None + + reuse = [checked(f, t) for f, t in zip(formulas, shown)] + todo = [k for k, r in enumerate(reuse) if r is None] + fresh = dict(zip(todo, render_many([(shown[k] or "\\text{–}", formulas[k].display) for k in todo], preset))) counts: dict[str, int] = {} secs: dict[str, list[float]] = {} cards = [] - for n, (f, tex, rr) in enumerate(zip(formulas, shown, rendered), 1): + for n, (f, tex) in enumerate(zip(formulas, shown), 1): + rr = fresh.get(n - 1) counts[f.status] = counts.get(f.status, 0) + 1 latest = latest_by_engine(f) for e, r in latest.items(): @@ -89,11 +98,14 @@ def main() -> int: try: o = out / "img" / f"{f.id}-o.png" if not o.exists(): # the original never changes; cutting it from the PDF is the slow part - save_png(original_png(source.original_bytes(_ref_for(source, f))), o) + save_png(original_png(original_bytes(source, ws, f)), o) orig = f"" except Exception: # noqa: BLE001 orig = "Original nicht lesbar" - if tex and rr.ok: + if tex and reuse[n - 1] is not None: + save_png(original_png(reuse[n - 1].read_bytes()), out / "img" / f"{f.id}-l.png") + latex = f"" + elif tex and rr is not None and rr.ok: g = (rr.rasterize(1.6) * 255).astype(np.uint8) save_png(Image.fromarray(255 - g), out / "img" / f"{f.id}-l.png") latex = f"" diff --git a/tools/vergleich.py b/tools/vergleich.py index 2c16477..56366e8 100644 --- a/tools/vergleich.py +++ b/tools/vergleich.py @@ -28,7 +28,7 @@ from PIL import Image sys.path.insert(0, str(Path(__file__).resolve().parents[1])) -from formula_img2latex.pipeline import _ref_for, _wanted, open_all, readings_of, run_consensus # noqa: E402 +from formula_img2latex.pipeline import _wanted, open_all, original_bytes, readings_of, run_consensus # noqa: E402 from formula_img2latex.render import render_many # noqa: E402 from formula_img2latex.tex import strip_delimiters # noqa: E402 @@ -88,7 +88,7 @@ def main() -> int: summary = [] for n, f in enumerate(formulas, 1): counts[f.status] = counts.get(f.status, 0) + 1 - orig = original(source.original_bytes(_ref_for(source, f))) + orig = original(original_bytes(source, ws, f)) lines = [f"Original"] reads = [] for r in readings_of(f):