feat: Formeln im Fließtext (pdf2md --formulas), etwa dreimal schneller
- pdf2md-Formelliste lesen: jede Formel mit Seite/Rechteck/PDF-Text, 300-dpi-Bild - eingebettete Formeln: Satzzeichen am Ende abgetrennt, als Text dahinter (final_suffix) - Normalform: \underset-Grenzwerte, Gleichungsnummer als \tag, \textbf, \cfrac, wörtliches \n - Tempo: Verweis- und Ausschnitt-Cache, pix2text gebündelt nach Pixelbudget, Lesen und Prüfen überlappend (forkserver), Qwen 8 Plätze, Render-Vorfilter, kein neuer Scan, je Paket nur dessen Datensätze, Zeiten je Stufe - Bildprüfung: max. 1600 px Breite, 2,5 MP, Proportionen-Abbruch - gemessen und abgeschaltet: PDF-Textebene als Bestätigung (pdftext=0) - Corral 10 %: 79 % automatisch, Pilot 869 Formeln 70 → ~24 min Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
2478bc5cb9
commit
6373db73a4
15 changed files with 479 additions and 112 deletions
31
CHANGELOG.md
31
CHANGELOG.md
|
|
@ -1,5 +1,36 @@
|
|||
# Changelog
|
||||
|
||||
## Unveröffentlicht
|
||||
|
||||
Alle Formeln eines PDF-Buchs, auch im Fließtext, und etwa dreimal schneller.
|
||||
|
||||
- **pdf2md --formulas** (Nachbarprojekt): Die Formelerkennung von pix2text (MFD) findet jede Formel auf der
|
||||
Seite, eingebettet oder abgesetzt, auch in Merkkästen. Sie wird zeichengenau aus dem Text geschnitten
|
||||
(300 dpi) und mit Seite, Rechteck und PDF-Text in `formulas-*.json` vermerkt. "Formel"-Boxen mit Text
|
||||
werden zu Text, solche mit Kurven zu Grafiken. formula_img2latex liest die Liste
|
||||
(`sources/pdf2md.listed`). Corral, 10 % der 9239 Formeln: **79 % automatisch** (im Text 83 %).
|
||||
- **Eingebettete Formeln:** Ein Satzzeichen am Ende wird vor dem Vergleich abgetrennt und als Text hinter
|
||||
die Formel gesetzt (`final_suffix`).
|
||||
- **Normalform:** `\underset{…}{\lim}` = `\lim_{…}`, Gleichungsnummer am Ende = `\tag`, `\textbf` als
|
||||
Fettdruck, `\cfrac` = `\frac` (pix2text schreibt `\dfrac`), wörtliches `\n` aus Modellantworten entfällt.
|
||||
- **Tempo** (Pilot 869 Formeln: 70 → ~24 min):
|
||||
- Markdown-Verweise und Ausschnitte nur einmal lesen (`crops/`);
|
||||
- pix2text gebündelt nach Pixelbudget, auf der freiesten GPU, einmal geladen;
|
||||
- Lesen und Prüfen überlappen, die Prüfprozesse starten per `forkserver`;
|
||||
- Qwen mit 8 parallelen Anfragen (`llama-server --parallel 8 -c 32768`);
|
||||
- pdflatex beim Entscheiden nur, wenn gleiche Bilder möglich sind;
|
||||
- kein neuer Scan beim Fortsetzen;
|
||||
- je Paket nur dessen Datensätze laden;
|
||||
- Zeiten je Stufe in der Statuszeile.
|
||||
- **Bildprüfung:** Originale höchstens 1600 px breit, über 2,5 MP nicht, Lesarten mit mehr als 2,5-fach
|
||||
falschen Proportionen sofort abgelehnt. Vorher hielten einzelne Formeln den Lauf über 10 Minuten auf.
|
||||
- **Gemessen und verworfen:**
|
||||
- PDF-Textebene als Bestätigung: 12 von 319 widersprachen Qwen, fehlende Punkte und Indizes; Schalter
|
||||
`pdftext` bleibt aus.
|
||||
- pix2texts eigene Sicherheit: Fehler auch bei 1,0.
|
||||
- Qwen3-VL-8B: nicht schneller als das MoE-30B-Modell (3B aktiv), etwas schlechter.
|
||||
- **Offen:** Formeln in Tabellen.
|
||||
|
||||
## 0.4.0 – 2026-10-09
|
||||
|
||||
Bessere Normalform: Lesarten, die sich nur in der Schreibweise unterscheiden, gelten als gleich.
|
||||
|
|
|
|||
|
|
@ -185,11 +185,11 @@ def _dispatch(args: argparse.Namespace) -> int:
|
|||
source, ws = pipeline.open_all(args.source, args.workspace, create=create, pdf=getattr(args, "pdf", None))
|
||||
q = args.quiet
|
||||
if args.command == "scan":
|
||||
pipeline.scan(source, ws, quiet=q)
|
||||
pipeline.scan(source, ws, quiet=q, force=True) # explicit scan: always re-read
|
||||
elif args.command == "recognize":
|
||||
pipeline.recognize(source, ws, args.engine, args.model, args.url, args.only_unresolved,
|
||||
args.include_figures, args.limit,
|
||||
args.workers or {"local": 4, "p2t": 1}.get(args.engine, 8), q)
|
||||
args.workers or {"p2t": 1}.get(args.engine, 8), q)
|
||||
elif args.command == "verify":
|
||||
pipeline.verify_all(source, ws, None if args.preset == "auto" else args.preset, args.include_figures,
|
||||
args.workers, args.redo, q)
|
||||
|
|
|
|||
|
|
@ -31,6 +31,7 @@ from .imageio import trim
|
|||
from .render import Rendered
|
||||
|
||||
BIN_THRESHOLD = 0.4
|
||||
MAX_ASPECT_FACTOR = 2.5 # accepted readings: 0.6 to 1.54 (Corral); beyond this no comparison at all
|
||||
MAX_ASPECT_DEVIATION = 0.25
|
||||
TARGET_STROKE = 2.0 # strokes thicker than this (px) are scaled down before comparing
|
||||
OVERSAMPLE = 4
|
||||
|
|
@ -220,6 +221,10 @@ def compare(orig_ink: np.ndarray, rendered: Rendered, tolerance: float | None =
|
|||
sy, sx = h / hi.shape[0], w / hi.shape[1]
|
||||
aspect = sx / sy
|
||||
metrics: dict[str, float] = {"aspect": aspect, "stroke": sw, "norm": norm}
|
||||
if not 1 / MAX_ASPECT_FACTOR <= aspect <= MAX_ASPECT_FACTOR:
|
||||
# another layout altogether (a three-line derivation read as one line): it cannot pass, and its
|
||||
# canvas would be huge (one comparison took over 10 minutes)
|
||||
return Comparison(False, f"Proportionen weichen stark ab (Faktor {max(aspect, 1 / aspect):.1f})", metrics)
|
||||
if abs(aspect - 1.0) > MAX_ASPECT_DEVIATION:
|
||||
# Different layout (e.g. another typesetting engine); still compare at the height scale so a
|
||||
# human gets an overlay, but the metrics will keep it out of automatic acceptance.
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@ from __future__ import annotations
|
|||
|
||||
import os
|
||||
import re
|
||||
from collections import Counter
|
||||
|
||||
import numpy as np
|
||||
|
||||
|
|
@ -22,7 +23,7 @@ from .compare import _register
|
|||
from .confusables import tokenize
|
||||
from .imageio import trim
|
||||
from .render import render_many
|
||||
from .tex import fix_literal_newlines, prepare_for_render, split_tag, strip_delimiters
|
||||
from .tex import fix_literal_newlines, prepare_for_render, split_tag, strip_delimiters, trailing_number_to_tag
|
||||
|
||||
# tokens that carry no symbol
|
||||
_DROP = {"\\,", "\\;", "\\:", "\\!", "\\ ", "~", "\\quad", "\\qquad", "\\left", "\\right", "\\big", "\\Big", "\\bigg",
|
||||
|
|
@ -35,7 +36,7 @@ _SYNONYM = {
|
|||
"\\overrightarrow": "\\vec", "\\overline": "\\bar", "\\widehat": "\\hat", "\\widetilde": "\\tilde",
|
||||
"\\rightarrow": "\\to", "\\longrightarrow": "\\to", "\\dfrac": "\\frac", "\\tfrac": "\\frac", "\\cfrac": "\\frac",
|
||||
"\\leq": "\\le", "\\geq": "\\ge", "\\neq": "\\ne", "\\ldots": "\\dots", "\\prime": "'",
|
||||
"\\mathbf": "\\boldsymbol", "\\bm": "\\boldsymbol", "\\lbrace": "\\{", "\\rbrace": "\\}", "\\vert": "|",
|
||||
"\\mathbf": "\\boldsymbol", "\\bm": "\\boldsymbol", "\\textbf": "\\boldsymbol", "\\lbrace": "\\{", "\\rbrace": "\\}", "\\vert": "|",
|
||||
"\\lvert": "|", "\\rvert": "|", "\\Vert": "\\|", "\\lVert": "\\|", "\\rVert": "\\|", "\\gets": "\\leftarrow",
|
||||
"\\land": "\\wedge", "\\lor": "\\vee", "\\lnot": "\\neg", "\\leqslant": "\\le", "\\geqslant": "\\ge",
|
||||
"\\varnothing": "\\emptyset", "\\colon": ":", "\\cdotp": "\\cdot", "\\ast": "*",
|
||||
|
|
@ -57,6 +58,10 @@ _OPERATORS = {"sin", "cos", "tan", "cot", "sec", "csc", "arcsin", "arccos", "arc
|
|||
# upright text groups; one level of nested braces (\mathrm{f \ddot{u} r})
|
||||
_TEXT_GROUP = re.compile(r"\\(operatorname\*?|mathrm|text|textrm|mbox|textnormal|textup)\s*\{((?:[^{}]|\{[^{}]*\})*)\}")
|
||||
_BOLD = {"\\boldsymbol", "\\mathbf", "\\bm", "\\pmb"}
|
||||
# a limit written below/above an operator with \underset/\overset is the same as its sub-/superscript in
|
||||
# display style (pix2text: \underset{x\to0}{\operatorname*{lim}}); for other symbols the position differs
|
||||
_UNDERSET = re.compile(r"\\(under|over)set\s*\{((?:[^{}]|\{(?:[^{}]|\{[^{}]*\})*\})*)\}" # 2 brace levels
|
||||
r"\s*\{\s*(\\(?:lim|liminf|limsup|sum|prod|coprod|max|min|sup|inf|bigcup|bigcap|det|gcd|Pr))\s*\}")
|
||||
# number of arguments of a command (after synonyms); every other command takes none, so a group after
|
||||
# \Delta or after another group is free-standing (\Delta{x_{n}} = \Delta x_n, t^{2}\,{\sqrt{2}} = t^2\sqrt{2})
|
||||
_ARITY = {c: 2 for c in ("\\frac", "\\cfrac", "\\binom", "\\dbinom", "\\tbinom", "\\overset", "\\underset",
|
||||
|
|
@ -78,9 +83,10 @@ def canonical(tex: str, bold: bool = False) -> str:
|
|||
|
||||
*bold*: whether bold vs. normal counts (False: \\boldsymbol{v} and v are the same reading).
|
||||
"""
|
||||
body, tag = split_tag(strip_delimiters(fix_literal_newlines(tex)))
|
||||
body, tag = split_tag(trailing_number_to_tag(strip_delimiters(fix_literal_newlines(tex))))
|
||||
body = prepare_for_render(body)
|
||||
body = _TEXT_GROUP.sub(_text_group, body)
|
||||
body = _UNDERSET.sub(lambda m: f"{m.group(3)}{'_' if m.group(1) == 'under' else '^'}{{{m.group(2)}}}", body)
|
||||
body = re.sub(r'\{?\\"\{?([aouAOU])\}?\}?', lambda m: _UMLAUT[m.group(1)], body) # f{\"u}r -> für
|
||||
body = re.sub(r"\^\s*\{\s*((?:\\prime\s*)+)\}", lambda m: "'" * m.group(1).count("\\prime"), body) # f^{\prime}
|
||||
body = re.sub(r"\^\s*\\prime(?![A-Za-z])", "'", body)
|
||||
|
|
@ -215,6 +221,19 @@ def _flatten_groups(toks: list[str]) -> list[str]:
|
|||
return _emit(_parse(toks)[0])
|
||||
|
||||
|
||||
def _letters(canon: str) -> Counter:
|
||||
"""Latin letters and digits a canonical form prints (operator names spelled out, \\sin -> s i n). Two
|
||||
readings whose counts differ cannot render to the same picture."""
|
||||
out: Counter = Counter()
|
||||
for tok in canon.split("#tag(")[0].split(" "):
|
||||
if tok.startswith("\\"):
|
||||
if tok[1:] in _OPERATORS:
|
||||
out.update(tok[1:])
|
||||
elif len(tok) == 1 and tok.isascii() and tok.isalnum():
|
||||
out[tok] += 1
|
||||
return out
|
||||
|
||||
|
||||
def render_equal(a: str, b: str, display: bool, preset: str = "cm", zoom: float = 4.0) -> bool:
|
||||
"""True if both readings render to the same picture (render vs. render)."""
|
||||
ra, rb = render_many([(a, display), (b, display)], preset)
|
||||
|
|
@ -241,7 +260,8 @@ def same_reading(a: str, b: str, display: bool, preset: str = "cm", bold: bool =
|
|||
"""
|
||||
if not a or not b:
|
||||
return None
|
||||
if canonical(a, bold) == canonical(b, bold):
|
||||
ca, cb = canonical(a, bold), canonical(b, bold)
|
||||
if ca == cb:
|
||||
return "normalform"
|
||||
if structural is None:
|
||||
structural = os.environ.get("FORMULA_STRUCTURAL", "0") == "1"
|
||||
|
|
@ -249,6 +269,8 @@ def same_reading(a: str, b: str, display: bool, preset: str = "cm", bold: bool =
|
|||
from .structure import structural_equal
|
||||
if structural_equal(a, b, bold):
|
||||
return "strukturell"
|
||||
if _letters(ca) != _letters(cb): # different letters or digits: pdflatex need not even try
|
||||
return None
|
||||
if render_equal(strip_delimiters(a), strip_delimiters(b), display, preset):
|
||||
return "render"
|
||||
return None
|
||||
|
|
|
|||
|
|
@ -77,6 +77,7 @@ CONSENSUS_DEFAULTS: dict[str, float] = {
|
|||
"require_third": 1, # a third reading (cloud, other model family) must agree as well
|
||||
"arbiter": 0, # 1: Qwen = Gemma against pix2text is accepted if the image clearly shows their
|
||||
"arbiter_margin": 5.0, # reading: evidence (compare.contrast) at least this large (rule p2t)
|
||||
"pdftext": 0, # 1: the PDF text layer of a flat formula may confirm pix2text instead of Qwen
|
||||
"structural": 0, # 1: readings with the same unevaluated sympy tree agree (structure.py). Off:
|
||||
# after the multi-line normal form it found nothing more (Göllmann, 0.4.0)
|
||||
}
|
||||
|
|
@ -200,7 +201,8 @@ def decide_pair_tiebreak(readings: list[dict[str, Any] | None], agree: list[tupl
|
|||
return "not_formula", "Qwen" + (" und Gemma" if g else "") + ": keine Formel", None
|
||||
first = 1 if p.get("tex") else None # reading shown for review: pix2text's
|
||||
if frozenset((0, 1)) in pairs:
|
||||
how, who, confirm = pairs[frozenset((0, 1))], "pix2text und Qwen einig", q
|
||||
who = "pix2text und PDF-Text einig" if q.get("engine") == "pdftext" else "pix2text und Qwen einig"
|
||||
how, confirm = pairs[frozenset((0, 1))], q
|
||||
elif g is None:
|
||||
return "needs_review", f"pix2text und Qwen uneinig, {TIEBREAK_MISSING}", first
|
||||
elif frozenset((1, 2)) in pairs:
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@ import time
|
|||
from PIL import Image
|
||||
|
||||
from ..errors import EXIT_TOOL, FormulaError
|
||||
from ..tex import trailing_number_to_tag
|
||||
from .vlm import Recognition, prepare_image
|
||||
|
||||
DEFAULT_MODEL = "mfr-1.5"
|
||||
|
|
@ -38,15 +39,22 @@ def tidy_p2t(tex: str) -> str:
|
|||
tex = _BOLDMATH.sub(lambda m: "\\boldsymbol{" + re.sub(r"\s+", "", m.group(1)) + "}", tex)
|
||||
tex = _BF.sub(lambda m: "\\mathbf{" + re.sub(r"\s+", "", m.group(2)) + "}", tex)
|
||||
tex = re.sub(r"\\cfrac(?![A-Za-z])", r"\\dfrac", tex)
|
||||
return post_process(tex).strip()
|
||||
return trailing_number_to_tag(post_process(tex).strip())
|
||||
|
||||
|
||||
def resolve_device(device: str) -> str:
|
||||
"""Device "auto": the first visible GPU if onnxruntime has CUDA, else the CPU."""
|
||||
"""Device "auto": the visible GPU with the most free memory (not the one the VLM server fills), else the CPU."""
|
||||
if device != "auto":
|
||||
return device
|
||||
import onnxruntime
|
||||
return "cuda" if "CUDAExecutionProvider" in onnxruntime.get_available_providers() else "cpu"
|
||||
if "CUDAExecutionProvider" not in onnxruntime.get_available_providers():
|
||||
return "cpu"
|
||||
try:
|
||||
import torch
|
||||
free = [torch.cuda.mem_get_info(i)[0] for i in range(torch.cuda.device_count())]
|
||||
except Exception: # no torch or no driver access: the first GPU
|
||||
return "cuda"
|
||||
return f"cuda:{max(range(len(free)), key=free.__getitem__)}" if free else "cuda"
|
||||
|
||||
|
||||
def ort_provider(device: str) -> tuple[str, dict]:
|
||||
|
|
@ -61,6 +69,10 @@ def ort_provider(device: str) -> tuple[str, dict]:
|
|||
raise ValueError(f"unbekanntes Gerät {device!r} (cpu, cuda, cuda:N)")
|
||||
|
||||
|
||||
BATCH_MAX = 32 # images per bundle at most
|
||||
BATCH_PIXELS = 4_000_000 # largest image x count per bundle (the GPU is shared with the VLM servers)
|
||||
|
||||
|
||||
class Pix2TextEngine:
|
||||
def __init__(self, model: str = DEFAULT_MODEL, device: str = "auto") -> None:
|
||||
self.name = "p2t"
|
||||
|
|
@ -115,3 +127,48 @@ class Pix2TextEngine:
|
|||
tex = tidy_p2t(raw)
|
||||
return Recognition(self.name, self.model, tex or None, None, raw[:2000], None, time.time() - t0,
|
||||
p_mean=score)
|
||||
|
||||
def recognize_many(self, items: list[tuple[bytes, str]], batch: int = BATCH_MAX) -> list[Recognition]:
|
||||
"""Read many formula images (data, name), one model call per bundle: on the GPU many times faster than
|
||||
one call per image. Bundles are cut by a pixel budget (largest image x count; a bundle is padded to its
|
||||
largest image), so big images go alone and the GPU, shared with the VLM servers, does not run out of
|
||||
memory. Falls back to single images if a bundle fails."""
|
||||
try:
|
||||
self.check()
|
||||
except Exception as e:
|
||||
return [Recognition(self.name, self.model, None, None, "", str(e)[:300], 0.0) for _ in items]
|
||||
out: list[Recognition | None] = [None] * len(items)
|
||||
ready = []
|
||||
for i, (data, name) in enumerate(items):
|
||||
try:
|
||||
ready.append((i, Image.open(io.BytesIO(prepare_image(data, name))).convert("RGB")))
|
||||
except Exception as e: # unreadable image: an error reading, the others go on
|
||||
out[i] = Recognition(self.name, self.model, None, None, "", str(e)[:300], 0.0)
|
||||
ready.sort(key=lambda t: t[1].size[0] * t[1].size[1])
|
||||
bundles: list[list] = []
|
||||
cur: list = []
|
||||
width = height = 0
|
||||
for i, img in ready:
|
||||
w, h = max(width, img.size[0]), max(height, img.size[1])
|
||||
if cur and (len(cur) >= batch or w * h * (len(cur) + 1) > BATCH_PIXELS):
|
||||
bundles.append(cur)
|
||||
cur, w, h = [], img.size[0], img.size[1]
|
||||
cur.append((i, img))
|
||||
width, height = w, h
|
||||
if cur:
|
||||
bundles.append(cur)
|
||||
for bundle in bundles:
|
||||
t0 = time.time()
|
||||
try:
|
||||
with self._lock:
|
||||
results = self._ocr._one_batch([img for _, img in bundle], None)
|
||||
except Exception: # a model error (e.g. GPU memory): read this bundle one by one
|
||||
for i, _ in bundle:
|
||||
out[i] = self.recognize(*items[i])
|
||||
continue
|
||||
each = (time.time() - t0) / len(bundle)
|
||||
for (i, _), res in zip(bundle, results):
|
||||
raw = self._ocr.post_process(res["text"]).strip()
|
||||
out[i] = Recognition(self.name, self.model, tidy_p2t(raw) or None, None, raw[:2000], None, each,
|
||||
p_mean=float(res.get("score") or 0.0) or None)
|
||||
return [r for r in out if r is not None]
|
||||
|
|
|
|||
|
|
@ -3,12 +3,15 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import multiprocessing
|
||||
import io
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import statistics
|
||||
import threading
|
||||
import time
|
||||
import sys
|
||||
from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor, as_completed
|
||||
from pathlib import Path
|
||||
|
|
@ -30,10 +33,11 @@ from .render import PRESETS
|
|||
from .sources import open_source
|
||||
from .sources.base import ImageRef, Source
|
||||
from .store import HUMAN_STATUSES, REPLACE_STATUSES, Formula, Workspace, now_iso, sha256_bytes
|
||||
from .tex import normalize_for_compare, split_tag, strip_delimiters
|
||||
from .tex import normalize_for_compare, split_tag, strip_delimiters, text_to_tex
|
||||
from .verify import verify
|
||||
|
||||
PRESET_SAMPLE = 12
|
||||
RECOGNIZE_BATCH = 32 # images per pix2text call
|
||||
_SAVE_LOCK = threading.Lock()
|
||||
|
||||
|
||||
|
|
@ -44,7 +48,21 @@ def log(msg: str, quiet: bool = False) -> None:
|
|||
|
||||
# --------------------------------------------------------------------------------------- scan
|
||||
|
||||
def scan(source: Source, ws: Workspace, quiet: bool = False) -> dict[str, int]:
|
||||
def _markdown_fingerprint(source: Source) -> str:
|
||||
"""Changes whenever a Markdown file of the source changes (size, time) or the source description does."""
|
||||
parts = [json.dumps(source.describe(), sort_keys=True)]
|
||||
for md in source.markdown_files():
|
||||
st = md.stat()
|
||||
parts.append(f"{md}:{st.st_size}:{st.st_mtime_ns}")
|
||||
return hashlib.sha256("\n".join(parts).encode()).hexdigest()[:16]
|
||||
|
||||
|
||||
def scan(source: Source, ws: Workspace, quiet: bool = False, force: bool = False) -> dict[str, int]:
|
||||
fingerprint = _markdown_fingerprint(source)
|
||||
last = ws.meta.get("last_scan") or {}
|
||||
if not force and last.get("fingerprint") == fingerprint: # resuming: nothing changed, skip the slow part
|
||||
log(f"scan: unverändert seit {last.get('at', '?')}, {last.get('formulas', '?')} Bilder", quiet)
|
||||
return {k: last.get(k, 0) for k in ("formula", "maybe", "figure")}
|
||||
refs = source.image_refs()
|
||||
groups: dict[str, tuple[Formula, list[ImageRef]]] = {}
|
||||
for ref in refs:
|
||||
|
|
@ -67,19 +85,41 @@ def scan(source: Source, ws: Workspace, quiet: bool = False) -> dict[str, int]:
|
|||
f.alt = ref.alt
|
||||
f.occurrences = [{"file": r.file.relative_to(source.root).as_posix(), "line": r.line, "md": r.md,
|
||||
"context": r.context} for r in rs]
|
||||
data = source.original_bytes(ref)
|
||||
_add_pdf_text_reading(source, f, ref)
|
||||
data = original_bytes(source, ws, f, ref)
|
||||
f.width, f.height = _image_size(data)
|
||||
f.classify = classify(data, f.asset, f.known_formula, inline=not f.display)
|
||||
counts[f.classify["label"]] += 1
|
||||
ws.save(f)
|
||||
ws.meta["source"] = source.describe()
|
||||
ws.meta["last_scan"] = {"at": now_iso(), "images": len(refs), "formulas": len(groups), **counts}
|
||||
ws.meta["last_scan"] = {"at": now_iso(), "images": len(refs), "formulas": len(groups), **counts,
|
||||
"fingerprint": fingerprint}
|
||||
ws.save_meta()
|
||||
log(f"scan: {len(refs)} Bildverweise, {len(groups)} verschiedene Bilder: {counts['formula']} Formeln, "
|
||||
f"{counts['maybe']} unklar, {counts['figure']} Abbildungen", quiet)
|
||||
return counts
|
||||
|
||||
|
||||
def _add_pdf_text_reading(source: Source, f: Formula, ref: ImageRef) -> None:
|
||||
"""`pdf2md --formulas` lists the PDF text of each formula: a "flat" one (one baseline, one size: no lost
|
||||
super-/subscript, fraction or limit) becomes the reading "pdftext", which may confirm pix2text instead of Qwen."""
|
||||
entry = source.listed(ref) if hasattr(source, "listed") else None
|
||||
if not entry or not entry.get("flat") or any(c.get("engine") == "pdftext" for c in f.candidates):
|
||||
return
|
||||
tex = text_to_tex(entry.get("text") or "")
|
||||
if tex:
|
||||
f.candidates.append({"engine": "pdftext", "model": "PDF-Textebene", "tex": tex, "is_formula": True,
|
||||
"error": None, "seconds": 0.0, "created_at": now_iso()})
|
||||
|
||||
|
||||
def pdftext_confirms(f: Formula, bold: bool = False) -> bool:
|
||||
"""The PDF text layer reads the same formula as pix2text (same canonical form)."""
|
||||
from .consensus import canonical
|
||||
latest = latest_by_engine(f)
|
||||
p, t = latest.get("p2t"), latest.get("pdftext")
|
||||
return bool(p and t and p.get("tex") and canonical(p["tex"], bold) == canonical(t["tex"], bold))
|
||||
|
||||
|
||||
def _image_size(data: bytes) -> tuple[int, int]:
|
||||
try:
|
||||
with Image.open(io.BytesIO(data)) as im:
|
||||
|
|
@ -95,17 +135,45 @@ def _wanted(f: Formula, include_figures: bool) -> bool:
|
|||
return include_figures or label != "figure"
|
||||
|
||||
|
||||
def _records(ws: Workspace, only_ids: set[str] | None) -> list[Formula]:
|
||||
"""The records of *only_ids* (one part of a run), or all: a big book has 10 000 of them, loading all of them
|
||||
for every part of 50 cost seconds each time."""
|
||||
if only_ids is None:
|
||||
return ws.all()
|
||||
return [f for f in (ws.load(i) for i in sorted(only_ids)) if f is not None]
|
||||
|
||||
|
||||
_REFS: dict[tuple[str, int], tuple[dict, dict]] = {} # (file, mtime) -> ({(line, md): ref}, {md: ref})
|
||||
|
||||
|
||||
def _ref_for(source: Source, f: Formula) -> ImageRef:
|
||||
"""The image reference of a formula's first occurrence (each Markdown file is parsed once while unchanged)."""
|
||||
occ = f.occurrences[0]
|
||||
md = source.root / occ["file"]
|
||||
key = (str(md), md.stat().st_mtime_ns)
|
||||
if key not in _REFS:
|
||||
from .sources.base import scan_markdown
|
||||
for r in scan_markdown(md):
|
||||
if r.line == occ["line"] and r.md == occ["md"]:
|
||||
return r
|
||||
for r in scan_markdown(md):
|
||||
if r.md == occ["md"]:
|
||||
return r
|
||||
refs = scan_markdown(md)
|
||||
_REFS[key] = ({(r.line, r.md): r for r in refs}, {r.md: r for r in reversed(refs)})
|
||||
by_line, by_md = _REFS[key]
|
||||
ref = by_line.get((occ["line"], occ["md"])) or by_md.get(occ["md"])
|
||||
if ref is None:
|
||||
raise FileNotFoundError(f"Bildverweis nicht mehr gefunden: {occ['file']}:{occ['line']}")
|
||||
return ref
|
||||
|
||||
|
||||
def original_bytes(source: Source, ws: Workspace, f: Formula, ref: ImageRef | None = None) -> bytes:
|
||||
"""The original image of a formula. Crops from a PDF are expensive (600 dpi, or template matching):
|
||||
they are cut once and kept in <workspace>/crops/."""
|
||||
if getattr(source, "pdf", None) is None:
|
||||
return source.original_bytes(ref or _ref_for(source, f))
|
||||
path = ws.root / "crops" / f"{f.id}.png"
|
||||
if path.exists():
|
||||
return path.read_bytes()
|
||||
data = source.original_bytes(ref or _ref_for(source, f))
|
||||
path.parent.mkdir(exist_ok=True)
|
||||
path.write_bytes(data)
|
||||
return data
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------------- recognize
|
||||
|
|
@ -120,15 +188,26 @@ def needs_tiebreak(f: Formula) -> bool:
|
|||
return f.status == "needs_review" and TIEBREAK_MISSING in (f.reason or "")
|
||||
|
||||
|
||||
_ENGINES: dict[tuple, Any] = {}
|
||||
|
||||
|
||||
def _engine(name: str, model: str | None, url: str | None) -> Any:
|
||||
"""One engine per name/model/url for the whole run: pix2text would otherwise load its model for every part."""
|
||||
key = (name, model, url)
|
||||
if key not in _ENGINES:
|
||||
_ENGINES[key] = make_engine(name, model, url)
|
||||
return _ENGINES[key]
|
||||
|
||||
|
||||
def recognize(source: Source, ws: Workspace, engine_name: str, model: str | None = None, url: str | None = None,
|
||||
only_unresolved: bool = False, include_figures: bool = False, limit: int = 0,
|
||||
workers: int = 4, quiet: bool = False, only_ids: set[str] | None = None,
|
||||
workers: int = 8, quiet: bool = False, only_ids: set[str] | None = None,
|
||||
only_needing_third: bool = False, only_needing_tiebreak: bool = False) -> dict[str, int]:
|
||||
engine = make_engine(engine_name, model, url)
|
||||
engine = _engine(engine_name, model, url)
|
||||
engine.check()
|
||||
todo = []
|
||||
for f in ws.all():
|
||||
if not _wanted(f, include_figures) or (only_ids is not None and f.id not in only_ids):
|
||||
for f in _records(ws, only_ids):
|
||||
if not _wanted(f, include_figures):
|
||||
continue
|
||||
if only_unresolved and f.status in ("auto_accepted", "not_formula"):
|
||||
continue
|
||||
|
|
@ -146,14 +225,10 @@ def recognize(source: Source, ws: Workspace, engine_name: str, model: str | None
|
|||
stats = {"asked": 0, "latex": 0, "not_formula": 0, "error": 0, "tokens_in": 0, "tokens_out": 0}
|
||||
|
||||
def job(f: Formula) -> tuple[Formula, Any]:
|
||||
ref = _ref_for(source, f)
|
||||
ctx = f.occurrences[0].get("context", "")
|
||||
return f, engine.recognize(source.original_bytes(ref), f.asset, ctx, f.display)
|
||||
return f, engine.recognize(original_bytes(source, ws, f), f.asset, ctx, f.display)
|
||||
|
||||
with ThreadPoolExecutor(max_workers=max(1, workers)) as pool:
|
||||
futures = [pool.submit(job, f) for f in todo]
|
||||
for i, fut in enumerate(as_completed(futures), 1):
|
||||
f, rec = fut.result()
|
||||
def store(i: int, f: Formula, rec: Any) -> None:
|
||||
stats["asked"] += 1
|
||||
stats["tokens_in"] += rec.tokens_in
|
||||
stats["tokens_out"] += rec.tokens_out
|
||||
|
|
@ -173,6 +248,19 @@ def recognize(source: Source, ws: Workspace, engine_name: str, model: str | None
|
|||
stats["not_formula"] += 1
|
||||
if not quiet and (i % 25 == 0 or i == len(todo)):
|
||||
log(f" {i}/{len(todo)}")
|
||||
|
||||
if hasattr(engine, "recognize_many"): # pix2text: batches on the GPU instead of one call per image
|
||||
i = 0
|
||||
for start in range(0, len(todo), RECOGNIZE_BATCH):
|
||||
part = todo[start:start + RECOGNIZE_BATCH]
|
||||
for f, rec in zip(part, engine.recognize_many([(original_bytes(source, ws, f), f.asset) for f in part])):
|
||||
i += 1
|
||||
store(i, f, rec)
|
||||
else:
|
||||
with ThreadPoolExecutor(max_workers=max(1, workers)) as pool:
|
||||
futures = [pool.submit(job, f) for f in todo]
|
||||
for i, fut in enumerate(as_completed(futures), 1):
|
||||
store(i, *fut.result())
|
||||
log(f"recognize: {stats['latex']} LaTeX-Kandidaten, {stats['not_formula']} „keine Formel“, "
|
||||
f"{stats['error']} Fehler; Tokens {stats['tokens_in']} ein / {stats['tokens_out']} aus", quiet)
|
||||
return stats
|
||||
|
|
@ -237,7 +325,7 @@ def choose_preset(source: Source, ws: Workspace, formulas: list[Formula], quiet:
|
|||
sample = [f for f in formulas if _distinct_candidates(f)][:PRESET_SAMPLE]
|
||||
if not sample:
|
||||
return "cm"
|
||||
inks = [load_ink(source.original_bytes(_ref_for(source, f)), f.asset) for f in sample]
|
||||
inks = [load_ink(original_bytes(source, ws, f), f.asset) for f in sample]
|
||||
scores: dict[str, float] = {}
|
||||
for preset in PRESETS:
|
||||
rs = render_many([(_distinct_candidates(f)[0], f.display) for f in sample], preset)
|
||||
|
|
@ -252,8 +340,7 @@ def choose_preset(source: Source, ws: Workspace, formulas: list[Formula], quiet:
|
|||
def verify_all(source: Source, ws: Workspace, preset: str | None = None, include_figures: bool = False,
|
||||
workers: int = 0, redo: bool = False, quiet: bool = False,
|
||||
progress: Callable[[int, int], None] | None = None, only_ids: set[str] | None = None) -> dict[str, int]:
|
||||
formulas = [f for f in ws.all() if _wanted(f, include_figures) and _distinct_candidates(f)
|
||||
and (only_ids is None or f.id in only_ids)]
|
||||
formulas = [f for f in _records(ws, only_ids) if _wanted(f, include_figures) and _distinct_candidates(f)]
|
||||
mode = decide_mode(ws)
|
||||
if mode == "consensus" and consensus_rule(ws) == "vlm3":
|
||||
mode = "consensus-vlm3"
|
||||
|
|
@ -266,15 +353,16 @@ def verify_all(source: Source, ws: Workspace, preset: str | None = None, include
|
|||
ws.save_meta()
|
||||
jobs = []
|
||||
for f in formulas:
|
||||
ref = _ref_for(source, f)
|
||||
jobs.append((f.id, source.original_bytes(ref), f.asset, _verify_texes(f, mode), f.display, preset,
|
||||
jobs.append((f.id, original_bytes(source, ws, f), f.asset, _verify_texes(f, mode), f.display, preset,
|
||||
mode == "pixel"))
|
||||
log(f"verify: {len(jobs)} Formeln, Schrift {preset}", quiet)
|
||||
by_id = {f.id: f for f in formulas}
|
||||
ws.review_dir.mkdir(parents=True, exist_ok=True)
|
||||
workers = workers or max(1, (os.cpu_count() or 2) - 2)
|
||||
done = 0
|
||||
with ProcessPoolExecutor(max_workers=workers) as pool:
|
||||
# forkserver, not fork: the reading thread (run_consensus) may be inside pix2text/CUDA at that moment, and a
|
||||
# forked copy of a process mid-way in another thread inherits half-held locks (workers died, the run hung)
|
||||
with ProcessPoolExecutor(max_workers=workers, mp_context=multiprocessing.get_context("forkserver")) as pool:
|
||||
for fut in as_completed([pool.submit(_verify_job, j) for j in jobs]):
|
||||
fid, results, pngs = fut.result()
|
||||
f = by_id[fid]
|
||||
|
|
@ -385,6 +473,14 @@ def decide_all(ws: Workspace, quiet: bool = False, preset: str | None = None,
|
|||
return counts
|
||||
|
||||
|
||||
def _split_final_punct(tex: str) -> tuple[str, str]:
|
||||
"""(formula without a final period, comma or semicolon, that punctuation or "")."""
|
||||
m = re.search(r"(?:\\[,;!]|\\quad|\\qquad|\s)*([.,;])\s*$", tex)
|
||||
if not m or not tex[:m.start()].strip():
|
||||
return tex, ""
|
||||
return tex[:m.start()].rstrip(), m.group(1)
|
||||
|
||||
|
||||
def _strip_final_punct(tex: str) -> str:
|
||||
return re.sub(r"(\\[,;!]|\\quad|\\qquad|\s)*[.,;]\s*$", "", strip_delimiters(tex or "")).strip()
|
||||
|
||||
|
|
@ -399,14 +495,15 @@ def decide_all_consensus(ws: Workspace, quiet: bool, preset: str, only_ids: set[
|
|||
th = consensus_thresholds(ws.settings.get("consensus_thresholds"))
|
||||
rule = consensus_rule(ws)
|
||||
decide_fn = decide_pair_tiebreak if rule == "p2t" else decide_consensus
|
||||
formulas = [f for f in ws.all() if f.candidates and f.status not in HUMAN_STATUSES
|
||||
and (only_ids is None or f.id in only_ids)]
|
||||
formulas = [f for f in _records(ws, only_ids) if f.candidates and f.status not in HUMAN_STATUSES]
|
||||
plan = []
|
||||
for f in formulas:
|
||||
bold = bool(th.get("bold_relevant"))
|
||||
if rule == "p2t":
|
||||
latest = latest_by_engine(f)
|
||||
rs = [latest.get("local"), latest.get("p2t"), latest.get("local2")]
|
||||
if th.get("pdftext") and pdftext_confirms(f, bold): # the PDF text layer confirms pix2text
|
||||
rs[0] = latest["pdftext"]
|
||||
pairs = [(0, 1), (1, 2), (0, 2)]
|
||||
else:
|
||||
rs = readings_of(f)
|
||||
|
|
@ -416,9 +513,10 @@ def decide_all_consensus(ws: Workspace, quiet: bool, preset: str, only_ids: set[
|
|||
if rs[i] is None or rs[j] is None:
|
||||
continue
|
||||
a, b = rs[i].get("tex") or "", rs[j].get("tex") or ""
|
||||
# rule vlm3: a confirming reading (j >= 2) may lack the final period or comma. Rule p2t: strict,
|
||||
# pix2text's reading is the one used, so its final punctuation must be confirmed too
|
||||
if rule != "p2t" and j >= 2:
|
||||
# rule vlm3: a confirming reading (j >= 2) may lack the final period or comma. Rule p2t: strict for
|
||||
# displayed formulas; an inline one is cut out of a sentence, its final comma or period belongs to the
|
||||
# sentence (the readers see it or not): compared without, written after the formula (final_suffix)
|
||||
if (rule != "p2t" and j >= 2) or (rule == "p2t" and not f.display):
|
||||
a, b = _strip_final_punct(a), _strip_final_punct(b)
|
||||
how = same_reading(a, b, f.display, preset, bold, bool(th.get("structural")))
|
||||
if how:
|
||||
|
|
@ -445,6 +543,9 @@ def decide_all_consensus(ws: Workspace, quiet: bool, preset: str, only_ids: set[
|
|||
status, reason, use = decide_fn(rs, agree, veto, pok, book_ppt, th)
|
||||
f.status, f.reason = status, reason
|
||||
f.final_tex = _with_tag(f, tex) if status == "auto_accepted" and tex else None
|
||||
f.final_suffix = ""
|
||||
if f.final_tex and rule == "p2t" and not f.display:
|
||||
f.final_tex, f.final_suffix = _split_final_punct(f.final_tex)
|
||||
f.decided_at = now_iso()
|
||||
ws.save(f)
|
||||
counts[status] = counts.get(status, 0) + 1
|
||||
|
|
@ -476,7 +577,7 @@ def apply(source: Source, ws: Workspace, dry_run: bool = False, quiet: bool = Fa
|
|||
by_occ: dict[tuple[str, int, str], str] = {}
|
||||
for f in ws.all():
|
||||
if f.status in REPLACE_STATUSES and f.final_tex:
|
||||
math = source.format_math(f.final_tex, f.display)
|
||||
math = source.format_math(f.final_tex, f.display) + f.final_suffix
|
||||
for occ in f.occurrences:
|
||||
by_occ[(occ["file"], occ["line"], occ["md"])] = math
|
||||
state = ws.meta.setdefault("applied", {})
|
||||
|
|
@ -578,22 +679,45 @@ def run_consensus(source: Source, ws: Workspace, chunk: int = 50, preset: str |
|
|||
tiebreak_ok = True
|
||||
totals: dict[str, int] = {}
|
||||
done: list[str] = []
|
||||
for start in range(0, len(todo), chunk):
|
||||
part = set(todo[start:start + chunk])
|
||||
if rule == "p2t": # pix2text first (fast), then Qwen
|
||||
for eng in ("p2t", "local"):
|
||||
recognize(source, ws, eng, only_ids=part, include_figures=include_figures, quiet=True)
|
||||
parts = [set(todo[start:start + chunk]) for start in range(0, len(todo), chunk)]
|
||||
use_pdftext = bool(consensus_thresholds(ws.settings.get("consensus_thresholds")).get("pdftext"))
|
||||
secs: list[dict[str, float]] = [{} for _ in parts]
|
||||
|
||||
def timed(n: int, label: str, fn: Callable, *args: Any, **kwargs: Any) -> Any:
|
||||
t0 = time.time()
|
||||
try:
|
||||
return fn(*args, **kwargs)
|
||||
finally:
|
||||
secs[n][label] = secs[n].get(label, 0.0) + time.time() - t0
|
||||
|
||||
def read(n: int) -> None:
|
||||
"""GPU stage of part n: runs while the CPU checks part n-1 against the images."""
|
||||
if rule == "p2t": # pix2text first (fast), then Qwen – unless the PDF text layer already confirms it
|
||||
timed(n, "pix2text", recognize, source, ws, "p2t", only_ids=parts[n], include_figures=include_figures,
|
||||
quiet=True)
|
||||
ask = parts[n]
|
||||
if use_pdftext:
|
||||
ask = {fid for fid in parts[n] if not ((f := ws.load(fid)) and pdftext_confirms(f))}
|
||||
timed(n, "Qwen", recognize, source, ws, "local", only_ids=ask, include_figures=include_figures, quiet=True)
|
||||
else: # both VLMs run on their own GPU: ask them at the same time
|
||||
with ThreadPoolExecutor(max_workers=2) as both:
|
||||
for fut in [both.submit(recognize, source, ws, eng, only_ids=part, include_figures=include_figures,
|
||||
for fut in [both.submit(recognize, source, ws, eng, only_ids=parts[n], include_figures=include_figures,
|
||||
quiet=True) for eng in ("local", "local2")]:
|
||||
fut.result()
|
||||
verify_all(source, ws, preset, include_figures, quiet=True, only_ids=part)
|
||||
|
||||
with ThreadPoolExecutor(max_workers=1) as ahead:
|
||||
reading = ahead.submit(read, 0) if parts else None
|
||||
for n, part in enumerate(parts):
|
||||
t_part = time.time()
|
||||
reading.result()
|
||||
if n + 1 < len(parts):
|
||||
reading = ahead.submit(read, n + 1)
|
||||
timed(n, "Prüfung", verify_all, source, ws, preset, include_figures, quiet=True, only_ids=part)
|
||||
if rule == "p2t" and tiebreak_ok and any(needs_tiebreak(f) for f in map(ws.load, part) if f):
|
||||
try:
|
||||
recognize(source, ws, "local2", only_ids=part, only_needing_tiebreak=True,
|
||||
try: # Gemma runs on the other GPU, alongside Qwen reading the next part
|
||||
timed(n, "Gemma", recognize, source, ws, "local2", only_ids=part, only_needing_tiebreak=True,
|
||||
include_figures=include_figures, quiet=True)
|
||||
verify_all(source, ws, preset, include_figures, quiet=True, only_ids=part)
|
||||
timed(n, "Prüfung", verify_all, source, ws, preset, include_figures, quiet=True, only_ids=part)
|
||||
except FormulaError as e: # Gemma not running: the disputed formulas stay in review
|
||||
tiebreak_ok = False
|
||||
say(f"Stichwahl (Gemma) nicht möglich, strittige Formeln bleiben im Review: {e}")
|
||||
|
|
@ -611,5 +735,6 @@ def run_consensus(source: Source, ws: Workspace, chunk: int = 50, preset: str |
|
|||
image = sum(totals.get(k, 0) for k in ("rejected_keep_image", "not_formula", "no_candidate", "render_failed"))
|
||||
review = totals.get("needs_review", 0)
|
||||
say(f"{len(done):5d}/{len(todo)} verarbeitet: automatisch ersetzt {auto} ({auto / len(done):.0%}), "
|
||||
f"Review {review}, bleibt Bild/keine Formel {image}")
|
||||
f"Review {review}, bleibt Bild/keine Formel {image} [Paket {time.time() - t_part:.0f} s; "
|
||||
+ ", ".join(f"{k} {v:.0f} s" for k, v in secs[n].items()) + "]")
|
||||
return totals
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@ from urllib.parse import unquote, urlparse
|
|||
|
||||
from ..checks import pandoc_mathml_ok
|
||||
from ..imageio import load_ink
|
||||
from ..pipeline import _ref_for, comparison_pngs, readings_of
|
||||
from ..pipeline import comparison_pngs, original_bytes, readings_of
|
||||
from ..sources.base import Source
|
||||
from ..store import Formula, Workspace, now_iso
|
||||
from ..tex import strip_delimiters, unsafe_reason
|
||||
|
|
@ -114,7 +114,7 @@ def serve(source: Source, ws: Workspace, port: int = 8765, show_all: bool = Fals
|
|||
reason = unsafe_reason(tex)
|
||||
if reason:
|
||||
return {"ok": False, "error": reason}
|
||||
orig = load_ink(source.original_bytes(_ref_for(source, f)), f.asset)
|
||||
orig = load_ink(original_bytes(source, ws, f), f.asset)
|
||||
v = verify(orig, tex, f.display, preset, max_climb=0)
|
||||
pngs = comparison_pngs(v.comparison)
|
||||
pandoc = pandoc_mathml_ok([(tex, f.display)])[0]
|
||||
|
|
@ -132,7 +132,7 @@ def serve(source: Source, ws: Workspace, port: int = 8765, show_all: bool = Fals
|
|||
return {"error": "Pandoc kann diese Formel nicht nach MathML umsetzen"}
|
||||
best = ((f.verification or {}).get("best") or {}).get("tex", "")
|
||||
f.status = "accepted_by_human" if tex == best else "corrected_by_human"
|
||||
f.final_tex = tex
|
||||
f.final_tex, f.final_suffix = tex, "" # the human's LaTeX is used as it is
|
||||
elif action == "keep":
|
||||
f.status, f.final_tex = "kept_by_human", None
|
||||
elif action == "figure":
|
||||
|
|
|
|||
|
|
@ -1,19 +1,22 @@
|
|||
"""pdf2md output: a single <name>.md with <name>_images/, or a --split folder with index.md and images/.
|
||||
|
||||
pymupdf-layout classifies displayed formulas as boxes of class "formula"; pymupdf4llm writes them
|
||||
as images named "<pdf name>-<page:04d>-<box:02d>.<ext>" at low resolution. If the PDF is
|
||||
available, the formula is cut out of the vector page again at high resolution for comparison
|
||||
(see pdfcrop.py); otherwise the stored image is used.
|
||||
as images named "<pdf name>-<page:04d>-<box:02d>.<ext>" at low resolution. With `pdf2md --formulas`
|
||||
every formula found on the page (also inside text and boxes) is an image "<pdf name>-<page:04d>-f<n:03d>.png"
|
||||
(300 dpi from the vector page), listed with page, rectangle and PDF text in <images>/formulas-*.json; it is
|
||||
used as it is. Box images (150 dpi) are cut out of the vector page again at 600 dpi if the PDF is
|
||||
available (see pdfcrop.py), by locating the stored image; otherwise the stored image is used.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
from .base import ImageRef, Source
|
||||
|
||||
PDF_IMAGE_RE = re.compile(r"^(?P<pdf>.+)-(?P<page>\d{4})-(?P<box>\d{2})\.(?:png|jpe?g)$", re.I)
|
||||
PDF_IMAGE_RE = re.compile(r"^(?P<pdf>.+)-(?P<page>\d{4})-(?P<box>\d{2}|f\d{3})\.(?:png|jpe?g)$", re.I)
|
||||
|
||||
|
||||
class Pdf2MdSource(Source):
|
||||
|
|
@ -23,6 +26,7 @@ class Pdf2MdSource(Source):
|
|||
super().__init__(path)
|
||||
self.pdf = pdf
|
||||
self._cropper = None
|
||||
self._listed: dict[str, dict] | None = None
|
||||
|
||||
@classmethod
|
||||
def detect(cls, path: Path) -> bool:
|
||||
|
|
@ -40,13 +44,30 @@ class Pdf2MdSource(Source):
|
|||
|
||||
def page_box(self, ref: ImageRef) -> tuple[int, int] | None:
|
||||
m = PDF_IMAGE_RE.match(Path(ref.target).name)
|
||||
return (int(m.group("page")), int(m.group("box"))) if m else None
|
||||
if not m or m.group("box").startswith("f"):
|
||||
return None
|
||||
return int(m.group("page")), int(m.group("box"))
|
||||
|
||||
def listed(self, ref: ImageRef) -> dict | None:
|
||||
"""Entry of a formula found by `pdf2md --formulas`: {"page", "rect", "type", "score"}."""
|
||||
if self._listed is None:
|
||||
self._listed = {}
|
||||
for f in sorted(ref.asset_path().parent.glob("formulas-*.json")):
|
||||
for key, entry in json.loads(f.read_text(encoding="utf-8")).items():
|
||||
self._listed[Path(key).name] = entry # early pdf2md versions wrote the full path
|
||||
return self._listed.get(Path(ref.target).name)
|
||||
|
||||
def source_name(self, ref: ImageRef) -> str:
|
||||
entry = self.listed(ref)
|
||||
if entry:
|
||||
kind = "abgesetzt" if entry["type"] == "isolated" else "im Text"
|
||||
return f"Seite {entry['page']}, Formel {Path(ref.target).stem.rsplit('-f', 1)[-1]} ({kind})"
|
||||
pb = self.page_box(ref)
|
||||
return f"Seite {pb[0]}, Box {pb[1]}" if pb else ref.target
|
||||
|
||||
def known_formula(self, ref: ImageRef) -> bool:
|
||||
if self.listed(ref):
|
||||
return True # found by the formula detector on the page
|
||||
if self.pdf is None or self.page_box(ref) is None:
|
||||
return False
|
||||
crop = self._crop(ref)
|
||||
|
|
@ -58,6 +79,10 @@ class Pdf2MdSource(Source):
|
|||
if self._cropper is None:
|
||||
from ..pdfcrop import PdfCropper
|
||||
self._cropper = PdfCropper(self.pdf)
|
||||
if self.listed(ref):
|
||||
# pdf2md --formulas saved it at 300 dpi from the vector page: sharp enough (measured on the Corral
|
||||
# pilot: 78 % automatic), a 600-dpi crop only makes the models and the image check slower
|
||||
return ref.asset_path().read_bytes(), "formula"
|
||||
pb = self.page_box(ref)
|
||||
return self._cropper.crop(pb[0], pb[1], ref.asset_path()) if pb else None
|
||||
|
||||
|
|
|
|||
|
|
@ -87,6 +87,7 @@ class Formula:
|
|||
status: str = "pending"
|
||||
reason: str = ""
|
||||
final_tex: str | None = None
|
||||
final_suffix: str = "" # text after the formula: the sentence's punctuation cut out with an inline formula
|
||||
reviewer: str | None = None
|
||||
decided_at: str | None = None
|
||||
|
||||
|
|
|
|||
|
|
@ -115,6 +115,52 @@ def fix_literal_newlines(tex: str) -> str:
|
|||
_TAG_RE = re.compile(r"\\tag\*?\s*\{([^{}]*)\}")
|
||||
|
||||
|
||||
# PDF text layer -> LaTeX (only symbols whose spelling is unambiguous); anything else makes it unusable
|
||||
_UNICODE_TEX = {
|
||||
"\u2212": "-", "\u2264": "\\le ", "\u2265": "\\ge ", "\u2260": "\\ne ", "\u221e": "\\infty ", "\u2192": "\\to ",
|
||||
"\u00b7": "\\cdot ", "\u22c5": "\\cdot ", "\u00d7": "\\times ", "\u2208": "\\in ", "\u00b1": "\\pm ", "\u2032": "'",
|
||||
"\u2026": "\\dots ", "\u2248": "\\approx ", "\u2202": "\\partial ", "\u2207": "\\nabla ", "\u2205": "\\emptyset ",
|
||||
"\u2229": "\\cap ", "\u222a": "\\cup ", "\u2282": "\\subset ", "\u2286": "\\subseteq ", "\u21d2": "\\Rightarrow ",
|
||||
"\u21d4": "\\Leftrightarrow ", "\u2200": "\\forall ", "\u2203": "\\exists ", "\u2218": "\\circ ", "\u00b0": "^{\\circ}",
|
||||
}
|
||||
_GREEK_NAMES = ("alpha beta gamma delta epsilon zeta eta theta iota kappa lambda mu nu xi omicron pi rho varsigma sigma tau "
|
||||
"upsilon phi chi psi omega").split()
|
||||
for _i, _n in enumerate(_GREEK_NAMES):
|
||||
_UNICODE_TEX[chr(0x3B1 + _i)] = "\\" + _n + " " if _n != "omicron" else "o"
|
||||
if _n not in ("varsigma", "omicron"):
|
||||
_UNICODE_TEX[chr(0x391 + _i)] = ("\\" + _n.capitalize() + " ") if _n in (
|
||||
"gamma", "delta", "theta", "lambda", "xi", "pi", "sigma", "upsilon", "phi", "psi", "omega") else None
|
||||
_UNICODE_TEX = {k: v for k, v in _UNICODE_TEX.items() if v is not None}
|
||||
|
||||
|
||||
def text_to_tex(text: str) -> str | None:
|
||||
"""A formula of the PDF text layer ("R < 1", "0 \u2264 a") as LaTeX, or None if it contains a symbol without
|
||||
a safe spelling (font-specific glyphs such as "P" for a sum sign cannot be told apart and simply won't match)."""
|
||||
out = []
|
||||
for c in text:
|
||||
if c.isascii() and (c.isalnum() or c in " +-=<>()[],.;:/|'!*"):
|
||||
out.append(c)
|
||||
elif c in _UNICODE_TEX:
|
||||
out.append(_UNICODE_TEX[c])
|
||||
else:
|
||||
return None
|
||||
tex = "".join(out).strip()
|
||||
return tex or None
|
||||
|
||||
|
||||
# an equation number written as text after a wide gap: "...\\qquad\\qquad(7.16)" (pix2text)
|
||||
_TRAILING_NUMBER_RE = re.compile(
|
||||
r"(?:\\qquad|\\quad|(?:\\[;,: ]|~){3,})(?:\\q?quad|\\[;,:! ]|~|\s)*\(\s*(\d+(?:\.\d+)*[a-z]?)\s*\)\s*$")
|
||||
|
||||
|
||||
def trailing_number_to_tag(tex: str) -> str:
|
||||
"""`a=b\\qquad(7.16)` -> `a=b\\tag{7.16}`: the same equation number as a model's \\tag. Only after a wide
|
||||
gap (\\quad, \\qquad or three small spaces), so f(2) stays a formula."""
|
||||
if _TAG_RE.search(tex):
|
||||
return tex
|
||||
return _TRAILING_NUMBER_RE.sub(lambda m: f"\\tag{{{m.group(1)}}}", tex.rstrip())
|
||||
|
||||
|
||||
def split_tag(tex: str) -> tuple[str, str | None]:
|
||||
"""Remove a \\tag{...}; return (formula, tag text or None)."""
|
||||
m = _TAG_RE.search(tex)
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ from __future__ import annotations
|
|||
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
import cv2
|
||||
import numpy as np
|
||||
|
||||
from .compare import Comparison, compare, contrast
|
||||
|
|
@ -19,8 +20,10 @@ from .render import render_many
|
|||
SWITCH_MARGIN = 2.0 # evidence (see compare.contrast) by which a variant must win to replace the candidate
|
||||
MAX_CLIMB = 3
|
||||
MIN_NCC_FOR_VARIANTS = 0.6 # hopeless candidates are not worth the variant test
|
||||
MAX_ORIG_PIXELS = 4_000_000 # larger crops are figures or text boxes (largest accepted formula: 1.7 MP);
|
||||
# the elastic comparison of a full-page figure killed its worker process
|
||||
MAX_COMPARE_WIDTH = 1600 # wider originals are shrunk first: the elastic comparison grows with the square of the
|
||||
# width (a 3500-px derivation kept one process busy for minutes); a rough check needs no more
|
||||
MAX_ORIG_PIXELS = 2_500_000 # larger crops are figures or text boxes (largest accepted formula: 1.7 MP); a full-page
|
||||
# figure killed its worker process, a 4-MP box kept one busy for over 10 minutes
|
||||
|
||||
|
||||
@dataclass
|
||||
|
|
@ -75,6 +78,15 @@ def _check_variants(orig: np.ndarray, cand: Comparison, tex: str, display: bool,
|
|||
return min_c, worst, worst_kind, n, eq, rival
|
||||
|
||||
|
||||
def _shrink(orig: np.ndarray) -> np.ndarray:
|
||||
"""The original at most MAX_COMPARE_WIDTH wide (ink image, values 0..1)."""
|
||||
h, w = orig.shape[:2]
|
||||
if w <= MAX_COMPARE_WIDTH:
|
||||
return orig
|
||||
f = MAX_COMPARE_WIDTH / w
|
||||
return cv2.resize(orig, (MAX_COMPARE_WIDTH, max(1, round(h * f))), interpolation=cv2.INTER_AREA)
|
||||
|
||||
|
||||
def image_arbiter(orig: np.ndarray, a: str, b: str, display: bool, preset: str) -> float | None:
|
||||
"""Evidence that the original shows reading *a* rather than *b* (compare.contrast: positive = a).
|
||||
|
||||
|
|
@ -83,6 +95,7 @@ def image_arbiter(orig: np.ndarray, a: str, b: str, display: bool, preset: str)
|
|||
"""
|
||||
if orig.size > MAX_ORIG_PIXELS:
|
||||
return None
|
||||
orig = _shrink(orig)
|
||||
ra, rb = render_many([(a, display), (b, display)], preset)
|
||||
ca = compare(orig, ra)
|
||||
if not ca.ok:
|
||||
|
|
@ -96,6 +109,7 @@ def verify(orig: np.ndarray, tex: str, display: bool, preset: str, max_climb: in
|
|||
"""Verify *tex* against the trimmed ink image *orig*."""
|
||||
if orig.size > MAX_ORIG_PIXELS:
|
||||
return Verification(tex, False, f"Ausschnitt zu groß für den Bildvergleich ({orig.size / 1e6:.1f} MP)")
|
||||
orig = _shrink(orig)
|
||||
history: list[str] = []
|
||||
current = tex
|
||||
while True:
|
||||
|
|
|
|||
|
|
@ -315,11 +315,20 @@ def test_canonical_form() -> None:
|
|||
(r"\begin{aligned}\displaystyle a&\displaystyle=0\,.\\ \displaystyle\end{aligned}", r"a=0."),
|
||||
# pix2text's \cfrac; model readings with literal \n line breaks (JSON escape gone wrong)
|
||||
(r"\cfrac{df}{dx}=\cfrac{df}{du}\cdot\cfrac{du}{dx}", r"\frac{df}{dx}=\frac{df}{du}\cdot\frac{du}{dx}"),
|
||||
(r"\begin{aligned}\n\lim_{x\to0} f &= 1 \\\nk_2 &= 2\n\end{aligned}", r"\lim_{x\to 0}f=1\\k_2=2")]
|
||||
(r"\begin{aligned}\n\lim_{x\to0} f &= 1 \\\nk_2 &= 2\n\end{aligned}", r"\lim_{x\to 0}f=1\\k_2=2"),
|
||||
# pix2text (Corral): limits with \underset, equation number as text, \textbf
|
||||
(r"\underset{x\to-\infty}{\operatorname*{lim}}\ f(x)\,=\,L", r"\lim_{x \to -\infty} f(x) = L"),
|
||||
(r"\underset{\underset{m/n~\mathrm{rational}}{m/n\to\sqrt{2}}}{\operatorname*{lim}}3^{m/n}",
|
||||
r"\lim_{\underset{m/n~\mathrm{rational}}{m/n\to\sqrt{2}}}3^{m/n}"),
|
||||
(r"y\ =\ x^{\prime}\sin\theta\qquad\qquad(7.12)", r"y = x' \sin \theta \tag{7.12}"),
|
||||
(r"A=1\;\;\;\;\;\;(8.1)", r"A=1\tag{8.1}"),
|
||||
(r"A_{v}~=~\frac{A_{o}}{\textbf{1}-\textbf{T}}", r"A_v = \frac{A_o}{1 - T}")]
|
||||
differ += [(r"\begin{pmatrix}1&2\end{pmatrix}", r"\begin{pmatrix}12\end{pmatrix}"), (r"a\\b\\c", r"a\\bc"),
|
||||
(r"f(x)=\begin{cases}1\\0\end{cases}", r"f(x)=\begin{aligned}1\\0\end{aligned}"),
|
||||
(r"x^{2}y", r"x^{2y}"), (r"\frac{a}{b}c", r"\frac{a}{bc}"), (r"\cdot\cdot", r"\dots"),
|
||||
(r"\nu", "u"), (r"a\neq b", "a b"), (r"a\\\nu", r"a\\u")] # real \n commands stay
|
||||
(r"\nu", "u"), (r"a\neq b", "a b"), (r"a\\\nu", r"a\\u"), # real \n commands stay
|
||||
(r"y=f(2)", r"y=f\tag{2}"), (r"y=x\qquad(7.12)", r"y=x"), # number only after a wide gap
|
||||
(r"\underset{x}{a}", "a_x"), (r"\overset{n}{\sum}", r"\sum_n")]
|
||||
assert [p for p in same if canonical(p[0]) != canonical(p[1])] == []
|
||||
assert [p for p in differ if canonical(p[0]) == canonical(p[1])] == []
|
||||
|
||||
|
|
@ -366,6 +375,7 @@ def test_tidy_p2t() -> None:
|
|||
r"{\mathrm{{\bf~ r o t}}} \, v": r"{\mathbf{rot}}\,v",
|
||||
r"\mathrm{\bf~ r o t} \, E": r"\mathbf{rot}\,E",
|
||||
r"\cfrac{d f}{d x}": r"\dfrac{d f}{d x}", # Pandoc has no \cfrac
|
||||
r"y=x\qquad\qquad(7.12)": r"y=x\tag{7.12}", # equation number as text
|
||||
}
|
||||
for raw, tidy in cases.items():
|
||||
assert tidy_p2t(raw) == tidy, (raw, tidy_p2t(raw))
|
||||
|
|
@ -423,3 +433,20 @@ def test_verify_skips_formulas_without_current_reading(epub2md_dir: Path) -> Non
|
|||
{"engine": "local", "model": "A", "tex": None, "is_formula": False, "prompt": 2}] # current: none
|
||||
ws.save(f)
|
||||
pipeline.verify_all(source, ws, preset="cm", workers=1, quiet=True) # must not crash
|
||||
|
||||
|
||||
def test_inline_final_punctuation() -> None:
|
||||
from formula_img2latex.pipeline import _split_final_punct
|
||||
assert _split_final_punct("a^{x},") == ("a^{x}", ",")
|
||||
assert _split_final_punct(r"u=\pi-2x\,.") == (r"u=\pi-2x", ".")
|
||||
assert _split_final_punct("x") == ("x", "")
|
||||
assert _split_final_punct(".") == (".", "") # nothing but punctuation stays as it is
|
||||
|
||||
|
||||
def test_pdf_text_to_tex() -> None:
|
||||
from formula_img2latex.consensus import canonical
|
||||
from formula_img2latex.tex import text_to_tex
|
||||
assert canonical(text_to_tex("R < 1")) == canonical("R<1")
|
||||
assert canonical(text_to_tex("0 ≤ a")) == canonical(r"0\le a")
|
||||
assert canonical(text_to_tex("[1,∞)")) == canonical(r"[1,\infty)")
|
||||
assert text_to_tex("ℝ") is None # no safe spelling: not usable
|
||||
|
|
|
|||
|
|
@ -23,9 +23,9 @@ from PIL import Image
|
|||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
|
||||
from formula_img2latex.pipeline import _ref_for, latest_by_engine, open_all # noqa: E402
|
||||
from formula_img2latex.pipeline import latest_by_engine, open_all, original_bytes # noqa: E402
|
||||
from formula_img2latex.render import render_many # noqa: E402
|
||||
from formula_img2latex.tex import strip_delimiters # noqa: E402
|
||||
from formula_img2latex.tex import split_tag, strip_delimiters # noqa: E402
|
||||
|
||||
LABEL = {"local": "Qwen", "p2t": "pix2text", "local2": "Gemma", "cloud": "GLM"}
|
||||
STATUS = {"auto_accepted": ("automatisch ersetzt", "good"), "accepted_by_human": ("von Hand übernommen", "good"),
|
||||
|
|
@ -75,12 +75,21 @@ def main() -> int:
|
|||
p = latest_by_engine(f).get("p2t")
|
||||
tex = p.get("tex") if p else None
|
||||
shown.append(strip_delimiters(tex) if tex else None)
|
||||
rendered = render_many([(t or "\\text{–}", f.display) for t, f in zip(shown, formulas)], preset)
|
||||
# already set during the image check (same LaTeX): reuse that picture instead of running pdflatex again
|
||||
def checked(f, tex: str | None) -> Path | None:
|
||||
best = (f.verification or {}).get("best") or {}
|
||||
rigid = ws.review_dir / f"{f.id}-rigid.png"
|
||||
return rigid if tex and best.get("tex") == split_tag(tex)[0] and rigid.exists() else None
|
||||
|
||||
reuse = [checked(f, t) for f, t in zip(formulas, shown)]
|
||||
todo = [k for k, r in enumerate(reuse) if r is None]
|
||||
fresh = dict(zip(todo, render_many([(shown[k] or "\\text{–}", formulas[k].display) for k in todo], preset)))
|
||||
|
||||
counts: dict[str, int] = {}
|
||||
secs: dict[str, list[float]] = {}
|
||||
cards = []
|
||||
for n, (f, tex, rr) in enumerate(zip(formulas, shown, rendered), 1):
|
||||
for n, (f, tex) in enumerate(zip(formulas, shown), 1):
|
||||
rr = fresh.get(n - 1)
|
||||
counts[f.status] = counts.get(f.status, 0) + 1
|
||||
latest = latest_by_engine(f)
|
||||
for e, r in latest.items():
|
||||
|
|
@ -89,11 +98,14 @@ def main() -> int:
|
|||
try:
|
||||
o = out / "img" / f"{f.id}-o.png"
|
||||
if not o.exists(): # the original never changes; cutting it from the PDF is the slow part
|
||||
save_png(original_png(source.original_bytes(_ref_for(source, f))), o)
|
||||
save_png(original_png(original_bytes(source, ws, f)), o)
|
||||
orig = f"<img loading=lazy src='img/{f.id}-o.png'>"
|
||||
except Exception: # noqa: BLE001
|
||||
orig = "<i>Original nicht lesbar</i>"
|
||||
if tex and rr.ok:
|
||||
if tex and reuse[n - 1] is not None:
|
||||
save_png(original_png(reuse[n - 1].read_bytes()), out / "img" / f"{f.id}-l.png")
|
||||
latex = f"<img loading=lazy src='img/{f.id}-l.png'>"
|
||||
elif tex and rr is not None and rr.ok:
|
||||
g = (rr.rasterize(1.6) * 255).astype(np.uint8)
|
||||
save_png(Image.fromarray(255 - g), out / "img" / f"{f.id}-l.png")
|
||||
latex = f"<img loading=lazy src='img/{f.id}-l.png'>"
|
||||
|
|
|
|||
|
|
@ -28,7 +28,7 @@ from PIL import Image
|
|||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
|
||||
from formula_img2latex.pipeline import _ref_for, _wanted, open_all, readings_of, run_consensus # noqa: E402
|
||||
from formula_img2latex.pipeline import _wanted, open_all, original_bytes, readings_of, run_consensus # noqa: E402
|
||||
from formula_img2latex.render import render_many # noqa: E402
|
||||
from formula_img2latex.tex import strip_delimiters # noqa: E402
|
||||
|
||||
|
|
@ -88,7 +88,7 @@ def main() -> int:
|
|||
summary = []
|
||||
for n, f in enumerate(formulas, 1):
|
||||
counts[f.status] = counts.get(f.status, 0) + 1
|
||||
orig = original(source.original_bytes(_ref_for(source, f)))
|
||||
orig = original(original_bytes(source, ws, f))
|
||||
lines = [f"<tr><th>Original</th><td><img src='{uri(orig)}'></td><td></td></tr>"]
|
||||
reads = []
|
||||
for r in readings_of(f):
|
||||
|
|
|
|||
Loading…
Reference in a new issue