diff --git a/formula_img2latex/cli.py b/formula_img2latex/cli.py index a1f07dd..febc1fa 100644 --- a/formula_img2latex/cli.py +++ b/formula_img2latex/cli.py @@ -73,7 +73,7 @@ def build_parser() -> argparse.ArgumentParser: sp = sub.add_parser("recognize", help="LaTeX-Kandidaten mit einem Vision-Modell erzeugen") common(sp) recog_opts(sp) - sp.add_argument("--engine", choices=["local", "local2", "cloud"], default="local") + sp.add_argument("--engine", choices=["local", "local2", "cloud", "p2t"], default="local") sp.add_argument("--only-unresolved", action="store_true", help="nur Formeln, die noch nicht automatisch angenommen sind") @@ -182,7 +182,8 @@ def _dispatch(args: argparse.Namespace) -> int: pipeline.scan(source, ws, quiet=q) elif args.command == "recognize": pipeline.recognize(source, ws, args.engine, args.model, args.url, args.only_unresolved, - args.include_figures, args.limit, args.workers or (4 if args.engine == "local" else 8), q) + args.include_figures, args.limit, + args.workers or {"local": 4, "p2t": 1}.get(args.engine, 8), q) elif args.command == "verify": pipeline.verify_all(source, ws, None if args.preset == "auto" else args.preset, args.include_figures, args.workers, args.redo, q) diff --git a/formula_img2latex/consensus.py b/formula_img2latex/consensus.py index 010098a..eb0ca3e 100644 --- a/formula_img2latex/consensus.py +++ b/formula_img2latex/consensus.py @@ -47,6 +47,12 @@ _ENV_DELIMS = {"pmatrix": ("(", ")"), "bmatrix": ("[", "]"), "Bmatrix": ("\\{", _UMLAUT = {"a": "ä", "o": "ö", "u": "ü", "A": "Ä", "O": "Ö", "U": "Ü"} +# \operatorname{l i m} (Pix2Text spells upright words letter by letter) is the same symbol as \lim +_OPERATORS = {"sin", "cos", "tan", "cot", "sec", "csc", "arcsin", "arccos", "arctan", "sinh", "cosh", "tanh", "coth", + "lim", "liminf", "limsup", "log", "ln", "lg", "exp", "max", "min", "sup", "inf", "det", "dim", "ker", + "deg", "gcd", "arg", "hom", "Pr"} +# upright text groups; one level of nested braces (\mathrm{f \ddot{u} r}) +_TEXT_GROUP = re.compile(r"\\(operatorname\*?|mathrm|text|textrm|mbox|textnormal|textup)\s*\{((?:[^{}]|\{[^{}]*\})*)\}") _BOLD = {"\\boldsymbol", "\\mathbf", "\\bm", "\\pmb"} # commands that never take an argument: a group right after them is free-standing _NOARG = {"\\cdot", "\\times", "\\pm", "\\mp", "\\le", "\\ge", "\\ne", "\\in", "\\to", "\\Rightarrow", @@ -60,7 +66,11 @@ def canonical(tex: str, bold: bool = False) -> str: """ body, tag = split_tag(strip_delimiters(tex)) body = prepare_for_render(body) + body = _TEXT_GROUP.sub(_text_group, body) body = re.sub(r'\{?\\"\{?([aouAOU])\}?\}?', lambda m: _UMLAUT[m.group(1)], body) # f{\"u}r -> für + body = re.sub(r"\^\s*\{\s*((?:\\prime\s*)+)\}", lambda m: "'" * m.group(1).count("\\prime"), body) # f^{\prime} + body = re.sub(r"\^\s*\\prime(?![A-Za-z])", "'", body) + body = re.sub(r"\\\\\s*(?=\\end\{)", "", body) # a final \\ before \end{...} adds no row body = re.sub(r"\{?\\ss\{?\}?\}?", "ß", body) body = re.sub(r"\{([^{}]*)\}\s*\\atop\s*\{([^{}]*)\}", r"\\substack{\1\\\\\2}", body) body = re.sub(r"\\begin\{array\}\{[^{}]*\}", r"\\begin{matrix}", body).replace("\\end{array}", "\\end{matrix}") @@ -77,6 +87,15 @@ def canonical(tex: str, bold: bool = False) -> str: return s + (f"#tag({tag})" if tag else "") +def _text_group(m: re.Match[str]) -> str: + """\\operatorname{l i m} -> \\lim; \\ddot{u} inside upright text -> ü (Pix2Text's spelling of umlauts).""" + inner = re.sub(r"\\ddot\s*\{?\s*([aouAOU])\s*\}?", lambda mm: _UMLAUT[mm.group(1)], m.group(2)) + word = re.sub(r"[\s~]|\\[,;: ]", "", inner) + if word in _OPERATORS: + return f"\\{word} " + return f"\\{m.group(1)}{{{inner}}}" + + def _parse(toks: list[str], i: int = 0) -> tuple[list, int]: """Token list -> nested lists for {...} groups.""" out: list = [] diff --git a/formula_img2latex/engines/__init__.py b/formula_img2latex/engines/__init__.py index 79d65c0..b10e61b 100644 --- a/formula_img2latex/engines/__init__.py +++ b/formula_img2latex/engines/__init__.py @@ -2,13 +2,14 @@ Engines only propose; verify.py decides. Every engine speaks the OpenAI chat-completions API with an image part, so a local llama.cpp server (llama-server with --mmproj) and OpenRouter use -the same code. +the same code. The exception is 'p2t' (Pix2Text), a dedicated formula OCR model running in-process. """ from __future__ import annotations import os +from .p2t import DEFAULT_MODEL as DEFAULT_P2T_MODEL, Pix2TextEngine from .vlm import VisionEngine DEFAULT_LOCAL_URL = "http://127.0.0.1:8011/v1" @@ -20,8 +21,8 @@ DEFAULT_CLOUD_URL = "https://openrouter.ai/api/v1" DEFAULT_CLOUD_MODEL = "z-ai/glm-4.6v" -def make_engine(name: str, model: str | None = None, url: str | None = None) -> VisionEngine: - """'local'/'local2' (two llama.cpp servers with different models) or 'cloud' (OpenRouter).""" +def make_engine(name: str, model: str | None = None, url: str | None = None) -> VisionEngine | Pix2TextEngine: + """'local'/'local2' (two llama.cpp servers with different models), 'cloud' (OpenRouter) or 'p2t'.""" if name in ("local", "local2"): suffix = "" if name == "local" else "2" return VisionEngine( @@ -44,4 +45,7 @@ def make_engine(name: str, model: str | None = None, url: str | None = None) -> # GLM-4.6V with reasoning garbled formulas it reads correctly without (and took 25x the tokens) thinking=os.environ.get("FORMULA_CLOUD_THINKING", "0") == "1", ) - raise ValueError(f"unbekannte Engine {name!r} (local, local2, cloud)") + if name == "p2t": + return Pix2TextEngine(model=model or os.environ.get("FORMULA_P2T_MODEL", DEFAULT_P2T_MODEL), + device=os.environ.get("FORMULA_P2T_DEVICE", "cpu")) + raise ValueError(f"unbekannte Engine {name!r} (local, local2, cloud, p2t)") diff --git a/formula_img2latex/engines/p2t.py b/formula_img2latex/engines/p2t.py new file mode 100644 index 0000000..f543fb6 --- /dev/null +++ b/formula_img2latex/engines/p2t.py @@ -0,0 +1,59 @@ +"""Pix2Text formula OCR (MFR model) as a local candidate generator. + +Not a vision-language model but a dedicated image-to-LaTeX transformer: another kind of model than +the VLM engines, so its errors are likely to be different ones. Runs on the CPU (ONNX), no server. +""" + +from __future__ import annotations + +import io +import threading +import time + +from PIL import Image + +from ..errors import EXIT_TOOL, FormulaError +from .vlm import Recognition, prepare_image + +DEFAULT_MODEL = "mfr-1.5" + + +class Pix2TextEngine: + def __init__(self, model: str = DEFAULT_MODEL, device: str = "cpu") -> None: + self.name = "p2t" + self.model_name = model + self.model = f"pix2text-{model}" + self.device = device + self.thinking = None # part of the cache key in pipeline.recognize; meaningless here + self._ocr = None + self._lock = threading.Lock() # one model instance; inference is not shared between threads + + def check(self) -> None: + """Load the model (downloads it to ~/.pix2text on first use).""" + if self._ocr is not None: + return + try: + from pix2text import LatexOCR + except ImportError as e: + raise FormulaError(f"pix2text ist nicht installiert ({e}): pip install 'pix2text[multilingual]'", + EXIT_TOOL) from e + try: + self._ocr = LatexOCR(model_name=self.model_name, model_backend="onnx", device=self.device) + except Exception as e: # download or model files broken: any exception type is possible here + raise FormulaError(f"Pix2Text-Modell {self.model_name} nicht ladbar ({e})", EXIT_TOOL) from e + + def recognize(self, data: bytes, name: str = "", context: str = "", display: bool = False) -> Recognition: + """Read one formula image. *context* and *display* are not used (the model sees only the image).""" + t0 = time.time() + try: + self.check() + img = Image.open(io.BytesIO(prepare_image(data, name))).convert("RGB") + with self._lock: + # _one_batch instead of recognize(): the latter draws a tqdm bar for every image + out = self._ocr._one_batch([img], None)[0] + tex = self._ocr.post_process(out["text"]).strip() + except Exception as e: # model not loadable or one unreadable image: must not stop the run + return Recognition(self.name, self.model, None, None, "", str(e)[:300], time.time() - t0) + score = float(out.get("score") or 0.0) or None + return Recognition(self.name, self.model, tex or None, None, tex[:2000], None, time.time() - t0, + p_mean=score) diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index 2f59a26..4a032c7 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -185,13 +185,25 @@ def test_canonical_form() -> None: from formula_img2latex.consensus import canonical same = [(r"x^{2}+y^{2}", "x^2 + y^2"), (r"\vec{b}", r"\overrightarrow{b}"), (r"\left(a\right)", "(a)"), (r"\mathrm{d}x", "dx"), (r"\frac{1}{2}", r"\dfrac12"), (r"{\mathrm{e}}^{x}", "e^x"), (r"x\le1", r"x \leq 1"), - (r"\mathbf{v}", r"\boldsymbol{v}")] + (r"\mathbf{v}", r"\boldsymbol{v}"), + # Pix2Text spellings + (r"\operatorname{l i m}_{x \to0} \frac{\operatorname{s i n} x} {x}", r"\lim_{x\to 0}\frac{\sin x}{x}"), + (r"\mathrm{f \ddot{u} r ~ a l l e ~} x", r"\text{für alle } x"), (r"f^{\prime\prime}(x)", "f''(x)"), + (r"\left( \begin{matrix} {1} & {2} \\ \end{matrix} \right)", r"\begin{pmatrix}1&2\end{pmatrix}"), + (r"{\int}_{0}^{\infty} \! e^{-x^{2}} \, d x \!=\! \frac{\sqrt\pi} 2", r"\int_0^\infty e^{-x^2}\,dx=\frac{\sqrt{\pi}}{2}")] differ = [(r"x_{2,k}", r"x_{2;k}"), ("x^{2}", "x_{2}"), (r"\mathbb{R}", "R"), ("a_{n+1}", "a_n+1"), - (r"\frac{a}{b+c}", r"\frac{a}{b}+c"), (r"x \in t", r"x \int"), (r"\sqrt[3]{x}", r"\sqrt{x}")] + (r"\frac{a}{b+c}", r"\frac{a}{b}+c"), (r"x \in t", r"x \int"), (r"\sqrt[3]{x}", r"\sqrt{x}"), + (r"\ddot{u}", "ü"), (r"\mathrm{lin}", r"\lim"), (r"f^{\prime 2}", "f'"), (r"f^{\prime}", "f")] assert all(canonical(a) == canonical(b) for a, b in same) assert all(canonical(a) != canonical(b) for a, b in differ) +def test_p2t_engine_is_lazy() -> None: + from formula_img2latex.engines import make_engine + e = make_engine("p2t") + assert (e.name, e.model, e._ocr) == ("p2t", "pix2text-mfr-1.5", None) # model loads only in check() + + def test_logprob_confidence() -> None: import math