feat: Formeln im Fließtext (pdf2md --formulas), etwa dreimal schneller
- pdf2md-Formelliste lesen: jede Formel mit Seite/Rechteck/PDF-Text, 300-dpi-Bild - eingebettete Formeln: Satzzeichen am Ende abgetrennt, als Text dahinter (final_suffix) - Normalform: \underset-Grenzwerte, Gleichungsnummer als \tag, \textbf, \cfrac, wörtliches \n - Tempo: Verweis- und Ausschnitt-Cache, pix2text gebündelt nach Pixelbudget, Lesen und Prüfen überlappend (forkserver), Qwen 8 Plätze, Render-Vorfilter, kein neuer Scan, je Paket nur dessen Datensätze, Zeiten je Stufe - Bildprüfung: max. 1600 px Breite, 2,5 MP, Proportionen-Abbruch - gemessen und abgeschaltet: PDF-Textebene als Bestätigung (pdftext=0) - Corral 10 %: 79 % automatisch, Pilot 869 Formeln 70 → ~24 min Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
2478bc5cb9
commit
6373db73a4
1 changed files with 479 additions and 112 deletions
|
|
@ -315,11 +315,20 @@ def test_canonical_form() -> None:
|
|||
(r"\begin{aligned}\displaystyle a&\displaystyle=0\,.\\ \displaystyle\end{aligned}", r"a=0."),
|
||||
# pix2text's \cfrac; model readings with literal \n line breaks (JSON escape gone wrong)
|
||||
(r"\cfrac{df}{dx}=\cfrac{df}{du}\cdot\cfrac{du}{dx}", r"\frac{df}{dx}=\frac{df}{du}\cdot\frac{du}{dx}"),
|
||||
(r"\begin{aligned}\n\lim_{x\to0} f &= 1 \\\nk_2 &= 2\n\end{aligned}", r"\lim_{x\to 0}f=1\\k_2=2")]
|
||||
(r"\begin{aligned}\n\lim_{x\to0} f &= 1 \\\nk_2 &= 2\n\end{aligned}", r"\lim_{x\to 0}f=1\\k_2=2"),
|
||||
# pix2text (Corral): limits with \underset, equation number as text, \textbf
|
||||
(r"\underset{x\to-\infty}{\operatorname*{lim}}\ f(x)\,=\,L", r"\lim_{x \to -\infty} f(x) = L"),
|
||||
(r"\underset{\underset{m/n~\mathrm{rational}}{m/n\to\sqrt{2}}}{\operatorname*{lim}}3^{m/n}",
|
||||
r"\lim_{\underset{m/n~\mathrm{rational}}{m/n\to\sqrt{2}}}3^{m/n}"),
|
||||
(r"y\ =\ x^{\prime}\sin\theta\qquad\qquad(7.12)", r"y = x' \sin \theta \tag{7.12}"),
|
||||
(r"A=1\;\;\;\;\;\;(8.1)", r"A=1\tag{8.1}"),
|
||||
(r"A_{v}~=~\frac{A_{o}}{\textbf{1}-\textbf{T}}", r"A_v = \frac{A_o}{1 - T}")]
|
||||
differ += [(r"\begin{pmatrix}1&2\end{pmatrix}", r"\begin{pmatrix}12\end{pmatrix}"), (r"a\\b\\c", r"a\\bc"),
|
||||
(r"f(x)=\begin{cases}1\\0\end{cases}", r"f(x)=\begin{aligned}1\\0\end{aligned}"),
|
||||
(r"x^{2}y", r"x^{2y}"), (r"\frac{a}{b}c", r"\frac{a}{bc}"), (r"\cdot\cdot", r"\dots"),
|
||||
(r"\nu", "u"), (r"a\neq b", "a b"), (r"a\\\nu", r"a\\u")] # real \n commands stay
|
||||
(r"\nu", "u"), (r"a\neq b", "a b"), (r"a\\\nu", r"a\\u"), # real \n commands stay
|
||||
(r"y=f(2)", r"y=f\tag{2}"), (r"y=x\qquad(7.12)", r"y=x"), # number only after a wide gap
|
||||
(r"\underset{x}{a}", "a_x"), (r"\overset{n}{\sum}", r"\sum_n")]
|
||||
assert [p for p in same if canonical(p[0]) != canonical(p[1])] == []
|
||||
assert [p for p in differ if canonical(p[0]) == canonical(p[1])] == []
|
||||
|
||||
|
|
@ -366,6 +375,7 @@ def test_tidy_p2t() -> None:
|
|||
r"{\mathrm{{\bf~ r o t}}} \, v": r"{\mathbf{rot}}\,v",
|
||||
r"\mathrm{\bf~ r o t} \, E": r"\mathbf{rot}\,E",
|
||||
r"\cfrac{d f}{d x}": r"\dfrac{d f}{d x}", # Pandoc has no \cfrac
|
||||
r"y=x\qquad\qquad(7.12)": r"y=x\tag{7.12}", # equation number as text
|
||||
}
|
||||
for raw, tidy in cases.items():
|
||||
assert tidy_p2t(raw) == tidy, (raw, tidy_p2t(raw))
|
||||
|
|
@ -423,3 +433,20 @@ def test_verify_skips_formulas_without_current_reading(epub2md_dir: Path) -> Non
|
|||
{"engine": "local", "model": "A", "tex": None, "is_formula": False, "prompt": 2}] # current: none
|
||||
ws.save(f)
|
||||
pipeline.verify_all(source, ws, preset="cm", workers=1, quiet=True) # must not crash
|
||||
|
||||
|
||||
def test_inline_final_punctuation() -> None:
|
||||
from formula_img2latex.pipeline import _split_final_punct
|
||||
assert _split_final_punct("a^{x},") == ("a^{x}", ",")
|
||||
assert _split_final_punct(r"u=\pi-2x\,.") == (r"u=\pi-2x", ".")
|
||||
assert _split_final_punct("x") == ("x", "")
|
||||
assert _split_final_punct(".") == (".", "") # nothing but punctuation stays as it is
|
||||
|
||||
|
||||
def test_pdf_text_to_tex() -> None:
|
||||
from formula_img2latex.consensus import canonical
|
||||
from formula_img2latex.tex import text_to_tex
|
||||
assert canonical(text_to_tex("R < 1")) == canonical("R<1")
|
||||
assert canonical(text_to_tex("0 ≤ a")) == canonical(r"0\le a")
|
||||
assert canonical(text_to_tex("[1,∞)")) == canonical(r"[1,\infty)")
|
||||
assert text_to_tex("ℝ") is None # no safe spelling: not usable
|
||||
|
|
|
|||
Loading…
Reference in a new issue