From eb72d0b0f2c8a0933af08f6d5daf1660de0df06d Mon Sep 17 00:00:00 2001 From: dschlueter Date: Wed, 7 Oct 2026 03:00:40 +0200 Subject: [PATCH] =?UTF-8?q?feat:=20formula=5Fimg2latex=200.1.0=20=E2=80=93?= =?UTF-8?q?=20Formelbilder=20gepr=C3=BCft=20durch=20LaTeX=20ersetzen?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Nachbearbeitung für epub2md- und pdf2md-Ausgaben: scan → recognize (Vision-LLM lokal/OpenRouter) → verify → review → apply/revert. - Rendern mit pdflatex (Batch, Fehlerisolierung, unsichere Befehle gesperrt), Schrift-Presets cm/times/txfonts/palatino/stix/euler, automatische Wahl je Buch - Bildvergleich in Originalauflösung mit ¼-Pixel-Ausrichtung, Pixel/Punkt-Konsistenz - Kontrasttest gegen layouttreue Verwechslungsvarianten, Selbstkorrektur per Hill-Climbing - Pandoc/MathML-Prüfung vor jeder Annahme - pdf2md: Formelbild per Template-Matching auf der Seite finden, 600-dpi-Neuausschnitt - Review-Galerie auf localhost (Token-geschützt), idempotentes apply mit Sicherung, revert - calibrate: Messung an Formelbildern mit TeX im alt-Text (0/600 Falsch-Annahmen) Co-Authored-By: Claude Opus 5.5 --- .gitignore | 26 ++ BEDIENUNGSANLEITUNG.md | 173 +++++++ Idee/01_Lage.md | 10 + Idee/02_IdeeFormulaImgToLaTeX.md | 649 ++++++++++++++++++++++++++ README.md | 159 +++++++ formula_img2latex.py | 9 + formula_img2latex/__init__.py | 3 + formula_img2latex/__main__.py | 5 + formula_img2latex/calibrate.py | 196 ++++++++ formula_img2latex/checks.py | 46 ++ formula_img2latex/classify.py | 40 ++ formula_img2latex/cli.py | 202 ++++++++ formula_img2latex/compare.py | 326 +++++++++++++ formula_img2latex/confusables.py | 212 +++++++++ formula_img2latex/decide.py | 60 +++ formula_img2latex/engines/__init__.py | 36 ++ formula_img2latex/engines/vlm.py | 174 +++++++ formula_img2latex/errors.py | 15 + formula_img2latex/imageio.py | 91 ++++ formula_img2latex/pdfcrop.py | 81 ++++ formula_img2latex/pipeline.py | 398 ++++++++++++++++ formula_img2latex/render.py | 176 +++++++ formula_img2latex/review/__init__.py | 0 formula_img2latex/review/page.html | 157 +++++++ formula_img2latex/review/server.py | 161 +++++++ formula_img2latex/sources/__init__.py | 24 + formula_img2latex/sources/base.py | 111 +++++ formula_img2latex/sources/epub2md.py | 68 +++ formula_img2latex/sources/pdf2md.py | 67 +++ formula_img2latex/store.py | 168 +++++++ formula_img2latex/tex.py | 131 ++++++ formula_img2latex/verify.py | 92 ++++ pytest.ini | 3 + requirements-dev.txt | 2 + requirements.txt | 7 + tests/conftest.py | 68 +++ tests/test_core.py | 109 +++++ tests/test_pipeline.py | 108 +++++ 38 files changed, 4363 insertions(+) create mode 100644 .gitignore create mode 100644 BEDIENUNGSANLEITUNG.md create mode 100644 Idee/01_Lage.md create mode 100644 Idee/02_IdeeFormulaImgToLaTeX.md create mode 100644 README.md create mode 100755 formula_img2latex.py create mode 100644 formula_img2latex/__init__.py create mode 100644 formula_img2latex/__main__.py create mode 100644 formula_img2latex/calibrate.py create mode 100644 formula_img2latex/checks.py create mode 100644 formula_img2latex/classify.py create mode 100644 formula_img2latex/cli.py create mode 100644 formula_img2latex/compare.py create mode 100644 formula_img2latex/confusables.py create mode 100644 formula_img2latex/decide.py create mode 100644 formula_img2latex/engines/__init__.py create mode 100644 formula_img2latex/engines/vlm.py create mode 100644 formula_img2latex/errors.py create mode 100644 formula_img2latex/imageio.py create mode 100644 formula_img2latex/pdfcrop.py create mode 100644 formula_img2latex/pipeline.py create mode 100644 formula_img2latex/render.py create mode 100644 formula_img2latex/review/__init__.py create mode 100644 formula_img2latex/review/page.html create mode 100644 formula_img2latex/review/server.py create mode 100644 formula_img2latex/sources/__init__.py create mode 100644 formula_img2latex/sources/base.py create mode 100644 formula_img2latex/sources/epub2md.py create mode 100644 formula_img2latex/sources/pdf2md.py create mode 100644 formula_img2latex/store.py create mode 100644 formula_img2latex/tex.py create mode 100644 formula_img2latex/verify.py create mode 100644 pytest.ini create mode 100644 requirements-dev.txt create mode 100644 requirements.txt create mode 100644 tests/conftest.py create mode 100644 tests/test_core.py create mode 100644 tests/test_pipeline.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..352c1ac --- /dev/null +++ b/.gitignore @@ -0,0 +1,26 @@ +# Python +__pycache__/ +*.py[cod] +*.egg-info/ +.venv/ +venv/ +.pytest_cache/ +.mypy_cache/ +.ruff_cache/ +.coverage +htmlcov/ +build/ +dist/ + +# Urheberrechtlich geschützte Beispiel-Bücher und Arbeitsdaten +Beipiele/ +Beispiele/ +output/ +*.formulas/ +.calibration/ + +# Editor/OS +.idea/ +.vscode/ +*.swp +.DS_Store diff --git a/BEDIENUNGSANLEITUNG.md b/BEDIENUNGSANLEITUNG.md new file mode 100644 index 0000000..2d36903 --- /dev/null +++ b/BEDIENUNGSANLEITUNG.md @@ -0,0 +1,173 @@ +# Bedienungsanleitung formula_img2latex + +Dieses Programm ersetzt Formelbilder in den Markdown-Dateien von epub2md und pdf2md durch echte +LaTeX-Formeln. Dadurch passen die Formeln beim Neubau als EPUB oder PDF in Größe und Schrift zum Text. + +Es ersetzt eine Formel nur, wenn das gerenderte LaTeX genauso aussieht wie das Bild. Bei Zweifeln fragt +es dich in einer Review-Seite im Browser. Ohne deine Zustimmung bleibt die Formel ein Bild. + +--- + +## 1. Einrichtung (einmalig) + +```bash +sudo apt install texlive-latex-extra texlive-fonts-extra texlive-science pandoc +cd ~/Python_Programs/formula_img2latex +pip install -r requirements.txt +python3 formula_img2latex.py --version +``` + +### Lokales Bildmodell starten + +Die Formelerkennung macht ein Vision-Sprachmodell. Lokal läuft es mit llama.cpp auf einer Grafikkarte: + +```bash +M=~/endeavour_home/lm_studio/models/lmstudio-community/Qwen3-VL-30B-A3B-Instruct-GGUF +CUDA_VISIBLE_DEVICES=1 ~/.local/bin/llama-server \ + -m $M/Qwen3-VL-30B-A3B-Instruct-Q4_K_M.gguf --mmproj $M/mmproj-Qwen3-VL-30B-A3B-Instruct-F16.gguf \ + --host 127.0.0.1 --port 8011 -ngl 99 -c 16384 --parallel 4 +``` + +Der Server belegt etwa 20 GB auf GPU 1. Beenden mit Strg+C. + +### Cloud-Stufe (optional) + +Formeln, die lokal nicht sicher erkannt werden, kann ein Cloud-Modell über OpenRouter versuchen. Dafür muss +`OPENROUTER_API_KEY` gesetzt sein. Ein Buch mit 2000 Formeln kostet mit dem Standardmodell +`google/gemini-2.5-flash` deutlich unter 1 $. Ohne Cloud: `--no-cloud`. + +--- + +## 2. Ein Buch bearbeiten + +### Schritt 1: Buch wie gewohnt umwandeln + +```bash +cd ~/Python_Programs/epub2md +python3 epub2md.py buch.epub output/buch +``` + +Bei PDFs mit pdf2md immer `--images` (oder `--epub`) angeben, sonst gibt es keine Formelbilder. + +### Schritt 2: Formeln erkennen und prüfen + +```bash +cd ~/Python_Programs/formula_img2latex +python3 formula_img2latex.py run ~/Python_Programs/epub2md/output/buch +``` + +Bei pdf2md zusätzlich die Original-PDF angeben. Dann schneidet das Programm jede Formel scharf aus dem PDF: + +```bash +python3 formula_img2latex.py run Output/buch.md --pdf buch.pdf +``` + +Am Ende steht eine Übersicht, zum Beispiel: + +```text + auto_accepted 412 ← sicher, wird ersetzt + needs_review 380 ← bitte im Browser ansehen + rejected_keep_image 95 ← passt nicht, bleibt Bild + not_formula 117 ← Abbildung/Symbol +``` + +Neben dem Buchordner entsteht ein Arbeitsordner `buch.formulas/`. Er enthält alle Ergebnisse. Lösch ihn nicht, +solange du am Buch arbeitest. + +### Schritt 3: Zweifelsfälle im Browser entscheiden + +```bash +python3 formula_img2latex.py review ~/Python_Programs/epub2md/output/buch +``` + +Der Browser öffnet `http://127.0.0.1:8765/`. Für jede Formel siehst du: + +- **Original:** das Bild aus dem Buch. +- **Overlay:** Original und LaTeX übereinander. Mit dem Regler blendest du zwischen beiden. Springt etwas, + stimmt etwas nicht (Exponent, Vorzeichen, Klammer). +- **Diff:** schwarz = beide gleich, rot = nur im Original, blau = nur im LaTeX, gelb = deutliche Abweichung. +- **LaTeX:** Text im Eingabefeld, darunter eventuell weitere Kandidaten zum Anklicken. + +Schaltflächen: + +| Schaltfläche | Wirkung | +| --- | --- | +| LaTeX prüfen | rendert dein (korrigiertes) LaTeX neu und zeigt Overlay und Messwerte | +| LaTeX übernehmen | die Formel wird beim nächsten `apply` ersetzt | +| Bild behalten | die Formel bleibt ein Bild | +| Keine Formel | das Bild ist eine Abbildung oder ein Symbol | +| Zurücksetzen | Entscheidung zurücknehmen | + +Jede Entscheidung wird sofort gespeichert. Du kannst jederzeit aufhören (Strg+C im Terminal) und später +weitermachen. `--all` zeigt auch die automatisch angenommenen Formeln. + +### Schritt 4: Formeln ins Markdown schreiben + +```bash +python3 formula_img2latex.py apply ~/Python_Programs/epub2md/output/buch --dry-run # nur zählen +python3 formula_img2latex.py apply ~/Python_Programs/epub2md/output/buch +``` + +Aus `![…](../media/images/0042-…jpg)` wird `$L = \{-\sqrt{2};\sqrt{2}\}$` im Text bzw. `$$…$$` als eigener +Absatz. Die Bilddateien bleiben liegen. + +**Danach die Gesamtdatei neu bauen**, sie enthält sonst noch die Bilder: + +```bash +cd ~/Python_Programs/epub2md +python3 epub2md_merge.py output/buch --force +``` + +Für EPUB und PDF mit Formeln gelten die Pandoc-Befehle aus der epub2md-Anleitung mit `--mathml`. + +### Rückgängig machen + +```bash +python3 formula_img2latex.py revert ~/Python_Programs/epub2md/output/buch +``` + +Das stellt die Kapiteldateien wie vor dem ersten `apply` wieder her. Hast du eine Datei seitdem von Hand +geändert, wird sie übersprungen. `--force` setzt sie trotzdem zurück, dann ist die Handänderung weg. + +--- + +## 3. Einzelne Schritte statt `run` + +```bash +python3 formula_img2latex.py scan AUSGABE # Bilder finden +python3 formula_img2latex.py recognize AUSGABE --engine local # lokal erkennen +python3 formula_img2latex.py verify AUSGABE # prüfen und entscheiden +python3 formula_img2latex.py recognize AUSGABE --engine cloud --only-unresolved +python3 formula_img2latex.py verify AUSGABE +python3 formula_img2latex.py status AUSGABE +``` + +Nützliche Optionen: + +| Option | Wirkung | +| --- | --- | +| `--limit 50` | nur die ersten 50 Bilder (zum Ausprobieren) | +| `--preset cm` | Schrift fest vorgeben (`cm`, `times`, `txfonts`, `palatino`, `stix`, `euler`); Standard: automatisch | +| `--include-figures` | auch Bilder versuchen, die wie Abbildungen aussehen | +| `--model NAME` | anderes Modell (lokal: Servername, Cloud: OpenRouter-ID) | +| `--workspace ORDNER` | Arbeitsordner an anderem Ort | +| `--redo` (verify) | bereits geprüfte Formeln neu prüfen, z. B. nach einem Programm-Update | + +--- + +## 4. Häufige Fragen + +**Das Buch wurde mit epub2md `--force` neu umgewandelt. Ist meine Arbeit weg?** +Nein. Die Entscheidungen hängen am Inhalt der Bilder, nicht an Dateinamen. `scan`, dann `apply`. + +**Warum landet so viel im Review?** +Das Programm ersetzt nur, was es beweisen kann. Häufige Gründe: Das Bild ist sehr klein (Ziffern unter +8 Pixeln). Das Buch wurde mit einem anderen Satzsystem gesetzt. Oder die Erkennung schreibt eine Formel +typografisch anders (`\vec` statt langem Pfeil). Im Review genügt dann meist ein Blick auf das Overlay. + +**Geht die blaue Hinterlegung mancher Formeln verloren?** +Ja. Das Bild hatte die Farbe eingebaut, LaTeX-Formeln haben keinen Hintergrund. + +**Werden Buchinhalte ins Internet geschickt?** +Nur mit der Cloud-Stufe, und nur die einzelnen Formelbilder samt eines kurzen Textausschnitts davor. Mit +`--no-cloud` bleibt alles lokal. diff --git a/Idee/01_Lage.md b/Idee/01_Lage.md new file mode 100644 index 0000000..b7d28d4 --- /dev/null +++ b/Idee/01_Lage.md @@ -0,0 +1,10 @@ +# Wie gehe ich mit mathematischen Formeln in übersetzten Büchern um, die nicht als LaTeX-Formeln vorliegen? + +Ich habe zwei Tools geschrieben, die EPUB-Bücher und PDF-Bücher kapitelweise in MD-Dateien übersetzen. + +Du findest die Repositories lokal unter unter /home/dschlueter/Python_Programs/epub2md und /home/dschlueter/Python_Programs/pdf2md , remote unter https://kitux.de/forgejo/dschlueter/epub2md beziehungsweise https://kitux.de/forgejo/dschlueter/pdf2md . + +Wie erzeuge ich aus den Bildern (PNG, JPG, SVG etc.) der Formeln zuverlässig korrekte LaTeX-Formeln und setze sie in so in die MD-Dateien ein, dass sie die Bilder der Formeln ersetze? + + + diff --git a/Idee/02_IdeeFormulaImgToLaTeX.md b/Idee/02_IdeeFormulaImgToLaTeX.md new file mode 100644 index 0000000..d3ac976 --- /dev/null +++ b/Idee/02_IdeeFormulaImgToLaTeX.md @@ -0,0 +1,649 @@ + + +# Ist bei Mathe-Lehrbüchern die Wahrscheinlichkeit höher, dass die LaTeX-Übersetzung der enthaltenen Formeln eher in der EPUB-Ausgabe oder in der PDF-Ausgabe enthalten sind, größer? + +Eher in der **EPUB-Ausgabe** – allerdings mit einer wichtigen Einschränkung: Dort ist es technisch deutlich wahrscheinlicher, dass Formeln als strukturierte Daten vorliegen, typischerweise MathML und gelegentlich zusätzlich TeX/LaTeX als Alternativ- oder Quelldarstellung. In einer gewöhnlichen PDF sind Formeln meist nur als positionierte Glyphen oder Vektoren gespeichert, nicht als rekonstruierbarer LaTeX-Quelltext.[^1_1][^1_2] + +## Warum EPUB eher geeignet ist + +EPUB 3 basiert im Kern auf Web-Technologien wie XHTML, CSS und XML. Für Mathematik ist MathML vorgesehen. Ein Verlag kann daher pro Formel unter anderem speichern: + +- sichtbare Formel als MathML, +- Textalternative für Screenreader, +- ggf. semantische Annotationen, +- manchmal den ursprünglichen LaTeX-Code, etwa in einem `annotation`-Element. + +Ein sauber produziertes, zugängliches EPUB kann mathematische Formeln also wesentlich maschinenlesbarer machen. Aktuelle Empfehlungen zur EPUB-Produktion nennen MathML als Quelldatenformat; daraus können Darstellung und zugängliche Textrepräsentationen abgeleitet werden.[^1_3] + +Beispielhaft könnte eine Formel intern ungefähr so aussehen: + +```xml + + + + x2 + +y2 + =r2 + + + x^2 + y^2 = r^2 + + + +``` + +Der `annotation`-Teil wäre dann tatsächlich ein extrahierbarer LaTeX-/TeX-String. Seine Existenz ist aber nicht garantiert. + +## Warum PDF meist schlechter ist + +Ein PDF ist primär ein festes Seitenlayout. Selbst wenn es ursprünglich mit LaTeX erzeugt wurde, enthält das End-PDF in der Regel nicht mehr die ursprünglichen Eingaben wie `\frac{a}{b}` oder `\int_0^1`. Stattdessen enthält es Anweisungen wie: zeichne dieses Zeichen mit dieser Schrift an dieser Position. + +Das führt zu mehreren Problemen: + +- Kopierter Formeltext ist oft unvollständig oder in falscher Reihenfolge. +- Brüche, Indizes, Matrizen und mehrzeilige Gleichungen verlieren ihre Struktur. +- Formeln können als Vektorgrafik, Bild oder Kombination aus Text und Grafik vorliegen. +- Selbst „getaggte“ PDFs enthalten häufig nur begrenzte mathematische Semantik; TeX-Quellcode ist selten eingebettet. + +Bei kommerziellen PDF-E-Books werden Formeln häufig nicht einmal als passende Formel-Tags oder strukturierte Inhalte bereitgestellt.[^1_2][^1_4] + +## Die praktische Einschränkung + +„EPUB ist besser geeignet“ bedeutet nicht automatisch: „Das konkrete EPUB enthält LaTeX.“ + +Viele Verlage nutzen leider weiterhin: + +- Formelbilder, etwa SVG oder PNG, +- gerenderte Formelgrafiken ohne aussagekräftigen Alternativtext, +- MathML ohne eingebettete TeX-Annotation, +- proprietäre Konvertierungspipelines, die Struktur beim Export verlieren. + +Der EPUB-Standard unterstützt MathML zwar, doch Reader-Unterstützung ist uneinheitlich. Deshalb empfehlen Branchenleitfäden teils weiterhin, Formeln visuell als Bild einzubinden und parallel aus MathML abgeleitete Textalternativen zu hinterlegen.[^1_3] + +## Faustregel + +| Was du suchst | Wahrscheinlicheres Format | Begründung | +| :-- | :-- | :-- | +| Originaler LaTeX-Quelltext | Weder PDF noch EPUB zuverlässig | Meist nur beim Autor, Verlag oder in einem Open-Source-Repository erhältlich | +| Maschinenlesbare Formelstruktur | EPUB | EPUB kann MathML und semantische Metadaten enthalten | +| Visuell originalgetreue Satzseite | PDF | PDF konserviert das gedruckte Layout | +| Extraktion für eigene Verarbeitung | EPUB, aber prüfen | XHTML/MathML ist oft besser zugänglich als PDF-Glyphen | +| LaTeX aus gekauften Lehrbüchern | Selten in beiden | Eine eingebettete TeX-Annotation ist eher Ausnahme als Regel | + +Wenn dein Ziel die Weiterverarbeitung von Formeln ist, ist die sinnvollste Reihenfolge meist: + +1. EPUB entpacken – eine `.epub` ist technisch ein ZIP-Archiv. +2. Nach `.xhtml`, `.html`, `.svg`, `[^1_10][^1_5][^1_6][^1_7][^1_8][^1_9] + +
⁂
+ +[^1_1]: https://ethz.ch/staffnet/de/lehre/accessibility-and-inclusion-in-teaching/accessible-learning-materials/accessible-documents---presentations/latex.html + +[^1_2]: https://www.anleitungen.rrze.fau.de/files/2019/11/Mathematische-Formeln-erstellen-und-vorlesen.pdf + +[^1_3]: https://www.boersenverein.de/beratung-service/barrierefreiheit/leitfaden-barrierefreie-epub3-e-books-seite-2/ + +[^1_4]: https://www.anleitungen.rrze.fau.de/files/2019/08/Mathematische-Formeln-erstellen-und-vorlesen-18a.pdf + +[^1_5]: https://www.anleitungen.rrze.fau.de/files/2019/11/E-Book-Beispiel-Deep-Learning-Technische-Eigenschaften.pdf + +[^1_6]: https://wr.informatik.uni-hamburg.de/\_media/teaching/sommersemester_2021/siw-21latex_und_bibtex_stahl_ausarbeitung-neu.pdf + +[^1_7]: https://www.uni-frankfurt.de/69481937/handout.pdf + +[^1_8]: https://www.stochastik.uni-freiburg.de/de/professoren/peter-pfaffelhuber/inhalte/schreiben_math + +[^1_9]: https://pressbooks.pub/ebooksgrundlagen/chapter/e-books-formate/ + +[^1_10]: https://archiv.dante.de/DTK/PDF/komoedie_2015_4.pdf + + +--- + +# Ich habe jetzt Tools gebaut, die kapitelweise EPUB-Bücher in MDs übersetzen und PDF-Bücher in MDs übersetzen. Wie kann ich automatisieren, dass Formeln, wenn sie noch nicht in LaTeX-Schreibweise vorliegen (sondern als BIld oder SVG etc.) in LaTeX-Schreibweise konvertiert werden. Aber ACHTUNG: Genaue Kontrolle, ob die Formel genauso gerendert wird, wie sie im Original steht! Begründung: So bekommen die Formeln beim erneuten Erstellen der Bücher als MD, EPUB oder PDF eine zum Text passende Größe und das Layout sieht viel besser aus. + +Ja: Baue dafür keine einfache „Formelbild → LaTeX“-Ersetzung, sondern eine **mehrstufige, revisionssichere Pipeline**. Die automatische Erkennung erzeugt nur einen Kandidaten; erst eine visuelle und strukturelle Gegenprüfung entscheidet, ob das Bild durch LaTeX ersetzt werden darf. + +Das passt besonders gut zu deinem kapitelweisen EPUB-/PDF-nach-Markdown-Workflow: Du kannst jede Formel mit Herkunft, OCR-Ergebnis, Rendervergleich und Review-Status versionieren. So bleiben fehlerhafte Kandidaten als Originalbild erhalten und werden nicht stillschweigend in fachlich falsches LaTeX verwandelt. + +## Zielarchitektur + +Trenne strikt zwischen vier Dingen: + +1. **Original-Asset:** Die exakt ausgeschnittene Formelgrafik bzw. der Formelbereich aus der PDF. +2. **Kandidat:** vom OCR-Modell erzeugter LaTeX-String plus Metadaten. +3. **Prüfrendering:** Aus dem Kandidaten mit einer festgelegten Rendering-Engine erzeugtes SVG/PNG. +4. **Entscheidung:** automatischer Vergleich und bei Unsicherheit menschliche Freigabe. + +Die Kernregel sollte sein: + +> Ersetze eine Originalformel nur, wenn der LaTeX-Kandidat syntaktisch kompiliert, seine gerenderte Darstellung hinreichend ähnlich ist und kein Regelwerk einen semantischen Risikofall erkennt. + +Nicht bestandene oder zweifelhafte Fälle bleiben im Markdown als Bild, zusammen mit einer maschinenlesbaren Markierung. Dadurch erzeugst du zwar zunächst nicht hundertprozentig „reines LaTeX-Markdown“, aber du vermeidest den viel gravierenderen Fehler, mathematische Aussagen unbemerkt zu verfälschen. + +## Extraktionsreihenfolge + +Bevor du irgendein OCR-Modell auf ein Bild loslässt, solltest du die Formelquelle in dieser Reihenfolge prüfen. Das spart Kosten, erhöht Genauigkeit und liefert häufig die bessere Semantik. + + +| Priorität | Quelle im Buch | Vorgehen | Vertrauen | +| :-- | :-- | :-- | :-- | +| 1 | Vorhandenes MathML im EPUB | MathML direkt übernehmen oder nach LaTeX konvertieren | Sehr hoch | +| 2 | TeX-Annotation im MathML/SVG | Nach `application/x-tex`, `annotation`, ``, `<desc>` und Metadaten suchen | Sehr hoch | +| 3 | Textbasierte PDF-Formel | PDF-Textobjekte und Tags extrahieren; nur bei sauberer Reihenfolge verwenden | Mittel bis hoch | +| 4 | SVG mit echten Textknoten | `<text>`-, `<tspan>`- und MathJax-/KaTeX-Struktur analysieren | Mittel | +| 5 | SVG-Pfade oder Rasterbild | Formel beschneiden und Math-OCR verwenden | Variabel | +| 6 | Bild mit schlechter Qualität | Hochskalieren, entzerren, ggf. manuell nachbearbeiten | Niedrig | + +EPUB 3 kann mathematische Inhalte als MathML enthalten. Bei sauber produzierten Büchern ist das wertvoller als eine optische Rückerkennung. MathML kann zusätzlich TeX als Annotation führen, beispielsweise in einem Element mit `encoding="application/x-tex"`. Für solche Fälle gilt: extrahiere den vorhandenen Inhalt; mache keine OCR. Die EPUB-Empfehlungen verlangen für Formeln MathML statt bloßer Bildformeln, auch wenn die Verlagsrealität leider oft schlechter ist.[^2_1][^2_2] + +## Formelquellen erkennen + +Für deine EPUB-Pipeline würde ich pro XHTML-Datei zuerst eine Klassifikation durchführen: + +```python +FORMULA_HINTS = ( + "math", + "mathml", + "equation", + "formula", + "latex", + "tex", + "mathjax", + "katex", +) + +def classify_formula_node(node) -> str | None: + tag = getattr(node, "name", "") or "" + classes = " ".join(node.get("class", [])) + node_id = node.get("id", "") + probe = f"{tag} {classes} {node_id}".lower() + + if tag.lower() == "math" or "mathml" in probe: + return "mathml" + if tag.lower() == "svg" or node.find("svg"): + return "svg" + if tag.lower() == "img" and any(hint in probe for hint in FORMULA_HINTS): + return "image" + return None +``` + +Für jedes gefundene Objekt solltest du eine eindeutige Formel-ID erzeugen. Sie muss stabil sein, damit du Korrekturen oder Freigaben bei einem erneuten Lauf wieder zuordnen kannst: + +```text +book-id/ + chapter-003/ + formulas/ + f-003-001-original.svg + f-003-001-source.png + f-003-001-candidate.tex + f-003-001-rendered.svg + f-003-001-rendered.png + f-003-001-diff.png + f-003-001.json +``` + +Eine mögliche ID wäre beispielsweise: + +```text +f-003-001 +``` + +oder, wenn du Stabilität bei erneuten Konvertierungen brauchst: + +```text +f-003-001-8b7f2e1c +``` + +Der Hash sollte sich aus Kapitelpfad, Elementposition und einem Hash des Originalassets zusammensetzen. + +## LaTeX-OCR einsetzen + +Für gedruckte, komplexe Lehrbuchformeln ist Mathpix als Qualitätsreferenz sinnvoll: Die API akzeptiert Bilder sowie ganze PDF-, EPUB- und Dokumentdateien und kann LaTeX, MathML, Markdown und strukturierte Zeilendaten liefern. Die zeilenbasierten Ergebnisse enthalten Formel-/Texttyp, Bounding Boxes, LaTeX und Konfidenzwerte.[^2_3][^2_4] + +Als lokale und datenschutzfreundliche Alternative kannst du pix2tex/LaTeX-OCR einsetzen. Das Projekt verwendet ein Vision-Transformer-Modell, das Formelfotos bzw. -bilder in LaTeX übersetzt; es lässt sich als Kommandozeilenprogramm, Python-Bibliothek oder Docker/HTTP-API betreiben. Für deinen Server-Workflow wäre die Python- oder Docker-Variante passend.[^2_5][^2_6] + +Ich würde nicht „Mathpix oder pix2tex“ wählen, sondern beide als abgestufte Kandidatengeneratoren verwenden: + + +| Stufe | Werkzeug | Einsatz | +| :-- | :-- | :-- | +| Lokal, schnell, kostenlos | pix2tex | Erste Erkennung bei klaren Rasterformeln | +| Cloud-Referenz, hohe Qualität | Mathpix | Komplexe, mehrzeilige oder lokal unsichere Formeln | +| Deterministisch | vorhandenes MathML / TeX | Immer bevorzugen | +| Menschliche Bearbeitung | lokales Review-UI | Alle Grenzfälle und unklare Unterschiede | + +Mathpix kann durch `confidence_threshold` und `confidence_rate_threshold` Fehler bei niedriger mathematischer Konfidenz signalisieren. Die Konfidenz ist nützlich für die Priorisierung, aber keine Freigabe: Ein optisch plausibles OCR-Ergebnis kann mathematisch dennoch falsch sein.[^2_7][^2_8] + +Ein praktikables Entscheidungsraster könnte sein: + +```text +MathML/TeX vorhanden? + Ja → extrahieren, rendern, optional vergleichen + Nein → pix2tex OCR + +pix2tex kompiliert nicht? + Ja → Mathpix versuchen oder manueller Review + Nein → rendern und Bildvergleich + +Bildvergleich sehr gut + OCR-Konfidenz hoch + keine Risikomuster? + Ja → automatisch übernehmen + Nein → Review-Warteschlange + +Mensch bestätigt? + Ja → LaTeX übernehmen + Nein → Originalbild beibehalten +``` + + +## Rendervergleich als Pflichtprüfung + +Die wichtigste Kontrollstufe ist nicht „LaTeX ist kompilierbar“, sondern: + +> Originalformel und gerenderter LaTeX-Kandidat müssen bei gleicher Geometrie optisch übereinstimmen. + +Hierfür brauchst du eine standardisierte Renderumgebung. Entscheide dich pro Projekt für genau eine Rendering-Engine, beispielsweise: + +- LaTeX → PDF/SVG mit LuaLaTeX oder XeLaTeX und `dvisvgm`, +- alternativ LaTeX → SVG mit MathJax, +- anschließend SVG/PDF → PNG mit fixer Auflösung und denselben Renderparametern. + +MathJax kann TeX direkt nach SVG transformieren; SVG ist vorteilhaft, weil du es in EPUB und HTML wiederverwenden und verlustarm in PNG für den Vergleich rasterisieren kannst.[^2_9][^2_10] + +Der entscheidende Punkt: Ein Bildvergleich funktioniert erst sinnvoll, wenn du vorab normalisierst: + +1. Transparente oder weiße Ränder an beiden Bildern entfernen. +2. Beide Bilder in Graustufen konvertieren. +3. Invertieren, falls das Original helle Formel auf dunklem Hintergrund enthält. +4. Baseline, Höhe und Breite möglichst anhand sichtbarer Zeichen ausrichten. +5. Renderbild skalieren und geringfügig verschieben, bis es die beste Überlappung erreicht. +6. Dann mindestens zwei Metriken berechnen: Formähnlichkeit und Pixeldifferenz. +7. Bei Abweichung eine Heatmap bzw. Diff-PNG erzeugen. + +ImageMagick kann Differenzbilder erzeugen und Metriken wie RMSE, NCC, SSIM und DSSIM berechnen. Bei NCC bedeutet 1 vollständige Übereinstimmung; SSIM misst strukturelle Ähnlichkeit. Das ist für leicht abweichendes Antialiasing und unterschiedliche Schrift-Renderings besser als ein strikter Pixelvergleich.[^2_11][^2_12][^2_13] + +Ein Beispiel mit ImageMagick 7: + +```bash +mkdir -p normalized review + +magick original.png \ + -alpha off -background white -flatten \ + -colorspace Gray -trim +repage \ + normalized/original.png + +magick rendered.png \ + -alpha off -background white -flatten \ + -colorspace Gray -trim +repage \ + normalized/rendered.png + +magick compare \ + -metric SSIM \ + normalized/original.png \ + normalized/rendered.png \ + review/f-003-001-diff.png +``` + +In der Praxis sind die Bilder vor diesem Vergleich meist unterschiedlich groß. Deshalb solltest du nicht schlicht beide auf dieselbe Pixelgröße strecken: Das würde geometrische Fehler verschleiern. Besser: + +- Skaliere den Renderkandidaten nur proportional. +- Suche innerhalb enger Grenzen nach optimalem Skalierungsfaktor und Versatz. +- Speichere die gefundene Transformation als Metadaten. +- Bewerte Skalierung außerhalb eines erlaubten Bereichs als Warnsignal. + +Beispielsweise: + +```text +erlaubte Skalierung: 0,85 bis 1,15 +erlaubter horizontaler Versatz: ±12 Pixel +erlaubter vertikaler Versatz: ±12 Pixel +SSIM für automatische Annahme: mindestens 0,992 +SSIM für Review: 0,960 bis unter 0,992 +SSIM unter 0,960: ablehnen oder zwingender Review +``` + +Diese Werte sind Startwerte, keine universellen Wahrheiten. Du solltest sie an einem manuell geprüften Korpus aus deinen Büchern kalibrieren: etwa 100 bis 300 Formeln, aufgeteilt nach Inline-Formeln, Gleichungen, Matrizen, Summen, Integralen, Chemie, Tabellenformeln und Diagrammnotationen. + +## Semantische Risiken erkennen + +Ein hoher SSIM-Wert allein reicht nicht. Manche Fehler sind optisch klein, mathematisch aber gravierend. Daher sollten bestimmte Muster immer in den Review-Queue gehen: + +- Hoch- und Tiefstellungen, insbesondere `x^2` gegenüber `x_2`. +- Minus `-`, Gedankenstrich `–`, Bindestrich `-` und Negationszeichen. +- Punkt `.` gegenüber Multiplikationspunkt `\cdot`. +- `1`, `l`, `I`, `|` und `\ell`. +- `0`, `O`, `o`. +- `\nu`, `v`, `\upsilon`. +- `\rho`, `p`, `\varrho`. +- `\epsilon`, `\varepsilon`. +- `\phi`, `\varphi`. +- `\le`, `<`, `\leq`, `\ge`, `>`. +- `\in`, `\notin`, `\subset`, `\subseteq`. +- Vektorpfeile, Hüte, Überstriche und Tilden. +- Grenzen von Summen, Produkten und Integralen. +- Matrixklammern, Fälle, Ausrichtung und Gleichungssysteme. +- Gleichungsnummern und Referenzlabels. +- Text in Formeln, etwa `\text{für }`, Einheiten, Variablenbedingungen und Indizes. + +Du kannst den Kandidaten per LaTeX-Parser oder notfalls per Heuristik mit Risikopunkten versehen: + +```python +RISK_TOKENS = ( + r"\frac", r"\sqrt", r"\sum", r"\prod", r"\int", + r"\begin{matrix}", r"\begin{pmatrix}", r"\begin{cases}", + r"^", r"_", r"\overline", r"\vec", r"\hat", + r"\not", r"\le", r"\ge", r"\subset", r"\in", +) + +def risk_score(latex: str) -> int: + score = 0 + for token in RISK_TOKENS: + score += latex.count(token) + if "\n" in latex or r"\\" in latex: + score += 2 + return score +``` + +Das ist keine semantische Validierung. Es sorgt lediglich dafür, dass komplexe Formeln nicht wegen eines guten Bildvergleichs automatisch durchgewinkt werden. + +## Robustes Manifest + +Lege zu jeder Formel ein JSON-Manifest ab. Es ist der Schlüssel zu Reproduzierbarkeit, Git-Diffs, Review und späteren Verbesserungen deiner OCR-Pipeline. + +```json +{ + "formula_id": "f-003-001-8b7f2e1c", + "book_id": "operations-research-01", + "chapter_id": "003", + "source": { + "format": "epub", + "xhtml_file": "OEBPS/chapter03.xhtml", + "element_xpath": "/html/body/div[^2_4]/p[^2_7]/img[^2_1]", + "page": null, + "original_asset": "formulas/f-003-001-original.svg", + "source_sha256": "..." + }, + "extraction": { + "method": "pix2tex", + "model_version": "pix2tex-v1", + "latex_raw": "\\sum_{i=1}^{n} x_i", + "latex_normalized": "\\sum_{i=1}^{n} x_i", + "ocr_confidence": null, + "created_at": "2026-10-06T22:33:00+02:00" + }, + "validation": { + "latex_compiles": true, + "renderer": "MathJax-SVG", + "normalization": { + "scale": 1.018, + "offset_x_px": 2, + "offset_y_px": -1 + }, + "ssim": 0.9961, + "rmse": 0.023, + "diff_image": "formulas/f-003-001-diff.png", + "risk_score": 3 + }, + "decision": { + "status": "needs_review", + "reason": "complex_operator", + "reviewer": null, + "reviewed_at": null + } +} +``` + +Mögliche Statuswerte: + +```text +source_tex +auto_accepted +needs_review +accepted_by_human +rejected_keep_image +corrected_by_human +render_failed +ocr_failed +``` + +Der Markdown-Konverter setzt abhängig davon entweder LaTeX oder ein Bild ein: + +```markdown +<!-- formula-id: f-003-001-8b7f2e1c | status: accepted_by_human --> +\[ +\sum_{i=1}^{n} x_i +\] +``` + +Oder, bewusst ohne stillschweigende Ersetzung: + +```markdown +<!-- formula-id: f-003-001-8b7f2e1c | status: rejected_keep_image --> +![Formel f-003-001](assets/formulas/f-003-001-source.png) +``` + + +## Review-Seite erzeugen + +Für eine schnelle Prüfung ist eine statische HTML-Review-Galerie ideal. Pro Formel zeigst du nebeneinander: + +- Originalbild oder Original-SVG, +- gerenderte LaTeX-Fassung, +- transparentes Overlay, +- Differenzbild, +- den LaTeX-Quelltext, +- Erkennungsmethode und Modellversion, +- OCR-Konfidenz, +- SSIM/RMSE, +- Risikoscore, +- Link zur Kapitel-MD-Datei beziehungsweise PDF-Seite, +- Schaltflächen bzw. Statusdatei für „annehmen“, „LaTeX korrigieren“, „Bild behalten“. + +Ein besonders gutes Prüfmittel ist ein Umschalt-Overlay: + +```text +Original ↔ Render-Kandidat ↔ Überlagerung ↔ Differenz +``` + +Bei Überlagerung mit etwa 50 Prozent Transparenz springen falsche Exponenten, verschobene Integrationsgrenzen, fehlende Punkte und vertauschte Klammern sofort ins Auge. Bei komplexen Matrizengleichungen ist das wesentlich zuverlässiger als eine reine Zahl. + +Für lokale Nutzung kannst du das als rein statische HTML-Seite plus JavaScript bauen. Die Entscheidung speicherst du anschließend etwa in `review_decisions.json` oder direkt im jeweiligen Formelmanifest. So brauchst du weder Cloud noch Datenbank und kannst den Stand gut mit Git versionieren. + +## Empfohlene Automationslogik + +Ich würde mit folgenden konservativen Regeln anfangen: + +```text +1. Vorhandenes TeX/MathML: + automatisch übernehmen; + dennoch rendern und protokollieren. + +2. SVG mit eindeutiger MathJax-/KaTeX-Struktur: + strukturell nach TeX/MathML extrahieren; + nicht optisch per OCR rekonstruieren. + +3. Klar isoliertes Formelbild: + pix2tex ausführen; + wenn Ergebnis gültig kompiliert, rendern und vergleichen. + +4. Pix2tex unsicher, nicht kompilierbar oder schlechter Vergleich: + Mathpix als zweites OCR-System ausführen. + +5. Beide Systeme liefern semantisch denselben normalisierten LaTeX-AST + und der Rendervergleich ist sehr gut: + automatisch übernehmen, falls kein Risikomuster vorliegt. + +6. Abweichende OCR-Kandidaten, komplexe Struktur, niedrige Konfidenz, + schlechter Vergleich oder auffällige Zeichen: + zwingend in die Review-Galerie. + +7. Negativ geprüfte Formel: + Originalbild im Markdown erhalten; + Status und Gründe speichern; + niemals „best guess“ ungeprüft veröffentlichen. +``` + +Die zwei OCR-Modelle liefern bewusst nicht denselben Code. Deshalb solltest du LaTeX nicht als String vergleichen. Normalisiere vor dem Vergleich wenigstens: + +- Leerzeichen, +- `\left`/`\right`, falls semantisch entbehrlich, +- äquivalente Befehle wie `\leq` und `\le`, +- Klammerformatierungen, +- überflüssige Gruppierungen. + +Noch besser wäre eine kleine Zwischenrepräsentation bzw. ein LaTeX-AST. Zwei verschiedene, aber äquivalente Schreibweisen sollten als gleich gelten: + +```latex +\frac{a}{b} +``` + +und + +```latex +{a \over b} +``` + +Visuell und mathematisch können sie gleich sein, aber als Strings unterscheiden sie sich. + +## Größe und Layout beim Neubau + +Dein Motiv ist richtig: Semantisches LaTeX statt Formelpixel verbessert Reflow, Schriftgröße, Zoom, Kontrast, responsives EPUB-Layout und konsistente PDF-Typografie erheblich. + +Für Markdown solltest du unterscheiden: + +- Inline-Formeln in $...$, nur wenn sie wirklich in eine Textzeile gehören. +- Abgesetzte Formeln in [ ... ]. +- Nummerierte Gleichungen und Labels als eigene Blockstruktur. +- Lange Ausdrücke, Matrizen und Fallunterscheidungen stets als Display-Mathematik. +- Originale Gleichungsnummern separat erhalten, statt sie aus Versehen als Formelteil zu OCRen. + +Beispiel: + +```markdown +Die Zielfunktion lautet + +\[ +\min \; z = \sum_{i=1}^{n} c_i x_i +\tag{3.7} +\] + +unter den Nebenbedingungen + +\[ +\sum_{i=1}^{n} a_{ji}x_i \le b_j, +\qquad j = 1, \ldots, m. +\] +``` + +Damit kann dein EPUB-Renderer die Formel flexibel an die Bildschirmbreite anpassen, und die PDF-Ausgabe kann die mathematische Schriftgröße konsistent an den umgebenden Satz koppeln. + +## Konkrete Toolwahl + +Für deinen Linux-/Python-Stack würde ich so beginnen: + +- `lxml` oder Beautiful Soup für EPUB-XHTML, MathML und SVG-Analyse. +- `zipfile` für EPUB-Entpacken und Neubauen. +- PyMuPDF/Fitz für PDF-Seiten, Bildextraktion und Formel-Crops. +- `pix2tex` als lokaler erster OCR-Pass. +- Mathpix nur als optionalen zweiten Pass für schwerere Fälle oder als Qualitätsreferenz. Es kann Bilder, PDFs und EPUBs verarbeiten und liefert LaTeX/MathML/Markdown zurück.[^2_4][^2_3] +- MathJax oder LuaLaTeX plus `dvisvgm` als fest definierter TeX-Renderer. MathJax unterstützt die Konvertierung von TeX nach SVG direkt.[^2_10][^2_9] +- `cairosvg`, Inkscape oder `rsvg-convert`, um SVG reproduzierbar nach PNG zu rastern. +- ImageMagick 7 für Vorverarbeitung, Vergleich und Diff-Bilder. `magick compare` kann mathematische und visuell interpretierbare Differenzen ausgeben.[^2_12][^2_11] +- Eine statische HTML-Review-Galerie, statt in der ersten Version eine vollständige Web-App zu bauen. + +Die Qualitätsregel sollte dabei immer konservativ bleiben: Automatisiere Erkennung, Rendering, Vergleich, Sortierung und Dokumentation. Automatisiere die endgültige Akzeptanz nur für sehr einfache, extrem sichere Fälle. + +<span style="display:none">[^2_14][^2_15][^2_16][^2_17][^2_18][^2_19][^2_20][^2_21][^2_22][^2_23][^2_24][^2_25][^2_26][^2_27][^2_28][^2_29][^2_30][^2_31][^2_32][^2_33][^2_34][^2_35][^2_36][^2_37][^2_38][^2_39][^2_40][^2_41][^2_42]</span> + +<div align="center">⁂</div> + +[^2_1]: https://www.boersenverein.de/beratung-service/barrierefreiheit/leitfaden-barrierefreie-epub3-e-books-seite-2/ + +[^2_2]: https://ethz.ch/staffnet/de/lehre/accessibility-and-inclusion-in-teaching/accessible-learning-materials/accessible-documents---presentations/latex.html + +[^2_3]: https://mathpix.com/ocr + +[^2_4]: https://mathpix.com/docs/mpxpy/images + +[^2_5]: https://github.com/lukas-blecher/LaTeX-OCR + +[^2_6]: https://lukas-blecher.github.io/LaTeX-OCR/ + +[^2_7]: https://docs.mathpix.com/reference/post-v3-text + +[^2_8]: https://apis.io/schemas/mathpix/mathpix-image-result/ + +[^2_9]: https://docs.mathjax.org/en/v4.0/web/convert.html + +[^2_10]: https://docs.mathjax.org/en/latest/output/svg.html + +[^2_11]: https://imagemagick.org/compare/ + +[^2_12]: https://imagemagick.org/command-line-options/ + +[^2_13]: https://www.baeldung.com/linux/image-comparison + +[^2_14]: https://wr.informatik.uni-hamburg.de/\_media/teaching/sommersemester_2021/siw-21latex_und_bibtex_stahl_ausarbeitung-neu.pdf + +[^2_15]: https://www.stochastik.uni-freiburg.de/de/professoren/peter-pfaffelhuber/inhalte/schreiben_math + +[^2_16]: https://arxiv.org/html/2408.04015v1 + +[^2_17]: https://mathpix.com/blog + +[^2_18]: https://mathpix.com/ + +[^2_19]: https://mathpix.com/blog/files-api + +[^2_20]: https://github.com/topics/latex-ocr + +[^2_21]: https://www.underleaf.ai/learn/latex/accessibility + +[^2_22]: https://github.com/api-evangelist/mathpix + +[^2_23]: https://tex.stackexchange.com/questions/116863/image-equation-to-tex + +[^2_24]: https://mcpmarket.com/tools/skills/mathpix-ocr-latex-pipeline + +[^2_25]: https://man.archlinux.org/man/pdftocairo.1.en + +[^2_26]: https://usage.imagemagick.org/compare/ + +[^2_27]: https://man.archlinux.org/man/compare.1.en + +[^2_28]: https://mathjax.github.io/MathJax-demos-web/input/tex2svg.html + +[^2_29]: https://mathjax.github.io/MathJax-demos-web/input/tex-mml2svg.html + +[^2_30]: https://blog.jeffterrace.com/2012/09/comparing-image-comparison-algorithms.html?m=1 + +[^2_31]: https://viereck.ch/latex-to-svg/ + +[^2_32]: https://tex.stackexchange.com/questions/255470/compile-tex-directly-into-svg-using-the-command-line + +[^2_33]: https://www.simplified.guide/imagemagick/compare-images + +[^2_34]: https://github.com/ImageMagick/ImageMagick/discussions/6814 + +[^2_35]: tools.math_formula_ocr + +[^2_36]: projects.textbook.operations_research_trading + +[^2_37]: tools.ebook_conversion + +[^2_38]: tools.markdown.latex_conversion + +[^2_39]: https://www.perplexity.ai/search/7483c59a-1497-4d41-93b2-8f564ddbf224 + +[^2_40]: https://www.perplexity.ai/search/b03e509e-2dd2-4821-b2b0-03f4cdc5328a + +[^2_41]: https://www.perplexity.ai/search/7a6c1e3b-efb1-40ca-a9a1-c10ec14ba3e2 + +[^2_42]: https://www.perplexity.ai/search/e6f21cb6-7c55-4bcb-8f4f-5af8a75e59a2 + diff --git a/README.md b/README.md new file mode 100644 index 0000000..5844a11 --- /dev/null +++ b/README.md @@ -0,0 +1,159 @@ +# formula_img2latex + +Nachbearbeitungs-Werkzeug für die Markdown-Ausgaben von +[epub2md](https://kitux.de/forgejo/dschlueter/epub2md) und +[pdf2md](https://kitux.de/forgejo/dschlueter/pdf2md): Formeln, die nur als Bild vorliegen, werden durch +LaTeX ersetzt – **aber nur, wenn das gerenderte LaTeX nachweislich so aussieht wie das Original**. +Zweifelsfälle gehen in eine Review-Galerie, alles andere bleibt Bild. + +Schritt-für-Schritt-Anleitung: [BEDIENUNGSANLEITUNG.md](BEDIENUNGSANLEITUNG.md). Hintergrund und +ursprüngliche Planung: `Idee/`. + +> **Stand 0.1.0 (Prototyp).** Der Ablauf ist vollständig (scan → erkennen → prüfen → Review → apply/revert) +> und getestet. Die Annahmeschwellen sind noch nicht endgültig kalibriert; an echten Büchern landet +> derzeit der größte Teil im Review (siehe [Stand und Grenzen](#stand-und-grenzen)). + +## Ablauf + +```text +scan Bildverweise in chapters/*.md (bzw. pdf2md-Markdown) finden, Formel/Abbildung vorklassifizieren +recognize LaTeX-Kandidaten von einem Vision-Modell holen: lokal (llama.cpp) zuerst, Cloud (OpenRouter) nur für Ungelöste +verify Kandidat rendern (pdflatex), mit dem Original vergleichen, Verwechslungsvarianten prüfen, entscheiden +review Zweifelsfälle im Browser: Original / Render / Overlay / Diff, LaTeX korrigieren und neu prüfen +apply angenommene Formeln ins Markdown schreiben (aus einer Sicherung, idempotent) +revert Markdown-Dateien auf den Stand vor apply zurücksetzen +run scan + recognize lokal + verify + recognize Cloud (nur Ungelöste) + verify +calibrate Messung an Formelbildern mit bekanntem TeX (Springer-EPUBs: TeX im alt-Text) +``` + +```bash +python3 formula_img2latex.py run output/buch # epub2md-Ausgabeordner +python3 formula_img2latex.py review output/buch +python3 formula_img2latex.py apply output/buch +python3 epub2md_merge.py output/buch --force # Gesamtdatei danach neu bauen +``` + +## Wie geprüft wird + +Die OCR liefert nur einen **Kandidaten**. Entschieden wird über das Bild: + +1. **Rendern** mit pdflatex (Paket `preview`, viele Formeln je Lauf, `-no-shell-escape`, + `openout_any=p`). Unsichere Befehle (`\input`, `\write`, `\def`, `\catcode`, …) werden vorher abgelehnt. + Die Schrift wird je Buch automatisch gewählt: `cm`, `times`, `txfonts`, `palatino`, `stix`, `euler`. +2. **Vergleichen** in der Auflösung des Originals: Tinte normalisieren (Transparenz, JPEG-Rauschen, + getönter Hintergrund), gleichmäßig skalieren (nie strecken), auf ¼ Pixel ausrichten. + Kennzahlen: Korrelation (NCC), SSIM, Abweichungsflecken, Größenabweichung, **Pixel pro Punkt**. + Pixel pro Punkt ist innerhalb eines Buches konstant; Abweichungen verraten falsche Stile + (`\frac` statt `\tfrac`, `\displaystyle`). +3. **Kontrasttest gegen Verwechslungsvarianten** (`confusables.py`). Erzeugt werden ~50–160 Varianten, + die sich durch je einen typischen Erkennungsfehler unterscheiden: andere Ziffer, hoch statt tief, + `,`/`;`, `-`/`+`/`=`, ähnliche Buchstaben, Groß/klein, `\mathcal`/`\mathbb`/…, fehlendes Vorzeichen, Strich + oder Akzent. Varianten werden **layouttreu** gerendert (`\mathrlap{neu}\mathbin{\phantom{alt}}`), damit + sich nur das eine Zeichen ändert. Verglichen wird nur dort, wo sich Kandidat und Variante unterscheiden + (Likelihood-Test). Der Kandidat muss **jede** unterscheidbare Variante mit Abstand schlagen. +4. **Selbstkorrektur:** Erklärt eine Variante das Bild deutlich besser, wird sie neuer Kandidat + (höchstens 3 Schritte). So werden einzelne OCR-Fehler ohne erneute Modellanfrage repariert. +5. **Pandoc-Prüfung:** Jede angenommene Formel muss mit `pandoc --mathml` umsetzbar sein, sonst + bliebe sie im späteren EPUB/PDF roher TeX-Text. + +Status je Formel: `auto_accepted`, `needs_review`, `rejected_keep_image`, `render_failed`, `no_candidate`, +`not_formula`, nach Review `accepted_by_human`, `corrected_by_human`, `kept_by_human`. + +## Arbeitsordner + +```text +<ausgabe>.formulas/ # neben der Ausgabe, weil epub2md --force die Ausgabe ersetzt +├── workspace.json # Quelle, Schrift, Pixel/Punkt, Schwellen, apply-Zustand +├── formulas/f-<sha12>-<d|i>.json # je Bild und Darstellung: Kandidaten, Prüfung, Entscheidung +├── review/<id>-{orig,render,diff}.png +└── backup/ # Markdown-Dateien vor dem ersten apply +``` + +Die Datensätze sind über den SHA-256 des Bildes verknüpft. Ein erneuter epub2md-Lauf mit `--force` verliert +deshalb keine Entscheidungen. `scan` und `apply` spielen sie ohne neue OCR wieder ein. Modellantworten +werden je Modell gespeichert; dasselbe Modell wird nie zweimal nach demselben Bild gefragt. + +## Quellen + +| Quelle | Erkennung | Begrenzer | +| --- | --- | --- | +| epub2md-Ordner | `conversion-report.json` (`tool: epub2md`); Warnungen `formula-image` markieren sichere Formeln | `--math auto`: `$…$`/`$$…$$`, `--math tex`: `\(…\)`/`\[…\]` | +| pdf2md-Datei `<name>.md` + `<name>_images/` oder `--split`-Ordner | Bildname `<pdf>-<seite>-<box>`; mit `--pdf` wird jede Formel per Template-Matching auf der Seite gefunden, mit 600 dpi aus dem Vektor-PDF neu ausgeschnitten und an Mathe-Schriften (CMMI, EURM, …) als Formel erkannt | `$…$`/`$$…$$` | + +Abgesetzte Bildformeln gelten als Display-Mathematik, Bilder im Fließtext als Inline-Mathematik. +Codeblöcke werden nie verändert. Inline-Formeln, die pdf2md als Glyphensalat im Fließtext ausgibt, +sind nicht Teil dieser Version. + +## Engines + +Jede Engine spricht die OpenAI-Chat-API mit Bildteil: + +| `--engine` | Standard | Konfiguration | +| --- | --- | --- | +| `local` | llama.cpp-Server `http://127.0.0.1:8011/v1`, Modell vom Server | `FORMULA_LOCAL_URL`, `FORMULA_LOCAL_MODEL`, `FORMULA_LOCAL_API_KEY` | +| `cloud` | OpenRouter, `google/gemini-2.5-flash` | `OPENROUTER_API_KEY`, `FORMULA_CLOUD_MODEL`, `FORMULA_CLOUD_URL` | + +Getestet: **Qwen3-VL-30B-A3B-Instruct** (Q4_K_M + mmproj) auf einer RTX 3090 mit 0,2–0,5 s je Formel; +Gehrke Brückenkurs (1329 Bilder) in 9 Minuten. + +## Stand und Grenzen + +Messungen vom 07.10.2026 (Details: `calibrate`): + +- **Synthetisch** (Computer Modern, ~1,4 px/pt, JPEG – wie Gehrke): Die korrekte Formel schlägt in + 98 % der Fälle alle Verwechslungsvarianten; ein einzelner falscher Startkandidat wird meist repariert. +- **Gehrke Brückenkurs** (EPUB, ~1400 Formel-JPGs, CM): Die OCR ist gut. Abgelehnt oder ins Review + geschickt wird überwiegend wegen **typografischer** Abweichungen der OCR, z. B. + `\xrightarrow{…}` statt `\stackrel{…}{\longrightarrow}`, `\vec` statt `\overrightarrow`, Text neben statt + über `=`. Bei einer Pfeillänge verrutscht alles dahinter. Abhilfe in Arbeit: Korrekturrunde, in der das + Modell Original und eigenes Render nebeneinander sieht. +- **Springer-Bücher** (Liesen, Göllmann): Was im alt-Text kein TeX hat, sind meist komplexe mehrzeilige + Formeln eines fremden Satzsystems. Abstände und Klammergrößen weichen ab, der starre Vergleich bestätigt + sie nicht → Review. +- **PDF** (Hefferon, Euler-Schrift): Einfache Formeln passen mit `euler` sehr gut (NCC 0,96). Wo Ziffern + oder Operatornamen in anderen Schnitten gesetzt sind (serifenloses `sgn`), wird der Kontrasttest + unzuverlässig → Review. +- Formeln auf farbig hinterlegten Kästen verlieren beim Ersetzen die Hinterlegung. + +Nächste Schritte: Schwellen mit `calibrate` an den layouttreuen Varianten neu festlegen; Korrekturrunde +mit Diff-Bild; abschnittsweiser Vergleich (Zeilen/Spalten getrennt ausrichten) für mehrzeilige Formeln +fremder Satzsysteme; Inline-Formeln in PDFs. + +## Installation + +```bash +sudo apt install texlive-latex-extra texlive-fonts-extra texlive-science pandoc # pdflatex, Schriften, Pandoc +pip install -r requirements.txt # pymupdf, numpy, scipy, scikit-image, opencv, Pillow, lxml, requests +pip install -r requirements-dev.txt # pytest +``` + +## Tests + +```bash +python3 -m pytest -q # ca. 30 s; synthetische Bilder, kein Modellserver, keine Buchinhalte +``` + +Rückgabecodes: `0` Erfolg, `1` Bedienfehler, `2` Eingabe ist keine epub2md-/pdf2md-Ausgabe, +`3` externes Werkzeug fehlt (pdflatex, pandoc, Modellserver), `4` interner Fehler. + +## Architektur + +```text +formula_img2latex/ +├── cli.py Unterbefehle, Exit-Codes +├── pipeline.py scan, recognize, verify (Prozess-Pool), Entscheidung, apply/revert +├── sources/ epub2md.py, pdf2md.py, base.py (Markdown-Scan, Begrenzer) +├── pdfcrop.py Formelbild auf der PDF-Seite finden, 600-dpi-Neuausschnitt, Mathe-Schrift-Erkennung +├── classify.py Formel / unklar / Abbildung (Farbe der Tinte, Graustufen, Größe, Kontext) +├── engines/ Vision-Modelle über die OpenAI-API (llama.cpp, OpenRouter) +├── render.py pdflatex-Batch, Schrift-Presets, Rasterung +├── imageio.py Originale laden (PNG/GIF/JPG/SVG, Transparenz, Tönung) +├── compare.py Ausrichtung, Kennzahlen, Kontrasttest, Diff-Bilder +├── confusables.py Verwechslungsvarianten, layouttreue Prüfvarianten +├── verify.py Prüfung + Selbstkorrektur +├── decide.py Schwellen und Status +├── checks.py Pandoc/MathML-Prüfung +├── calibrate.py Messung an Formelbildern mit bekanntem TeX +├── store.py Arbeitsordner, Datensätze +└── review/ lokaler Review-Server + Seite +``` diff --git a/formula_img2latex.py b/formula_img2latex.py new file mode 100755 index 0000000..1196e25 --- /dev/null +++ b/formula_img2latex.py @@ -0,0 +1,9 @@ +#!/usr/bin/env python3 +"""Entry point: python3 formula_img2latex.py BEFEHL [Optionen]""" + +import sys + +from formula_img2latex.cli import main + +if __name__ == "__main__": + sys.exit(main()) diff --git a/formula_img2latex/__init__.py b/formula_img2latex/__init__.py new file mode 100644 index 0000000..8711171 --- /dev/null +++ b/formula_img2latex/__init__.py @@ -0,0 +1,3 @@ +"""formula_img2latex: Formelbilder in Markdown geprüft durch LaTeX ersetzen.""" + +__version__ = "0.1.0" diff --git a/formula_img2latex/__main__.py b/formula_img2latex/__main__.py new file mode 100644 index 0000000..dd8a8c9 --- /dev/null +++ b/formula_img2latex/__main__.py @@ -0,0 +1,5 @@ +import sys + +from .cli import main + +sys.exit(main()) diff --git a/formula_img2latex/calibrate.py b/formula_img2latex/calibrate.py new file mode 100644 index 0000000..dcc6def --- /dev/null +++ b/formula_img2latex/calibrate.py @@ -0,0 +1,196 @@ +"""Calibration on formula images whose true TeX is known (e.g. Springer EPUBs: TeX in the alt text). + +For every sampled image: + * the true TeX is verified -> would it be accepted automatically? (coverage) + * up to three simulated recognition errors (random confusable variants of the truth) are + verified as starting candidates -> are they repaired, sent to review, or (the failure that + must not happen) accepted although they render differently from the truth? +With --synthetic the same formulas are additionally rendered as small JPEGs in Computer Modern +(like the Gehrke books), so the low-resolution case is measured too. +""" + +from __future__ import annotations + +import html +import io +import json +import os +import posixpath +import random +import re +import statistics +import sys +import zipfile +from concurrent.futures import ProcessPoolExecutor, as_completed +from pathlib import Path +from typing import Any + +import cv2 +import numpy as np +from PIL import Image + +from .compare import compare, contrast +from .confusables import variants +from .decide import decide, thresholds +from .imageio import load_ink +from .render import PRESETS, render_many +from .tex import strip_delimiters, unsafe_reason +from .verify import verify + +SYNTH_PX_PER_PT = 1.41 + + +def corpus(epub: Path) -> list[tuple[str, bytes, str, bool]]: + """(name, image bytes, true TeX, display) for every image with TeX in its alt text.""" + out = [] + with zipfile.ZipFile(epub) as z: + names = set(z.namelist()) + for n in z.namelist(): + if not n.lower().endswith((".xhtml", ".html", ".htm")): + continue + s = z.read(n).decode("utf-8", "replace") + base = n.rsplit("/", 1)[0] if "/" in n else "" + for m in re.finditer(r"<img\b[^>]*>", s): + tag = m.group(0) + src, alt = re.search(r'src="([^"]+)"', tag), re.search(r'alt="([^"]*)"', tag) + if not (src and alt and alt.group(1).lstrip().startswith("$")): + continue + path = posixpath.normpath(posixpath.join(base, src.group(1))) + tex = strip_delimiters(html.unescape(alt.group(1))) + if path in names and not unsafe_reason(tex): + display = "IEq" not in path and "inline" not in tag.lower() + out.append((path, z.read(path), tex, display)) + return out + + +def synth_jpeg(tex: str, display: bool, rng: random.Random) -> bytes | None: + r = render_many([(tex, display)], "cm")[0] + if not r.ok: + return None + ppt = SYNTH_PX_PER_PT * rng.uniform(0.97, 1.03) + gray = 1.0 - r.rasterize(ppt * 4) + pad = np.ones((gray.shape[0] + 8, gray.shape[1] + 8), np.float32) + oy, ox = rng.randrange(4), rng.randrange(4) + pad[oy:oy + gray.shape[0], ox:ox + gray.shape[1]] = gray + h, w = pad.shape[0] // 4, pad.shape[1] // 4 + small = cv2.resize(pad[: h * 4, : w * 4], (w, h), interpolation=cv2.INTER_AREA) + buf = io.BytesIO() + Image.fromarray((small * 255).astype(np.uint8)).save(buf, "JPEG", quality=75) + return buf.getvalue() + + +def _equivalent(orig: np.ndarray, a: str, b: str, display: bool, preset: str) -> bool: + """True if a and b render identically at the original's scale (e.g. \\le vs \\leq).""" + ra, rb = render_many([(a, display), (b, display)], preset) + if not (ra.ok and rb.ok): + return False + ca = compare(orig, ra) + cb = compare(orig, rb, px_per_pt=ca.metrics.get("px_per_pt")) + return ca.ok and cb.ok and contrast(ca, cb) is None + + +def _job(args: tuple[str, bytes, str, bool, str, int]) -> dict[str, Any]: + name, data, truth, display, preset, seed = args + os.environ.setdefault("OMP_NUM_THREADS", "1") + rng = random.Random(seed) + orig = load_ink(data, name) + th = thresholds() + res: dict[str, Any] = {"name": name, "truth": truth, "display": display} + vt = verify(orig, truth, display, preset) + res["truth_ver"] = vt.to_json() + res["truth_status"] = decide(res["truth_ver"], True, None, th)[0] + starts = [v.tex for v in variants(truth)] + rng.shuffle(starts) + res["errors"] = [] + for wrong in starts[:3]: + if _equivalent(orig, wrong, truth, display, preset): + continue + vw = verify(orig, wrong, display, preset) + st = decide(vw.to_json(), True, None, th)[0] + final_ok = vw.tex == truth or _equivalent(orig, vw.tex, truth, display, preset) + res["errors"].append({"start": wrong, "final": vw.tex, "status": st, "repaired": final_ok, + "min_contrast": vw.min_contrast}) + return res + + +def calibrate(epubs: list[Path], samples: int = 200, preset: str | None = None, synthetic: bool = False, + report: Path | None = None, workers: int = 0, quiet: bool = False) -> dict[str, Any]: + def log(m: str) -> None: + if not quiet: + print(m, file=sys.stderr, flush=True) + + rng = random.Random(42) + sets: list[tuple[str, list[tuple[str, bytes, str, bool]], str]] = [] + for epub in epubs: + items = corpus(epub) + log(f"{epub.name}: {len(items)} Formelbilder mit TeX im alt-Text") + if not items: + continue + smp = rng.sample(items, min(samples, len(items))) + p = preset or _best_preset(smp[:12]) + sets.append((f"{epub.stem[:40]} [{p}]", smp, p)) + if synthetic: + syn = [] + for name, _data, tex, display in smp: + img = synth_jpeg(tex, display, rng) + if img: + syn.append((name + "#synth.jpg", img, tex, display)) + sets.append((f"{epub.stem[:40]} [synthetisch, cm]", syn, "cm")) + summary: dict[str, Any] = {"sets": []} + for label, items, p in sets: + log(f"prüfe {label}: {len(items)} Formeln …") + jobs = [(n, d, t, disp, p, i) for i, (n, d, t, disp) in enumerate(items)] + results = [] + with ProcessPoolExecutor(max_workers=workers or max(1, (os.cpu_count() or 2) - 2)) as pool: + for fut in as_completed([pool.submit(_job, j) for j in jobs]): + results.append(fut.result()) + summary["sets"].append(_summarize(label, results)) + summary["sets"][-1]["results"] = results + _print(summary["sets"][-1], log) + if report: + report.write_text(json.dumps(summary, ensure_ascii=False, indent=1), encoding="utf-8") + log(f"Bericht: {report}") + return summary + + +def _best_preset(items: list[tuple[str, bytes, str, bool]]) -> str: + inks = [load_ink(d, n) for n, d, _t, _disp in items] + best, best_v = "cm", -1.0 + for p in PRESETS: + rs = render_many([(t, disp) for _n, _d, t, disp in items], p) + v = statistics.median([compare(i, r).metrics.get("ncc", 0.0) if r.ok else 0.0 for i, r in zip(inks, rs)]) + if v > best_v: + best, best_v = p, v + return best + + +def _summarize(label: str, results: list[dict[str, Any]]) -> dict[str, Any]: + n = len(results) + truth_auto = sum(r["truth_status"] == "auto_accepted" for r in results) + errs = [e for r in results for e in r["errors"]] + wrong_accepted = [e for e in errs if e["status"] == "auto_accepted" and not e["repaired"]] + repaired_accepted = sum(e["status"] == "auto_accepted" and e["repaired"] for e in errs) + mcs = [r["truth_ver"]["min_contrast"] for r in results if r["truth_ver"].get("min_contrast") is not None] + return { + "label": label, "formulas": n, + "truth_auto_accepted": truth_auto, + "truth_min_contrast_p5_p50": [round(float(np.percentile(mcs, q)), 3) for q in (5, 50)] if mcs else None, + "simulated_errors": len(errs), + "errors_repaired_and_accepted": repaired_accepted, + "errors_to_review_or_rejected": sum(e["status"] != "auto_accepted" for e in errs), + "errors_wrongly_accepted": len(wrong_accepted), + "wrongly_accepted_examples": [{"start": e["start"], "final": e["final"]} for e in wrong_accepted[:10]], + } + + +def _print(s: dict[str, Any], log: Any) -> None: + n = max(1, s["formulas"]) + e = max(1, s["simulated_errors"]) + log(f" {s['label']}") + log(f" korrekte Formeln automatisch angenommen: {s['truth_auto_accepted']}/{s['formulas']} " + f"({s['truth_auto_accepted'] / n:.0%}); Kontrast p5/p50: {s['truth_min_contrast_p5_p50']}") + log(f" simulierte Erkennungsfehler: {s['simulated_errors']}: repariert+angenommen " + f"{s['errors_repaired_and_accepted']} ({s['errors_repaired_and_accepted'] / e:.0%}), " + f"Review/Bild {s['errors_to_review_or_rejected']}, FALSCH ANGENOMMEN {s['errors_wrongly_accepted']}") + for ex in s["wrongly_accepted_examples"]: + log(f" falsch: {ex['final']} (Start {ex['start']})") diff --git a/formula_img2latex/checks.py b/formula_img2latex/checks.py new file mode 100644 index 0000000..d3f0166 --- /dev/null +++ b/formula_img2latex/checks.py @@ -0,0 +1,46 @@ +"""Checks that a formula survives the later book build: Pandoc (texmath) must turn it into MathML. + +epub2md's README builds HTML/EPUB with `pandoc --mathml` and PDF via HTML + xelatex. A formula +that Pandoc cannot convert stays raw TeX in the book, so such candidates are never accepted. +""" + +from __future__ import annotations + +import re +import shutil +import subprocess + +from .errors import EXIT_TOOL, FormulaError +from .tex import strip_delimiters + +_MARK = "F2LMARK" + + +def pandoc_available() -> bool: + return shutil.which("pandoc") is not None + + +def pandoc_mathml_ok(formulas: list[tuple[str, bool]]) -> list[bool]: + """For each (tex, display): True if `pandoc --mathml` converts it to MathML.""" + if not formulas: + return [] + if not pandoc_available(): + raise FormulaError("pandoc nicht gefunden – installieren mit: sudo apt install pandoc", EXIT_TOOL) + parts = [] + for i, (tex, display) in enumerate(formulas): + body = strip_delimiters(tex).replace("\n", " ") + math = f"$${body}$$" if display else f"${body}$" + parts.append(f"{_MARK}{i}\n\n{math}\n") + proc = subprocess.run( + ["pandoc", "-f", "markdown", "-t", "html", "--mathml"], + input="\n".join(parts), capture_output=True, text=True, timeout=300, check=False, + ) + if proc.returncode != 0: + raise FormulaError(f"pandoc-Fehler: {proc.stderr.strip()[:300]}", EXIT_TOOL) + chunks = re.split(rf"<p>{_MARK}(\d+)</p>", proc.stdout) + ok = [False] * len(formulas) + for k in range(1, len(chunks) - 1, 2): + i = int(chunks[k]) + html = chunks[k + 1] + ok[i] = "<math" in html and 'class="math' not in html + return ok diff --git a/formula_img2latex/classify.py b/formula_img2latex/classify.py new file mode 100644 index 0000000..61534a9 --- /dev/null +++ b/formula_img2latex/classify.py @@ -0,0 +1,40 @@ +"""Formula or figure? A cheap pre-filter before any model is asked. + +Wrong guesses are not dangerous: a figure that slips through will not match any rendered +formula and is rejected by the comparison; a formula labelled "figure" simply stays an image +(and can be forced with --include-figures). +""" + +from __future__ import annotations + +from typing import Any + +from .imageio import color_stats, gray_to_ink, load_gray, trim + +MAX_FORMULA_HEIGHT = 600 # px; higher images are almost always figures (unless the converter says formula) +MAX_COLOR_SHARE = 0.10 # share of coloured ink pixels; formulas are printed in black +MAX_MIDTONES = 0.30 # photos and shaded diagrams have many gray levels; formulas only antialiasing + + +def classify(data: bytes, name: str, known_formula: bool, inline: bool) -> dict[str, Any]: + """Return {"label": "formula"|"maybe"|"figure", "why": str, features...}.""" + gray = load_gray(data, name) + ink = trim(gray_to_ink(gray)) + h, w = (ink.shape if ink.size else (0, 0)) + stats = color_stats(data, name) + feats: dict[str, Any] = {"height": int(gray.shape[0]), "width": int(gray.shape[1]), + "ink_height": int(h), "ink_width": int(w), + "colorfulness": round(stats["colorfulness"], 4), "midtones": round(stats["midtones"], 4)} + if h == 0: + return {"label": "figure", "why": "leeres Bild", **feats} + if known_formula: + return {"label": "formula", "why": "vom Konverter als Formel gemeldet", **feats} + if stats["colorfulness"] > MAX_COLOR_SHARE: + return {"label": "figure", "why": "farbig", **feats} + if stats["midtones"] > MAX_MIDTONES: + return {"label": "figure", "why": "viele Graustufen (Foto/Grafik)", **feats} + if h > MAX_FORMULA_HEIGHT: + return {"label": "figure", "why": "sehr hoch", **feats} + if inline: + return {"label": "formula", "why": "steht im Fließtext, schwarz-weiß", **feats} + return {"label": "maybe", "why": "abgesetztes Schwarz-Weiß-Bild", **feats} diff --git a/formula_img2latex/cli.py b/formula_img2latex/cli.py new file mode 100644 index 0000000..fd11edb --- /dev/null +++ b/formula_img2latex/cli.py @@ -0,0 +1,202 @@ +"""Command line interface.""" + +from __future__ import annotations + +import argparse +import json +import sys +import traceback +from pathlib import Path +from typing import NoReturn + +from . import __version__ +from .errors import EXIT_INPUT, EXIT_INTERNAL, EXIT_OK, FormulaError +from .render import PRESETS + +DESCRIPTION = """\ +Ersetzt Formelbilder in Markdown-Ausgaben von epub2md und pdf2md durch LaTeX – aber nur, wenn +das gerenderte LaTeX nachweislich so aussieht wie das Original. Zweifelsfälle gehen in eine +Review-Galerie, alles andere bleibt Bild. +""" + +EPILOG = """\ +Typischer Ablauf: + formula_img2latex.py run output/buch # scan, erkennen (lokal, dann Cloud), prüfen + formula_img2latex.py review output/buch # Zweifelsfälle im Browser entscheiden + formula_img2latex.py apply output/buch # geprüfte Formeln ins Markdown schreiben + formula_img2latex.py revert output/buch # alles rückgängig machen + +Rückgabecodes: + 0 Erfolgreich + 1 Ungültige Parameter, Eingabe/Arbeitsordner nicht lesbar oder nicht beschreibbar + 2 Eingabe ist keine epub2md-/pdf2md-Ausgabe + 3 Externes Werkzeug fehlt oder versagt (pdflatex, pandoc, Modellserver) + 4 Unerwarteter interner Fehler +""" + + +class _Parser(argparse.ArgumentParser): + def error(self, message: str) -> NoReturn: + self.print_usage(sys.stderr) + print(f"{self.prog}: Fehler: {message}", file=sys.stderr) + sys.exit(EXIT_INPUT) + + +def build_parser() -> argparse.ArgumentParser: + p = _Parser(prog="formula_img2latex.py", description=DESCRIPTION, epilog=EPILOG, + formatter_class=argparse.RawDescriptionHelpFormatter) + p.add_argument("--version", action="version", version=f"%(prog)s {__version__}") + sub = p.add_subparsers(dest="command", required=True, parser_class=_Parser) + + def common(sp: argparse.ArgumentParser) -> None: + sp.add_argument("source", type=Path, metavar="AUSGABE", + help="epub2md-Ausgabeordner, pdf2md-Markdown-Datei oder pdf2md-Split-Ordner") + sp.add_argument("--workspace", type=Path, default=None, + help="Arbeitsordner (Standard: <AUSGABE>.formulas neben der Ausgabe)") + sp.add_argument("--quiet", "-q", action="store_true", help="nur Fehler ausgeben") + + def recog_opts(sp: argparse.ArgumentParser) -> None: + sp.add_argument("--model", default=None, help="Modellname (Standard: local = Modell des Servers, " + "cloud = $FORMULA_CLOUD_MODEL oder google/gemini-2.5-flash)") + sp.add_argument("--url", default=None, help="API-Basis-URL (Standard: local http://127.0.0.1:8011/v1, " + "cloud https://openrouter.ai/api/v1)") + sp.add_argument("--include-figures", action="store_true", + help="auch Bilder versuchen, die wie Abbildungen aussehen") + sp.add_argument("--limit", type=int, default=0, help="höchstens N Bilder (zum Ausprobieren)") + sp.add_argument("--workers", type=int, default=0, help="parallele Anfragen bzw. Prüfprozesse") + + sp = sub.add_parser("scan", help="Formelbilder finden und klassifizieren") + common(sp) + sp.add_argument("--pdf", type=Path, default=None, + help="pdf2md: Original-PDF, um Formeln scharf neu auszuschneiden") + + sp = sub.add_parser("recognize", help="LaTeX-Kandidaten mit einem Vision-Modell erzeugen") + common(sp) + recog_opts(sp) + sp.add_argument("--engine", choices=["local", "cloud"], default="local") + sp.add_argument("--only-unresolved", action="store_true", + help="nur Formeln, die noch nicht automatisch angenommen sind") + + sp = sub.add_parser("verify", help="Kandidaten rendern, mit dem Original vergleichen, entscheiden") + common(sp) + sp.add_argument("--preset", choices=["auto", *PRESETS], default="auto", + help="Schrift für das Rendern (Standard: automatisch je Buch)") + sp.add_argument("--workers", type=int, default=0) + sp.add_argument("--include-figures", action="store_true") + sp.add_argument("--redo", action="store_true", help="auch bereits geprüfte Formeln neu prüfen") + + sp = sub.add_parser("run", help="scan + recognize (lokal, dann Cloud) + verify") + common(sp) + recog_opts(sp) + sp.add_argument("--pdf", type=Path, default=None) + sp.add_argument("--cloud-model", default=None, help="Modell für die Cloud-Stufe") + sp.add_argument("--no-local", action="store_true", help="lokale Stufe überspringen") + sp.add_argument("--no-cloud", action="store_true", help="Cloud-Stufe überspringen") + sp.add_argument("--preset", choices=["auto", *PRESETS], default="auto") + + sp = sub.add_parser("review", help="Review-Galerie im Browser (localhost)") + common(sp) + sp.add_argument("--port", type=int, default=8765) + sp.add_argument("--all", action="store_true", help="alle Formeln zeigen, nicht nur Zweifelsfälle") + sp.add_argument("--no-browser", action="store_true", help="Browser nicht automatisch öffnen") + + sp = sub.add_parser("apply", help="geprüfte Formeln ins Markdown schreiben") + common(sp) + sp.add_argument("--dry-run", action="store_true", help="nur zählen, nichts schreiben") + + sp = sub.add_parser("revert", help="Markdown-Dateien auf den Stand vor apply zurücksetzen") + common(sp) + sp.add_argument("--force", action="store_true", help="auch seit apply veränderte Dateien zurücksetzen") + + sp = sub.add_parser("status", help="Stand des Arbeitsordners anzeigen") + common(sp) + sp.add_argument("--json", action="store_true") + + sp = sub.add_parser("calibrate", help="Schwellen an Formelbildern mit bekanntem TeX prüfen (QA)") + sp.add_argument("epubs", type=Path, nargs="+", metavar="EPUB", + help="EPUBs mit TeX im alt-Text der Formelbilder (z. B. Springer)") + sp.add_argument("--samples", type=int, default=200, help="Formeln je Buch (Standard 200)") + sp.add_argument("--preset", choices=["auto", *PRESETS], default="auto") + sp.add_argument("--synthetic", action="store_true", + help="zusätzlich kleine JPEG-Nachbildungen (wie Gehrke) aus denselben Formeln erzeugen") + sp.add_argument("--report", type=Path, default=None, help="Bericht als JSON speichern") + sp.add_argument("--workers", type=int, default=0) + sp.add_argument("--quiet", "-q", action="store_true") + return p + + +def _status(ws, quiet: bool, as_json: bool) -> None: + from .store import REPLACE_STATUSES + counts = ws.stats() + if as_json: + print(json.dumps({"workspace": str(ws.root), "source": ws.meta.get("source"), "settings": ws.settings, + "status": counts}, ensure_ascii=False, indent=1)) + return + total = sum(counts.values()) + rep = sum(counts.get(s, 0) for s in REPLACE_STATUSES) + print(f"Arbeitsordner: {ws.root}") + print(f"Schrift: {ws.settings.get('preset') or '–'} Pixel/Punkt (Buch): " + f"{ws.settings.get('book_px_per_pt') and round(ws.settings['book_px_per_pt'], 3) or '–'}") + for k in sorted(counts): + print(f" {k:22s} {counts[k]:6d}") + print(f" {'gesamt':22s} {total:6d} davon ersetzbar: {rep}") + + +def main(argv: list[str] | None = None) -> int: + args = build_parser().parse_args(argv) + try: + return _dispatch(args) + except FormulaError as e: + print(f"formula_img2latex: Fehler: {e}", file=sys.stderr) + return e.code + except KeyboardInterrupt: + print("formula_img2latex: abgebrochen", file=sys.stderr) + return EXIT_INTERNAL + except Exception: # noqa: BLE001 - last resort, report and exit with the internal error code + traceback.print_exc() + return EXIT_INTERNAL + + +def _dispatch(args: argparse.Namespace) -> int: + from . import pipeline + + if args.command == "calibrate": + from .calibrate import calibrate + calibrate(args.epubs, samples=args.samples, preset=None if args.preset == "auto" else args.preset, + synthetic=args.synthetic, report=args.report, workers=args.workers, quiet=args.quiet) + return EXIT_OK + if not args.source.exists(): + raise FormulaError(f"Eingabe nicht gefunden: {args.source}") + create = args.command in ("scan", "run") + source, ws = pipeline.open_all(args.source, args.workspace, create=create, pdf=getattr(args, "pdf", None)) + q = args.quiet + if args.command == "scan": + pipeline.scan(source, ws, quiet=q) + elif args.command == "recognize": + pipeline.recognize(source, ws, args.engine, args.model, args.url, args.only_unresolved, + args.include_figures, args.limit, args.workers or (4 if args.engine == "local" else 8), q) + elif args.command == "verify": + pipeline.verify_all(source, ws, None if args.preset == "auto" else args.preset, args.include_figures, + args.workers, args.redo, q) + elif args.command == "run": + preset = None if args.preset == "auto" else args.preset + pipeline.scan(source, ws, quiet=q) + if not args.no_local: + pipeline.recognize(source, ws, "local", args.model, args.url, False, args.include_figures, + args.limit, args.workers or 4, q) + pipeline.verify_all(source, ws, preset, args.include_figures, 0, False, q) + if not args.no_cloud: + pipeline.recognize(source, ws, "cloud", args.cloud_model, None, True, args.include_figures, + args.limit, args.workers or 8, q) + pipeline.verify_all(source, ws, preset, args.include_figures, 0, False, q) + _status(ws, q, False) + elif args.command == "review": + from .review.server import serve + serve(source, ws, port=args.port, show_all=args.all, open_browser=not args.no_browser) + elif args.command == "apply": + pipeline.apply(source, ws, args.dry_run, q) + elif args.command == "revert": + pipeline.revert(source, ws, args.force, q) + elif args.command == "status": + _status(ws, q, args.json) + return EXIT_OK diff --git a/formula_img2latex/compare.py b/formula_img2latex/compare.py new file mode 100644 index 0000000..526aa58 --- /dev/null +++ b/formula_img2latex/compare.py @@ -0,0 +1,326 @@ +"""Compare an original formula image with a rendered candidate. + +Comparison happens at the original's resolution (after normalizing very thick strokes), never by +blowing up a small original. The rendered vector page is rasterized with oversampling, scaled +uniformly (no stretching: that would hide geometric errors) and aligned to a quarter pixel. + +Metrics: + ncc Pearson correlation of the aligned ink images (1 = identical) + ssim structural similarity of the aligned images + max_blob largest connected area of ink that exists in only one image, in units of + stroke width² – a missing minus or a misplaced exponent is one compact blob, + while font/antialiasing differences are thin scattered fringes + worst_cover smallest share of any glyph component that is matched by the other image + chamfer mean distance (px) of ink to the nearest ink of the other image (max of both ways) + aspect width/height scale ratio before alignment (1 = same proportions) + px_per_pt original pixels per TeX point – constant within a book, so deviations reveal + wrong styles (\\displaystyle vs. inline, \\frac vs. \\tfrac) +""" + +from __future__ import annotations + +from dataclasses import dataclass, field + +import cv2 +import numpy as np +from scipy import ndimage +from skimage.metrics import structural_similarity +from skimage.morphology import skeletonize + +from .imageio import trim +from .render import Rendered + +BIN_THRESHOLD = 0.4 +MAX_ASPECT_DEVIATION = 0.25 +TARGET_STROKE = 2.0 # strokes thicker than this (px) are scaled down before comparing +OVERSAMPLE = 4 +SUBPIXEL_MAX_HEIGHT = 150 # quarter-pixel alignment only for small images (px) + + +@dataclass +class Comparison: + ok: bool + reason: str | None = None + metrics: dict[str, float] = field(default_factory=dict) + original: np.ndarray | None = None # aligned canvases, for diff images + candidate: np.ndarray | None = None + mismatch: np.ndarray | None = None + + +def stroke_width(ink: np.ndarray) -> float: + b = ink > BIN_THRESHOLD + if not b.any(): + return 1.0 + skel = skeletonize(b) + n = int(skel.sum()) + return float(ink.sum() / n) if n else 1.0 + + +def glyph_size(binary: np.ndarray) -> int: + """Typical glyph extent (px): median of the larger side of the ink components, odd, >= 3.""" + lab, n = ndimage.label(binary, structure=np.ones((3, 3))) + if n == 0: + return 3 + sl = ndimage.find_objects(lab) + sizes = [max(s[0].stop - s[0].start, s[1].stop - s[1].start) for s in sl if s is not None] + k = int(round(float(np.median(sizes)))) + return max(3, k | 1) + + +def _resize(img: np.ndarray, w: int, h: int) -> np.ndarray: + return cv2.resize(img, (max(1, w), max(1, h)), interpolation=cv2.INTER_AREA) + + +def _pad(img: np.ndarray, m: int) -> np.ndarray: + return np.pad(img, m, mode="constant") + + +def _place(canvas_shape: tuple[int, int], img: np.ndarray, y: int, x: int) -> np.ndarray: + out = np.zeros(canvas_shape, dtype=np.float32) + h, w = img.shape + y0, x0 = max(0, y), max(0, x) + y1, x1 = min(canvas_shape[0], y + h), min(canvas_shape[1], x + w) + if y1 > y0 and x1 > x0: + out[y0:y1, x0:x1] = img[y0 - y: y1 - y, x0 - x: x1 - x] + return out + + +def _ncc(a: np.ndarray, b: np.ndarray) -> float: + a = a - a.mean() + b = b - b.mean() + den = float(np.sqrt((a * a).sum() * (b * b).sum())) + return float((a * b).sum() / den) if den > 0 else 0.0 + + +def _aligned_candidate(orig: np.ndarray, hi: np.ndarray, scale: float, margin: int) -> tuple[np.ndarray, float]: + """Scale *hi* (oversampled render) by *scale*, align on the padded original with ¼-px precision.""" + hs = max(1, int(round(hi.shape[0] * scale))) + ws = max(1, int(round(hi.shape[1] * scale))) + h4, w4 = hs * OVERSAMPLE, ws * OVERSAMPLE + big = _resize(hi, w4, h4) # candidate at 4x the original's resolution + opad = _pad(orig, margin) + small = _resize(big, ws, hs) + if small.shape[0] > opad.shape[0] or small.shape[1] > opad.shape[1]: + return np.zeros_like(opad), -np.inf + res = cv2.matchTemplate(opad, small, cv2.TM_CCORR_NORMED) + _, _, _, (bx, by) = cv2.minMaxLoc(res) + best, best_score = _place(opad.shape, small, by, bx), -1.0 + if hs > SUBPIXEL_MAX_HEIGHT: # large images: integer alignment is precise enough + return best, _ncc(opad, best) + half = OVERSAMPLE // 2 + for dy in range(-half, half + 1): # quarter-pixel refinement, ±½ px + for dx in range(-half, half + 1): + shifted = np.zeros((h4 + 2 * OVERSAMPLE, w4 + 2 * OVERSAMPLE), dtype=np.float32) + oy, ox = OVERSAMPLE + dy, OVERSAMPLE + dx + shifted[oy: oy + h4, ox: ox + w4] = big + s = _resize(shifted, ws + 2, hs + 2) + cand = _place(opad.shape, s, by - 1, bx - 1) + score = _ncc(opad, cand) + if score > best_score: + best, best_score = cand, score + return best, best_score + + +def compare(orig_ink: np.ndarray, rendered: Rendered, tolerance: float | None = None, + px_per_pt: float | None = None) -> Comparison: + """Compare a trimmed original ink image with a rendered candidate. + + *px_per_pt* fixes the scale (e.g. for confusable variants of an already aligned candidate: + same font size, so only the position is searched). + """ + if not rendered.ok: + return Comparison(False, rendered.error or "nicht gerendert") + orig = orig_ink.astype(np.float32) + if orig.size == 0: + return Comparison(False, "Originalbild enthält keine Tinte") + sw = stroke_width(orig) + norm = 1.0 + if sw > TARGET_STROKE * 1.25: + norm = TARGET_STROKE / sw + orig = trim(_resize(orig, int(round(orig.shape[1] * norm)), int(round(orig.shape[0] * norm)))) + sw = stroke_width(orig) + h, w = orig.shape + zoom = max(2.0, OVERSAMPLE * h / max(rendered.page_height_pt(), 1.0)) + hi = trim(rendered.rasterize(zoom)) + if hi.size == 0: + return Comparison(False, "Kandidat ist leer") + sy, sx = h / hi.shape[0], w / hi.shape[1] + aspect = sx / sy + metrics: dict[str, float] = {"aspect": aspect, "stroke": sw, "norm": norm} + if abs(aspect - 1.0) > MAX_ASPECT_DEVIATION: + # Different layout (e.g. another typesetting engine); still compare at the height scale so a + # human gets an overlay, but the metrics will keep it out of automatic acceptance. + sx = sy + + margin = int(np.ceil(max(h, w) * 0.06)) + 3 + if px_per_pt: + lo = up = px_per_pt * norm / zoom + rounds, step = 2, lo * 0.004 + grid = [lo] + else: + lo, up = min(sx, sy) * 0.97, max(sx, sy) * 1.03 + rounds, step = 3, (up - lo) / 8 + grid = list(np.linspace(lo, up, 9)) + # the scaled candidate must always fit onto the padded original + margin = max(margin, int(np.ceil(max(hi.shape[1] * up - w, hi.shape[0] * up - h, 0))) // 2 + 3) + best_scale, best_score, best_cand = lo, -np.inf, None + for _round in range(rounds): # coarse grid, then finer grids around the best scale + for s in grid: + cand, score = _aligned_candidate(orig, hi, float(s), margin) + if score > best_score: + best_scale, best_score, best_cand = float(s), score, cand + step /= 4 + grid = [best_scale + k * step for k in (-3, -2, -1, 1, 2, 3)] + if best_cand is None: + return Comparison(False, "Kandidat passt nicht auf das Original", metrics) + opad = _pad(orig, margin) + + tol = tolerance if tolerance is not None else max(1.0, 0.6 * sw) + ob, rb = opad > BIN_THRESHOLD, best_cand > BIN_THRESHOLD + d_to_r = ndimage.distance_transform_edt(~rb) if rb.any() else np.full(ob.shape, 1e9) + d_to_o = ndimage.distance_transform_edt(~ob) if ob.any() else np.full(ob.shape, 1e9) + miss_o = ob & (d_to_r > tol) # original ink without candidate ink nearby + miss_r = rb & (d_to_o > tol) # candidate ink without original ink nearby + mismatch = miss_o | miss_r + lab, n = ndimage.label(mismatch, structure=np.ones((3, 3))) + max_blob = float(np.bincount(lab.ravel())[1:].max()) / (sw * sw) if n else 0.0 + + def worst_cover(binary: np.ndarray, dist: np.ndarray) -> float: + lab2, n2 = ndimage.label(binary, structure=np.ones((3, 3))) + if n2 == 0: + return 0.0 + idx = np.arange(1, n2 + 1) + sizes = ndimage.sum(np.ones_like(dist), lab2, idx) + good = ndimage.sum(dist <= tol, lab2, idx) + keep = sizes >= 2 + return float((good[keep] / sizes[keep]).min()) if keep.any() else 1.0 + + # Local error density: mismatch summed over a glyph-sized window. A wrong digit at 7 px height + # leaves several small fragments; together they stand out against thin antialiasing fringes. + k = glyph_size(ob) + win_mis = cv2.boxFilter(mismatch.astype(np.float32), -1, (k, k), normalize=False, borderType=cv2.BORDER_CONSTANT) + absdiff = np.abs(opad - best_cand) + win_abs = cv2.boxFilter(absdiff, -1, (k, k), normalize=False, borderType=cv2.BORDER_CONSTANT) + ch_o = float(d_to_r[ob].mean()) if ob.any() else 99.0 + ch_r = float(d_to_o[rb].mean()) if rb.any() else 99.0 + m = min(opad.shape) + win = min(7, m if m % 2 else m - 1) + ssim = float(structural_similarity(1 - opad, 1 - best_cand, data_range=1.0, win_size=win)) if win >= 3 else 0.0 + metrics.update({ + "ncc": _ncc(opad, best_cand), + "ssim": ssim, + "max_blob": max_blob, + "local_mis": float(win_mis.max()) / (sw * sw), + "local_abs": float(win_abs.max()) / (sw * sw), + "glyph": float(k), + "worst_cover": min(worst_cover(ob, d_to_r), worst_cover(rb, d_to_o)), + "chamfer": max(ch_o, ch_r), + "ink_ratio": float(best_cand.sum() / max(opad.sum(), 1e-6)), + # size mismatch after alignment, with 1.5 px slack for trimming at low resolution + "aspect_dev": max(0.0, abs(hi.shape[1] * best_scale - w) - 1.5) / w + + max(0.0, abs(hi.shape[0] * best_scale - h) - 1.5) / h, + "px_per_pt": best_scale * zoom / norm, + "tolerance": tol, + }) + return Comparison(True, None, metrics, opad, best_cand, mismatch) + + +def contrast(a: Comparison, b: Comparison, max_shift: float = 0.5) -> float | None: + """How much better candidate *a* explains the original than candidate *b*, in [-1, 1]. + + Only the pixels where the two renders differ are looked at (a likelihood-ratio test on the + discriminating region), so a wrong subscript digit is not drowned by the rest of the formula. + Returns None if the two renders are identical at this resolution (equivalent spellings such as + \\le and \\leq). + """ + if not (a.ok and b.ok) or a.original is None or b.original is None: + return 1.0 if a.ok and not b.ok else (-1.0 if b.ok and not a.ok else 0.0) + assert a.candidate is not None and b.candidate is not None + a_orig, a_cand, b_cand = a.original, a.candidate, b.candidate + if a.original.shape != b.original.shape: + # Both canvases are the same original with different symmetric margins: crop to the smaller. + hh = min(a.original.shape[0], b.original.shape[0]) + ww = min(a.original.shape[1], b.original.shape[1]) + + def crop(x: np.ndarray) -> np.ndarray: + dy, dx = (x.shape[0] - hh) // 2, (x.shape[1] - ww) // 2 + return x[dy:dy + hh, dx:dx + ww] + + a_orig, a_cand, b_cand = crop(a.original), crop(a.candidate), crop(b.candidate) + b_cand = _register(b_cand, a_cand) + region = np.abs(a_cand - b_cand) > 0.2 + if not region.any(): + return None + # Window around the differing pixels; the variants are rendered layout-preserving (see + # confusables.probe), so the difference stays local to the changed symbol. + g = glyph_size(a_orig > BIN_THRESHOLD) + ys, xs = np.nonzero(region) + y0, y1 = max(0, ys.min() - g), min(region.shape[0], ys.max() + g + 1) + x0, x1 = max(0, xs.min() - g), min(region.shape[1], xs.max() + g + 1) + o = a_orig[y0:y1, x0:x1] + ea = _best_shift_error(o, a_cand[y0:y1, x0:x1], max_shift) + eb = _best_shift_error(o, b_cand[y0:y1, x0:x1], max_shift) + return (eb - ea) / (eb + ea) if eb + ea > 0 else 0.0 + + +def _shift(img: np.ndarray, dy: float, dx: float) -> np.ndarray: + m = np.float32([[1, 0, dx], [0, 1, dy]]) + return cv2.warpAffine(img, m, (img.shape[1], img.shape[0]), flags=cv2.INTER_LINEAR, + borderMode=cv2.BORDER_CONSTANT, borderValue=0) + + +def _register(b: np.ndarray, a: np.ndarray, max_px: int = 3) -> np.ndarray: + """Shift render *b* onto render *a* (both of nearly the same formula) so identical parts coincide.""" + best, best_err = b, float(np.abs(a - b).sum()) + by = bx = 0.0 + for dy in range(-max_px, max_px + 1): + for dx in range(-max_px, max_px + 1): + if dy == 0 and dx == 0: + continue + cand = _shift(b, dy, dx) + err = float(np.abs(a - cand).sum()) + if err < best_err: + best, best_err, by, bx = cand, err, float(dy), float(dx) + for fy in (-0.5, -0.25, 0.0, 0.25, 0.5): + for fx in (-0.5, -0.25, 0.0, 0.25, 0.5): + if fy == 0 and fx == 0: + continue + cand = _shift(b, by + fy, bx + fx) + err = float(np.abs(a - cand).sum()) + if err < best_err: + best, best_err = cand, err + return best + + +_LOCAL_SHIFTS = [(dy * 0.25, dx * 0.25) for dy in range(-4, 5) for dx in range(-4, 5)] + + +def _best_shift_error(o: np.ndarray, win: np.ndarray, max_shift: float) -> float: + """Smallest squared error of window *win* against *o* over small sub-pixel shifts.""" + pad = 2 + big = np.pad(win, pad) + best = float("inf") + for dy, dx in _LOCAL_SHIFTS: + if abs(dy) > max_shift or abs(dx) > max_shift: + continue + shifted = _shift(big, dy, dx)[pad:-pad, pad:-pad] + best = min(best, float(((o - shifted) ** 2).sum())) + return best + + +def diff_image(c: Comparison, upscale: int = 0) -> np.ndarray: + """RGB overlay: black = both, red = only original, blue = only candidate, yellow = mismatch blob.""" + assert c.original is not None and c.candidate is not None and c.mismatch is not None + o, r = c.original, c.candidate + rgb = np.ones(o.shape + (3,), dtype=np.float32) + rgb[..., 0] -= r + rgb[..., 1] -= np.maximum(o, r) + rgb[..., 2] -= o + rgb = np.clip(rgb, 0, 1) + rgb[c.mismatch] = rgb[c.mismatch] * 0.4 + np.array([1.0, 0.85, 0.0]) * 0.6 + out = (rgb * 255).astype(np.uint8) + if upscale <= 0: + upscale = max(1, int(np.ceil(120 / max(o.shape[0], 1)))) + if upscale > 1: + out = cv2.resize(out, (out.shape[1] * upscale, out.shape[0] * upscale), interpolation=cv2.INTER_NEAREST) + return out diff --git a/formula_img2latex/confusables.py b/formula_img2latex/confusables.py new file mode 100644 index 0000000..cea3878 --- /dev/null +++ b/formula_img2latex/confusables.py @@ -0,0 +1,212 @@ +"""Generate visually confusable variants of a TeX formula. + +A candidate is only trusted if it explains the original image clearly better than every variant +that differs by one typical recognition error: another digit, sub- instead of superscript, a +similar letter or symbol, a missing sign or prime. Variants that render identically (\\le vs. +\\leq) are filtered out later by the comparison itself. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass + +MAX_VARIANTS = 160 + +_TOKEN_RE = re.compile(r"\\[A-Za-z]+\*?|\\.|\s+|.", re.S) + +# Groups of mutually confusable tokens. A token is replaced by every other member of its groups. +GROUPS: list[tuple[str, ...]] = [ + tuple("0123456789"), + ("+", "-", "=", "\\pm", "\\mp", "\\div"), + ("<", "\\le", "\\leqslant", "\\ll", "\\prec"), + (">", "\\ge", "\\geqslant", "\\gg", "\\succ"), + ("\\le", "\\leq"), ("\\ge", "\\geq"), ("\\ne", "\\neq", "="), + ("\\cdot", ".", "\\times", "\\ast", "*", "\\circ", "\\bullet"), + (",", ";", ".", ":"), + ("(", "[", "\\{", "\\langle", "|"), + (")", "]", "\\}", "\\rangle", "|"), + ("|", "\\|", "/", "l", "1", "I"), + ("\\to", "\\rightarrow", "\\Rightarrow", "\\mapsto", "\\longrightarrow"), + ("\\leftarrow", "\\Leftarrow", "\\gets"), + ("\\Leftrightarrow", "\\leftrightarrow", "\\iff"), + ("\\subset", "\\subseteq", "\\subsetneq", "\\in", "\\sqsubset"), + ("\\supset", "\\supseteq", "\\ni"), + ("\\in", "\\epsilon", "\\varepsilon", "\\notin"), + ("\\cup", "\\cap", "\\sqcup", "\\uplus"), + ("\\vee", "\\wedge", "v", "\\lor", "\\land"), + ("\\sim", "\\approx", "\\simeq", "\\cong", "\\equiv", "="), + ("\\infty", "\\propto", "\\alpha"), + ("x", "y", "\\chi", "\\times", "X"), + ("v", "u", "\\nu", "\\upsilon", "V", "U"), + ("n", "m", "h", "\\eta", "r"), + ("i", "j", "l", "1", "\\iota"), + ("o", "0", "O", "\\circ", "\\sigma", "a"), + ("p", "\\rho", "q", "P"), + ("e", "c", "\\varepsilon", "\\epsilon", "\\in", "E"), + ("k", "\\kappa", "K"), + ("w", "\\omega", "W"), + ("a", "\\alpha", "d"), + ("t", "\\tau", "f", "\\dagger"), + ("b", "6", "\\delta"), + ("s", "S", "5"), + ("z", "Z", "2"), + ("g", "9", "q"), + ("\\phi", "\\varphi", "\\Phi", "\\emptyset", "\\varnothing", "\\psi"), + ("\\theta", "\\vartheta", "\\Theta", "0"), + ("\\rho", "\\varrho"), ("\\sigma", "\\varsigma"), ("\\pi", "\\varpi", "n"), + ("\\Sigma", "\\sum"), ("\\Pi", "\\prod"), ("\\Delta", "\\triangle", "\\nabla"), + ("\\partial", "\\delta", "d"), + ("\\bar", "\\overline", "\\hat", "\\widehat", "\\tilde", "\\widetilde", "\\vec", "\\dot", "\\ddot", + "\\check", "\\breve", "\\underline"), + ("\\vec", "\\overrightarrow", "\\overleftarrow", "\\overline"), + ("\\mathbb", "\\mathbf", "\\mathcal", "\\mathrm", "\\mathit", "\\mathsf", "\\mathfrak", "\\mathscr", + "\\boldsymbol"), + ("\\sin", "\\cos", "\\tan", "\\sinh", "\\cosh"), + ("\\log", "\\ln", "\\lg", "\\exp"), + ("\\lim", "\\sup", "\\inf", "\\max", "\\min", "\\limsup", "\\liminf"), + ("\\int", "\\oint", "\\iint"), + ("\\frac", "\\dfrac", "\\tfrac"), + ("\\dots", "\\cdots", "\\ldots", "\\vdots"), + ("'", "\\prime", ","), +] + +_INDEX: dict[str, list[str]] = {} +for _g in GROUPS: + for _t in _g: + _INDEX.setdefault(_t, []) + _INDEX[_t].extend(x for x in _g if x != _t and x not in _INDEX[_t]) + +# Deleting these single tokens is a typical OCR error (missing sign, prime, dot, accent). +_DELETABLE = {"-", "+", "'", "\\prime", ",", ".", ";", "\\cdot", "!", "|", "\\,", "\\;", "\\!", + "\\bar", "\\overline", "\\hat", "\\tilde", "\\vec", "\\dot", "\\ast", "*", + "\\mathbb", "\\mathcal", "\\mathbf", "\\mathrm", "\\mathit", "\\mathsf", "\\mathfrak", + "\\mathscr", "\\boldsymbol", "\\bm"} +# A plain letter may really be printed in a special alphabet (N vs. \mathcal{N}, R vs. \mathbb{R}). +_FONTS_UPPER = ("\\mathcal", "\\mathbb", "\\mathbf", "\\mathrm", "\\mathscr", "\\mathfrak", "\\boldsymbol") +_FONTS_LOWER = ("\\mathbf", "\\mathrm", "\\boldsymbol") +_FONT_CMDS = {"\\mathbb", "\\mathcal", "\\mathbf", "\\mathrm", "\\mathit", "\\mathsf", "\\mathfrak", + "\\mathscr", "\\boldsymbol", "\\bm", "\\operatorname", "\\text", "\\textrm", "\\mbox"} +# Inserting a minus after these is a typical OCR error (lost sign). +_MINUS_AFTER = {"(", "[", "=", ",", "{", "^", "_", "\\le", "\\ge", "<", ">"} + + +@dataclass(frozen=True) +class Variant: + tex: str + kind: str # e.g. "subst 3->8", "sub<->sup", "delete -", "insert -" + priority: int # lower = tried first when capping + probe: str = "" # layout-preserving TeX for the contrast test ("" = render tex itself) + + @property + def render_tex(self) -> str: + return self.probe or self.tex + + +# TeX math class of a token, so a phantom keeps the original spacing. +_BIN = {"+", "-", "\\pm", "\\mp", "\\cdot", "\\times", "\\div", "\\ast", "*", "\\circ", "\\bullet", "\\cup", + "\\cap", "\\sqcup", "\\uplus", "\\vee", "\\wedge", "\\lor", "\\land", "\\setminus"} +_REL = {"=", "<", ">", ":", "\\le", "\\leq", "\\ge", "\\geq", "\\ne", "\\neq", "\\leqslant", "\\geqslant", "\\ll", + "\\gg", "\\prec", "\\succ", "\\to", "\\rightarrow", "\\Rightarrow", "\\mapsto", "\\longrightarrow", + "\\leftarrow", "\\Leftarrow", "\\gets", "\\Leftrightarrow", "\\leftrightarrow", "\\iff", "\\subset", + "\\subseteq", "\\subsetneq", "\\sqsubset", "\\supset", "\\supseteq", "\\ni", "\\in", "\\notin", "\\sim", + "\\approx", "\\simeq", "\\cong", "\\equiv", "\\propto"} +_PUNCT = {",", ";"} +_OPEN = {"(", "[", "\\{", "\\langle"} +_CLOSE = {")", "]", "\\}", "\\rangle"} +_OP = {"\\sum", "\\prod", "\\int", "\\oint", "\\iint", "\\lim", "\\sup", "\\inf", "\\max", "\\min", + "\\limsup", "\\liminf", "\\sin", "\\cos", "\\tan", "\\sinh", "\\cosh", "\\log", "\\ln", "\\lg", "\\exp"} +# Tokens that take arguments cannot be wrapped on their own; such variants are rendered as they are. +_TAKES_ARG = {"\\frac", "\\dfrac", "\\tfrac", "\\sqrt", "\\text", "\\operatorname", "\\mathbb", "\\mathcal", + "\\mathbf", "\\mathrm", "\\mathit", "\\mathsf", "\\mathfrak", "\\mathscr", "\\boldsymbol", "\\bm", + "\\bar", "\\overline", "\\hat", "\\widehat", "\\tilde", "\\widetilde", "\\vec", "\\dot", "\\ddot", + "\\check", "\\breve", "\\underline", "\\overrightarrow", "\\overleftarrow", "'", "\\prime"} + + +def _phantom(t: str) -> str: + cls = ("\\mathbin" if t in _BIN else "\\mathrel" if t in _REL else "\\mathpunct" if t in _PUNCT + else "\\mathopen" if t in _OPEN else "\\mathclose" if t in _CLOSE else "\\mathop" if t in _OP + else "\\mathord") + return f"{cls}{{\\phantom{{{t}}}}}" + + +def _probe_subst(t: str, new: str) -> str | None: + """Draw *new* where *t* was, keeping the width and spacing class of *t* (no layout shift).""" + if t in _TAKES_ARG or new.split("{")[0] in _TAKES_ARG - {"\\mathbb", "\\mathcal", "\\mathbf", "\\mathrm", + "\\mathscr", "\\mathfrak", "\\boldsymbol"}: + return None + return f"\\mathrlap{{{new}}}{_phantom(t)}" + + +def tokenize(tex: str) -> list[str]: + return _TOKEN_RE.findall(tex) + + +def _needs_space(prev: str, new: str, nxt: str) -> str: + """A replacement command followed by a letter needs a separating space (\\alpha x, not \\alphax).""" + if re.fullmatch(r"\\[A-Za-z]+", new) and re.match(r"[A-Za-z]", nxt or ""): + return new + " " + return new + + +def _in_font_arg(toks: list[str], i: int) -> bool: + """True if token i is (inside) the argument of a font/text command, e.g. the N in \\mathcal{N}.""" + depth = 0 + for j in range(i - 1, -1, -1): + t = toks[j] + if t == "}": + depth += 1 + elif t == "{": + if depth == 0: + k = j - 1 + while k >= 0 and toks[k].isspace(): + k -= 1 + return k >= 0 and toks[k] in _FONT_CMDS + depth -= 1 + elif depth == 0 and t in _FONT_CMDS: + return True + return False + + +def variants(tex: str, limit: int = MAX_VARIANTS) -> list[Variant]: + toks = tokenize(tex) + out: dict[str, Variant] = {} + + def add(new_toks: list[str], kind: str, prio: int, probe_toks: list[str] | None = None) -> None: + s = "".join(new_toks).strip() + if s and s != tex.strip() and s not in out: + out[s] = Variant(s, kind, prio, "".join(probe_toks).strip() if probe_toks else "") + + def probe(i: int, wrap: str | None) -> list[str] | None: + if wrap is None: + return None + nxt = toks[i + 1] if i + 1 < len(toks) else "" + return toks[:i] + [wrap + (" " if re.match(r"[A-Za-z]", nxt) else "")] + toks[i + 1:] + + for i, t in enumerate(toks): + nxt = toks[i + 1] if i + 1 < len(toks) else "" + prev = toks[i - 1] if i > 0 else "" + if t in ("^", "_"): + add(toks[:i] + ["_" if t == "^" else "^"] + toks[i + 1:], "sub<->sup", 0) + continue + for alt in _INDEX.get(t, []): + prio = 1 if t.isdigit() or not t.isalpha() else 2 + add(toks[:i] + [_needs_space(prev, alt, nxt)] + toks[i + 1:], f"subst {t}->{alt}", prio, + probe(i, _probe_subst(t, alt))) + if len(t) == 1 and t.isalpha() and not _in_font_arg(toks, i): + for font in (_FONTS_UPPER if t.isupper() else _FONTS_LOWER): + add(toks[:i] + [font + "{" + t + "}"] + toks[i + 1:], f"font {font}{{{t}}}", 3, + probe(i, _probe_subst(t, font + "{" + t + "}"))) + if len(t) == 1 and t.isalpha() and t.lower() not in "xvnioepkwatbszg": + add(toks[:i] + [t.swapcase()] + toks[i + 1:], f"case {t}", 3, probe(i, _probe_subst(t, t.swapcase()))) + if t in _DELETABLE: + add(toks[:i] + toks[i + 1:], f"delete {t}", 1, + probe(i, None if t in _TAKES_ARG or t.startswith("\\\\,") else _phantom(t))) + if t in _MINUS_AFTER and nxt not in ("-", "+"): + add(toks[:i + 1] + ["-"] + toks[i + 1:], f"insert - after {t}", 2) + if len(t) == 1 and t.isalpha() and nxt != "'": + add(toks[:i + 1] + ["'"] + toks[i + 1:], f"insert ' after {t}", 4) + if toks and toks[0] not in ("-", "+"): + add(["-"] + toks, "insert - at start", 2) + res = sorted(out.values(), key=lambda v: v.priority) + return res[:limit] diff --git a/formula_img2latex/decide.py b/formula_img2latex/decide.py new file mode 100644 index 0000000..31082b3 --- /dev/null +++ b/formula_img2latex/decide.py @@ -0,0 +1,60 @@ +"""Turn a verification into a status. All thresholds live here and can be overridden by `calibrate`. + +Accept only if + * the candidate compiles and Pandoc converts it to MathML, + * the absolute comparison is plausible (correlation, proportions, size consistent with the book), + * it beats every distinguishable confusable variant by at least `min_contrast`. +Everything plausible but not certain goes to review; everything else keeps the image. +""" + +from __future__ import annotations + +from typing import Any + +DEFAULTS: dict[str, float] = { + "min_contrast": 0.40, # accept: worst win over a confusable variant (calibrated: worst rival 0.35) + "review_contrast": -0.10, # below this a variant explains the image better -> reject + "min_ncc": 0.85, # accept + "review_ncc": 0.15, # below: nothing in common (wrong image/figure) -> keep image + "max_aspect_dev": 0.08, # accept: proportions within 8 % + "max_scale_dev": 0.08, # accept: px/pt within 8 % of the book median (wrong style, \frac vs \tfrac) +} + + +def thresholds(overrides: dict[str, Any] | None = None) -> dict[str, float]: + t = dict(DEFAULTS) + for k, v in (overrides or {}).items(): + if k in t and v is not None: + t[k] = float(v) + return t + + +def decide(ver: dict[str, Any], pandoc_ok: bool, book_px_per_pt: float | None, + th: dict[str, float]) -> tuple[str, str]: + """Return (status, German reason).""" + if not ver.get("ok"): + return "render_failed", f"Kandidat nicht vergleichbar: {ver.get('reason') or 'unbekannt'}" + m = ver.get("metrics", {}) + ncc = float(m.get("ncc", 0.0)) + aspect_dev = float(m.get("aspect_dev", abs(float(m.get("aspect", 1.0)) - 1.0))) + mc = ver.get("min_contrast") + problems: list[str] = [] + if not pandoc_ok: + problems.append("Pandoc kann die Formel nicht nach MathML umsetzen") + if ncc < th["min_ncc"]: + problems.append(f"Ähnlichkeit {ncc:.3f} < {th['min_ncc']}") + if aspect_dev > th["max_aspect_dev"]: + problems.append(f"Proportionen weichen um {aspect_dev:.0%} ab") + if book_px_per_pt: + dev = abs(float(m.get("px_per_pt", book_px_per_pt)) / book_px_per_pt - 1.0) + if dev > th["max_scale_dev"]: + problems.append(f"Schriftgröße weicht um {dev:.0%} vom Buch ab (Stil?)") + if mc is not None and mc < th["min_contrast"]: + problems.append(f"nicht eindeutig gegenüber „{ver.get('worst_variant')}“ (Kontrast {mc:.2f})") + if not problems: + why = "alle Prüfungen bestanden" + if mc is None: + why += " (keine unterscheidbaren Varianten)" + return "auto_accepted", why + hopeless = ncc < th["review_ncc"] or (mc is not None and mc < th["review_contrast"]) + return ("rejected_keep_image" if hopeless else "needs_review"), "; ".join(problems) diff --git a/formula_img2latex/engines/__init__.py b/formula_img2latex/engines/__init__.py new file mode 100644 index 0000000..9d0dc1f --- /dev/null +++ b/formula_img2latex/engines/__init__.py @@ -0,0 +1,36 @@ +"""Candidate generators: turn a formula image into a LaTeX candidate. + +Engines only propose; verify.py decides. Every engine speaks the OpenAI chat-completions API +with an image part, so a local llama.cpp server (llama-server with --mmproj) and OpenRouter use +the same code. +""" + +from __future__ import annotations + +import os + +from .vlm import VisionEngine + +DEFAULT_LOCAL_URL = "http://127.0.0.1:8011/v1" +DEFAULT_CLOUD_URL = "https://openrouter.ai/api/v1" +DEFAULT_CLOUD_MODEL = "google/gemini-2.5-flash" + + +def make_engine(name: str, model: str | None = None, url: str | None = None) -> VisionEngine: + """'local' (llama.cpp server) or 'cloud' (OpenRouter).""" + if name == "local": + return VisionEngine( + name="local", + base_url=url or os.environ.get("FORMULA_LOCAL_URL", DEFAULT_LOCAL_URL), + model=model or os.environ.get("FORMULA_LOCAL_MODEL", ""), + api_key=os.environ.get("FORMULA_LOCAL_API_KEY", ""), + ) + if name == "cloud": + return VisionEngine( + name="cloud", + base_url=url or os.environ.get("FORMULA_CLOUD_URL", DEFAULT_CLOUD_URL), + model=model or os.environ.get("FORMULA_CLOUD_MODEL", DEFAULT_CLOUD_MODEL), + api_key=os.environ.get("OPENROUTER_API_KEY", ""), + needs_key=True, + ) + raise ValueError(f"unbekannte Engine {name!r} (local, cloud)") diff --git a/formula_img2latex/engines/vlm.py b/formula_img2latex/engines/vlm.py new file mode 100644 index 0000000..1e44721 --- /dev/null +++ b/formula_img2latex/engines/vlm.py @@ -0,0 +1,174 @@ +"""Vision-language model over an OpenAI-compatible chat-completions endpoint.""" + +from __future__ import annotations + +import base64 +import io +import json +import re +import time +from dataclasses import dataclass + +import numpy as np +import requests +from PIL import Image + +from ..errors import EXIT_TOOL, FormulaError +from ..imageio import gray_to_ink, load_gray + +MIN_HEIGHT = 96 # small formula images are enlarged to at least this height for the model +MAX_UPSCALE = 8 +TIMEOUT_S = 120 + +PROMPT = """Transcribe the mathematical formula in the image into LaTeX (amsmath, amssymb). + +Rules: +- Reproduce exactly what is printed: every symbol, digit, sign, sub- and superscript, prime, accent and bracket. +- Do not simplify, complete, reorder or correct anything. Do not add a trailing period unless one is printed. +- Use \\mathbb, \\mathcal, \\mathrm, \\boldsymbol etc. where the print uses such fonts. +- Use \\frac for fractions, \\sqrt for roots, \\begin{pmatrix}/\\begin{array}/\\begin{cases} for matrices and cases, + \\begin{aligned} for several aligned lines. +- Write an equation number such as (3.7) at the right margin as \\tag{3.7}. +- Do not wrap the result in $ or \\[ \\]. +- If the image is not a formula (photo, diagram, plot, plain text), set is_formula to false and latex to "". +{context} +Answer with JSON only: {"is_formula": true, "latex": "..."}""" + +CONTEXT_LINE = "- The formula is {where} this text from the book (for orientation only, do not copy it): \"{text}\"\n" + + +@dataclass +class Recognition: + engine: str + model: str + tex: str | None + is_formula: bool | None + raw: str = "" + error: str | None = None + seconds: float = 0.0 + tokens_in: int = 0 + tokens_out: int = 0 + + def to_json(self) -> dict: + return {k: v for k, v in self.__dict__.items()} + + +def prepare_image(data: bytes, name: str = "") -> bytes: + """Flatten transparency onto white, enlarge tiny formulas, add a margin; PNG bytes.""" + ink = gray_to_ink(load_gray(data, name)) + gray = (255 - np.clip(ink * 255, 0, 255)).astype(np.uint8) + im = Image.fromarray(gray) + if im.height < MIN_HEIGHT: + f = min(MAX_UPSCALE, max(1, int(np.ceil(MIN_HEIGHT / max(im.height, 1))))) + im = im.resize((im.width * f, im.height * f), Image.LANCZOS) + pad = max(8, im.height // 6) + canvas = Image.new("L", (im.width + 2 * pad, im.height + 2 * pad), 255) + canvas.paste(im, (pad, pad)) + buf = io.BytesIO() + canvas.save(buf, "PNG") + return buf.getvalue() + + +def parse_answer(text: str) -> tuple[str | None, bool | None]: + """Extract (latex, is_formula) from a model answer (JSON, fenced JSON, or bare LaTeX).""" + s = text.strip() + s = re.sub(r"^```(?:json|latex|tex)?\s*|\s*```$", "", s, flags=re.S).strip() + m = re.search(r"\{.*\}", s, re.S) + if m: + blob = m.group(0) + # Models often write LaTeX with single backslashes inside JSON: "\frac" would become a + # form feed + "rac", "\nu" a newline + "u". Such control characters mean: re-read the + # blob with every lone backslash taken literally. + literal = re.sub(r'\\\\|\\"|\\', lambda mm: "\\\\" if mm.group(0) == "\\" else mm.group(0), blob) + for attempt in (blob, literal): + try: + data = json.loads(attempt) + except ValueError: + continue + if not isinstance(data, dict): + continue + tex = data.get("latex") + if attempt is blob and isinstance(tex, str) and re.search(r"[\x00-\x1f]", tex): + continue + isf = data.get("is_formula") + return (tex.strip() if isinstance(tex, str) and tex.strip() else None, + bool(isf) if isf is not None else None) + mm = re.search(r'"latex"\s*:\s*"(.*)"\s*\}?\s*$', blob, re.S) + if mm: + return mm.group(1).replace('\\"', '"').replace("\\\\", "\\").strip() or None, None + if s and "{" not in s[:1]: + return s, None + return None, None + + +class VisionEngine: + def __init__(self, name: str, base_url: str, model: str, api_key: str = "", needs_key: bool = False) -> None: + self.name = name + self.base_url = base_url.rstrip("/") + self.model = model + self.api_key = api_key + self.needs_key = needs_key + self.session = requests.Session() + + def check(self) -> None: + """Raise FormulaError if the endpoint is unusable; fills in the model name of a local server.""" + if self.needs_key and not self.api_key: + raise FormulaError("OPENROUTER_API_KEY ist nicht gesetzt (für --engine cloud nötig)", EXIT_TOOL) + if not self.model: + try: + r = self.session.get(f"{self.base_url}/models", headers=self._headers(), timeout=10) + r.raise_for_status() + self.model = r.json()["data"][0]["id"] + except (requests.RequestException, KeyError, IndexError, ValueError) as e: + raise FormulaError(f"Lokaler Modellserver unter {self.base_url} nicht erreichbar ({e}). " + "Server starten (siehe BEDIENUNGSANLEITUNG) oder --engine cloud verwenden.", + EXIT_TOOL) from e + + def _headers(self) -> dict[str, str]: + h = {"Content-Type": "application/json"} + if self.api_key: + h["Authorization"] = f"Bearer {self.api_key}" + if "openrouter.ai" in self.base_url: + h["X-Title"] = "formula_img2latex" + return h + + def recognize(self, data: bytes, name: str = "", context: str = "", display: bool = False) -> Recognition: + png = prepare_image(data, name) + ctx = "" + if context: + where = "displayed after" if display else "part of" + ctx = CONTEXT_LINE.format(where=where, text=context.replace('"', "'")[:240]) + body = { + "model": self.model, + "temperature": 0, + "max_tokens": 1500, + "messages": [{ + "role": "user", + "content": [ + {"type": "text", "text": PROMPT.replace("{context}", ctx)}, + {"type": "image_url", "image_url": {"url": "data:image/png;base64," + base64.b64encode(png).decode()}}, + ], + }], + } + t0 = time.time() + last_err = "" + for attempt in range(3): + try: + r = self.session.post(f"{self.base_url}/chat/completions", headers=self._headers(), + json=body, timeout=TIMEOUT_S) + if r.status_code in (429, 500, 502, 503, 504): + last_err = f"HTTP {r.status_code}" + time.sleep(2 * (attempt + 1)) + continue + r.raise_for_status() + js = r.json() + text = (js["choices"][0]["message"].get("content") or "").strip() + usage = js.get("usage") or {} + tex, isf = parse_answer(text) + return Recognition(self.name, js.get("model", self.model), tex, isf, text[:2000], None, + time.time() - t0, int(usage.get("prompt_tokens") or 0), + int(usage.get("completion_tokens") or 0)) + except (requests.RequestException, KeyError, IndexError, ValueError) as e: + last_err = str(e)[:300] + time.sleep(1 + attempt) + return Recognition(self.name, self.model, None, None, "", last_err or "unbekannter Fehler", time.time() - t0) diff --git a/formula_img2latex/errors.py b/formula_img2latex/errors.py new file mode 100644 index 0000000..868221c --- /dev/null +++ b/formula_img2latex/errors.py @@ -0,0 +1,15 @@ +"""Exit codes and the tool's own exception type.""" + +EXIT_OK = 0 +EXIT_INPUT = 1 # invalid parameters, missing/unreadable input, unwritable workspace +EXIT_SOURCE = 2 # input directory is not a known converter output +EXIT_TOOL = 3 # required external tool (pdflatex, pandoc) missing or failing +EXIT_INTERNAL = 4 # unexpected internal error + + +class FormulaError(Exception): + """Error with an exit code and a German message for the user.""" + + def __init__(self, message: str, code: int = EXIT_INPUT) -> None: + super().__init__(message) + self.code = code diff --git a/formula_img2latex/imageio.py b/formula_img2latex/imageio.py new file mode 100644 index 0000000..ead8ed9 --- /dev/null +++ b/formula_img2latex/imageio.py @@ -0,0 +1,91 @@ +"""Load original formula assets (PNG, GIF, JPG, SVG, …) as normalized ink images.""" + +from __future__ import annotations + +import io +from pathlib import Path + +import numpy as np +import pymupdf +from PIL import Image + +INK_THRESHOLD = 0.2 # pixels above this count as ink when trimming +NOISE_FLOOR = 0.08 # JPEG ringing on paper is set to zero below this +SVG_ZOOM = 4.0 + + +def _gray_from_pil(im: Image.Image) -> np.ndarray: + """Composite onto white (transparent PNG/GIF formulas are black ink on nothing) and return gray 0..1.""" + if im.mode == "P": + im = im.convert("RGBA") + if im.mode in ("RGBA", "LA", "PA") or "transparency" in im.info: + im = im.convert("RGBA") + bg = Image.new("RGBA", im.size, (255, 255, 255, 255)) + im = Image.alpha_composite(bg, im) + return np.asarray(im.convert("L"), dtype=np.float32) / 255.0 + + +def _gray_from_svg(data: bytes) -> np.ndarray: + doc = pymupdf.open(stream=data, filetype="svg") + pix = doc[0].get_pixmap(matrix=pymupdf.Matrix(SVG_ZOOM, SVG_ZOOM), colorspace=pymupdf.csGRAY, alpha=False) + gray = np.frombuffer(pix.samples, dtype=np.uint8).reshape(pix.height, pix.stride)[:, : pix.width] + return gray.astype(np.float32) / 255.0 + + +def load_gray(data: bytes, name: str = "") -> np.ndarray: + if name.lower().endswith((".svg", ".svgz")) or data.lstrip()[:5] in (b"<?xml", b"<svg "): + return _gray_from_svg(data) + with Image.open(io.BytesIO(data)) as im: + im.seek(0) + return _gray_from_pil(im) + + +def gray_to_ink(gray: np.ndarray) -> np.ndarray: + """Ink = 1 - gray, stretched between paper and darkest ink; inverted if the image is light on dark.""" + border = np.concatenate([gray[0], gray[-1], gray[:, 0], gray[:, -1]]) + if border.size and np.median(border) < 0.5: + gray = 1.0 - gray + ink = 1.0 - gray + paper = float(np.percentile(ink, 5)) + top = float(ink.max()) + if top - paper < 0.15: + return np.zeros_like(ink) + ink = np.clip((ink - paper) / (top - paper), 0.0, 1.0) + ink[ink < NOISE_FLOOR] = 0.0 + return ink + + +def trim(ink: np.ndarray, threshold: float = INK_THRESHOLD) -> np.ndarray: + ys, xs = np.nonzero(ink > threshold) + if ys.size == 0: + return ink[:0, :0] + return ink[ys.min(): ys.max() + 1, xs.min(): xs.max() + 1] + + +def load_ink(path_or_data: str | Path | bytes, name: str = "") -> np.ndarray: + """Trimmed ink image (float32 0..1) of an original formula asset.""" + if isinstance(path_or_data, (str, Path)): + name = name or str(path_or_data) + data = Path(path_or_data).read_bytes() + else: + data = path_or_data + return trim(gray_to_ink(load_gray(data, name))) + + +def color_stats(data: bytes, name: str = "") -> dict[str, float]: + """Simple statistics used to tell formulas (black/white, few gray levels) from figures.""" + if name.lower().endswith((".svg", ".svgz")): + return {"colorfulness": 0.0, "midtones": 0.0} + with Image.open(io.BytesIO(data)) as im: + im.seek(0) + if im.mode == "P": + im = im.convert("RGBA") + rgb = np.asarray(im.convert("RGB"), dtype=np.float32) / 255.0 + gray = _gray_from_pil(im) + sat = rgb.max(axis=2) - rgb.min(axis=2) + ink = gray_to_ink(gray) + mid = float(np.mean((ink > 0.25) & (ink < 0.75))) + # Colour of the *ink* only: formulas on a tinted box (light blue highlight) are still black. + dark = ink > 0.5 + colorful = float(np.mean(sat[dark] > 0.25)) if dark.any() else 0.0 + return {"colorfulness": colorful, "midtones": mid} diff --git a/formula_img2latex/pdfcrop.py b/formula_img2latex/pdfcrop.py new file mode 100644 index 0000000..1768ddf --- /dev/null +++ b/formula_img2latex/pdfcrop.py @@ -0,0 +1,81 @@ +"""Locate a pdf2md formula image on its PDF page and cut it out of the vector page again, sharply. + +pymupdf4llm stores displayed formulas as 150-dpi images. The comparison works much better on a +600-dpi crop of the vector original, so the stored image is found on the rendered page by +template matching (robust against the converter's internal box numbering) and re-cut. +""" + +from __future__ import annotations + +import re +from pathlib import Path + +import cv2 +import numpy as np +import pymupdf + +from .imageio import gray_to_ink, load_gray + +CROP_DPI = 600 +TRY_DPI = (150, 144, 200, 120, 96, 300, 72) +MIN_MATCH = 0.85 +_MATH_FONT_RE = re.compile(r"CMMI|CMSY|CMEX|CMBSY|MSBM|MSAM|EUFM|EURM|EUSM|RSFS|Math|math|Symbol|SY\d|rsfs|" + r"txsy|txex|pxsy|pxex|txmi|pxmi|NewTXMI|NewPXMI|Fourier-Math|STIX.*Math|Cambria Math|" + r"MTMI|MTSY|MTEX|LMMath|LMRoman.*Math|Latin Modern Math") + + +class PdfCropper: + def __init__(self, pdf: Path) -> None: + self.doc = pymupdf.open(pdf) + self._page_cache: dict[tuple[int, int], np.ndarray] = {} + + def _page_gray(self, pno: int, dpi: int) -> np.ndarray: + key = (pno, dpi) + if key not in self._page_cache: + if len(self._page_cache) > 8: + self._page_cache.clear() + pix = self.doc[pno].get_pixmap(dpi=dpi, colorspace=pymupdf.csGRAY, alpha=False) + arr = np.frombuffer(pix.samples, dtype=np.uint8).reshape(pix.height, pix.stride)[:, : pix.width] + self._page_cache[key] = arr.astype(np.float32) / 255.0 + return self._page_cache[key] + + def locate(self, page: int, image: Path) -> tuple[pymupdf.Rect, float] | None: + """PDF rectangle of *image* on 1-based *page* and the match score, or None.""" + if not 1 <= page <= self.doc.page_count: + return None + tmpl = load_gray(image.read_bytes(), image.name) + best: tuple[pymupdf.Rect, float] | None = None + for dpi in TRY_DPI: + g = self._page_gray(page - 1, dpi) + if tmpl.shape[0] > g.shape[0] or tmpl.shape[1] > g.shape[1]: + continue + res = cv2.matchTemplate(g, tmpl, cv2.TM_CCOEFF_NORMED) + _, score, _, (x, y) = cv2.minMaxLoc(res) + f = 72.0 / dpi + rect = pymupdf.Rect(x * f, y * f, (x + tmpl.shape[1]) * f, (y + tmpl.shape[0]) * f) + if best is None or score > best[1]: + best = (rect, float(score)) + if score >= 0.97: + break + return best if best and best[1] >= MIN_MATCH else None + + def is_math(self, page: int, rect: pymupdf.Rect) -> bool: + """True if text inside *rect* is set in math fonts (TeX-made PDFs).""" + d = self.doc[page - 1].get_text("dict", clip=rect) + fonts = {span["font"] for b in d.get("blocks", []) for ln in b.get("lines", []) for span in ln.get("spans", [])} + return any(_MATH_FONT_RE.search(f) for f in fonts) + + def crop(self, page: int, box: int, image: Path) -> tuple[bytes, str] | None: + """(PNG bytes of a 600-dpi crop, 'formula'|'picture') or None if the image is not found.""" + loc = self.locate(page, image) + if loc is None: + return None + rect, _score = loc + rect = rect + (-1, -1, 1, 1) + pix = self.doc[page - 1].get_pixmap(dpi=CROP_DPI, clip=rect, colorspace=pymupdf.csGRAY, alpha=False) + png = pix.tobytes("png") + ink = gray_to_ink(load_gray(png, "x.png")) + if not ink.any(): + return None + return png, ("formula" if self.is_math(page, rect) else "picture") + diff --git a/formula_img2latex/pipeline.py b/formula_img2latex/pipeline.py new file mode 100644 index 0000000..61da9d4 --- /dev/null +++ b/formula_img2latex/pipeline.py @@ -0,0 +1,398 @@ +"""Workflow steps: scan, recognize, verify, apply, revert.""" + +from __future__ import annotations + +import hashlib +import io +import os +import shutil +import statistics +import sys +from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor, as_completed +from pathlib import Path +from typing import Any, Callable + +import numpy as np +from PIL import Image + +from .checks import pandoc_mathml_ok +from .classify import classify +from .compare import diff_image +from .decide import decide, thresholds +from .engines import make_engine +from .imageio import load_ink +from .render import PRESETS +from .sources import open_source +from .sources.base import ImageRef, Source +from .store import HUMAN_STATUSES, REPLACE_STATUSES, Formula, Workspace, now_iso, sha256_bytes +from .tex import normalize_for_compare, split_tag, strip_delimiters +from .verify import verify + +PRESET_SAMPLE = 12 + + +def log(msg: str, quiet: bool = False) -> None: + if not quiet: + print(msg, file=sys.stderr, flush=True) + + +# --------------------------------------------------------------------------------------- scan + +def scan(source: Source, ws: Workspace, quiet: bool = False) -> dict[str, int]: + refs = source.image_refs() + groups: dict[str, tuple[Formula, list[ImageRef]]] = {} + for ref in refs: + p = ref.asset_path() + if not p.exists(): + continue + sha = sha256_bytes(p.read_bytes()) + fid = Formula.make_id(sha, ref.display) + if fid not in groups: + f = ws.load(fid) or Formula(id=fid, sha256=sha, asset=p.relative_to(source.root).as_posix(), + display=ref.display) + groups[fid] = (f, []) + groups[fid][1].append(ref) + counts = {"formula": 0, "maybe": 0, "figure": 0} + for fid, (f, rs) in groups.items(): + ref = rs[0] + f.asset = ref.asset_path().relative_to(source.root).as_posix() + f.source_name = source.source_name(ref) + f.known_formula = source.known_formula(ref) + f.alt = ref.alt + f.occurrences = [{"file": r.file.relative_to(source.root).as_posix(), "line": r.line, "md": r.md, + "context": r.context} for r in rs] + data = source.original_bytes(ref) + f.width, f.height = _image_size(data) + f.classify = classify(data, f.asset, f.known_formula, inline=not f.display) + counts[f.classify["label"]] += 1 + ws.save(f) + ws.meta["source"] = source.describe() + ws.meta["last_scan"] = {"at": now_iso(), "images": len(refs), "formulas": len(groups), **counts} + ws.save_meta() + log(f"scan: {len(refs)} Bildverweise, {len(groups)} verschiedene Bilder: {counts['formula']} Formeln, " + f"{counts['maybe']} unklar, {counts['figure']} Abbildungen", quiet) + return counts + + +def _image_size(data: bytes) -> tuple[int, int]: + try: + with Image.open(io.BytesIO(data)) as im: + return im.size + except OSError: # SVG + return 0, 0 + + +def _wanted(f: Formula, include_figures: bool) -> bool: + if f.status in HUMAN_STATUSES: + return False + label = (f.classify or {}).get("label", "maybe") + return include_figures or label != "figure" + + +def _ref_for(source: Source, f: Formula) -> ImageRef: + occ = f.occurrences[0] + md = source.root / occ["file"] + from .sources.base import scan_markdown + for r in scan_markdown(md): + if r.line == occ["line"] and r.md == occ["md"]: + return r + for r in scan_markdown(md): + if r.md == occ["md"]: + return r + raise FileNotFoundError(f"Bildverweis nicht mehr gefunden: {occ['file']}:{occ['line']}") + + +# --------------------------------------------------------------------------------- recognize + +def recognize(source: Source, ws: Workspace, engine_name: str, model: str | None = None, url: str | None = None, + only_unresolved: bool = False, include_figures: bool = False, limit: int = 0, + workers: int = 4, quiet: bool = False) -> dict[str, int]: + engine = make_engine(engine_name, model, url) + engine.check() + todo = [] + for f in ws.all(): + if not _wanted(f, include_figures): + continue + if only_unresolved and f.status in ("auto_accepted", "not_formula"): + continue + if any(c.get("engine") == engine.name and c.get("model") == engine.model for c in f.candidates): + continue # cached: never ask the same model twice + todo.append(f) + if limit: + todo = todo[:limit] + log(f"recognize: {len(todo)} Bilder mit {engine.name} ({engine.model})", quiet) + stats = {"asked": 0, "latex": 0, "not_formula": 0, "error": 0, "tokens_in": 0, "tokens_out": 0} + + def job(f: Formula) -> tuple[Formula, Any]: + ref = _ref_for(source, f) + ctx = f.occurrences[0].get("context", "") + return f, engine.recognize(source.original_bytes(ref), f.asset, ctx, f.display) + + with ThreadPoolExecutor(max_workers=max(1, workers)) as pool: + futures = [pool.submit(job, f) for f in todo] + for i, fut in enumerate(as_completed(futures), 1): + f, rec = fut.result() + stats["asked"] += 1 + stats["tokens_in"] += rec.tokens_in + stats["tokens_out"] += rec.tokens_out + cand = {"engine": engine.name, "model": engine.model, "tex": rec.tex, "is_formula": rec.is_formula, + "error": rec.error, "seconds": round(rec.seconds, 2), "created_at": now_iso()} + f.candidates.append(cand) + if rec.error: + stats["error"] += 1 + elif rec.tex: + stats["latex"] += 1 + else: + stats["not_formula"] += 1 + ws.save(f) + if not quiet and (i % 25 == 0 or i == len(todo)): + log(f" {i}/{len(todo)}") + log(f"recognize: {stats['latex']} LaTeX-Kandidaten, {stats['not_formula']} „keine Formel“, " + f"{stats['error']} Fehler; Tokens {stats['tokens_in']} ein / {stats['tokens_out']} aus", quiet) + return stats + + +# ------------------------------------------------------------------------------------ verify + +def comparison_pngs(c: Any) -> dict[str, bytes]: + """PNG images of an aligned comparison: original, candidate render and colored overlay.""" + out: dict[str, bytes] = {} + if c is None or c.original is None: + return out + for key, arr in (("orig", c.original), ("render", c.candidate)): + buf = io.BytesIO() + Image.fromarray(((1 - np.clip(arr, 0, 1)) * 255).astype(np.uint8), "L").save(buf, "PNG") + out[key] = buf.getvalue() + buf = io.BytesIO() + Image.fromarray(diff_image(c, upscale=1)).save(buf, "PNG") + out["diff"] = buf.getvalue() + return out + + +def _verify_job(args: tuple[str, bytes, str, list[str], bool, str]) -> tuple[str, list[dict], dict[str, bytes]]: + """Worker process: verify all distinct candidates of one formula; return records and images.""" + fid, data, name, texes, display, preset = args + os.environ.setdefault("OMP_NUM_THREADS", "1") + orig = load_ink(data, name) + results = [] + best = None + for tex in texes: + v = verify(orig, tex, display, preset) + results.append(v.to_json()) + key = (v.ok, v.min_contrast if v.min_contrast is not None else 1.0, v.metrics.get("ncc", 0.0)) + if best is None or key > best[0]: + best = (key, v) + return fid, results, comparison_pngs(best[1].comparison if best else None) + + +def _distinct_candidates(f: Formula) -> list[str]: + seen: dict[str, str] = {} + for c in f.candidates: + tex = c.get("tex") + if not tex: + continue + body, _tag = split_tag(strip_delimiters(tex)) + key = normalize_for_compare(body) + seen.setdefault(key, body) + return list(seen.values()) + + +def choose_preset(source: Source, ws: Workspace, formulas: list[Formula], quiet: bool = False) -> str: + """Pick the font preset under which a sample of candidates matches the originals best.""" + from .render import render_many + from .compare import compare + sample = [f for f in formulas if _distinct_candidates(f)][:PRESET_SAMPLE] + if not sample: + return "cm" + inks = [load_ink(source.original_bytes(_ref_for(source, f)), f.asset) for f in sample] + scores: dict[str, float] = {} + for preset in PRESETS: + rs = render_many([(_distinct_candidates(f)[0], f.display) for f in sample], preset) + vals = [compare(i, r).metrics.get("ncc", 0.0) if r.ok else 0.0 for i, r in zip(inks, rs)] + scores[preset] = float(statistics.median(vals)) + best = max(scores, key=lambda k: scores[k]) + log("Schrift: " + ", ".join(f"{k} {v:.3f}" for k, v in sorted(scores.items(), key=lambda kv: -kv[1])) + + f" → {best}", quiet) + return best + + +def verify_all(source: Source, ws: Workspace, preset: str | None = None, include_figures: bool = False, + workers: int = 0, redo: bool = False, quiet: bool = False, + progress: Callable[[int, int], None] | None = None) -> dict[str, int]: + formulas = [f for f in ws.all() if _wanted(f, include_figures) and _distinct_candidates(f)] + if not redo: + formulas = [f for f in formulas if not _verified_current(f)] + if preset is None: + preset = ws.settings.get("preset") or choose_preset(source, ws, formulas, quiet) + ws.settings["preset"] = preset + ws.save_meta() + jobs = [] + for f in formulas: + ref = _ref_for(source, f) + jobs.append((f.id, source.original_bytes(ref), f.asset, _distinct_candidates(f), f.display, preset)) + log(f"verify: {len(jobs)} Formeln, Schrift {preset}", quiet) + by_id = {f.id: f for f in formulas} + ws.review_dir.mkdir(parents=True, exist_ok=True) + workers = workers or max(1, (os.cpu_count() or 2) - 2) + done = 0 + with ProcessPoolExecutor(max_workers=workers) as pool: + for fut in as_completed([pool.submit(_verify_job, j) for j in jobs]): + fid, results, pngs = fut.result() + f = by_id[fid] + best = max(results, key=lambda r: (r["ok"], r["min_contrast"] if r["min_contrast"] is not None else 1.0, + r["metrics"].get("ncc", 0.0))) + f.verification = {"preset": preset, "candidates_key": _cands_key(f), "best": best, "all": results, + "at": now_iso()} + for key, png in pngs.items(): + (ws.review_dir / f"{fid}-{key}.png").write_bytes(png) + ws.save(f) + done += 1 + if progress: + progress(done, len(jobs)) + elif not quiet and (done % 25 == 0 or done == len(jobs)): + log(f" {done}/{len(jobs)}") + return decide_all(ws, quiet=quiet) + + +def _cands_key(f: Formula) -> str: + return hashlib.sha256("\n".join(sorted(_distinct_candidates(f))).encode()).hexdigest()[:16] + + +def _verified_current(f: Formula) -> bool: + v = f.verification or {} + return v.get("candidates_key") == _cands_key(f) + + +def decide_all(ws: Workspace, quiet: bool = False) -> dict[str, int]: + th = thresholds(ws.settings.get("thresholds")) + formulas = [f for f in ws.all() if f.verification and f.status not in HUMAN_STATUSES] + # Book-wide scale: all formula images of a book share one resolution, so px/pt is constant. + good = [f.verification["best"]["metrics"]["px_per_pt"] for f in formulas + if f.verification["best"].get("ok") and f.verification["best"]["metrics"].get("ncc", 0) >= th["min_ncc"] + and (f.verification["best"].get("min_contrast") or 1.0) >= th["min_contrast"]] + book_ppt = float(statistics.median(good)) if len(good) >= 5 else None + ws.settings["book_px_per_pt"] = book_ppt + texes = [(f.verification["best"]["tex"], f.display) for f in formulas] + pandoc = pandoc_mathml_ok(texes) if texes else [] + counts: dict[str, int] = {} + for f, pok in zip(formulas, pandoc): + best = f.verification["best"] + status, reason = decide(best, pok, book_ppt, th) + f.status, f.reason = status, reason + f.final_tex = _with_tag(f, best["tex"]) if status == "auto_accepted" else None + f.decided_at = now_iso() + ws.save(f) + counts[status] = counts.get(status, 0) + 1 + for f in ws.all(): + if not f.verification and f.status == "pending" and f.candidates and not _distinct_candidates(f): + f.status = "not_formula" if any(c.get("is_formula") is False for c in f.candidates) else "no_candidate" + f.reason = "Modell: keine Formel" if f.status == "not_formula" else "kein Kandidat" + ws.save(f) + counts[f.status] = counts.get(f.status, 0) + 1 + ws.save_meta() + log("Entscheidung: " + ", ".join(f"{k} {v}" for k, v in sorted(counts.items())), quiet) + return counts + + +def _with_tag(f: Formula, tex: str) -> str: + """Re-attach an equation number recognized as \\tag{...} – as visible text, like epub2md does.""" + for c in f.candidates: + if c.get("tex"): + _body, tag = split_tag(strip_delimiters(c["tex"])) + if tag: + return tex + f"\\qquad\\text{{({tag})}}" if f.display else tex + return tex + + +# ------------------------------------------------------------------------------ apply/revert + +def _file_sha(p: Path) -> str: + return sha256_bytes(p.read_bytes()) + + +def apply(source: Source, ws: Workspace, dry_run: bool = False, quiet: bool = False) -> dict[str, int]: + """Rewrite Markdown files from their backed-up originals; idempotent.""" + by_occ: dict[tuple[str, int, str], str] = {} + for f in ws.all(): + if f.status in REPLACE_STATUSES and f.final_tex: + math = source.format_math(f.final_tex, f.display) + for occ in f.occurrences: + by_occ[(occ["file"], occ["line"], occ["md"])] = math + state = ws.meta.setdefault("applied", {}) + files = {occ[0] for occ in by_occ} | set(state) + stats = {"files": 0, "replaced": 0} + for rel in sorted(files): + path = source.root / rel + backup = ws.backup_dir / rel + if not path.exists(): + continue + current = _file_sha(path) + known = state.get(rel, {}) + if backup.exists() and current not in (known.get("applied_sha"), known.get("original_sha")): + # File was regenerated (epub2md --force) or edited by hand: it is the new original. + if not dry_run: + backup.unlink() + if not backup.exists() and not dry_run: + backup.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(path, backup) + known = state[rel] = {"original_sha": current} + original = (backup if backup.exists() else path).read_text(encoding="utf-8").split("\n") + n = 0 + for (frel, line, md), math in by_occ.items(): + if frel != rel or line > len(original): + continue + if md in original[line - 1]: + original[line - 1] = original[line - 1].replace(md, math, 1) + n += 1 + text = "\n".join(original) + stats["replaced"] += n + if dry_run: + continue + if n == 0 and backup.exists(): + shutil.copy2(backup, path) + else: + path.write_text(text, encoding="utf-8") + state[rel] = {"original_sha": known.get("original_sha") or _file_sha(backup), + "applied_sha": _file_sha(path), "replaced": n, "at": now_iso()} + stats["files"] += 1 + if not dry_run: + ws.save_meta() + log(f"apply{' (Probelauf)' if dry_run else ''}: {stats['replaced']} Formelbilder in {stats['files']} Dateien " + "durch LaTeX ersetzt", quiet) + return stats + + +def revert(source: Source, ws: Workspace, force: bool = False, quiet: bool = False) -> dict[str, int]: + state = ws.meta.get("applied", {}) + stats = {"restored": 0, "skipped": 0} + restored: list[str] = [] + for rel, info in sorted(state.items()): + path, backup = source.root / rel, ws.backup_dir / rel + if not backup.exists(): + continue + if path.exists() and _file_sha(path) != info.get("applied_sha") and not force: + log(f" übersprungen (seit apply verändert, --force zum Überschreiben): {rel}", quiet) + stats["skipped"] += 1 + continue + shutil.copy2(backup, path) + backup.unlink() + restored.append(rel) + stats["restored"] += 1 + ws.meta["applied"] = {k: v for k, v in state.items() if k not in restored} + if not ws.meta["applied"]: + shutil.rmtree(ws.backup_dir, ignore_errors=True) + ws.save_meta() + log(f"revert: {stats['restored']} Dateien wiederhergestellt, {stats['skipped']} übersprungen", quiet) + return stats + + +def open_all(path: Path, workspace: Path | None, create: bool = False, pdf: Path | None = None + ) -> tuple[Source, Workspace]: + source = open_source(path) + if pdf is not None and hasattr(source, "pdf"): + source.pdf = pdf.resolve() + ws_root = workspace or Workspace.default_for(path) + ws = Workspace.open(ws_root, create=create, source=source.describe()) + if getattr(source, "pdf", "unset") is None and ws.meta.get("source", {}).get("pdf"): + source.pdf = Path(ws.meta["source"]["pdf"]) + return source, ws diff --git a/formula_img2latex/render.py b/formula_img2latex/render.py new file mode 100644 index 0000000..8c56657 --- /dev/null +++ b/formula_img2latex/render.py @@ -0,0 +1,176 @@ +"""Render TeX formulas with pdflatex (preview package, one tight page per formula) and rasterize them. + +Many formulas share one pdflatex run. Formulas that raise TeX errors are identified via the line +numbers in the log, removed, and the batch is compiled again, so one bad candidate never spoils +the others. +""" + +from __future__ import annotations + +import os +import re +import shutil +import subprocess +import tempfile +from dataclasses import dataclass +from pathlib import Path + +import numpy as np +import pymupdf + +from .errors import EXIT_TOOL, FormulaError +from .tex import prepare_for_render, unsafe_reason + +BATCH_SIZE = 150 +TIMEOUT_S = 180 + +_COMMON_HEAD = r"""\documentclass[10pt]{article} +\usepackage[T1]{fontenc} +\usepackage[utf8]{inputenc} +\usepackage{amsmath} +%(fonts)s +\usepackage{mathtools} +\usepackage{bm} +\usepackage{mathrsfs} +\usepackage[active,tightpage]{preview} +\setlength\PreviewBorder{1pt} +\pagestyle{empty} +\begin{document} +""" + +# Font presets: the renderer must use the same typeface as the book, otherwise a correct +# formula looks different from the original. +PRESETS: dict[str, str] = { + "cm": "\\usepackage{lmodern}\n\\usepackage{amssymb}", + "times": "\\usepackage{newtxtext}\n\\usepackage{newtxmath}", + "txfonts": "\\usepackage{txfonts}", + "palatino": "\\usepackage{newpxtext}\n\\usepackage{newpxmath}", + "stix": "\\usepackage{stix2}", + "euler": "\\usepackage{amssymb}\n\\usepackage{eulervm}", +} + + +@dataclass +class Rendered: + """One rendered formula: page *page* of *doc* (vector), or an error.""" + + tex: str + display: bool + doc: pymupdf.Document | None = None + page: int = -1 + error: str | None = None + + @property + def ok(self) -> bool: + return self.doc is not None + + def rasterize(self, zoom: float) -> np.ndarray: + """Ink image (float32, 0 = paper, 1 = full ink) at *zoom* (1.0 = 72 dpi).""" + assert self.doc is not None + pix = self.doc[self.page].get_pixmap(matrix=pymupdf.Matrix(zoom, zoom), colorspace=pymupdf.csGRAY, alpha=False) + gray = np.frombuffer(pix.samples, dtype=np.uint8).reshape(pix.height, pix.stride)[:, : pix.width] + return 1.0 - gray.astype(np.float32) / 255.0 + + def page_height_pt(self) -> float: + assert self.doc is not None + return float(self.doc[self.page].rect.height) + + +def pdflatex_available() -> bool: + return shutil.which("pdflatex") is not None + + +def _body_line(tex: str, display: bool) -> str: + inner = prepare_for_render(tex) + style = "\\displaystyle " if display else "" + return "\\begin{preview}$" + style + inner + "$\\end{preview}" + + +def _compile(lines: list[str], preset: str, workdir: Path) -> tuple[pymupdf.Document | None, str, list[int]]: + """Compile; return (document or None, log text, list of failing formula indices).""" + head = _COMMON_HEAD % {"fonts": PRESETS[preset]} + head_lines = head.count("\n") + src = [head] + starts: list[tuple[int, int]] = [] # (first line, last line) per formula, 1-based + line = head_lines + 1 + for body in lines: + n = body.count("\n") + 1 + starts.append((line, line + n - 1)) + src.append(body + "\n\n") + line += n + 1 + src.append("\\end{document}\n") + tex_path = workdir / "f.tex" + tex_path.write_text("".join(src), encoding="utf-8") + for old in ("f.pdf", "f.log", "f.aux"): + (workdir / old).unlink(missing_ok=True) + env = dict(os.environ, openout_any="p", openin_any="p", shell_escape="f") + try: + subprocess.run( + ["pdflatex", "-interaction=nonstopmode", "-no-shell-escape", "-file-line-error", "f.tex"], + cwd=workdir, env=env, stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, + timeout=TIMEOUT_S, check=False, + ) + except subprocess.TimeoutExpired: + return None, "Zeitüberschreitung", list(range(len(lines))) + log = (workdir / "f.log").read_text(encoding="latin-1") if (workdir / "f.log").exists() else "" + bad_lines = [int(m.group(1)) for m in re.finditer(r"^\./f\.tex:(\d+):", log, re.M)] + failing = sorted({i for ln in bad_lines for i, (a, b) in enumerate(starts) if a <= ln <= b}) + if bad_lines and not failing: + failing = list(range(len(lines))) # error outside a formula: preamble or runaway argument + pdf = workdir / "f.pdf" + if failing or not pdf.exists(): + return None, log, failing or list(range(len(lines))) + doc = pymupdf.open(stream=pdf.read_bytes(), filetype="pdf") + if doc.page_count != len(lines): + return None, log, list(range(len(lines))) + return doc, log, [] + + +def _log_error(log: str) -> str: + m = re.search(r"^\./f\.tex:\d+: (.+)$", log, re.M) + return m.group(1).strip() if m else "LaTeX-Fehler" + + +def render_many(items: list[tuple[str, bool]], preset: str = "cm") -> list[Rendered]: + """Render (tex, display) pairs. Never raises for bad formulas; check Rendered.ok.""" + if preset not in PRESETS: + raise FormulaError(f"unbekanntes Schrift-Preset {preset!r} (bekannt: {', '.join(PRESETS)})") + if not pdflatex_available(): + raise FormulaError("pdflatex nicht gefunden – installieren mit: sudo apt install texlive-latex-extra", EXIT_TOOL) + results = [Rendered(tex=t, display=d) for t, d in items] + todo = [] + for i, r in enumerate(results): + reason = unsafe_reason(prepare_for_render(r.tex)) + if reason: + r.error = reason + else: + todo.append(i) + with tempfile.TemporaryDirectory(prefix="f2l-render-") as tmp: + work = Path(tmp) + for start in range(0, len(todo), BATCH_SIZE): + _render_batch(results, todo[start:start + BATCH_SIZE], preset, work) + return results + + +def _render_batch(results: list[Rendered], idx: list[int], preset: str, work: Path) -> None: + pending = list(idx) + for _attempt in range(4): + if not pending: + return + lines = [_body_line(results[i].tex, results[i].display) for i in pending] + doc, log, failing = _compile(lines, preset, work) + if doc is not None: + for page, i in enumerate(pending): + results[i].doc, results[i].page = doc, page + return + if len(failing) == len(pending) and len(pending) > 1: + break # cannot attribute the error: fall back to one run per formula + for k in failing: + results[pending[k]].error = _log_error(log) + pending = [i for k, i in enumerate(pending) if k not in set(failing)] + for i in pending: + doc, log, failing = _compile([_body_line(results[i].tex, results[i].display)], preset, work) + if doc is not None: + results[i].doc, results[i].page, results[i].error = doc, 0, None + else: + results[i].error = _log_error(log) diff --git a/formula_img2latex/review/__init__.py b/formula_img2latex/review/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/formula_img2latex/review/page.html b/formula_img2latex/review/page.html new file mode 100644 index 0000000..682b224 --- /dev/null +++ b/formula_img2latex/review/page.html @@ -0,0 +1,157 @@ +<!doctype html> +<html lang="de"> +<head> +<meta charset="utf-8"> +<meta name="viewport" content="width=device-width, initial-scale=1"> +<title>Formel-Review + + + +
+

Formel-Review

+ + + + Overlay: Regler zwischen Original (links) und LaTeX-Render (rechts). Diff: rot = nur Original, blau = nur LaTeX, gelb = Abweichung. +
+
+ + + diff --git a/formula_img2latex/review/server.py b/formula_img2latex/review/server.py new file mode 100644 index 0000000..eaa5765 --- /dev/null +++ b/formula_img2latex/review/server.py @@ -0,0 +1,161 @@ +"""Local review gallery: http://127.0.0.1:PORT, decisions are written straight into the workspace. + +Only binds to localhost. Every POST must carry the per-session token embedded in the page, so +other web pages open in the same browser cannot send decisions. +""" + +from __future__ import annotations + +import base64 +import json +import secrets +import threading +import webbrowser +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from typing import Any +from urllib.parse import unquote, urlparse + +from ..checks import pandoc_mathml_ok +from ..imageio import load_ink +from ..pipeline import _ref_for, comparison_pngs +from ..sources.base import Source +from ..store import Formula, Workspace, now_iso +from ..tex import strip_delimiters, unsafe_reason +from ..verify import verify + +REVIEW_STATUSES = ("needs_review", "rejected_keep_image", "render_failed", "no_candidate") +PAGE = (Path(__file__).parent / "page.html").read_text(encoding="utf-8") +_LOCK = threading.Lock() + + +def _card(f: Formula) -> dict[str, Any]: + best = (f.verification or {}).get("best") or {} + cands = [c.get("tex") for c in f.candidates if c.get("tex")] + return { + "id": f.id, "status": f.status, "reason": f.reason, "display": f.display, "asset": f.asset, + "source_name": f.source_name, "occurrences": f.occurrences[:3], "n_occ": len(f.occurrences), + "tex": f.final_tex or best.get("tex") or (cands[0] if cands else ""), + "candidates": cands, "metrics": best.get("metrics", {}), "min_contrast": best.get("min_contrast"), + "worst_variant": best.get("worst_variant"), "climbed_from": best.get("climbed_from", []), + "classify": f.classify, + } + + +def serve(source: Source, ws: Workspace, port: int = 8765, show_all: bool = False, open_browser: bool = True) -> None: + token = secrets.token_urlsafe(16) + preset = ws.settings.get("preset") or "cm" + + class Handler(BaseHTTPRequestHandler): + def log_message(self, fmt: str, *args: Any) -> None: # keep the terminal quiet + return + + def _send(self, code: int, body: bytes, ctype: str) -> None: + self.send_response(code) + self.send_header("Content-Type", ctype) + self.send_header("Content-Length", str(len(body))) + self.send_header("Cache-Control", "no-store") + self.end_headers() + self.wfile.write(body) + + def _json(self, data: Any, code: int = 200) -> None: + self._send(code, json.dumps(data, ensure_ascii=False).encode(), "application/json; charset=utf-8") + + def _file(self, base: Path, rel: str) -> None: + path = (base / unquote(rel)).resolve() + if base.resolve() not in path.parents or not path.is_file(): + self._send(404, b"not found", "text/plain") + return + ctype = {".png": "image/png", ".gif": "image/gif", ".jpg": "image/jpeg", ".jpeg": "image/jpeg", + ".svg": "image/svg+xml"}.get(path.suffix.lower(), "application/octet-stream") + self._send(200, path.read_bytes(), ctype) + + def do_GET(self) -> None: + url = urlparse(self.path) + if url.path == "/": + formulas = [f for f in ws.all() if show_all or f.status in REVIEW_STATUSES] + formulas.sort(key=lambda f: (f.status != "needs_review", f.occurrences[0]["file"] if f.occurrences else "", + f.occurrences[0]["line"] if f.occurrences else 0)) + data = {"token": token, "cards": [_card(f) for f in formulas], "stats": ws.stats(), + "source": ws.meta.get("source", {})} + html = PAGE.replace("/*DATA*/null", json.dumps(data, ensure_ascii=False).replace(" None: + if self.headers.get("X-Token") != token: + self._json({"error": "ungültiges Token"}, 403) + return + length = int(self.headers.get("Content-Length") or 0) + try: + req = json.loads(self.rfile.read(min(length, 1_000_000)) or b"{}") + except ValueError: + self._json({"error": "ungültige Anfrage"}, 400) + return + with _LOCK: + f = ws.load(str(req.get("id", ""))) + if f is None: + self._json({"error": "Formel unbekannt"}, 404) + return + if self.path == "/check": + self._json(self._check(f, str(req.get("tex", "")))) + elif self.path == "/decide": + self._json(self._decide(f, str(req.get("action", "")), str(req.get("tex", "")))) + else: + self._json({"error": "unbekannter Pfad"}, 404) + + def _check(self, f: Formula, tex: str) -> dict[str, Any]: + tex = strip_delimiters(tex) + reason = unsafe_reason(tex) + if reason: + return {"ok": False, "error": reason} + orig = load_ink(source.original_bytes(_ref_for(source, f)), f.asset) + v = verify(orig, tex, f.display, preset, max_climb=0) + pngs = comparison_pngs(v.comparison) + pandoc = pandoc_mathml_ok([(tex, f.display)])[0] + return {"ok": v.ok, "error": v.reason, "metrics": {k: round(x, 3) for k, x in v.metrics.items()}, + "min_contrast": v.min_contrast, "worst_variant": v.worst_variant, "pandoc": pandoc, + "images": {k: "data:image/png;base64," + base64.b64encode(b).decode() for k, b in pngs.items()}} + + def _decide(self, f: Formula, action: str, tex: str) -> dict[str, Any]: + tex = strip_delimiters(tex) + if action in ("accept", "correct"): + reason = unsafe_reason(tex) + if reason: + return {"error": reason} + if not pandoc_mathml_ok([(tex, f.display)])[0]: + return {"error": "Pandoc kann diese Formel nicht nach MathML umsetzen"} + best = ((f.verification or {}).get("best") or {}).get("tex", "") + f.status = "accepted_by_human" if tex == best else "corrected_by_human" + f.final_tex = tex + elif action == "keep": + f.status, f.final_tex = "kept_by_human", None + elif action == "figure": + f.status, f.final_tex = "not_formula", None + elif action == "reset": + f.status, f.final_tex = "needs_review", None + else: + return {"error": "unbekannte Aktion"} + f.reviewer, f.decided_at = "review", now_iso() + f.reason = {"accept": "im Review bestätigt", "correct": "im Review korrigiert", "keep": "im Review: Bild behalten", + "figure": "im Review: keine Formel", "reset": "zurückgesetzt"}[action] + ws.save(f) + return {"ok": True, "status": f.status, "stats": ws.stats()} + + httpd = ThreadingHTTPServer(("127.0.0.1", port), Handler) + url = f"http://127.0.0.1:{port}/" + print(f"Review-Galerie: {url} (Beenden mit Strg+C)") + if open_browser: + threading.Timer(0.5, lambda: webbrowser.open(url)).start() + try: + httpd.serve_forever() + except KeyboardInterrupt: + pass + finally: + httpd.server_close() + print("Review beendet. Nächster Schritt: apply") diff --git a/formula_img2latex/sources/__init__.py b/formula_img2latex/sources/__init__.py new file mode 100644 index 0000000..07aca75 --- /dev/null +++ b/formula_img2latex/sources/__init__.py @@ -0,0 +1,24 @@ +"""Source adapters: find formula images in converter output and describe how to write math back. + +Every adapter yields `ImageRef`s; the Markdown scanning itself is shared (markdown.py). +""" + +from __future__ import annotations + +from pathlib import Path + +from ..errors import EXIT_SOURCE, FormulaError +from .base import Source + + +def open_source(path: Path) -> Source: + """Detect the converter that produced *path* and return the matching adapter.""" + from .epub2md import Epub2MdSource + from .pdf2md import Pdf2MdSource + + for cls in (Epub2MdSource, Pdf2MdSource): + if cls.detect(path): + return cls(path) + raise FormulaError( + f"{path} ist weder eine epub2md-Ausgabe (conversion-report.json) noch eine pdf2md-Ausgabe " + "(.md mit _images/ bzw. --split-Ordner mit index.md)", EXIT_SOURCE) diff --git a/formula_img2latex/sources/base.py b/formula_img2latex/sources/base.py new file mode 100644 index 0000000..595cce4 --- /dev/null +++ b/formula_img2latex/sources/base.py @@ -0,0 +1,111 @@ +"""Shared Markdown scanning and the Source interface.""" + +from __future__ import annotations + +import re +from dataclasses import dataclass +from pathlib import Path +from urllib.parse import unquote + +IMAGE_RE = re.compile(r"!\[((?:[^\]\\]|\\.)*)\]\(\s*]+)>?(?:\s+\"[^\"]*\")?\s*\)") +_FENCE_RE = re.compile(r"^\s*(```+|~~~+)") +_RASTER_EXT = (".png", ".gif", ".jpg", ".jpeg", ".svg", ".svgz", ".webp", ".bmp", ".tif", ".tiff") + + +@dataclass +class ImageRef: + file: Path # Markdown file (absolute) + line: int # 1-based line number + start: int # column span of the reference within the line + end: int + md: str # the exact reference text + alt: str + target: str # link target as written + display: bool # alone in its paragraph -> display math + context: str # nearby text, for prompts and review + + def asset_path(self) -> Path: + return (self.file.parent / unquote(self.target)).resolve() + + +def scan_markdown(md_file: Path) -> list[ImageRef]: + """All local raster/SVG image references outside code blocks.""" + lines = md_file.read_text(encoding="utf-8").split("\n") + refs: list[ImageRef] = [] + fence: str | None = None + for i, line in enumerate(lines): + m = _FENCE_RE.match(line) + if m: + mark = m.group(1)[0] * 3 + if fence is None: + fence = mark + elif mark == fence: + fence = None + continue + if fence is not None or "![" not in line: + continue + for im in IMAGE_RE.finditer(line): + target = im.group(2) + if "://" in target or target.startswith("data:") or not target.lower().endswith(_RASTER_EXT): + continue + rest = (line[: im.start()] + line[im.end():]).strip() + prev_blank = i == 0 or not lines[i - 1].strip() + next_blank = i + 1 >= len(lines) or not lines[i + 1].strip() + display = rest in ("", ">") and prev_blank and next_blank and not line.lstrip().startswith("|") + ctx = rest if rest and rest != ">" else (lines[i - 2].strip() if i >= 2 else "") + refs.append(ImageRef(md_file, i + 1, im.start(), im.end(), im.group(0), + im.group(1), target, display, ctx[:300])) + return refs + + +class Source: + """A converter output directory or file. Subclasses fill in the specifics.""" + + kind = "?" + + def __init__(self, path: Path) -> None: + self.path = path.resolve() + + @classmethod + def detect(cls, path: Path) -> bool: + raise NotImplementedError + + @property + def root(self) -> Path: + """Directory all relative paths in the workspace refer to.""" + return self.path if self.path.is_dir() else self.path.parent + + def markdown_files(self) -> list[Path]: + raise NotImplementedError + + def describe(self) -> dict: + return {"kind": self.kind, "path": str(self.path)} + + def image_refs(self) -> list[ImageRef]: + refs: list[ImageRef] = [] + for md in self.markdown_files(): + refs.extend(scan_markdown(md)) + return refs + + def known_formula(self, ref: ImageRef) -> bool: + return False + + def source_name(self, ref: ImageRef) -> str: + return ref.target + + def original_bytes(self, ref: ImageRef) -> bytes: + """Best available original of the formula (pdf2md re-crops from the vector PDF).""" + return ref.asset_path().read_bytes() + + def font_hint(self) -> str | None: + return None + + def math_delimiters(self) -> tuple[str, str, str, str]: + return "$", "$", "$$", "$$" + + def format_math(self, tex: str, display: bool) -> str: + il, ir, dl, dr = self.math_delimiters() + body = tex.strip() + if display: + return f"{dl}{body}{dr}" + return f"{il}{body}{ir}" diff --git a/formula_img2latex/sources/epub2md.py b/formula_img2latex/sources/epub2md.py new file mode 100644 index 0000000..1e95dda --- /dev/null +++ b/formula_img2latex/sources/epub2md.py @@ -0,0 +1,68 @@ +"""epub2md output directory: chapters/*.md, media/, conversion-report.json.""" + +from __future__ import annotations + +import json +import posixpath +import re +from pathlib import Path + +from .base import ImageRef, Source + +_WARN_NAME_RE = re.compile(r"\(([^()]+\.(?:png|gif|jpe?g|svgz?|webp|bmp|tiff?))\)", re.I) + + +class Epub2MdSource(Source): + kind = "epub2md" + + def __init__(self, path: Path) -> None: + super().__init__(path) + self.report = json.loads((self.path / "conversion-report.json").read_text(encoding="utf-8")) + files = (self.report.get("media") or {}).get("files") or [] + # output file (relative to root) -> original path inside the EPUB + self.origin = {f["file"]: f.get("source", "") for f in files if f.get("file")} + self.formula_names = { + m.group(1) + for w in self.report.get("warnings", []) + if w.get("code") in ("formula-image", "formula-svg") + for m in [_WARN_NAME_RE.search(w.get("message", ""))] if m + } + + @classmethod + def detect(cls, path: Path) -> bool: + rep = path / "conversion-report.json" + if not (path.is_dir() and rep.exists() and (path / "chapters").is_dir()): + return False + try: + return json.loads(rep.read_text(encoding="utf-8")).get("tool") == "epub2md" + except (OSError, ValueError): + return False + + def markdown_files(self) -> list[Path]: + # The merged book file (epub2md_merge.py) is derived; it must be rebuilt after `apply`. + return sorted((self.path / "chapters").glob("*.md")) + + def describe(self) -> dict: + inp = self.report.get("input") or {} + return {"kind": self.kind, "path": str(self.path), "book": inp.get("name", ""), + "book_sha256": inp.get("sha256", ""), "math": self.math_mode()} + + def _rel(self, ref: ImageRef) -> str: + return ref.asset_path().relative_to(self.path).as_posix() + + def source_name(self, ref: ImageRef) -> str: + try: + return self.origin.get(self._rel(ref), "") or ref.target + except ValueError: + return ref.target + + def known_formula(self, ref: ImageRef) -> bool: + return posixpath.basename(self.source_name(ref)) in self.formula_names + + def math_mode(self) -> str: + return str((self.report.get("options") or {}).get("math", "auto")) + + def math_delimiters(self) -> tuple[str, str, str, str]: + if self.math_mode() == "tex": + return "\\(", "\\)", "\\[", "\\]" + return "$", "$", "$$", "$$" diff --git a/formula_img2latex/sources/pdf2md.py b/formula_img2latex/sources/pdf2md.py new file mode 100644 index 0000000..d8d3cda --- /dev/null +++ b/formula_img2latex/sources/pdf2md.py @@ -0,0 +1,67 @@ +"""pdf2md output: a single .md with _images/, or a --split folder with index.md and images/. + +pymupdf-layout classifies displayed formulas as boxes of class "formula"; pymupdf4llm writes them +as images named "--." at low resolution. If the PDF is +available, the formula is cut out of the vector page again at high resolution for comparison +(see pdfcrop.py); otherwise the stored image is used. +""" + +from __future__ import annotations + +import re +from pathlib import Path + +from .base import ImageRef, Source + +PDF_IMAGE_RE = re.compile(r"^(?P.+)-(?P\d{4})-(?P\d{2})\.(?:png|jpe?g)$", re.I) + + +class Pdf2MdSource(Source): + kind = "pdf2md" + + def __init__(self, path: Path, pdf: Path | None = None) -> None: + super().__init__(path) + self.pdf = pdf + self._cropper = None + + @classmethod + def detect(cls, path: Path) -> bool: + if path.is_file() and path.suffix == ".md": + return (path.parent / f"{path.stem}_images").is_dir() or "_images/" in path.read_text(encoding="utf-8") + return path.is_dir() and (path / "index.md").exists() and (path / "images").is_dir() + + def markdown_files(self) -> list[Path]: + if self.path.is_file(): + return [self.path] + return sorted(p for p in self.path.glob("*.md") if p.name != "index.md") + + def describe(self) -> dict: + return {"kind": self.kind, "path": str(self.path), "pdf": str(self.pdf) if self.pdf else ""} + + def page_box(self, ref: ImageRef) -> tuple[int, int] | None: + m = PDF_IMAGE_RE.match(Path(ref.target).name) + return (int(m.group("page")), int(m.group("box"))) if m else None + + def source_name(self, ref: ImageRef) -> str: + pb = self.page_box(ref) + return f"Seite {pb[0]}, Box {pb[1]}" if pb else ref.target + + def known_formula(self, ref: ImageRef) -> bool: + if self.pdf is None or self.page_box(ref) is None: + return False + crop = self._crop(ref) + return crop is not None and crop[1] == "formula" + + def _crop(self, ref: ImageRef) -> tuple[bytes, str] | None: + if self.pdf is None: + return None + if self._cropper is None: + from ..pdfcrop import PdfCropper + self._cropper = PdfCropper(self.pdf) + pb = self.page_box(ref) + return self._cropper.crop(pb[0], pb[1], ref.asset_path()) if pb else None + + def original_bytes(self, ref: ImageRef) -> bytes: + crop = self._crop(ref) + return crop[0] if crop else ref.asset_path().read_bytes() + diff --git a/formula_img2latex/store.py b/formula_img2latex/store.py new file mode 100644 index 0000000..036d892 --- /dev/null +++ b/formula_img2latex/store.py @@ -0,0 +1,168 @@ +"""Workspace: one directory per converted book with a JSON record per formula. + + .formulas/ + ├── workspace.json source, settings (font preset, scale), statistics + ├── formulas/.json one record per (image, display/inline): candidates, verification, decision + ├── review/-diff.png overlay images for the review gallery + └── backup/… untouched Markdown files from before the first `apply` + +The directory lives next to the converter output because `epub2md --force` replaces the output. +Records are keyed by the image's SHA-256, so decisions survive a re-conversion. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import tempfile +from dataclasses import asdict, dataclass, field +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +from . import __version__ +from .errors import EXIT_INPUT, FormulaError + +STATUSES = ( + "pending", # found, nothing done yet + "no_candidate", # recognition produced nothing usable + "render_failed", # no candidate compiles + "auto_accepted", # passed all checks + "needs_review", # plausible, but a human must look + "rejected_keep_image", # does not match; image stays + "accepted_by_human", + "corrected_by_human", + "kept_by_human", # human decided: keep the image + "not_formula", # classified (or marked) as a figure +) +REPLACE_STATUSES = ("auto_accepted", "accepted_by_human", "corrected_by_human") +HUMAN_STATUSES = ("accepted_by_human", "corrected_by_human", "kept_by_human") + + +def now_iso() -> str: + return datetime.now(timezone.utc).astimezone().isoformat(timespec="seconds") + + +def sha256_bytes(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +def write_json_atomic(path: Path, data: Any) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + fd, tmp = tempfile.mkstemp(dir=path.parent, prefix=".tmp-", suffix=".json") + try: + with os.fdopen(fd, "w", encoding="utf-8") as f: + json.dump(data, f, ensure_ascii=False, indent=1) + f.write("\n") + os.replace(tmp, path) + except BaseException: + Path(tmp).unlink(missing_ok=True) + raise + + +@dataclass +class Occurrence: + file: str # Markdown file relative to the source root + line: int # 1-based + md: str # exact Markdown image reference, e.g. ![alt](../media/images/x.png) + context: str = "" # surrounding text (for OCR prompts and review) + + +@dataclass +class Formula: + id: str + sha256: str + asset: str # image path relative to the source root + display: bool + source_name: str = "" # name in the original book (EPUB path, PDF page/box) + known_formula: bool = False # converter said it is a formula (epub2md warning, layout class) + alt: str = "" + width: int = 0 + height: int = 0 + classify: dict[str, Any] = field(default_factory=dict) + occurrences: list[dict[str, Any]] = field(default_factory=list) + candidates: list[dict[str, Any]] = field(default_factory=list) # {engine, model, tex, created_at} + verification: dict[str, Any] | None = None + status: str = "pending" + reason: str = "" + final_tex: str | None = None + reviewer: str | None = None + decided_at: str | None = None + + @staticmethod + def make_id(sha: str, display: bool) -> str: + return f"f-{sha[:12]}-{'d' if display else 'i'}" + + def to_json(self) -> dict[str, Any]: + return asdict(self) + + @classmethod + def from_json(cls, data: dict[str, Any]) -> Formula: + known = {k: v for k, v in data.items() if k in cls.__dataclass_fields__} + return cls(**known) + + +class Workspace: + def __init__(self, root: Path) -> None: + self.root = root + self.formulas_dir = root / "formulas" + self.review_dir = root / "review" + self.backup_dir = root / "backup" + self.meta_path = root / "workspace.json" + self.meta: dict[str, Any] = {} + + @staticmethod + def default_for(source: Path) -> Path: + source = source.resolve() + name = source.name[:-3] if source.suffix == ".md" else source.name + return source.parent / f"{name}.formulas" + + @classmethod + def open(cls, root: Path, create: bool = False, source: dict[str, Any] | None = None) -> Workspace: + ws = cls(root) + if ws.meta_path.exists(): + ws.meta = json.loads(ws.meta_path.read_text(encoding="utf-8")) + elif create: + if root.exists() and any(root.iterdir()): + raise FormulaError(f"Arbeitsordner existiert und ist kein formula_img2latex-Ordner: {root}", EXIT_INPUT) + ws.meta = {"tool": "formula_img2latex", "version": __version__, "created_at": now_iso(), + "source": source or {}, "settings": {}} + ws.save_meta() + else: + raise FormulaError(f"Kein Arbeitsordner gefunden: {root} (zuerst 'scan' ausführen)", EXIT_INPUT) + return ws + + def save_meta(self) -> None: + self.meta["version"] = __version__ + self.meta["updated_at"] = now_iso() + write_json_atomic(self.meta_path, self.meta) + + @property + def settings(self) -> dict[str, Any]: + return self.meta.setdefault("settings", {}) + + def formula_path(self, fid: str) -> Path: + return self.formulas_dir / f"{fid}.json" + + def load(self, fid: str) -> Formula | None: + p = self.formula_path(fid) + if not p.exists(): + return None + return Formula.from_json(json.loads(p.read_text(encoding="utf-8"))) + + def save(self, f: Formula) -> None: + write_json_atomic(self.formula_path(f.id), f.to_json()) + + def all(self) -> list[Formula]: + if not self.formulas_dir.exists(): + return [] + out = [Formula.from_json(json.loads(p.read_text(encoding="utf-8"))) + for p in sorted(self.formulas_dir.glob("f-*.json"))] + return out + + def stats(self) -> dict[str, int]: + counts: dict[str, int] = {} + for f in self.all(): + counts[f.status] = counts.get(f.status, 0) + 1 + return counts diff --git a/formula_img2latex/tex.py b/formula_img2latex/tex.py new file mode 100644 index 0000000..505f7f9 --- /dev/null +++ b/formula_img2latex/tex.py @@ -0,0 +1,131 @@ +"""TeX string helpers: delimiter handling, safety screening, render preparation. + +Candidates come from OCR models and e-book alt texts, i.e. untrusted input. Before anything +reaches pdflatex we reject control sequences that could read/write files, run code or redefine +macros (which also rules out endless loops), and we require balanced braces. +""" + +from __future__ import annotations + +import re + +MAX_TEX_LENGTH = 4000 + +# Control sequences that are never needed for a formula but could do harm or loop forever. +UNSAFE_COMMANDS = frozenset({ + "input", "include", "includeonly", "InputIfFileExists", "openin", "openout", "read", "readline", + "write", "immediate", "closein", "closeout", "newwrite", "newread", + "catcode", "lccode", "uccode", "mathcode", "delcode", "sfcode", "endlinechar", "escapechar", + "def", "edef", "gdef", "xdef", "let", "futurelet", "global", "long", "outer", + "newcommand", "renewcommand", "providecommand", "DeclareRobustCommand", "DeclareMathOperator", + "newenvironment", "renewenvironment", "csname", "endcsname", "expandafter", "noexpand", + "afterassignment", "aftergroup", "loop", "repeat", "iterate", "special", "directlua", "latelua", + "usepackage", "RequirePackage", "documentclass", "makeatletter", "makeatother", "jobname", + "shipout", "output", "everypar", "everymath", "everydisplay", "everyhbox", "everyvbox", "errhelp", + "errmessage", "message", "batchmode", "nonstopmode", "scrollmode", "errorstopmode", "dump", + "pdfprimitive", "pdfobj", "pdfliteral", "pdfstartlink", "pdffiledump", "pdfmdfivesum", "pdffilesize", + "url", "href", "hyperref", "includegraphics", "verb", "lstinline", "tracingall", "showlists", +}) + +_CS_RE = re.compile(r"\\([A-Za-z@]+)") +_DELIMS = (("$$", "$$"), ("\\[", "\\]"), ("\\(", "\\)"), ("$", "$")) + +# Top-level display environments that cannot live inside $...$; mapped to their inner forms. +_ENV_MAP = { + "align": "aligned", "align*": "aligned", "flalign": "aligned", "flalign*": "aligned", + "alignat": "alignedat", "alignat*": "alignedat", + "eqnarray": "aligned", "eqnarray*": "aligned", + "gather": "gathered", "gather*": "gathered", + "multline": "gathered", "multline*": "gathered", +} + + +def strip_delimiters(tex: str) -> str: + """Remove one pair of outer math delimiters ($$..$$, \\[..\\], \\(..\\), $..$).""" + s = tex.strip() + for left, right in _DELIMS: + if s.startswith(left) and s.endswith(right) and len(s) >= len(left) + len(right): + inner = s[len(left):len(s) - len(right)] + if left == "$" and "$" in inner: + continue + return inner.strip() + return s + + +def braces_balanced(tex: str) -> bool: + depth = 0 + i = 0 + while i < len(tex): + c = tex[i] + if c == "\\": + i += 2 + continue + if c == "{": + depth += 1 + elif c == "}": + depth -= 1 + if depth < 0: + return False + i += 1 + return depth == 0 + + +def unsafe_reason(tex: str) -> str | None: + """Return a German reason if *tex* must not be rendered, else None.""" + if not tex.strip(): + return "leere Formel" + if len(tex) > MAX_TEX_LENGTH: + return f"Formel länger als {MAX_TEX_LENGTH} Zeichen" + if "@" in "".join(_CS_RE.findall(tex)): + return "interne Makros (@) sind nicht erlaubt" + for name in _CS_RE.findall(tex): + if name in UNSAFE_COMMANDS: + return f"Befehl \\{name} ist nicht erlaubt" + if "^^" in tex: + return "TeX-Zeichencodes (^^) sind nicht erlaubt" + if "$" in tex.replace("\\$", ""): + return "Formel enthält Mathe-Begrenzer ($)" + if re.search(r"\\(begin|end)\s*\{\s*document\s*\}", tex): + return "\\begin{document} ist nicht erlaubt" + if not braces_balanced(tex): + return "geschweifte Klammern unausgeglichen" + return None + + +_TAG_RE = re.compile(r"\\tag\*?\s*\{([^{}]*)\}") + + +def split_tag(tex: str) -> tuple[str, str | None]: + """Remove a \\tag{...}; return (formula, tag text or None).""" + m = _TAG_RE.search(tex) + if not m: + return tex, None + return (tex[:m.start()] + tex[m.end():]).strip(), m.group(1).strip() + + +def prepare_for_render(tex: str) -> str: + """Turn a candidate into something that compiles inside $...$ (no labels, tags, top-level envs).""" + s = strip_delimiters(tex) + s, _ = split_tag(s) + s = re.sub(r"\\(label|eqref|ref)\s*\{[^{}]*\}", "", s) + s = re.sub(r"\\(nonumber|notag)\b", "", s) + + def env(m: re.Match[str]) -> str: + kind, name = m.group(1), m.group(2) + if name in ("equation", "equation*", "displaymath", "math"): + return "" + return f"\\{kind}{{{_ENV_MAP.get(name, name)}}}" + + s = re.sub(r"\\(begin|end)\s*\{([A-Za-z]+\*?)\}", env, s) + return s.strip() + + +def normalize_for_compare(tex: str) -> str: + """Rough canonical form for comparing two candidates as strings (not a semantic check).""" + s = prepare_for_render(tex) + s = re.sub(r"\\(left|right|big|Big|bigg|Bigg)([lr]?)(?![A-Za-z])", "", s) + s = s.replace("\\leq", "\\le").replace("\\geq", "\\ge").replace("\\neq", "\\ne") + s = re.sub(r"\\[,;:! ]|~", "", s) + s = re.sub(r"\s+", "", s) + s = re.sub(r"([_^])\{(\\?[A-Za-z0-9])\}", r"\1\2", s) + return s diff --git a/formula_img2latex/verify.py b/formula_img2latex/verify.py new file mode 100644 index 0000000..3d9fb4b --- /dev/null +++ b/formula_img2latex/verify.py @@ -0,0 +1,92 @@ +"""Verify a TeX candidate against an original formula image. + +1. Render the candidate and compare it with the original (scale search, alignment, metrics). +2. Render all confusable variants (confusables.py) at the same scale and run the contrast test. +3. If a variant explains the image clearly better, it becomes the new candidate (local repair of + a recognition error) and the procedure repeats, at most *max_climb* times. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field + +import numpy as np + +from .compare import Comparison, compare, contrast +from .confusables import variants +from .render import render_many + +SWITCH_MARGIN = 0.25 # a variant must beat the candidate by this much to replace it +MAX_CLIMB = 3 +MIN_NCC_FOR_VARIANTS = 0.6 # hopeless candidates are not worth the variant test + + +@dataclass +class Verification: + tex: str + ok: bool # candidate rendered and was comparable + reason: str | None = None + metrics: dict[str, float] = field(default_factory=dict) + min_contrast: float | None = None # weakest win over a distinguishable variant + worst_variant: str | None = None + worst_kind: str | None = None + n_variants: int = 0 + n_equivalent: int = 0 # variants that render identically + climbed_from: list[str] = field(default_factory=list) + comparison: Comparison | None = None + + def to_json(self) -> dict: + return { + "tex": self.tex, "ok": self.ok, "reason": self.reason, + "metrics": {k: round(v, 5) for k, v in self.metrics.items()}, + "min_contrast": None if self.min_contrast is None else round(self.min_contrast, 4), + "worst_variant": self.worst_variant, "worst_kind": self.worst_kind, + "n_variants": self.n_variants, "n_equivalent": self.n_equivalent, + "climbed_from": self.climbed_from, + } + + +def _check_variants(orig: np.ndarray, cand: Comparison, tex: str, display: bool, preset: str + ) -> tuple[float | None, str | None, str | None, int, int, tuple[float, str] | None]: + """Return (min contrast, worst variant, its kind, #distinguishable, #equivalent, best rival).""" + vs = variants(tex) + if not vs: + return None, None, None, 0, 0, None + rendered = render_many([(v.render_tex, display) for v in vs], preset) + ppt = cand.metrics["px_per_pt"] + min_c, worst, worst_kind, n, eq = None, None, None, 0, 0 + rival: tuple[float, str] | None = None + for v, r in zip(vs, rendered): + if not r.ok: + continue + c = contrast(cand, compare(orig, r, px_per_pt=ppt)) + if c is None: + eq += 1 + continue + n += 1 + if min_c is None or c < min_c: + min_c, worst, worst_kind = c, v.tex, v.kind + if c < -SWITCH_MARGIN and (rival is None or c < rival[0]): + rival = (c, v.tex) + return min_c, worst, worst_kind, n, eq, rival + + +def verify(orig: np.ndarray, tex: str, display: bool, preset: str, max_climb: int = MAX_CLIMB, + check_variants: bool = True) -> Verification: + """Verify *tex* against the trimmed ink image *orig*.""" + history: list[str] = [] + current = tex + while True: + r = render_many([(current, display)], preset)[0] + comp = compare(orig, r) + if not comp.ok: + return Verification(current, False, comp.reason, comp.metrics, climbed_from=history, comparison=comp) + res = Verification(current, True, None, comp.metrics, climbed_from=list(history), comparison=comp) + if not check_variants or comp.metrics.get("ncc", 0.0) < MIN_NCC_FOR_VARIANTS: + return res + (res.min_contrast, res.worst_variant, res.worst_kind, res.n_variants, res.n_equivalent, + rival) = _check_variants(orig, comp, current, display, preset) + if rival is None or len(history) >= max_climb or rival[1] in history: + return res + history.append(current) + current = rival[1] diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 0000000..9ee0463 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,3 @@ +[pytest] +testpaths = tests +addopts = -p no:asyncio diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..a266747 --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,2 @@ +-r requirements.txt +pytest>=8.0 diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..44365ed --- /dev/null +++ b/requirements.txt @@ -0,0 +1,7 @@ +pymupdf>=1.24 +numpy>=1.26 +scipy>=1.11 +scikit-image>=0.22 +opencv-python-headless>=4.8 +Pillow>=10.0 +lxml>=5.0 diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..5eb7477 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,68 @@ +"""Shared fixtures: synthetic formula images and a minimal epub2md output directory.""" + +from __future__ import annotations + +import io +import json +import shutil +from pathlib import Path + +import cv2 +import numpy as np +import pytest +from PIL import Image + +from formula_img2latex.render import render_many + +needs_latex = pytest.mark.skipif(shutil.which("pdflatex") is None, reason="pdflatex fehlt") +needs_pandoc = pytest.mark.skipif(shutil.which("pandoc") is None, reason="pandoc fehlt") + + +def synth_image(tex: str, display: bool = False, px_per_pt: float = 1.41, fmt: str = "JPEG", + offset: tuple[int, int] = (1, 2), preset: str = "cm") -> bytes: + """Render *tex* like a small e-book formula image (Gehrke-like: ~1.4 px/pt, JPEG).""" + r = render_many([(tex, display)], preset)[0] + assert r.ok, r.error + gray = 1.0 - r.rasterize(px_per_pt * 4) + pad = np.ones((gray.shape[0] + 8, gray.shape[1] + 8), np.float32) + oy, ox = offset + pad[oy:oy + gray.shape[0], ox:ox + gray.shape[1]] = gray + h, w = pad.shape[0] // 4, pad.shape[1] // 4 + small = cv2.resize(pad[: h * 4, : w * 4], (w, h), interpolation=cv2.INTER_AREA) + buf = io.BytesIO() + im = Image.fromarray((small * 255).astype(np.uint8)) + if fmt == "PNG-ALPHA": # black ink on transparent background (Springer style) + rgba = Image.new("RGBA", im.size, (0, 0, 0, 0)) + rgba.putalpha(Image.fromarray(255 - np.asarray(im))) + rgba.save(buf, "PNG") + else: + im.save(buf, fmt, **({"quality": 75} if fmt == "JPEG" else {})) + return buf.getvalue() + + +@pytest.fixture +def epub2md_dir(tmp_path: Path) -> Path: + """Minimal epub2md output with two formula images (one inline, one display) and a figure.""" + root = tmp_path / "buch" + (root / "chapters").mkdir(parents=True) + (root / "media" / "images").mkdir(parents=True) + imgs = { + "0001-inline.jpg": synth_image(r"L=\{-\sqrt{2};\sqrt{2}\}", px_per_pt=2.2), + "0002-display.jpg": synth_image(r"x_{2,k}=\arcsin\left(-\frac{3}{5}\right)+2k\pi", display=True, px_per_pt=2.2), + } + for name, data in imgs.items(): + (root / "media" / "images" / name).write_bytes(data) + color = np.zeros((40, 60, 3), np.uint8) + color[:, :, 2] = 200 + Image.fromarray(color).save(root / "media" / "images" / "0003-figure.png") + (root / "chapters" / "001-kapitel.md").write_text( + "---\nchapter_title: Test\n---\n\n# Kapitel\n\n" + "Die Lösungsmenge lautet ![x](../media/images/0001-inline.jpg).\n\n" + "Allgemein gilt\n\n![y](../media/images/0002-display.jpg)\n\n" + "```\n![kein](../media/images/0001-inline.jpg)\n```\n\n" + "![Abbildung](../media/images/0003-figure.png)\n", encoding="utf-8") + report = {"tool": "epub2md", "options": {"math": "auto"}, "input": {"name": "buch.epub", "sha256": "x"}, + "media": {"files": [{"source": f"OEBPS/{n}", "file": f"media/images/{n}"} for n in imgs]}, + "warnings": [{"code": "formula-image", "message": "Formel liegt nur als Bild vor (0002-display.jpg)."}]} + (root / "conversion-report.json").write_text(json.dumps(report), encoding="utf-8") + return root diff --git a/tests/test_core.py b/tests/test_core.py new file mode 100644 index 0000000..65cf46e --- /dev/null +++ b/tests/test_core.py @@ -0,0 +1,109 @@ +"""Render, compare, confusables, verify: the core question "does this LaTeX look like the image?".""" + +from __future__ import annotations + +import pytest + +from conftest import needs_latex, needs_pandoc, synth_image +from formula_img2latex.checks import pandoc_mathml_ok +from formula_img2latex.compare import compare +from formula_img2latex.confusables import tokenize, variants +from formula_img2latex.decide import decide, thresholds +from formula_img2latex.imageio import load_ink +from formula_img2latex.render import render_many +from formula_img2latex.tex import prepare_for_render, split_tag, strip_delimiters, unsafe_reason +from formula_img2latex.verify import verify + + +# ----------------------------------------------------------------------------------- tex + +@pytest.mark.parametrize("tex", [r"\input{/etc/passwd}", r"\def\a{\a}\a", r"\write18{rm -rf /}", r"\csname x\endcsname", + r"\catcode`\^=7", r"a^^5cinput", r"\makeatletter\@tempa", r"\frac{a}{b", + r"x}{", r"$x$", r"\begin{document}", r"\immediate\openout1=x"]) +def test_unsafe_tex_is_rejected(tex: str) -> None: + assert unsafe_reason(tex) + + +@pytest.mark.parametrize("tex", [r"\frac{a}{b}", r"\begin{pmatrix}1&2\\3&4\end{pmatrix}", r"\text{für } x \in \mathbb{R}", + r"\sum_{i=1}^n x_i", r"\$5"]) +def test_normal_tex_is_safe(tex: str) -> None: + assert unsafe_reason(tex) is None + + +def test_delimiters_and_tags() -> None: + assert strip_delimiters("$$x^2$$") == "x^2" + assert strip_delimiters(r"\(a\)") == "a" + assert strip_delimiters("$a$ und $b$") == "$a$ und $b$" + assert split_tag(r"a=b \tag{3.7}") == ("a=b", "3.7") + assert prepare_for_render(r"\begin{align*}a&=b\\c&=d\end{align*}") == r"\begin{aligned}a&=b\\c&=d\end{aligned}" + + +# -------------------------------------------------------------------------------- render + +@needs_latex +def test_render_isolates_bad_formulas() -> None: + rs = render_many([(r"x^2", False), (r"\undefinedmacro", False), (r"\frac{1}{2}", True), (r"\input{x}", False)]) + assert [r.ok for r in rs] == [True, False, True, False] + assert "nicht erlaubt" in (rs[3].error or "") + + +# ------------------------------------------------------------------------------ compare + +@needs_latex +@pytest.mark.parametrize("fmt", ["JPEG", "PNG", "PNG-ALPHA", "GIF"]) +def test_true_render_matches_its_image(fmt: str) -> None: + tex = r"\sum_{k=1}^{n}(a_k+b_k)" + ink = load_ink(synth_image(tex, fmt=fmt), "x." + ("png" if "PNG" in fmt else fmt.lower())) + c = compare(ink, render_many([(tex, False)])[0]) + assert c.ok + assert c.metrics["ncc"] > 0.9 + assert c.metrics["aspect_dev"] < 0.03 + assert c.metrics["px_per_pt"] == pytest.approx(1.41, rel=0.04) + + +@needs_latex +def test_wrong_candidate_scores_lower() -> None: + ink = load_ink(synth_image(r"x^{2}+y^{2}=r^{2}"), "x.jpg") + good, bad = render_many([(r"x^{2}+y^{2}=r^{2}", False), (r"x_{2}+y^{2}=r^{2}", False)]) + assert compare(ink, good).metrics["ncc"] > compare(ink, bad).metrics["ncc"] + + +# --------------------------------------------------------------------------- confusables + +def test_tokenize_and_variants() -> None: + assert tokenize(r"\alpha_{2}") == ["\\alpha", "_", "{", "2", "}"] + vs = {v.tex for v in variants(r"x_{2}")} + assert r"x^{2}" in vs and r"x_{3}" in vs and r"y_{2}" in vs + vs = {v.tex for v in variants(r"N(\mu;4)")} + assert r"\mathcal{N}(\mu;4)" in vs and r"N(\mu,4)" in vs + assert all(v.tex != r"x_{2}" for v in variants(r"x_{2}")) + + +# -------------------------------------------------------------------------------- verify + +@needs_latex +def test_verify_accepts_truth_and_repairs_single_errors() -> None: + truth = r"L=\{-\sqrt{2};\sqrt{2}\}" + ink = load_ink(synth_image(truth), "x.jpg") + v = verify(ink, truth, False, "cm") + assert v.ok and v.tex == truth and v.min_contrast is not None and v.min_contrast > 0 + repaired = verify(ink, r"L=\{-\sqrt{2};\sqrt{3}\}", False, "cm") + assert repaired.tex == truth + assert repaired.climbed_from == [r"L=\{-\sqrt{2};\sqrt{3}\}"] + + +@needs_latex +def test_decision_rules() -> None: + th = thresholds() + ok = {"ok": True, "tex": "x", "metrics": {"ncc": 0.95, "aspect_dev": 0.0, "px_per_pt": 1.4}, "min_contrast": 0.6} + assert decide(ok, True, 1.4, th)[0] == "auto_accepted" + assert decide(ok, False, 1.4, th)[0] == "needs_review" # Pandoc fails + assert decide(ok, True, 1.7, th)[0] == "needs_review" # wrong size/style + assert decide({**ok, "min_contrast": 0.2}, True, 1.4, th)[0] == "needs_review" + assert decide({**ok, "min_contrast": -0.5}, True, 1.4, th)[0] == "rejected_keep_image" + assert decide({"ok": False, "reason": "x"}, True, None, th)[0] == "render_failed" + + +@needs_pandoc +def test_pandoc_check() -> None: + assert pandoc_mathml_ok([(r"\frac{a}{b}", False), (r"\mathinner{a\raise1pt\hbox{b}}", True)]) == [True, False] diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py new file mode 100644 index 0000000..80c3436 --- /dev/null +++ b/tests/test_pipeline.py @@ -0,0 +1,108 @@ +"""scan → (candidates) → verify → apply → revert on a synthetic epub2md output, without any model.""" + +from __future__ import annotations + +from pathlib import Path + +from conftest import needs_latex, needs_pandoc +from formula_img2latex import pipeline +from formula_img2latex.cli import main +from formula_img2latex.engines.vlm import parse_answer, prepare_image +from formula_img2latex.sources.base import scan_markdown +from formula_img2latex.store import Workspace + + +def _inject(ws: Workspace, by_asset: dict[str, str]) -> None: + """Pretend an engine answered (tests must not need a model server). + + The mechanics are tested here, not the calibrated thresholds (test_core.test_decision_rules), + so the contrast threshold is set low enough for the synthetic images. + """ + ws.settings["thresholds"] = {"min_contrast": 0.1} + ws.save_meta() + for f in ws.all(): + for name, tex in by_asset.items(): + if f.asset.endswith(name): + f.candidates.append({"engine": "test", "model": "fixture", "tex": tex, "is_formula": True}) + ws.save(f) + + +def test_scan_markdown_skips_code_and_detects_display(epub2md_dir: Path) -> None: + refs = scan_markdown(epub2md_dir / "chapters" / "001-kapitel.md") + assert [(r.target.rsplit("/", 1)[1], r.display) for r in refs] == [ + ("0001-inline.jpg", False), ("0002-display.jpg", True), ("0003-figure.png", True)] + + +def test_scan_classifies(epub2md_dir: Path) -> None: + source, ws = pipeline.open_all(epub2md_dir, None, create=True) + pipeline.scan(source, ws, quiet=True) + labels = {f.asset.rsplit("/", 1)[1]: (f.classify["label"], f.known_formula) for f in ws.all()} + assert labels["0001-inline.jpg"] == ("formula", False) + assert labels["0002-display.jpg"] == ("formula", True) # epub2md warning + assert labels["0003-figure.png"][0] == "figure" + assert ws.root == epub2md_dir.parent / "buch.formulas" + + +@needs_latex +@needs_pandoc +def test_verify_apply_revert_roundtrip(epub2md_dir: Path) -> None: + source, ws = pipeline.open_all(epub2md_dir, None, create=True) + pipeline.scan(source, ws, quiet=True) + _inject(ws, {"0001-inline.jpg": r"L = \{ - \sqrt{2} , \sqrt{2} \}", # comma: wrong, repaired to ; + "0002-display.jpg": r"x_{2,k}=\arcsin\left(-\frac{3}{5}\right)+2k\pi"}) + counts = pipeline.verify_all(source, ws, preset="cm", workers=2, quiet=True) + st = {f.asset.rsplit("/", 1)[1]: f for f in ws.all()} + inline = st["0001-inline.jpg"] + assert inline.verification["best"]["tex"].replace(" ", "") == r"L=\{-\sqrt{2};\sqrt{2}\}" + assert inline.status == "auto_accepted", inline.reason + assert sum(counts.values()) == 2 + md = epub2md_dir / "chapters" / "001-kapitel.md" + before = md.read_text(encoding="utf-8") + + pipeline.apply(source, ws, quiet=True) + after = md.read_text(encoding="utf-8") + assert "lautet $L = \\{ - \\sqrt{2} ; \\sqrt{2} \\}$." in after or "lautet $L=\\{-\\sqrt{2};\\sqrt{2}\\}$." in after + assert "```\n![kein](../media/images/0001-inline.jpg)\n```" in after # code block untouched + assert "0003-figure.png" in after + pipeline.apply(source, ws, quiet=True) # idempotent + assert md.read_text(encoding="utf-8") == after + + pipeline.revert(source, ws, quiet=True) + assert md.read_text(encoding="utf-8") == before + + +@needs_latex +def test_revert_refuses_hand_edited_files(epub2md_dir: Path) -> None: + source, ws = pipeline.open_all(epub2md_dir, None, create=True) + pipeline.scan(source, ws, quiet=True) + _inject(ws, {"0001-inline.jpg": r"L=\{-\sqrt{2};\sqrt{2}\}"}) + pipeline.verify_all(source, ws, preset="cm", workers=1, quiet=True) + pipeline.apply(source, ws, quiet=True) + md = epub2md_dir / "chapters" / "001-kapitel.md" + md.write_text(md.read_text(encoding="utf-8") + "\nHandnotiz\n", encoding="utf-8") + assert pipeline.revert(source, ws, quiet=True)["skipped"] == 1 + assert "Handnotiz" in md.read_text(encoding="utf-8") + + +def test_cli_errors(tmp_path: Path, capsys) -> None: + assert main(["status", str(tmp_path / "fehlt")]) == 1 + (tmp_path / "leer").mkdir() + assert main(["scan", str(tmp_path / "leer")]) == 2 + assert "weder eine epub2md-Ausgabe" in capsys.readouterr().err + + +def test_engine_answer_parsing() -> None: + assert parse_answer('{"is_formula": true, "latex": "x^2"}') == ("x^2", True) + assert parse_answer('```json\n{"is_formula": true, "latex": "\\frac{a}{b}"}\n```') == ("\\frac{a}{b}", True) + assert parse_answer('{"is_formula": true, "latex": "\\sqrt{2}"}') == ("\\sqrt{2}", True) # unescaped backslash + assert parse_answer('{"is_formula": false, "latex": ""}') == (None, False) + assert parse_answer("x_1 + x_2") == ("x_1 + x_2", None) + + +def test_prepare_image_enlarges_small_formulas(epub2md_dir: Path) -> None: + import io + + from PIL import Image + data = (epub2md_dir / "media" / "images" / "0001-inline.jpg").read_bytes() + with Image.open(io.BytesIO(prepare_image(data, "x.jpg"))) as im: + assert im.height >= 96 and im.mode == "L"