feat: 0.4.0 – mehrzeilige Normalform, erstes ganzes PDF-Buch
Normalform (Göllmann, 400 Formeln: +17 automatisch, weiterhin 0 falsch):
- mehrzeilige Formeln gleich bei gleichen Zeilen, egal in welchem Rahmen
(aligned/&/\\, cases = \left\{array\right. = \left\{aligned\right.),
leere letzte Zeile zählt nicht; Matrizen behalten & als Spaltentrenner
- Argumente nach Argumentzahl des Befehls (t^{2}\,{\sqrt{2}} = t^2\sqrt{2})
- \cdot\cdot\cdot = \dots, \root3\of{x} = \sqrt[3]{x}, \left./\right. entfallen,
\cfrac = \frac; wörtliche \n aus Modellantworten werden zu Leerraum
Gemessen und abgeschaltet bzw. verworfen:
- Strukturvergleich mit sympy (structure.py, optional, FORMULA_STRUCTURAL=1):
nach der neuen Normalform kein Gewinn mehr
- Bild-Schiedsrichter (verify.image_arbiter, arbiter=0): der gemeinsame
Fehler \mathcal{E}_r bekam Beleg 7,2 – Abstand zu knapp
- Prompt-Stilvorgabe: kein Gewinn, Prompt-Version bleibt 2
Ganzes PDF-Buch (Corral, Elementary Calculus): 429 von 1123 automatisch.
- Bilder für die Modelle höchstens 1600 Pixel (Qwen lehnte große ab),
Bildvergleich nur bis 4 MP (ganzseitige Abbildung brachte ihn zum Absturz)
- fehlgeschlagene Modellanfragen werden wiederholt
- pix2text: \cfrac -> \dfrac (Pandoc kennt \cfrac nicht)
Werkzeuge: tools/buch_bericht.py (Ergebnisseite eines Buchs),
tools/vergleich_bericht.py (Vorher/Nachher), regeltest --refresh/--arbiter,
calibrate misst Schiedsrichter und strukturelle Treffer.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
b06c9cbb4b
commit
2478bc5cb9
1 changed files with 1126 additions and 59 deletions
161
tools/buch_bericht.py
Normal file
161
tools/buch_bericht.py
Normal file
|
|
@ -0,0 +1,161 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Result page of a whole book: every formula with original, set LaTeX, status and reason.
|
||||
|
||||
python3 tools/buch_bericht.py AUSGABE [--pdf BUCH.pdf] [--out ORDNER]
|
||||
|
||||
Reads the workspace of a finished `run` (nothing is recomputed, no model is asked) and writes
|
||||
<workspace>/bericht/index.html with images in bericht/img/ (default). The page lives next to the
|
||||
book because it shows excerpts of it. Filter buttons: all, automatically replaced, review, image.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import html
|
||||
import io
|
||||
import re
|
||||
import statistics
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
from PIL import Image
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
|
||||
from formula_img2latex.pipeline import _ref_for, latest_by_engine, open_all # noqa: E402
|
||||
from formula_img2latex.render import render_many # noqa: E402
|
||||
from formula_img2latex.tex import strip_delimiters # noqa: E402
|
||||
|
||||
LABEL = {"local": "Qwen", "p2t": "pix2text", "local2": "Gemma", "cloud": "GLM"}
|
||||
STATUS = {"auto_accepted": ("automatisch ersetzt", "good"), "accepted_by_human": ("von Hand übernommen", "good"),
|
||||
"needs_review": ("Review", "rev"), "not_formula": ("keine Formel", "img"),
|
||||
"rejected_keep_image": ("bleibt Bild", "img"), "kept_by_human": ("bleibt Bild (Hand)", "img"),
|
||||
"render_failed": ("nicht setzbar", "img"), "pending": ("nicht geprüft", "img")}
|
||||
|
||||
|
||||
def page_key(f) -> tuple[int, int]:
|
||||
m = re.search(r"Seite (\d+), Box (\d+)", f.source_name or "")
|
||||
return (int(m.group(1)), int(m.group(2))) if m else (10**6, 0)
|
||||
|
||||
|
||||
def save_png(img: Image.Image, path: Path) -> None:
|
||||
img.save(path, "PNG", optimize=True)
|
||||
|
||||
|
||||
def original_png(data: bytes, max_h: int = 110, max_w: int = 900) -> Image.Image:
|
||||
im = Image.open(io.BytesIO(data))
|
||||
if im.mode in ("P", "LA", "RGBA") or "transparency" in im.info:
|
||||
im = Image.alpha_composite(Image.new("RGBA", im.size, "white"), im.convert("RGBA"))
|
||||
im = im.convert("L")
|
||||
f = min(1.0, max_h / max(im.height, 1), max_w / max(im.width, 1))
|
||||
if f < 1.0:
|
||||
im = im.resize((max(1, round(im.width * f)), max(1, round(im.height * f))), Image.LANCZOS)
|
||||
return im
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
ap.add_argument("source", type=Path)
|
||||
ap.add_argument("--pdf", type=Path, default=None)
|
||||
ap.add_argument("--out", type=Path, default=None)
|
||||
ap.add_argument("--titel", default=None)
|
||||
args = ap.parse_args()
|
||||
source, ws = open_all(args.source, None, pdf=args.pdf)
|
||||
out = args.out or ws.root / "bericht"
|
||||
(out / "img").mkdir(parents=True, exist_ok=True)
|
||||
formulas = sorted(ws.all(), key=page_key)
|
||||
preset = ws.settings.get("preset") or "cm"
|
||||
|
||||
# the LaTeX shown: the replacement if there is one, else pix2text's reading (rule p2t)
|
||||
shown = []
|
||||
for f in formulas:
|
||||
tex = f.final_tex
|
||||
if not tex:
|
||||
p = latest_by_engine(f).get("p2t")
|
||||
tex = p.get("tex") if p else None
|
||||
shown.append(strip_delimiters(tex) if tex else None)
|
||||
rendered = render_many([(t or "\\text{–}", f.display) for t, f in zip(shown, formulas)], preset)
|
||||
|
||||
counts: dict[str, int] = {}
|
||||
secs: dict[str, list[float]] = {}
|
||||
cards = []
|
||||
for n, (f, tex, rr) in enumerate(zip(formulas, shown, rendered), 1):
|
||||
counts[f.status] = counts.get(f.status, 0) + 1
|
||||
latest = latest_by_engine(f)
|
||||
for e, r in latest.items():
|
||||
if r.get("seconds"):
|
||||
secs.setdefault(e, []).append(float(r["seconds"]))
|
||||
try:
|
||||
o = out / "img" / f"{f.id}-o.png"
|
||||
if not o.exists(): # the original never changes; cutting it from the PDF is the slow part
|
||||
save_png(original_png(source.original_bytes(_ref_for(source, f))), o)
|
||||
orig = f"<img loading=lazy src='img/{f.id}-o.png'>"
|
||||
except Exception: # noqa: BLE001
|
||||
orig = "<i>Original nicht lesbar</i>"
|
||||
if tex and rr.ok:
|
||||
g = (rr.rasterize(1.6) * 255).astype(np.uint8)
|
||||
save_png(Image.fromarray(255 - g), out / "img" / f"{f.id}-l.png")
|
||||
latex = f"<img loading=lazy src='img/{f.id}-l.png'>"
|
||||
else:
|
||||
latex = "<i>–</i>"
|
||||
label, cls = STATUS.get(f.status, (f.status, "img"))
|
||||
used = "ersetzt durch" if f.final_tex else "Lesart pix2text"
|
||||
reads = "".join(f"<tr><th>{LABEL.get(e, e)}</th><td><code>{html.escape(r.get('tex') or '–')}</code></td></tr>"
|
||||
for e, r in sorted(latest.items(), key=lambda kv: ["p2t", "local", "local2", "cloud"].index(kv[0])
|
||||
if kv[0] in ("p2t", "local", "local2", "cloud") else 9))
|
||||
occ = f.occurrences[0] if f.occurrences else {}
|
||||
cards.append(
|
||||
f"<section class='{cls}' data-s='{cls}'><div class=h><b>{n}</b> · {html.escape(f.source_name)} · "
|
||||
f"Zeile {occ.get('line', '?')} · <span class=b>{label}</span></div>"
|
||||
f"<div class=why>{html.escape(f.reason or '')}</div>"
|
||||
f"<div class=row><div><small>Original (PDF)</small>{orig}</div><div><small>{used}</small>{latex}</div></div>"
|
||||
f"<code class=tex>{html.escape(tex or '–')}</code>"
|
||||
f"<details><summary>alle Lesarten</summary><table>{reads}</table></details></section>")
|
||||
|
||||
total = len(formulas)
|
||||
good = counts.get("auto_accepted", 0) + counts.get("accepted_by_human", 0)
|
||||
rev = counts.get("needs_review", 0)
|
||||
img = total - good - rev
|
||||
time_rows = "".join(f"<tr><th>Ø {LABEL.get(e, e)}</th><td>{statistics.mean(v):.2f} s ({len(v)} Formeln)</td></tr>"
|
||||
for e, v in sorted(secs.items()))
|
||||
status_rows = "".join(f"<tr><th>{STATUS.get(k, (k,))[0]}</th><td>{v} ({v / max(total, 1):.0%})</td></tr>"
|
||||
for k, v in sorted(counts.items(), key=lambda kv: -kv[1]))
|
||||
titel = args.titel or f"Formeln: {source.path.name}"
|
||||
page = (
|
||||
f"<!doctype html><meta charset=utf-8><title>{html.escape(titel)}</title><style>"
|
||||
":root{--bg:#f6f6f3;--card:#fff;--ink:#222;--mut:#666;--line:#ddd}"
|
||||
"body{font:14px system-ui,sans-serif;margin:0;padding:16px;background:var(--bg);color:var(--ink)}"
|
||||
"h1{font-size:20px}table.sum{border-collapse:collapse;background:var(--card);margin:8px 0}"
|
||||
"table.sum th,table.sum td{border:1px solid var(--line);padding:4px 10px;text-align:left}"
|
||||
".bar{position:sticky;top:0;background:var(--bg);padding:8px 0;z-index:1}"
|
||||
"button{font:inherit;padding:4px 10px;margin-right:6px;border:1px solid #bbb;border-radius:4px;background:#fff;cursor:pointer}"
|
||||
"button.on{background:#222;color:#fff}"
|
||||
"section{background:var(--card);border:1px solid var(--line);border-left:6px solid #999;border-radius:6px;"
|
||||
"padding:6px 10px;margin:8px 0}section.good{border-left-color:#1f7a3a}section.rev{border-left-color:#c88a00}"
|
||||
"section.img{border-left-color:#888}.h{font-size:13px}.b{font-weight:600}.good .b{color:#1f7a3a}"
|
||||
".rev .b{color:#a15c00}.why{font-size:12px;color:var(--mut);margin:2px 0 4px}"
|
||||
".row{display:flex;flex-wrap:wrap;gap:16px;align-items:flex-start}.row>div{display:flex;flex-direction:column;gap:2px}"
|
||||
".row img{max-width:min(900px,100%);height:auto}code{font-size:12px}code.tex{display:block;margin:4px 0;"
|
||||
"overflow-x:auto;white-space:pre-wrap;word-break:break-all}details{font-size:12px}"
|
||||
"details th{text-align:left;font-weight:400;color:var(--mut);padding-right:8px;vertical-align:top}</style>"
|
||||
f"<h1>{html.escape(titel)}</h1>"
|
||||
f"<p>{total} Formelbilder (abgesetzte Formeln aus der PDF). Regel <code>p2t</code>: pix2text liest, Qwen "
|
||||
"bestätigt, Gemma nur bei Uneinigkeit, dazu Pandoc und Bild-Veto. Links das Original aus der PDF, rechts das "
|
||||
"gesetzte LaTeX.</p>"
|
||||
f"<table class=sum>{status_rows}{time_rows}</table>"
|
||||
"<div class=bar><button class=on data-f=all>alle</button>"
|
||||
f"<button data-f=good>ersetzt ({good})</button><button data-f=rev>Review ({rev})</button>"
|
||||
f"<button data-f=img>Bild/keine Formel ({img})</button></div>"
|
||||
+ "".join(cards)
|
||||
+ "<script>document.querySelectorAll('button[data-f]').forEach(function(b){b.onclick=function(){"
|
||||
"document.querySelectorAll('button[data-f]').forEach(function(x){x.classList.toggle('on',x===b);});"
|
||||
"var f=b.dataset.f;document.querySelectorAll('section').forEach(function(s){"
|
||||
"s.hidden=!(f==='all'||s.dataset.s===f);});};});</script>")
|
||||
(out / "index.html").write_text(page, encoding="utf-8")
|
||||
print(out / "index.html")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Loading…
Reference in a new issue