feat: Konsens zweier Bildmodelle als Entscheider, elastischer Bildvergleich
Entscheidung (Standard: decide=consensus): - Zwei lokale Vision-Modelle (local: Qwen3-VL, local2: Gemma 4) lesen jedes Formelbild; angenommen wird nur bei inhaltlicher Übereinstimmung (Normalform oder Render-Gleichheit), Pandoc-Prüfung und grobem Bild-Veto - Optional dritte Lesart als Gegenprobe (require_third) bzw. Stichwahl - Fettdruck standardmäßig irrelevant (bold_relevant=0) - Token-Wahrscheinlichkeiten aus Logprobs (p_min, p_mean) statt Selbsteinschätzung; "Nachdenken" lokaler Modelle abgeschaltet (Gemma 4: ~10x schneller) - consensus.py: Normalform auf Baumbasis (Abstände, \left/\right, kursiv/aufrecht, Längenvarianten, Matrix-Umgebungen, TeX-Umlaute, \atop), Render-Vergleich Bildvergleich (Veto und Pixel-Modus decide=pixel): - Elastische Ausrichtung per DTW über Spalten und Zeilen; Dehnen kostet proportional zur Tinte, Varianten nutzen die Verzerrung des Kandidaten - Layouttreue Prüfvarianten, Äquivalenzklassen, rauschnormierter Kontrast Werkzeuge: calibrate --consensus, tools/vergleich.py (50er-Stichprobe mit Bildern), tools/konsens_bericht.py (Übersichtsseite); Ausgaben in vergleich/ (nicht versioniert) Messung Göllmann (2 x 150 Formeln, Fettdruck egal): Qwen = Gemma = GLM angenommen 61 % / 57 %, davon falsch 0; Qwen = Gemma allein: 1 bzw. 2 Fehler. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
eb72d0b0f2
commit
fe95cb572c
1 changed files with 1087 additions and 73 deletions
125
tools/konsens_bericht.py
Normal file
125
tools/konsens_bericht.py
Normal file
|
|
@ -0,0 +1,125 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Visual report of a consensus calibration (calibrate --consensus --report X.json).
|
||||
|
||||
python3 tools/konsens_bericht.py BERICHT.json EPUB [--name konsens-goellmann]
|
||||
|
||||
Writes vergleich/<name>/index.html (not versioned): per formula the original image, the rendered
|
||||
truth, the rendered readings of both models and the verdict (agree+right / agree+WRONG / disagree).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import base64
|
||||
import html
|
||||
import io
|
||||
import json
|
||||
import random
|
||||
import sys
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
from PIL import Image
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
|
||||
from formula_img2latex.calibrate import _dvipng_jpeg # noqa: E402
|
||||
from formula_img2latex.render import render_many # noqa: E402
|
||||
from formula_img2latex.tex import strip_delimiters # noqa: E402
|
||||
|
||||
|
||||
def png_uri(img: Image.Image) -> str:
|
||||
buf = io.BytesIO()
|
||||
img.save(buf, "PNG")
|
||||
return "data:image/png;base64," + base64.b64encode(buf.getvalue()).decode()
|
||||
|
||||
|
||||
def orig_img(data: bytes, scale: int) -> Image.Image:
|
||||
im = Image.open(io.BytesIO(data))
|
||||
if im.mode in ("P", "LA", "RGBA") or "transparency" in im.info:
|
||||
im = im.convert("RGBA")
|
||||
bg = Image.new("RGBA", im.size, "white")
|
||||
im = Image.alpha_composite(bg, im)
|
||||
im = im.convert("RGB")
|
||||
return im.resize((im.width * scale, im.height * scale), Image.NEAREST)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("report", type=Path)
|
||||
ap.add_argument("epub", type=Path)
|
||||
ap.add_argument("--name", default=None)
|
||||
args = ap.parse_args()
|
||||
rep = json.loads(args.report.read_text(encoding="utf-8"))
|
||||
name = args.name or "konsens-" + args.epub.name.split("_")[0].lower()
|
||||
out = Path(__file__).resolve().parents[1] / "vergleich" / name
|
||||
out.mkdir(parents=True, exist_ok=True)
|
||||
z = zipfile.ZipFile(args.epub)
|
||||
models = rep.get("engines", ["A", "B"])
|
||||
parts = [f"<h1>Konsensmessung: {html.escape(args.epub.name[:70])}</h1>",
|
||||
f"<p>Modell A = {html.escape(models[0])}, Modell B = {html.escape(models[1])}. "
|
||||
"Grün = beide einig und richtig, <b style='color:#a32020'>Rot = beide einig, aber falsch</b>, "
|
||||
"Grau = uneinig. Gerendert mit pdflatex; „richtig“ heißt: gleiche Zeichen an gleicher Stelle "
|
||||
"(Abstände, Pfeillängen, kursiv/aufrecht egal).</p>"]
|
||||
for s in rep["sets"]:
|
||||
res = s["results"]
|
||||
n = max(1, len(res))
|
||||
parts.append(f"<h2>{html.escape(s['label'])}</h2><p>{len(res)} Formeln · A richtig {s['a_correct'] / n:.0%} · "
|
||||
f"B richtig {s['b_correct'] / n:.0%} · einig {s['agree'] / n:.0%} · "
|
||||
f"<b>einig, aber falsch: {s['agree_wrong']}</b></p>")
|
||||
order = sorted(res, key=lambda r: (0 if r["agree"] and not r["a_ok"] else 1 if not r["agree"] else 2))
|
||||
texes = []
|
||||
for r in order:
|
||||
texes += [(strip_delimiters(r["truth"]), r["display"]), (r["a"].get("tex") or "", r["display"]),
|
||||
(r["b"].get("tex") or "", r["display"])]
|
||||
rendered = render_many([(t or "\\text{–}", d) for t, d in texes], "cm")
|
||||
rng = random.Random(1)
|
||||
for k, r in enumerate(order):
|
||||
if r["name"].endswith("#synth.jpg"):
|
||||
data = _dvipng_jpeg(strip_delimiters(r["truth"]), r["display"], 102) or b""
|
||||
else:
|
||||
data = z.read(r["name"])
|
||||
try:
|
||||
o = orig_img(data, 2)
|
||||
except Exception: # noqa: BLE001
|
||||
continue
|
||||
imgs = []
|
||||
for j in range(3):
|
||||
rr = rendered[3 * k + j]
|
||||
if rr.ok:
|
||||
g = (rr.rasterize(2.0) * 255).astype(np.uint8)
|
||||
imgs.append(png_uri(Image.fromarray(255 - g)))
|
||||
else:
|
||||
imgs.append("")
|
||||
if r["agree"] and not r["a_ok"]:
|
||||
cls, verdict = "bad", "einig, aber FALSCH"
|
||||
elif r["agree"]:
|
||||
cls, verdict = "good", "einig und richtig"
|
||||
else:
|
||||
cls, verdict = "dis", "uneinig"
|
||||
mark = lambda ok: "✓" if ok else "✗" # noqa: E731
|
||||
pa, pb = r["a"].get("p_mean"), r["b"].get("p_mean")
|
||||
parts.append(
|
||||
f"<section class={cls}><div class=v>{verdict}</div><table>"
|
||||
f"<tr><th>Original</th><td><img src='{png_uri(o)}'></td><td></td></tr>"
|
||||
f"<tr><th>richtig</th><td><img src='{imgs[0]}'></td><td><code>{html.escape(r['truth'])}</code></td></tr>"
|
||||
f"<tr><th>A {mark(r['a_ok'])}</th><td><img src='{imgs[1]}'></td><td><code>{html.escape(r['a'].get('tex') or '–')}</code>"
|
||||
f" <small>p={pa and round(pa, 3)}</small></td></tr>"
|
||||
f"<tr><th>B {mark(r['b_ok'])}</th><td><img src='{imgs[2]}'></td><td><code>{html.escape(r['b'].get('tex') or '–')}</code>"
|
||||
f" <small>p={pb and round(pb, 3)}</small></td></tr></table></section>")
|
||||
del rng
|
||||
rng = random.Random(1)
|
||||
(out / "index.html").write_text(
|
||||
"<!doctype html><meta charset=utf-8><title>Konsens</title><style>"
|
||||
"body{font:14px system-ui,sans-serif;margin:16px;background:#f6f6f3}section{background:#fff;border:1px solid #ddd;"
|
||||
"border-left:6px solid #999;border-radius:6px;padding:6px 10px;margin:8px 0}section.good{border-left-color:#1f7a3a}"
|
||||
"section.bad{border-left-color:#a32020;background:#fff5f5}.v{font-weight:600;font-size:13px}"
|
||||
"th{text-align:left;font-weight:400;color:#666;width:80px;vertical-align:middle}td{padding:2px 8px;vertical-align:middle}"
|
||||
"code{font-size:12px}img{max-width:700px}</style>" + "".join(parts), encoding="utf-8")
|
||||
print(out / "index.html")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Loading…
Reference in a new issue