dschlueter
388024a17b
- Dritte Lesart (GLM-4.6V über OpenRouter, ohne Nachdenken) muss zustimmen;
keine 2:1-Annahme; fehlender Schlusspunkt der dritten Lesart zählt nicht
- run in Paketen zu 50 mit Statuszeile, beide lokalen Modelle parallel,
GLM nur für Einige; nur die übernommene Lesart wird gegen das Bild geprüft
- Bild-Veto: Ähnlichkeit >= 0,65, Schriftgrößen-Prüfung abgeschaltet
- Antworten auf 800 Tokens begrenzt; Prompt-Version und Nachdenk-Einstellung
im Cache-Schlüssel; Verfahrensversion für Prüfergebnisse
- Behoben: Gleichungsnummern aus alten Lesarten ("(3.7)"), gleichzeitiges
Speichern zweier Engines, Absturz bei Formeln ohne aktuelle Lesart
- apply: deutsches Dezimalkomma 3{,}6; Review-Galerie zeigt aktuelle Lesarten
- tools/vergleich.py auf Konsens-Ablauf umgestellt
- Doku: README, BEDIENUNGSANLEITUNG, neues CHANGELOG
Gehrke-Brückenkurs: 942 von 1505 Formelbildern automatisch ersetzt (63 %),
62 von 62 von Hand geprüften korrekt; Pandoc/MathML 945 von 945.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
562 lines
25 KiB
Python
562 lines
25 KiB
Python
"""Workflow steps: scan, recognize, verify, apply, revert."""
|
||
|
||
from __future__ import annotations
|
||
|
||
import hashlib
|
||
import io
|
||
import os
|
||
import re
|
||
import shutil
|
||
import statistics
|
||
import threading
|
||
import sys
|
||
from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor, as_completed
|
||
from pathlib import Path
|
||
from typing import Any, Callable
|
||
|
||
import numpy as np
|
||
from PIL import Image
|
||
|
||
from .checks import pandoc_mathml_ok
|
||
from .classify import classify
|
||
from .compare import diff_image
|
||
from .decide import consensus_thresholds, decide, decide_consensus, thresholds
|
||
from .engines import make_engine
|
||
from .engines.vlm import PROMPT_VERSION
|
||
from .imageio import load_ink
|
||
from .render import PRESETS
|
||
from .sources import open_source
|
||
from .sources.base import ImageRef, Source
|
||
from .store import HUMAN_STATUSES, REPLACE_STATUSES, Formula, Workspace, now_iso, sha256_bytes
|
||
from .tex import normalize_for_compare, split_tag, strip_delimiters
|
||
from .verify import verify
|
||
|
||
PRESET_SAMPLE = 12
|
||
_SAVE_LOCK = threading.Lock()
|
||
|
||
|
||
def log(msg: str, quiet: bool = False) -> None:
|
||
if not quiet:
|
||
print(msg, file=sys.stderr, flush=True)
|
||
|
||
|
||
# --------------------------------------------------------------------------------------- scan
|
||
|
||
def scan(source: Source, ws: Workspace, quiet: bool = False) -> dict[str, int]:
|
||
refs = source.image_refs()
|
||
groups: dict[str, tuple[Formula, list[ImageRef]]] = {}
|
||
for ref in refs:
|
||
p = ref.asset_path()
|
||
if not p.exists():
|
||
continue
|
||
sha = sha256_bytes(p.read_bytes())
|
||
fid = Formula.make_id(sha, ref.display)
|
||
if fid not in groups:
|
||
f = ws.load(fid) or Formula(id=fid, sha256=sha, asset=p.relative_to(source.root).as_posix(),
|
||
display=ref.display)
|
||
groups[fid] = (f, [])
|
||
groups[fid][1].append(ref)
|
||
counts = {"formula": 0, "maybe": 0, "figure": 0}
|
||
for fid, (f, rs) in groups.items():
|
||
ref = rs[0]
|
||
f.asset = ref.asset_path().relative_to(source.root).as_posix()
|
||
f.source_name = source.source_name(ref)
|
||
f.known_formula = source.known_formula(ref)
|
||
f.alt = ref.alt
|
||
f.occurrences = [{"file": r.file.relative_to(source.root).as_posix(), "line": r.line, "md": r.md,
|
||
"context": r.context} for r in rs]
|
||
data = source.original_bytes(ref)
|
||
f.width, f.height = _image_size(data)
|
||
f.classify = classify(data, f.asset, f.known_formula, inline=not f.display)
|
||
counts[f.classify["label"]] += 1
|
||
ws.save(f)
|
||
ws.meta["source"] = source.describe()
|
||
ws.meta["last_scan"] = {"at": now_iso(), "images": len(refs), "formulas": len(groups), **counts}
|
||
ws.save_meta()
|
||
log(f"scan: {len(refs)} Bildverweise, {len(groups)} verschiedene Bilder: {counts['formula']} Formeln, "
|
||
f"{counts['maybe']} unklar, {counts['figure']} Abbildungen", quiet)
|
||
return counts
|
||
|
||
|
||
def _image_size(data: bytes) -> tuple[int, int]:
|
||
try:
|
||
with Image.open(io.BytesIO(data)) as im:
|
||
return im.size
|
||
except OSError: # SVG
|
||
return 0, 0
|
||
|
||
|
||
def _wanted(f: Formula, include_figures: bool) -> bool:
|
||
if f.status in HUMAN_STATUSES:
|
||
return False
|
||
label = (f.classify or {}).get("label", "maybe")
|
||
return include_figures or label != "figure"
|
||
|
||
|
||
def _ref_for(source: Source, f: Formula) -> ImageRef:
|
||
occ = f.occurrences[0]
|
||
md = source.root / occ["file"]
|
||
from .sources.base import scan_markdown
|
||
for r in scan_markdown(md):
|
||
if r.line == occ["line"] and r.md == occ["md"]:
|
||
return r
|
||
for r in scan_markdown(md):
|
||
if r.md == occ["md"]:
|
||
return r
|
||
raise FileNotFoundError(f"Bildverweis nicht mehr gefunden: {occ['file']}:{occ['line']}")
|
||
|
||
|
||
# --------------------------------------------------------------------------------- recognize
|
||
|
||
def needs_third(f: Formula) -> bool:
|
||
"""The two primary readings agree; only the confirming third reading is missing."""
|
||
return f.status == "needs_review" and "dritte Lesart fehlt" in (f.reason or "")
|
||
|
||
|
||
def recognize(source: Source, ws: Workspace, engine_name: str, model: str | None = None, url: str | None = None,
|
||
only_unresolved: bool = False, include_figures: bool = False, limit: int = 0,
|
||
workers: int = 4, quiet: bool = False, only_ids: set[str] | None = None,
|
||
only_needing_third: bool = False) -> dict[str, int]:
|
||
engine = make_engine(engine_name, model, url)
|
||
engine.check()
|
||
todo = []
|
||
for f in ws.all():
|
||
if not _wanted(f, include_figures) or (only_ids is not None and f.id not in only_ids):
|
||
continue
|
||
if only_unresolved and f.status in ("auto_accepted", "not_formula"):
|
||
continue
|
||
if only_needing_third and not needs_third(f):
|
||
continue
|
||
if any(c.get("engine") == engine.name and c.get("model") == engine.model
|
||
and c.get("prompt") == PROMPT_VERSION and c.get("thinking") == engine.thinking for c in f.candidates):
|
||
continue # cached: never ask the same model the same question twice
|
||
todo.append(f)
|
||
if limit:
|
||
todo = todo[:limit]
|
||
log(f"recognize: {len(todo)} Bilder mit {engine.name} ({engine.model})", quiet)
|
||
stats = {"asked": 0, "latex": 0, "not_formula": 0, "error": 0, "tokens_in": 0, "tokens_out": 0}
|
||
|
||
def job(f: Formula) -> tuple[Formula, Any]:
|
||
ref = _ref_for(source, f)
|
||
ctx = f.occurrences[0].get("context", "")
|
||
return f, engine.recognize(source.original_bytes(ref), f.asset, ctx, f.display)
|
||
|
||
with ThreadPoolExecutor(max_workers=max(1, workers)) as pool:
|
||
futures = [pool.submit(job, f) for f in todo]
|
||
for i, fut in enumerate(as_completed(futures), 1):
|
||
f, rec = fut.result()
|
||
stats["asked"] += 1
|
||
stats["tokens_in"] += rec.tokens_in
|
||
stats["tokens_out"] += rec.tokens_out
|
||
cand = {"engine": engine.name, "model": engine.model, "tex": rec.tex, "is_formula": rec.is_formula,
|
||
"p_min": rec.p_min, "p_mean": rec.p_mean, "self_conf": rec.self_conf, "prompt": PROMPT_VERSION,
|
||
"thinking": engine.thinking,
|
||
"error": rec.error, "seconds": round(rec.seconds, 2), "created_at": now_iso()}
|
||
with _SAVE_LOCK: # another engine may run in parallel: append to the current record
|
||
f = ws.load(f.id) or f
|
||
f.candidates.append(cand)
|
||
ws.save(f)
|
||
if rec.error:
|
||
stats["error"] += 1
|
||
elif rec.tex:
|
||
stats["latex"] += 1
|
||
else:
|
||
stats["not_formula"] += 1
|
||
if not quiet and (i % 25 == 0 or i == len(todo)):
|
||
log(f" {i}/{len(todo)}")
|
||
log(f"recognize: {stats['latex']} LaTeX-Kandidaten, {stats['not_formula']} „keine Formel“, "
|
||
f"{stats['error']} Fehler; Tokens {stats['tokens_in']} ein / {stats['tokens_out']} aus", quiet)
|
||
return stats
|
||
|
||
|
||
# ------------------------------------------------------------------------------------ verify
|
||
|
||
def comparison_pngs(c: Any) -> dict[str, bytes]:
|
||
"""PNG images of an aligned comparison: original, candidate render (elastic and rigid), overlay."""
|
||
out: dict[str, bytes] = {}
|
||
if c is None or c.original is None:
|
||
return out
|
||
for key, arr in (("orig", c.original), ("render", c.candidate), ("rigid", c.raw)):
|
||
if arr is None:
|
||
continue
|
||
buf = io.BytesIO()
|
||
Image.fromarray(((1 - np.clip(arr, 0, 1)) * 255).astype(np.uint8)).save(buf, "PNG")
|
||
out[key] = buf.getvalue()
|
||
buf = io.BytesIO()
|
||
Image.fromarray(diff_image(c, upscale=1)).save(buf, "PNG")
|
||
out["diff"] = buf.getvalue()
|
||
return out
|
||
|
||
|
||
def _verify_job(args: tuple) -> tuple[str, list[dict], dict[str, bytes]]:
|
||
"""Worker process: verify all distinct candidates of one formula; return records and images.
|
||
|
||
With variants=False (consensus mode) this is only the rough image check: render, elastic
|
||
comparison, scale – no confusable variants, no repair.
|
||
"""
|
||
fid, data, name, texes, display, preset = args[:6]
|
||
variants = args[6] if len(args) > 6 else True
|
||
os.environ.setdefault("OMP_NUM_THREADS", "1")
|
||
orig = load_ink(data, name)
|
||
results = []
|
||
best = None
|
||
for tex in texes:
|
||
v = verify(orig, tex, display, preset, check_variants=variants)
|
||
results.append(v.to_json())
|
||
key = (v.ok, v.min_contrast if v.min_contrast is not None else 1.0, v.metrics.get("ncc", 0.0))
|
||
if best is None or key > best[0]:
|
||
best = (key, v)
|
||
return fid, results, comparison_pngs(best[1].comparison if best else None)
|
||
|
||
|
||
def _distinct_candidates(f: Formula) -> list[str]:
|
||
seen: dict[str, str] = {}
|
||
for c in f.candidates:
|
||
tex = c.get("tex")
|
||
if not tex:
|
||
continue
|
||
body, _tag = split_tag(strip_delimiters(tex))
|
||
key = normalize_for_compare(body)
|
||
seen.setdefault(key, body)
|
||
return list(seen.values())
|
||
|
||
|
||
def choose_preset(source: Source, ws: Workspace, formulas: list[Formula], quiet: bool = False) -> str:
|
||
"""Pick the font preset under which a sample of candidates matches the originals best."""
|
||
from .render import render_many
|
||
from .compare import compare
|
||
sample = [f for f in formulas if _distinct_candidates(f)][:PRESET_SAMPLE]
|
||
if not sample:
|
||
return "cm"
|
||
inks = [load_ink(source.original_bytes(_ref_for(source, f)), f.asset) for f in sample]
|
||
scores: dict[str, float] = {}
|
||
for preset in PRESETS:
|
||
rs = render_many([(_distinct_candidates(f)[0], f.display) for f in sample], preset)
|
||
vals = [compare(i, r).metrics.get("ncc", 0.0) if r.ok else 0.0 for i, r in zip(inks, rs)]
|
||
scores[preset] = float(statistics.median(vals))
|
||
best = max(scores, key=lambda k: scores[k])
|
||
log("Schrift: " + ", ".join(f"{k} {v:.3f}" for k, v in sorted(scores.items(), key=lambda kv: -kv[1]))
|
||
+ f" → {best}", quiet)
|
||
return best
|
||
|
||
|
||
def verify_all(source: Source, ws: Workspace, preset: str | None = None, include_figures: bool = False,
|
||
workers: int = 0, redo: bool = False, quiet: bool = False,
|
||
progress: Callable[[int, int], None] | None = None, only_ids: set[str] | None = None) -> dict[str, int]:
|
||
formulas = [f for f in ws.all() if _wanted(f, include_figures) and _distinct_candidates(f)
|
||
and (only_ids is None or f.id in only_ids)]
|
||
mode = decide_mode(ws)
|
||
formulas = [f for f in formulas if _verify_texes(f, mode)]
|
||
if not redo:
|
||
formulas = [f for f in formulas if not _verified_current(f, mode)]
|
||
if preset is None:
|
||
preset = ws.settings.get("preset") or choose_preset(source, ws, formulas, quiet)
|
||
ws.settings["preset"] = preset
|
||
ws.save_meta()
|
||
jobs = []
|
||
for f in formulas:
|
||
ref = _ref_for(source, f)
|
||
jobs.append((f.id, source.original_bytes(ref), f.asset, _verify_texes(f, mode), f.display, preset,
|
||
mode == "pixel"))
|
||
log(f"verify: {len(jobs)} Formeln, Schrift {preset}", quiet)
|
||
by_id = {f.id: f for f in formulas}
|
||
ws.review_dir.mkdir(parents=True, exist_ok=True)
|
||
workers = workers or max(1, (os.cpu_count() or 2) - 2)
|
||
done = 0
|
||
with ProcessPoolExecutor(max_workers=workers) as pool:
|
||
for fut in as_completed([pool.submit(_verify_job, j) for j in jobs]):
|
||
fid, results, pngs = fut.result()
|
||
f = by_id[fid]
|
||
if not results:
|
||
continue
|
||
best = max(results, key=lambda r: (r["ok"], r["min_contrast"] if r["min_contrast"] is not None else 1.0,
|
||
r["metrics"].get("ncc", 0.0)))
|
||
f.verification = {"preset": preset, "candidates_key": _cands_key(f, mode), "best": best, "all": results,
|
||
"at": now_iso()}
|
||
for key, png in pngs.items():
|
||
(ws.review_dir / f"{fid}-{key}.png").write_bytes(png)
|
||
ws.save(f)
|
||
done += 1
|
||
if progress:
|
||
progress(done, len(jobs))
|
||
elif not quiet and (done % 25 == 0 or done == len(jobs)):
|
||
log(f" {done}/{len(jobs)}")
|
||
return decide_all(ws, quiet=quiet, preset=preset, only_ids=only_ids)
|
||
|
||
|
||
VERIFY_VERSION = 3 # bump when the comparison changes, so stored image checks are recomputed
|
||
|
||
|
||
def _verify_texes(f: Formula, mode: str) -> list[str]:
|
||
"""Readings to check against the image. Consensus: only the reading that would be used (the
|
||
first primary reading) – the veto needs nothing else, and a later third reading changes nothing."""
|
||
if mode != "consensus":
|
||
return _distinct_candidates(f)
|
||
for r in readings_of(f):
|
||
if r.get("tex"):
|
||
return [split_tag(strip_delimiters(r["tex"]))[0]]
|
||
return []
|
||
|
||
|
||
def _cands_key(f: Formula, mode: str = "pixel") -> str:
|
||
text = f"v{VERIFY_VERSION}\n" + "\n".join(sorted(_verify_texes(f, mode)))
|
||
return hashlib.sha256(text.encode()).hexdigest()[:16]
|
||
|
||
|
||
def _verified_current(f: Formula, mode: str = "pixel") -> bool:
|
||
v = f.verification or {}
|
||
return v.get("candidates_key") == _cands_key(f, mode)
|
||
|
||
|
||
def decide_mode(ws: Workspace) -> str:
|
||
return ws.settings.get("decide") or "consensus"
|
||
|
||
|
||
def readings_of(f: Formula) -> list[dict]:
|
||
"""Latest reading per model; local engines first (primary pair), then the rest (tie-break)."""
|
||
latest: dict[str, dict] = {}
|
||
for c in f.candidates:
|
||
if c.get("error"):
|
||
continue
|
||
latest[f"{c.get('engine')}|{c.get('model')}"] = c
|
||
order = {"local": 0, "local2": 1, "cloud": 2}
|
||
return sorted(latest.values(), key=lambda c: order.get(c.get("engine", ""), 3))
|
||
|
||
|
||
def decide_all(ws: Workspace, quiet: bool = False, preset: str | None = None,
|
||
only_ids: set[str] | None = None) -> dict[str, int]:
|
||
if decide_mode(ws) == "consensus":
|
||
return decide_all_consensus(ws, quiet, preset or ws.settings.get("preset") or "cm", only_ids)
|
||
th = thresholds(ws.settings.get("thresholds"))
|
||
formulas = [f for f in ws.all() if f.verification and f.status not in HUMAN_STATUSES]
|
||
# Book-wide scale: all formula images of a book share one resolution, so px/pt is constant.
|
||
good = [f.verification["best"]["metrics"]["px_per_pt"] for f in formulas
|
||
if f.verification["best"].get("ok") and f.verification["best"]["metrics"].get("ncc", 0) >= th["min_ncc"]
|
||
and (f.verification["best"].get("min_contrast") or 1.0) >= th["min_contrast"]]
|
||
book_ppt = float(statistics.median(good)) if len(good) >= 5 else None
|
||
ws.settings["book_px_per_pt"] = book_ppt
|
||
texes = [(f.verification["best"]["tex"], f.display) for f in formulas]
|
||
pandoc = pandoc_mathml_ok(texes) if texes else []
|
||
counts: dict[str, int] = {}
|
||
for f, pok in zip(formulas, pandoc):
|
||
best = f.verification["best"]
|
||
status, reason = decide(best, pok, book_ppt, th)
|
||
f.status, f.reason = status, reason
|
||
f.final_tex = _with_tag(f, best["tex"]) if status == "auto_accepted" else None
|
||
f.decided_at = now_iso()
|
||
ws.save(f)
|
||
counts[status] = counts.get(status, 0) + 1
|
||
for f in ws.all():
|
||
if not f.verification and f.status == "pending" and f.candidates and not _distinct_candidates(f):
|
||
f.status = "not_formula" if any(c.get("is_formula") is False for c in f.candidates) else "no_candidate"
|
||
f.reason = "Modell: keine Formel" if f.status == "not_formula" else "kein Kandidat"
|
||
ws.save(f)
|
||
counts[f.status] = counts.get(f.status, 0) + 1
|
||
ws.save_meta()
|
||
log("Entscheidung: " + ", ".join(f"{k} {v}" for k, v in sorted(counts.items())), quiet)
|
||
return counts
|
||
|
||
|
||
def _strip_final_punct(tex: str) -> str:
|
||
return re.sub(r"(\\[,;!]|\\quad|\\qquad|\s)*[.,;]\s*$", "", strip_delimiters(tex or "")).strip()
|
||
|
||
|
||
def _tex_key(tex: str) -> str:
|
||
"""Key under which verify stores a candidate (see _distinct_candidates)."""
|
||
return normalize_for_compare(split_tag(strip_delimiters(tex))[0])
|
||
|
||
|
||
def decide_all_consensus(ws: Workspace, quiet: bool, preset: str, only_ids: set[str] | None = None) -> dict[str, int]:
|
||
from .consensus import same_reading
|
||
th = consensus_thresholds(ws.settings.get("consensus_thresholds"))
|
||
formulas = [f for f in ws.all() if f.candidates and f.status not in HUMAN_STATUSES
|
||
and (only_ids is None or f.id in only_ids)]
|
||
plan = []
|
||
for f in formulas:
|
||
rs = readings_of(f)
|
||
bold = bool(th.get("bold_relevant"))
|
||
agree = []
|
||
for i in range(len(rs)):
|
||
for j in range(i + 1, len(rs)):
|
||
a, b = rs[i].get("tex") or "", rs[j].get("tex") or ""
|
||
if j >= 2: # the confirming reading: a missing final period/comma is no contradiction
|
||
a, b = _strip_final_punct(a), _strip_final_punct(b)
|
||
how = same_reading(a, b, f.display, preset, bold)
|
||
if how:
|
||
agree.append((i, j, how))
|
||
plan.append((f, rs, agree))
|
||
# Book-wide scale from agreed readings that resemble their original well.
|
||
by_tex = {f.id: {_tex_key(r["tex"]): r for r in (f.verification or {}).get("all", [])} for f, _, _ in plan}
|
||
good = []
|
||
for f, rs, agree in plan:
|
||
for i, j, _ in agree[:1]:
|
||
v = by_tex[f.id].get(_tex_key(rs[i].get("tex") or ""))
|
||
if v and v.get("ok") and v["metrics"].get("ncc", 0) >= 0.85:
|
||
good.append(v["metrics"]["px_per_pt"])
|
||
book_ppt = float(statistics.median(good)) if len(good) >= 5 else ws.settings.get("book_px_per_pt")
|
||
ws.settings["book_px_per_pt"] = book_ppt
|
||
chosen = []
|
||
for f, rs, agree in plan:
|
||
status, reason, use = decide_consensus(rs, agree, None, True, book_ppt, th) # preliminary: which reading
|
||
chosen.append(rs[use]["tex"] if use is not None and rs[use].get("tex") else None)
|
||
pandoc = pandoc_mathml_ok([(t or "x", f.display) for (f, _, _), t in zip(plan, chosen)]) if plan else []
|
||
counts: dict[str, int] = {}
|
||
for (f, rs, agree), tex, pok in zip(plan, chosen, pandoc):
|
||
veto = by_tex[f.id].get(_tex_key(tex)) if tex else None
|
||
status, reason, use = decide_consensus(rs, agree, veto, pok, book_ppt, th)
|
||
f.status, f.reason = status, reason
|
||
f.final_tex = _with_tag(f, tex) if status == "auto_accepted" and tex else None
|
||
f.decided_at = now_iso()
|
||
ws.save(f)
|
||
counts[status] = counts.get(status, 0) + 1
|
||
ws.save_meta()
|
||
log("Entscheidung (Konsens): " + ", ".join(f"{k} {v}" for k, v in sorted(counts.items())), quiet)
|
||
return counts
|
||
|
||
|
||
def _with_tag(f: Formula, tex: str) -> str:
|
||
"""Turn an equation number of the *chosen* reading (\\tag{3.7}) into visible text, like epub2md does.
|
||
|
||
Only the accepted reading counts – never other or older readings. In consensus mode the number
|
||
is part of the compared canonical form, so all agreeing readings carry the same number.
|
||
"""
|
||
body, tag = split_tag(strip_delimiters(tex))
|
||
if not tag:
|
||
return body
|
||
return body + f"\\qquad\\text{{({tag})}}" if f.display else body
|
||
|
||
|
||
# ------------------------------------------------------------------------------ apply/revert
|
||
|
||
def _file_sha(p: Path) -> str:
|
||
return sha256_bytes(p.read_bytes())
|
||
|
||
|
||
def apply(source: Source, ws: Workspace, dry_run: bool = False, quiet: bool = False) -> dict[str, int]:
|
||
"""Rewrite Markdown files from their backed-up originals; idempotent."""
|
||
by_occ: dict[tuple[str, int, str], str] = {}
|
||
for f in ws.all():
|
||
if f.status in REPLACE_STATUSES and f.final_tex:
|
||
math = source.format_math(f.final_tex, f.display)
|
||
for occ in f.occurrences:
|
||
by_occ[(occ["file"], occ["line"], occ["md"])] = math
|
||
state = ws.meta.setdefault("applied", {})
|
||
files = {occ[0] for occ in by_occ} | set(state)
|
||
stats = {"files": 0, "replaced": 0}
|
||
for rel in sorted(files):
|
||
path = source.root / rel
|
||
backup = ws.backup_dir / rel
|
||
if not path.exists():
|
||
continue
|
||
current = _file_sha(path)
|
||
known = state.get(rel, {})
|
||
if backup.exists() and current not in (known.get("applied_sha"), known.get("original_sha")):
|
||
# File was regenerated (epub2md --force) or edited by hand: it is the new original.
|
||
if not dry_run:
|
||
backup.unlink()
|
||
if not backup.exists() and not dry_run:
|
||
backup.parent.mkdir(parents=True, exist_ok=True)
|
||
shutil.copy2(path, backup)
|
||
known = state[rel] = {"original_sha": current}
|
||
original = (backup if backup.exists() else path).read_text(encoding="utf-8").split("\n")
|
||
n = 0
|
||
for (frel, line, md), math in by_occ.items():
|
||
if frel != rel or line > len(original):
|
||
continue
|
||
if md in original[line - 1]:
|
||
original[line - 1] = original[line - 1].replace(md, math, 1)
|
||
n += 1
|
||
text = "\n".join(original)
|
||
stats["replaced"] += n
|
||
if dry_run:
|
||
continue
|
||
if n == 0 and backup.exists():
|
||
shutil.copy2(backup, path)
|
||
else:
|
||
path.write_text(text, encoding="utf-8")
|
||
state[rel] = {"original_sha": known.get("original_sha") or _file_sha(backup),
|
||
"applied_sha": _file_sha(path), "replaced": n, "at": now_iso()}
|
||
stats["files"] += 1
|
||
if not dry_run:
|
||
ws.save_meta()
|
||
log(f"apply{' (Probelauf)' if dry_run else ''}: {stats['replaced']} Formelbilder in {stats['files']} Dateien "
|
||
"durch LaTeX ersetzt", quiet)
|
||
return stats
|
||
|
||
|
||
def revert(source: Source, ws: Workspace, force: bool = False, quiet: bool = False) -> dict[str, int]:
|
||
state = ws.meta.get("applied", {})
|
||
stats = {"restored": 0, "skipped": 0}
|
||
restored: list[str] = []
|
||
for rel, info in sorted(state.items()):
|
||
path, backup = source.root / rel, ws.backup_dir / rel
|
||
if not backup.exists():
|
||
continue
|
||
if path.exists() and _file_sha(path) != info.get("applied_sha") and not force:
|
||
log(f" übersprungen (seit apply verändert, --force zum Überschreiben): {rel}", quiet)
|
||
stats["skipped"] += 1
|
||
continue
|
||
shutil.copy2(backup, path)
|
||
backup.unlink()
|
||
restored.append(rel)
|
||
stats["restored"] += 1
|
||
ws.meta["applied"] = {k: v for k, v in state.items() if k not in restored}
|
||
if not ws.meta["applied"]:
|
||
shutil.rmtree(ws.backup_dir, ignore_errors=True)
|
||
ws.save_meta()
|
||
log(f"revert: {stats['restored']} Dateien wiederhergestellt, {stats['skipped']} übersprungen", quiet)
|
||
return stats
|
||
|
||
|
||
def open_all(path: Path, workspace: Path | None, create: bool = False, pdf: Path | None = None
|
||
) -> tuple[Source, Workspace]:
|
||
source = open_source(path)
|
||
if pdf is not None and hasattr(source, "pdf"):
|
||
source.pdf = pdf.resolve()
|
||
ws_root = workspace or Workspace.default_for(path)
|
||
ws = Workspace.open(ws_root, create=create, source=source.describe())
|
||
if getattr(source, "pdf", "unset") is None and ws.meta.get("source", {}).get("pdf"):
|
||
source.pdf = Path(ws.meta["source"]["pdf"])
|
||
return source, ws
|
||
|
||
|
||
# ------------------------------------------------------------------------- run in chunks
|
||
|
||
def run_consensus(source: Source, ws: Workspace, chunk: int = 50, preset: str | None = None,
|
||
include_figures: bool = False, limit: int = 0, cloud_model: str | None = None,
|
||
use_cloud: bool = True, ids: list[str] | None = None, quiet: bool = False,
|
||
report: Callable[[str], None] | None = None) -> dict[str, int]:
|
||
"""Process formulas in chunks: local, local2, verify; cloud only where both agree; verify again.
|
||
|
||
After every chunk a status line (processed / accepted / review / image) is reported.
|
||
"""
|
||
say = report or (lambda m: print(m, flush=True))
|
||
todo = ids or [f.id for f in ws.all() if _wanted(f, include_figures)]
|
||
if limit:
|
||
todo = todo[:limit]
|
||
totals: dict[str, int] = {}
|
||
done: list[str] = []
|
||
for start in range(0, len(todo), chunk):
|
||
part = set(todo[start:start + chunk])
|
||
# both local models run on their own GPU: ask them at the same time
|
||
with ThreadPoolExecutor(max_workers=2) as both:
|
||
for fut in [both.submit(recognize, source, ws, eng, only_ids=part, include_figures=include_figures,
|
||
quiet=True) for eng in ("local", "local2")]:
|
||
fut.result()
|
||
verify_all(source, ws, preset, include_figures, quiet=True, only_ids=part)
|
||
if use_cloud:
|
||
recognize(source, ws, "cloud", cloud_model, only_ids=part, only_needing_third=True,
|
||
include_figures=include_figures, workers=8, quiet=True)
|
||
verify_all(source, ws, preset, include_figures, quiet=True, only_ids=part)
|
||
done.extend(sorted(part))
|
||
totals = {}
|
||
for fid in done:
|
||
f = ws.load(fid)
|
||
if f is not None:
|
||
totals[f.status] = totals.get(f.status, 0) + 1
|
||
auto = totals.get("auto_accepted", 0)
|
||
image = sum(totals.get(k, 0) for k in ("rejected_keep_image", "not_formula", "no_candidate", "render_failed"))
|
||
review = totals.get("needs_review", 0)
|
||
say(f"{len(done):5d}/{len(todo)} verarbeitet: automatisch ersetzt {auto} ({auto / len(done):.0%}), "
|
||
f"Review {review}, bleibt Bild/keine Formel {image}")
|
||
return totals
|