218 lines
10 KiB
Python
218 lines
10 KiB
Python
|
"""Unabhängiger Validator für epub2md-Ausgaben (nutzt markdown-it-py, nicht epub2md/linkcheck.py).
|
|||
|
|
|
|||
|
|
Aufruf: python tools/qa/validate_output.py AUSGABEVERZEICHNIS [...] -> JSON auf stdout
|
|||
|
|
"""
|
|||
|
|
|
|||
|
|
from __future__ import annotations
|
|||
|
|
|
|||
|
|
import ast
|
|||
|
|
import hashlib
|
|||
|
|
import json
|
|||
|
|
import posixpath
|
|||
|
|
import re
|
|||
|
|
import sys
|
|||
|
|
import unicodedata
|
|||
|
|
from pathlib import Path
|
|||
|
|
from urllib.parse import unquote
|
|||
|
|
|
|||
|
|
import yaml
|
|||
|
|
from markdown_it import MarkdownIt
|
|||
|
|
from mdit_py_plugins.dollarmath import dollarmath_plugin
|
|||
|
|
from mdit_py_plugins.footnote import footnote_plugin
|
|||
|
|
|
|||
|
|
MD = MarkdownIt("commonmark").enable(["table", "strikethrough"]).use(footnote_plugin).use(dollarmath_plugin)
|
|||
|
|
ATTR = re.compile(r"""\b(href|src|id)\s*=\s*"([^"]*)\"""")
|
|||
|
|
NAV_RESIDUE = re.compile(r"^\s*(next|previous|prev|zurück|weiter|vorherige seite|nächste seite|table of contents|back to top)\s*$", re.I)
|
|||
|
|
ENTITY_IN_CODE = re.compile(r"&(lt|gt|amp|quot|#\d+|#x[0-9a-f]+);", re.I)
|
|||
|
|
|
|||
|
|
|
|||
|
|
def gh_slug(text: str) -> str:
|
|||
|
|
text = unicodedata.normalize("NFC", text).strip().lower()
|
|||
|
|
text = re.sub(r"[^\w\- ]", "", text)
|
|||
|
|
return text.replace(" ", "-")
|
|||
|
|
|
|||
|
|
|
|||
|
|
def split_frontmatter(text: str) -> tuple[dict | None, str, str | None]:
|
|||
|
|
if not text.startswith("---\n"):
|
|||
|
|
return None, text, None
|
|||
|
|
end = text.find("\n---\n", 4)
|
|||
|
|
if end < 0:
|
|||
|
|
return None, text, "Frontmatter nicht geschlossen"
|
|||
|
|
try:
|
|||
|
|
data = yaml.safe_load(text[4:end])
|
|||
|
|
except yaml.YAMLError as exc:
|
|||
|
|
return None, text[end + 5 :], f"YAML ungültig: {exc}"
|
|||
|
|
return data, text[end + 5 :], None
|
|||
|
|
|
|||
|
|
|
|||
|
|
def walk(tokens):
|
|||
|
|
for t in tokens:
|
|||
|
|
yield t
|
|||
|
|
if t.children:
|
|||
|
|
yield from walk(t.children)
|
|||
|
|
|
|||
|
|
|
|||
|
|
def analyse(rel: str, body: str) -> dict:
|
|||
|
|
tokens = MD.parse(body)
|
|||
|
|
info: dict = {"links": [], "anchors": set(), "fn_refs": set(), "fn_defs": set(), "code": [], "issues": []}
|
|||
|
|
lines = body.split("\n")
|
|||
|
|
heading_next = False
|
|||
|
|
for t in walk(tokens):
|
|||
|
|
if t.type == "heading_open":
|
|||
|
|
heading_next = True
|
|||
|
|
elif t.type == "inline" and heading_next:
|
|||
|
|
heading_next = False
|
|||
|
|
plain = re.sub(r"<[^>]+>", "", "".join(c.content for c in (t.children or []) if c.type in ("text", "code_inline")))
|
|||
|
|
info["anchors"].add(gh_slug(plain))
|
|||
|
|
if t.type == "link_open":
|
|||
|
|
info["links"].append(t.attrs.get("href", ""))
|
|||
|
|
elif t.type == "image":
|
|||
|
|
info["links"].append(t.attrs.get("src", ""))
|
|||
|
|
elif t.type in ("html_inline", "html_block"):
|
|||
|
|
for kind, value in ATTR.findall(t.content):
|
|||
|
|
if kind == "id":
|
|||
|
|
info["anchors"].add(value)
|
|||
|
|
else:
|
|||
|
|
info["links"].append(value)
|
|||
|
|
elif t.type == "footnote_ref":
|
|||
|
|
info["fn_refs"].add(t.meta.get("label"))
|
|||
|
|
elif t.type == "footnote_block_open":
|
|||
|
|
pass
|
|||
|
|
elif t.type == "footnote_open":
|
|||
|
|
info["fn_defs"].add(t.meta.get("label"))
|
|||
|
|
elif t.type == "fence":
|
|||
|
|
info["code"].append((t.info.strip(), t.content, t.map))
|
|||
|
|
# unclosed fence: markdown-it lets it run to the end of the document
|
|||
|
|
if t.map and t.map[1] >= len(lines) and not re.match(r"^\s*(`{3,}|~{3,})\s*$", lines[t.map[1] - 1] if t.map[1] - 1 < len(lines) else ""):
|
|||
|
|
info["issues"].append(("high", "unclosed-fence", f"Code-Fence ab Zeile {t.map[0] + 1} nicht geschlossen"))
|
|||
|
|
# raw footnote definitions that markdown-it did not pick up (e.g. broken indentation)
|
|||
|
|
for m in re.finditer(r"^\[\^([^\]]+)\]:", body, re.M):
|
|||
|
|
info["fn_defs"].add(m.group(1))
|
|||
|
|
for ln in lines:
|
|||
|
|
if NAV_RESIDUE.match(ln):
|
|||
|
|
info["issues"].append(("medium", "nav-residue", f"Navigationsrest: {ln.strip()!r}"))
|
|||
|
|
for lang, code, rng in info["code"]:
|
|||
|
|
if ENTITY_IN_CODE.search(code):
|
|||
|
|
info["issues"].append(("high", "entity-in-code", f"HTML-Entity im Codeblock (Zeile {rng[0] + 1}): {ENTITY_IN_CODE.search(code).group(0)}"))
|
|||
|
|
if lang == "python":
|
|||
|
|
try:
|
|||
|
|
ast.parse(code)
|
|||
|
|
except TabError as exc: # mixed tabs/spaces: inherited from the source, not a conversion defect
|
|||
|
|
info["issues"].append(("info", "python-tabs", f"Tab/Leerzeichen gemischt (Zeile {rng[0] + 1}): {exc.msg}"))
|
|||
|
|
except IndentationError as exc:
|
|||
|
|
info["issues"].append(("high", "python-indentation", f"Python-Einrückung kaputt (Zeile {rng[0] + 1}): {exc.msg}"))
|
|||
|
|
except SyntaxError:
|
|||
|
|
info["issues"].append(("info", "python-syntax", f"Python-Snippet nicht vollständig parsebar (Zeile {rng[0] + 1})"))
|
|||
|
|
if "<EFBFBD>" in body:
|
|||
|
|
info["issues"].append(("medium", "replacement-char", "Ersatzzeichen U+FFFD in der Ausgabe"))
|
|||
|
|
return info
|
|||
|
|
|
|||
|
|
|
|||
|
|
def validate(out: Path) -> dict:
|
|||
|
|
result: dict = {"output": str(out), "issues": [], "stats": {}}
|
|||
|
|
|
|||
|
|
def issue(sev: str, code: str, msg: str, file: str = "") -> None:
|
|||
|
|
result["issues"].append({"severity": sev, "code": code, "message": msg, "file": file})
|
|||
|
|
|
|||
|
|
md_files = sorted(p for p in out.rglob("*.md") if "diagnostics" not in p.parts)
|
|||
|
|
analysed: dict[str, dict] = {}
|
|||
|
|
bodies: dict[str, str] = {}
|
|||
|
|
for p in md_files:
|
|||
|
|
rel = p.relative_to(out).as_posix()
|
|||
|
|
text = p.read_text(encoding="utf-8")
|
|||
|
|
fm, body, err = split_frontmatter(text)
|
|||
|
|
if err:
|
|||
|
|
issue("high", "frontmatter", err, rel)
|
|||
|
|
if rel.startswith("chapters/") and fm is not None:
|
|||
|
|
for key in ("book_title", "chapter_number", "chapter_title", "source_epub", "conversion_status"):
|
|||
|
|
if key not in fm:
|
|||
|
|
issue("medium", "frontmatter-key", f"Frontmatter-Schlüssel fehlt: {key}", rel)
|
|||
|
|
if rel.startswith("chapters/") and not body.strip():
|
|||
|
|
issue("high", "empty-chapter", "Kapiteldatei ohne Inhalt", rel)
|
|||
|
|
bodies[rel] = body
|
|||
|
|
a = analyse(rel, body)
|
|||
|
|
analysed[rel] = a
|
|||
|
|
for sev, code, msg in a["issues"]:
|
|||
|
|
issue(sev, code, msg, rel)
|
|||
|
|
# links
|
|||
|
|
existing = {p.relative_to(out).as_posix() for p in out.rglob("*") if p.is_file()}
|
|||
|
|
n_links = 0
|
|||
|
|
for rel, a in analysed.items():
|
|||
|
|
for link in a["links"]:
|
|||
|
|
n_links += 1
|
|||
|
|
if not link or re.match(r"^[a-z][a-z0-9+.-]*:", link, re.I):
|
|||
|
|
continue
|
|||
|
|
target, _, frag = link.partition("#")
|
|||
|
|
target, frag = unquote(target), unquote(frag)
|
|||
|
|
resolved = posixpath.normpath(posixpath.join(posixpath.dirname(rel), target)) if target else rel
|
|||
|
|
if resolved.startswith(".."):
|
|||
|
|
issue("critical", "link-outside", f"Link verlässt Ausgabeordner: {link}", rel)
|
|||
|
|
elif resolved not in existing:
|
|||
|
|
issue("high", "broken-link", f"Ziel fehlt: {link}", rel)
|
|||
|
|
elif frag and resolved in analysed and frag not in analysed[resolved]["anchors"] \
|
|||
|
|
and not frag.startswith("fn"): # markdown-it footnote ids are renderer-specific
|
|||
|
|
issue("high", "broken-anchor", f"Anker fehlt: {link}", rel)
|
|||
|
|
missing_defs = a["fn_refs"] - a["fn_defs"]
|
|||
|
|
unused_defs = a["fn_defs"] - a["fn_refs"]
|
|||
|
|
if missing_defs:
|
|||
|
|
issue("high", "footnote-missing-def", f"Fußnotenverweise ohne Definition: {sorted(missing_defs)}", rel)
|
|||
|
|
if unused_defs:
|
|||
|
|
issue("medium", "footnote-unused-def", f"Fußnotendefinitionen ohne Verweis: {sorted(unused_defs)}", rel)
|
|||
|
|
# duplicate chapter bodies
|
|||
|
|
seen: dict[str, str] = {}
|
|||
|
|
for rel, body in bodies.items():
|
|||
|
|
norm = re.sub(r"\s+", " ", body).strip()
|
|||
|
|
if rel.startswith("chapters/") and len(norm) > 200:
|
|||
|
|
h = hashlib.sha1(norm.encode()).hexdigest()
|
|||
|
|
if h in seen:
|
|||
|
|
issue("medium", "duplicate-chapter", f"Inhalt identisch mit {seen[h]}", rel)
|
|||
|
|
seen[h] = rel
|
|||
|
|
# JSON / YAML files
|
|||
|
|
for name in ("manifest.json", "conversion-report.json"):
|
|||
|
|
p = out / name
|
|||
|
|
if not p.exists():
|
|||
|
|
issue("high", "missing-file", f"{name} fehlt")
|
|||
|
|
continue
|
|||
|
|
try:
|
|||
|
|
data = json.loads(p.read_text(encoding="utf-8"))
|
|||
|
|
except json.JSONDecodeError as exc:
|
|||
|
|
issue("high", "json-invalid", f"{name}: {exc}")
|
|||
|
|
continue
|
|||
|
|
for ch in data.get("chapters", []):
|
|||
|
|
if not (out / ch["file"]).is_file():
|
|||
|
|
issue("high", "json-ref-missing", f"{name}: Kapitel fehlt {ch['file']}")
|
|||
|
|
media = data.get("media", [])
|
|||
|
|
media = media.get("files", []) if isinstance(media, dict) else media
|
|||
|
|
for m in media:
|
|||
|
|
if not (out / m["file"]).is_file():
|
|||
|
|
issue("high", "json-ref-missing", f"{name}: Medium fehlt {m['file']}")
|
|||
|
|
if name == "conversion-report.json":
|
|||
|
|
n_files = len([p for p in md_files if p.parts[-2] == "chapters"])
|
|||
|
|
if len(data["chapters"]) != n_files:
|
|||
|
|
issue("medium", "report-mismatch", f"Bericht nennt {len(data['chapters'])} Kapitel, vorhanden {n_files}")
|
|||
|
|
if data.get("warnings") and data.get("lossless") not in (False,):
|
|||
|
|
issue("medium", "report-claims", "Bericht mit Warnungen behauptet nicht lossless=false")
|
|||
|
|
result["report_status"] = data.get("status")
|
|||
|
|
result["report_warnings"] = len(data.get("warnings", []))
|
|||
|
|
if (out / "metadata.yaml").exists():
|
|||
|
|
try:
|
|||
|
|
yaml.safe_load((out / "metadata.yaml").read_text(encoding="utf-8"))
|
|||
|
|
except yaml.YAMLError as exc:
|
|||
|
|
issue("high", "yaml-invalid", f"metadata.yaml: {exc}")
|
|||
|
|
result["stats"] = {
|
|||
|
|
"markdown_files": len(md_files),
|
|||
|
|
"links_checked": n_links,
|
|||
|
|
"code_blocks": sum(len(a["code"]) for a in analysed.values()),
|
|||
|
|
"footnote_refs": sum(len(a["fn_refs"]) for a in analysed.values()),
|
|||
|
|
"media_files": len([e for e in existing if e.startswith("media/")]),
|
|||
|
|
}
|
|||
|
|
counts: dict[str, int] = {}
|
|||
|
|
for i in result["issues"]:
|
|||
|
|
counts[i["severity"]] = counts.get(i["severity"], 0) + 1
|
|||
|
|
result["severity_counts"] = counts
|
|||
|
|
return result
|
|||
|
|
|
|||
|
|
|
|||
|
|
if __name__ == "__main__":
|
|||
|
|
results = [validate(Path(p)) for p in sys.argv[1:]]
|
|||
|
|
print(json.dumps(results, ensure_ascii=False, indent=2))
|