dschlueter
95bdc660ca
- „Bahrs“ → „Bahr’s“ ist jetzt eine Änderung; ’ ' ` gelten als gleich, Anführungszeichen am Wortrand zählen weiterhin nicht - Prüfmarken auch englisch: [check: …] - output/ (eigene Ergebnisse und Testläufe) in .gitignore Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
242 lines
8.9 KiB
Python
Executable file
242 lines
8.9 KiB
Python
Executable file
#!/usr/bin/env python3
|
||
"""Stellt Rohabschrift und korrigierten Text gegenüber: welche Wörter hat das Modell
|
||
ersetzt, eingefügt oder gestrichen? Ausgabe als HTML-Bericht."""
|
||
from collections import Counter
|
||
from dataclasses import dataclass, field
|
||
from difflib import SequenceMatcher
|
||
from pathlib import Path
|
||
import argparse
|
||
import html
|
||
import re
|
||
|
||
from vtt2txt import vtt_to_lines
|
||
|
||
# "[prüfen: …]" in German speeches, "[check: …]" in English ones
|
||
REVIEW_MARK = re.compile(r"\[(?:prüfen|check):[^\]]*\]")
|
||
TOKEN = re.compile(rf"{REVIEW_MARK.pattern}|\S+")
|
||
NON_WORD = re.compile(r"[^\w]+")
|
||
NON_WORD_KEEP_APOSTROPHE = re.compile(r"[^\w']+")
|
||
APOSTROPHES = str.maketrans("’‘`´", "''''")
|
||
ABBREVIATION = re.compile(r"\w\.") # one part of "z. B.", "d. h."
|
||
ABBREVIATION_END = re.compile(r"(?:^|\s)\w\.$")
|
||
EMPTY = "∅"
|
||
|
||
|
||
def add_token(words, token):
|
||
"""Append a token; attach punctuation-only tokens ("–") to the previous word
|
||
and join split abbreviations ("z. B."), so neither counts as a word change."""
|
||
if words and (
|
||
not NON_WORD.sub("", token)
|
||
or (ABBREVIATION.fullmatch(token) and ABBREVIATION_END.search(words[-1]))
|
||
):
|
||
words[-1] += " " + token
|
||
else:
|
||
words.append(token)
|
||
|
||
|
||
def normalize(word):
|
||
"""Compare words without case and punctuation; those changes are expected.
|
||
|
||
Apostrophes inside a word count ("Bahr’s" is not "Bahrs"), quotes at its
|
||
edges do not.
|
||
"""
|
||
word = word.lower().translate(APOSTROPHES)
|
||
return NON_WORD_KEEP_APOSTROPHE.sub("", word).strip("'") or word
|
||
|
||
|
||
@dataclass
|
||
class Comparison:
|
||
raw: list # words of the raw transcript
|
||
words: list # words of the corrected text, without review marks
|
||
paragraph: list # paragraph number of each corrected word
|
||
marks: dict = field(default_factory=dict) # word index -> review marks after it
|
||
opcodes: list = field(default_factory=list)
|
||
|
||
def changes(self):
|
||
"""Yield (tag, raw words, corrected words) for every difference."""
|
||
for tag, i1, i2, j1, j2 in self.opcodes:
|
||
if tag != "equal":
|
||
yield tag, self.raw[i1:i2], self.words[j1:j2]
|
||
|
||
|
||
def compare(raw, corrected):
|
||
"""Word-level comparison of the raw transcript and the corrected text."""
|
||
words, paragraph, marks = [], [], {}
|
||
for number, text in enumerate(re.split(r"\n\s*\n", corrected.strip())):
|
||
start = len(words)
|
||
for token in TOKEN.findall(text):
|
||
# Review marks stay out of the comparison, but are shown after their word
|
||
if REVIEW_MARK.fullmatch(token):
|
||
marks.setdefault(len(words) - 1, []).append(token)
|
||
# Punctuation is never merged across paragraphs
|
||
elif len(words) == start:
|
||
words.append(token)
|
||
else:
|
||
add_token(words, token)
|
||
paragraph += [number] * (len(words) - start)
|
||
|
||
raw_words = []
|
||
for token in raw.split():
|
||
add_token(raw_words, token)
|
||
matcher = SequenceMatcher(
|
||
None, [normalize(w) for w in raw_words], [normalize(w) for w in words],
|
||
autojunk=False,
|
||
)
|
||
return Comparison(raw_words, words, paragraph, marks, matcher.get_opcodes())
|
||
|
||
|
||
def change_summary(comparison):
|
||
"""[(old, new, count)] of all changes, most frequent first."""
|
||
counts = Counter(
|
||
(" ".join(old), " ".join(new)) for _, old, new in comparison.changes()
|
||
)
|
||
return [(old, new, count) for (old, new), count in counts.most_common()]
|
||
|
||
|
||
STYLE = """
|
||
:root { --bg: #fff; --fg: #1d1d1f; --muted: #6e6e73; --rep: #fff3b0; --ins: #d9f7d9;
|
||
--del: #fde2e2; --check: #ffe0b3; --line: #ddd; }
|
||
@media (prefers-color-scheme: dark) {
|
||
:root { --bg: #1c1c1e; --fg: #f2f2f7; --muted: #a1a1a6; --rep: #5c4d00; --ins: #1f4d1f;
|
||
--del: #5c1f1f; --check: #5c3a00; --line: #3a3a3c; }
|
||
}
|
||
body { background: var(--bg); color: var(--fg); font: 16px/1.6 system-ui, sans-serif;
|
||
max-width: 50rem; margin: 0 auto; padding: 1.5rem 1rem; }
|
||
h1 { font-size: 1.4rem; } h2 { font-size: 1.15rem; margin-top: 2rem; }
|
||
.muted { color: var(--muted); }
|
||
table { border-collapse: collapse; width: 100%; font-size: 0.95rem; }
|
||
td, th { border-bottom: 1px solid var(--line); padding: 0.2rem 0.5rem; text-align: left; }
|
||
td.n { text-align: right; width: 3rem; }
|
||
.rep { background: var(--rep); }
|
||
ins { background: var(--ins); text-decoration: none; }
|
||
del { background: var(--del); }
|
||
.orig { color: var(--muted); font-size: 0.8em; }
|
||
.check { background: var(--check); font-size: 0.85em; }
|
||
"""
|
||
|
||
|
||
def render_html(comparison, title):
|
||
esc = html.escape
|
||
summary = change_summary(comparison)
|
||
counts = Counter(tag for tag, _, _ in comparison.changes())
|
||
review_marks = sum(len(m) for m in comparison.marks.values())
|
||
|
||
rows = "\n".join(
|
||
f"<tr><td class=n>{count}</td><td><del>{esc(old) or EMPTY}</del></td>"
|
||
f"<td><ins>{esc(new) or EMPTY}</ins></td></tr>"
|
||
for old, new, count in summary
|
||
)
|
||
|
||
# Corrected text with its paragraphs; each word knows its paragraph
|
||
parts = []
|
||
current = 0
|
||
|
||
def emit(j, piece):
|
||
nonlocal current
|
||
if comparison.paragraph[j] != current:
|
||
current = comparison.paragraph[j]
|
||
parts.append("</p>\n<p>")
|
||
parts.append(piece)
|
||
for mark in comparison.marks.get(j, []):
|
||
parts.append(f"<span class=check>{esc(mark)}</span>")
|
||
|
||
for tag, i1, i2, j1, j2 in comparison.opcodes:
|
||
old = " ".join(comparison.raw[i1:i2])
|
||
if tag == "delete":
|
||
parts.append(f"<del title='gestrichen'>{esc(old)}</del>")
|
||
continue
|
||
for j in range(j1, j2):
|
||
word = esc(comparison.words[j])
|
||
if tag == "equal":
|
||
emit(j, word)
|
||
elif tag == "insert":
|
||
emit(j, f"<ins title='eingefügt'>{word}</ins>")
|
||
else:
|
||
emit(j, f"<span class=rep title='Original: {esc(old, quote=True)}'>{word}</span>")
|
||
if tag == "replace":
|
||
parts.append(f"<span class=orig>({esc(old)})</span>")
|
||
|
||
text = " ".join(parts).replace(" </p>\n<p> ", "</p>\n<p>")
|
||
|
||
return f"""<!doctype html>
|
||
<html lang="de"><head><meta charset="utf-8">
|
||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||
<title>Änderungen: {esc(title)}</title><style>{STYLE}</style></head>
|
||
<body>
|
||
<h1>Änderungen: {esc(title)}</h1>
|
||
<p class=muted>{len(comparison.raw)} Wörter Rohtext, {len(comparison.words)} Wörter korrigiert.
|
||
{counts['replace']} ersetzt, {counts['insert']} eingefügt, {counts['delete']} gestrichen,
|
||
{review_marks} Prüfmarken. Groß-/Kleinschreibung und Satzzeichen sind nicht markiert.</p>
|
||
<p><span class=rep>ersetzt</span> <span class=orig>(Original)</span>
|
||
<ins>eingefügt</ins> <del>gestrichen</del> <span class=check>[prüfen: …]</span></p>
|
||
<h2>Alle Änderungen nach Häufigkeit</h2>
|
||
<table><tr><th class=n>×</th><th>Rohtext</th><th>korrigiert</th></tr>
|
||
{rows}
|
||
</table>
|
||
<h2>Korrigierter Text</h2>
|
||
<p>{text}</p>
|
||
</body></html>
|
||
"""
|
||
|
||
|
||
def print_summary(comparison, limit=15):
|
||
summary = change_summary(comparison)
|
||
print(f"Änderungen gegenüber dem Rohtext: {sum(c for _, _, c in summary)}")
|
||
for old, new, count in summary[:limit]:
|
||
print(f" {count}× „{old or EMPTY}“ → „{new or EMPTY}“")
|
||
if len(summary) > limit:
|
||
print(f" … und {len(summary) - limit} weitere, siehe Bericht")
|
||
|
||
|
||
def write_report(raw, corrected, target, title):
|
||
comparison = compare(raw, corrected)
|
||
target.write_text(render_html(comparison, title), encoding="utf-8")
|
||
return comparison
|
||
|
||
|
||
def report_path(corrected_path):
|
||
"""rede.korrigiert.txt -> rede.aenderungen.html"""
|
||
stem = corrected_path.stem.removesuffix(".korrigiert")
|
||
return corrected_path.with_name(f"{stem}.aenderungen.html")
|
||
|
||
|
||
def read_raw(path):
|
||
text = path.read_text(encoding="utf-8-sig")
|
||
if path.suffix.lower() == ".vtt":
|
||
text = "\n".join(vtt_to_lines(text))
|
||
return text
|
||
|
||
|
||
def parse_args():
|
||
parser = argparse.ArgumentParser(
|
||
description="Stellt Rohabschrift und korrigierten Text als HTML-Bericht gegenüber."
|
||
)
|
||
parser.add_argument(
|
||
"-a", "--roh", required=True, type=Path, help="Rohabschrift (.txt oder .vtt)"
|
||
)
|
||
parser.add_argument(
|
||
"-b", "--korrigiert", required=True, type=Path, help="korrigierter Text"
|
||
)
|
||
parser.add_argument(
|
||
"-o", "--output", type=Path,
|
||
help="HTML-Bericht (Standard: <korrigiert ohne .korrigiert.txt>.aenderungen.html)",
|
||
)
|
||
args = parser.parse_args()
|
||
for path in (args.roh, args.korrigiert):
|
||
if not path.is_file():
|
||
parser.error(f"Datei nicht gefunden: {path}")
|
||
if args.output is None:
|
||
args.output = report_path(args.korrigiert)
|
||
return args
|
||
|
||
|
||
def main():
|
||
args = parse_args()
|
||
corrected = args.korrigiert.read_text(encoding="utf-8")
|
||
comparison = write_report(read_raw(args.roh), corrected, args.output, args.korrigiert.name)
|
||
print_summary(comparison)
|
||
print(f"Bericht: {args.output}")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|