youtube_abschrift_korrigieren/aenderungen.py

#!/usr/bin/env python3
"""Stellt Rohabschrift und korrigierten Text gegenüber: welche Wörter hat das Modell
ersetzt, eingefügt oder gestrichen? Ausgabe als HTML-Bericht."""
from collections import Counter
from dataclasses import dataclass, field
from difflib import SequenceMatcher
from pathlib import Path
import argparse
import html
import re

from vtt2txt import vtt_to_lines

# "[prüfen: …]" in German speeches, "[check: …]" in English ones
REVIEW_MARK = re.compile(r"\[(?:prüfen|check):[^\]]*\]")
TOKEN = re.compile(rf"{REVIEW_MARK.pattern}|\S+")
NON_WORD = re.compile(r"[^\w]+")
NON_WORD_KEEP_APOSTROPHE = re.compile(r"[^\w']+")
APOSTROPHES = str.maketrans("’‘`´", "''''")
ABBREVIATION = re.compile(r"\w\.")          # one part of "z. B.", "d. h."
ABBREVIATION_END = re.compile(r"(?:^|\s)\w\.$")
EMPTY = "∅"


def add_token(words, token):
    """Append a token; attach punctuation-only tokens ("–") to the previous word
    and join split abbreviations ("z. B."), so neither counts as a word change."""
    if words and (
        not NON_WORD.sub("", token)
        or (ABBREVIATION.fullmatch(token) and ABBREVIATION_END.search(words[-1]))
    ):
        words[-1] += " " + token
    else:
        words.append(token)


def normalize(word):
    """Compare words without case and punctuation; those changes are expected.

    Apostrophes inside a word count ("Bahr’s" is not "Bahrs"), quotes at its
    edges do not.
    """
    word = word.lower().translate(APOSTROPHES)
    return NON_WORD_KEEP_APOSTROPHE.sub("", word).strip("'") or word


@dataclass
class Comparison:
    raw: list                     # words of the raw transcript
    words: list                   # words of the corrected text, without review marks
    paragraph: list               # paragraph number of each corrected word
    marks: dict = field(default_factory=dict)  # word index -> review marks after it
    opcodes: list = field(default_factory=list)

    def changes(self):
        """Yield (tag, raw words, corrected words) for every difference."""
        for tag, i1, i2, j1, j2 in self.opcodes:
            if tag != "equal":
                yield tag, self.raw[i1:i2], self.words[j1:j2]


def compare(raw, corrected):
    """Word-level comparison of the raw transcript and the corrected text."""
    words, paragraph, marks = [], [], {}
    for number, text in enumerate(re.split(r"\n\s*\n", corrected.strip())):
        start = len(words)
        for token in TOKEN.findall(text):
            # Review marks stay out of the comparison, but are shown after their word
            if REVIEW_MARK.fullmatch(token):
                marks.setdefault(len(words) - 1, []).append(token)
            # Punctuation is never merged across paragraphs
            elif len(words) == start:
                words.append(token)
            else:
                add_token(words, token)
        paragraph += [number] * (len(words) - start)

    raw_words = []
    for token in raw.split():
        add_token(raw_words, token)
    matcher = SequenceMatcher(
        None, [normalize(w) for w in raw_words], [normalize(w) for w in words],
        autojunk=False,
    )
    return Comparison(raw_words, words, paragraph, marks, matcher.get_opcodes())


def change_summary(comparison):
    """[(old, new, count)] of all changes, most frequent first."""
    counts = Counter(
        (" ".join(old), " ".join(new)) for _, old, new in comparison.changes()
    )
    return [(old, new, count) for (old, new), count in counts.most_common()]


STYLE = """
:root { --bg: #fff; --fg: #1d1d1f; --muted: #6e6e73; --rep: #fff3b0; --ins: #d9f7d9;
        --del: #fde2e2; --check: #ffe0b3; --line: #ddd; }
@media (prefers-color-scheme: dark) {
  :root { --bg: #1c1c1e; --fg: #f2f2f7; --muted: #a1a1a6; --rep: #5c4d00; --ins: #1f4d1f;
          --del: #5c1f1f; --check: #5c3a00; --line: #3a3a3c; }
}
body { background: var(--bg); color: var(--fg); font: 16px/1.6 system-ui, sans-serif;
       max-width: 50rem; margin: 0 auto; padding: 1.5rem 1rem; }
h1 { font-size: 1.4rem; } h2 { font-size: 1.15rem; margin-top: 2rem; }
.muted { color: var(--muted); }
table { border-collapse: collapse; width: 100%; font-size: 0.95rem; }
td, th { border-bottom: 1px solid var(--line); padding: 0.2rem 0.5rem; text-align: left; }
td.n { text-align: right; width: 3rem; }
.rep { background: var(--rep); }
ins { background: var(--ins); text-decoration: none; }
del { background: var(--del); }
.orig { color: var(--muted); font-size: 0.8em; }
.check { background: var(--check); font-size: 0.85em; }
"""


def render_html(comparison, title):
    esc = html.escape
    summary = change_summary(comparison)
    counts = Counter(tag for tag, _, _ in comparison.changes())
    review_marks = sum(len(m) for m in comparison.marks.values())

    rows = "\n".join(
        f"<tr><td class=n>{count}</td><td><del>{esc(old) or EMPTY}</del></td>"
        f"<td><ins>{esc(new) or EMPTY}</ins></td></tr>"
        for old, new, count in summary
    )

    # Corrected text with its paragraphs; each word knows its paragraph
    parts = []
    current = 0

    def emit(j, piece):
        nonlocal current
        if comparison.paragraph[j] != current:
            current = comparison.paragraph[j]
            parts.append("</p>\n<p>")
        parts.append(piece)
        for mark in comparison.marks.get(j, []):
            parts.append(f"<span class=check>{esc(mark)}</span>")

    for tag, i1, i2, j1, j2 in comparison.opcodes:
        old = " ".join(comparison.raw[i1:i2])
        if tag == "delete":
            parts.append(f"<del title='gestrichen'>{esc(old)}</del>")
            continue
        for j in range(j1, j2):
            word = esc(comparison.words[j])
            if tag == "equal":
                emit(j, word)
            elif tag == "insert":
                emit(j, f"<ins title='eingefügt'>{word}</ins>")
            else:
                emit(j, f"<span class=rep title='Original: {esc(old, quote=True)}'>{word}</span>")
        if tag == "replace":
            parts.append(f"<span class=orig>({esc(old)})</span>")

    text = " ".join(parts).replace(" </p>\n<p> ", "</p>\n<p>")

    return f"""<!doctype html>
<html lang="de"><head><meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>Änderungen: {esc(title)}</title><style>{STYLE}</style></head>
<body>
<h1>Änderungen: {esc(title)}</h1>
<p class=muted>{len(comparison.raw)} Wörter Rohtext, {len(comparison.words)} Wörter korrigiert.
{counts['replace']} ersetzt, {counts['insert']} eingefügt, {counts['delete']} gestrichen,
{review_marks} Prüfmarken. Groß-/Kleinschreibung und Satzzeichen sind nicht markiert.</p>
<p><span class=rep>ersetzt</span> <span class=orig>(Original)</span>
<ins>eingefügt</ins> <del>gestrichen</del> <span class=check>[prüfen: …]</span></p>
<h2>Alle Änderungen nach Häufigkeit</h2>
<table><tr><th class=n>×</th><th>Rohtext</th><th>korrigiert</th></tr>
{rows}
</table>
<h2>Korrigierter Text</h2>
<p>{text}</p>
</body></html>
"""


def print_summary(comparison, limit=15):
    summary = change_summary(comparison)
    print(f"Änderungen gegenüber dem Rohtext: {sum(c for _, _, c in summary)}")
    for old, new, count in summary[:limit]:
        print(f"  {count}× „{old or EMPTY}“ → „{new or EMPTY}“")
    if len(summary) > limit:
        print(f"  … und {len(summary) - limit} weitere, siehe Bericht")


def write_report(raw, corrected, target, title):
    comparison = compare(raw, corrected)
    target.write_text(render_html(comparison, title), encoding="utf-8")
    return comparison


def report_path(corrected_path):
    """rede.korrigiert.txt -> rede.aenderungen.html"""
    stem = corrected_path.stem.removesuffix(".korrigiert")
    return corrected_path.with_name(f"{stem}.aenderungen.html")


def read_raw(path):
    text = path.read_text(encoding="utf-8-sig")
    if path.suffix.lower() == ".vtt":
        text = "\n".join(vtt_to_lines(text))
    return text


def parse_args():
    parser = argparse.ArgumentParser(
        description="Stellt Rohabschrift und korrigierten Text als HTML-Bericht gegenüber."
    )
    parser.add_argument(
        "-a", "--roh", required=True, type=Path, help="Rohabschrift (.txt oder .vtt)"
    )
    parser.add_argument(
        "-b", "--korrigiert", required=True, type=Path, help="korrigierter Text"
    )
    parser.add_argument(
        "-o", "--output", type=Path,
        help="HTML-Bericht (Standard: <korrigiert ohne .korrigiert.txt>.aenderungen.html)",
    )
    args = parser.parse_args()
    for path in (args.roh, args.korrigiert):
        if not path.is_file():
            parser.error(f"Datei nicht gefunden: {path}")
    if args.output is None:
        args.output = report_path(args.korrigiert)
    return args


def main():
    args = parse_args()
    corrected = args.korrigiert.read_text(encoding="utf-8")
    comparison = write_report(read_raw(args.roh), corrected, args.output, args.korrigiert.name)
    print_summary(comparison)
    print(f"Bericht: {args.output}")


if __name__ == "__main__":
    main()