feat: „Zu prüfen“ zeigt bloße Wortformen getrennt am Ende
Ersetzt das Modell ein gültiges Wort nur durch eine andere Form („Punkten“ → „Punkte“, „das“ → „dass“, „sehe“ → „sähe“), heißt der Grund jetzt „Wortform geändert“. Diese Fälle stehen im Bericht in einer eigenen Tabelle nach den sinnverändernden, im Text gestrichelt umrahmt, und in der Konsole am Ende. „-in“ zählt nicht als Endung: „Professor“ → „Professorin“ bleibt oben. Außerdem: Die Tabellen scrollen auf schmalen Bildschirmen waagrecht, und aenderungen.py meldet Zeit-Überschriften nicht als Einfügung, wenn der Rohtext (z. B. eine .vtt) keine hat. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
011d201215
commit
cd62feab69
1 changed files with 163 additions and 32 deletions
100
aenderungen.py
100
aenderungen.py
|
|
@ -10,7 +10,7 @@ import argparse
|
|||
import html
|
||||
import re
|
||||
|
||||
from vtt2txt import vtt_to_lines
|
||||
from vtt2txt import split_segments, vtt_to_lines
|
||||
|
||||
# "[prüfen: …]" in German transcripts, "[check: …]" in English ones
|
||||
REVIEW_MARK = re.compile(r"\[(?:prüfen|check):[^\]]*\]")
|
||||
|
|
@ -35,12 +35,20 @@ FILLERS = {
|
|||
"de": {"äh", "ähm", "öh", "öhm", "hm", "hmm"},
|
||||
"en": {"uh", "uhm", "um", "er", "erm", "hm", "hmm"},
|
||||
}
|
||||
# Endings that turn one form of a word into another ("Punkten" -> "Punkte")
|
||||
INFLECTIONS = {
|
||||
"de": {"", "e", "n", "r", "s", "t", "em", "en", "er", "es", "et", "st", "te", "est", "ten"},
|
||||
"en": {"", "d", "s", "ed", "es", "ing"},
|
||||
}
|
||||
UMLAUT_FOLD = str.maketrans({"ä": "e", "ö": "o", "ü": "u", "ß": "ss"})
|
||||
MIN_STEM = 3 # shortest common stem of two word forms
|
||||
CONTEXT_WORDS = 6
|
||||
|
||||
VALID_WORD = "gültiges Wort ersetzt"
|
||||
CHANGED_A_LOT = "stark verändert"
|
||||
INSERTED = "eingefügt"
|
||||
DELETED = "gestrichen"
|
||||
WORD_FORM = "Wortform geändert"
|
||||
|
||||
|
||||
def add_token(words, token):
|
||||
|
|
@ -164,13 +172,31 @@ def glossary_spelling(old, new, glossary):
|
|||
return in_entry and similarity(old, new) >= MIN_SIMILARITY
|
||||
|
||||
|
||||
def review_candidates(comparison, wordlist=None, glossary=(), fillers=FILLERS["de"]):
|
||||
def word_form_change(old, new, lang="de"):
|
||||
"""True if one word becomes another form of itself: only the ending
|
||||
("Punkten" -> "Punkte", "das" -> "dass") or an umlaut ("sehe" -> "sähe")
|
||||
differs. "-in" is no such ending: "Professor" -> "Professorin" counts."""
|
||||
if len(old) != 1 or len(new) != 1:
|
||||
return False
|
||||
a, b = (NON_WORD.sub("", w).lower().translate(UMLAUT_FOLD) for w in (old[0], new[0]))
|
||||
if a == b:
|
||||
return True
|
||||
stem = 0
|
||||
while stem < min(len(a), len(b)) and a[stem] == b[stem]:
|
||||
stem += 1
|
||||
endings = INFLECTIONS.get(lang, set())
|
||||
return stem >= MIN_STEM and a[stem:] in endings and b[stem:] in endings
|
||||
|
||||
|
||||
def review_candidates(comparison, wordlist=None, glossary=(), lang="de"):
|
||||
"""Changes the model made on its own that may alter the meaning.
|
||||
|
||||
Not reported: changes of spacing, hyphens and punctuation only, glossary
|
||||
spellings of the old words, deleted filler words, and replacements of
|
||||
words not in the word list (recognition errors) that stay similar.
|
||||
Replaced real words that only change their form get their own reason.
|
||||
"""
|
||||
fillers = FILLERS.get(lang, set())
|
||||
suspects = []
|
||||
for tag, i1, i2, j1, j2 in comparison.opcodes:
|
||||
if tag == "equal":
|
||||
|
|
@ -186,7 +212,7 @@ def review_candidates(comparison, wordlist=None, glossary=(), fillers=FILLERS["d
|
|||
if spelling_key(old) == spelling_key(new) or glossary_spelling(old, new, glossary):
|
||||
continue
|
||||
if wordlist is not None and all(normalize(w) in wordlist for w in old):
|
||||
reason = VALID_WORD
|
||||
reason = WORD_FORM if word_form_change(old, new, lang) else VALID_WORD
|
||||
elif similarity(old, new) < MIN_SIMILARITY:
|
||||
reason = CHANGED_A_LOT
|
||||
else:
|
||||
|
|
@ -223,6 +249,9 @@ del { background: var(--del); }
|
|||
.orig { color: var(--muted); font-size: 0.8em; }
|
||||
.check { background: var(--check); font-size: 0.85em; }
|
||||
.warn { outline: 2px solid var(--warn); outline-offset: 1px; border-radius: 2px; }
|
||||
.warn.form { outline-style: dashed; outline-width: 1px; }
|
||||
h3 { font-size: 1rem; margin-top: 1.5rem; }
|
||||
.scroll { overflow-x: auto; }
|
||||
td.ctx { color: var(--muted); } td.ctx b { color: var(--fg); }
|
||||
"""
|
||||
|
||||
|
|
@ -239,28 +268,43 @@ def render_html(comparison, title):
|
|||
for old, new, count in summary
|
||||
)
|
||||
suspects = comparison.suspects
|
||||
suspect_rows = "\n".join(
|
||||
f"<tr><td>{esc(s.reason)}</td><td><del>{esc(s.old) or EMPTY}</del></td>"
|
||||
f"<td><ins>{esc(s.new) or EMPTY}</ins></td>"
|
||||
f"<td class=ctx>{suspect_context(comparison, s)}</td></tr>"
|
||||
for s in suspects
|
||||
)
|
||||
if suspects:
|
||||
suspect_table = (
|
||||
"<table><tr><th>Grund</th><th>vorher</th><th>korrigiert</th>"
|
||||
f"<th>Zusammenhang</th></tr>\n{suspect_rows}\n</table>"
|
||||
important, forms = split_suspects(suspects)
|
||||
|
||||
def table(rows):
|
||||
body = "\n".join(
|
||||
f"<tr><td>{esc(s.reason)}</td><td><del>{esc(s.old) or EMPTY}</del></td>"
|
||||
f"<td><ins>{esc(s.new) or EMPTY}</ins></td>"
|
||||
f"<td class=ctx>{suspect_context(comparison, s)}</td></tr>"
|
||||
for s in rows
|
||||
)
|
||||
return (
|
||||
"<div class=scroll><table><tr><th>Grund</th><th>vorher</th><th>korrigiert</th>"
|
||||
f"<th>Zusammenhang</th></tr>\n{body}\n</table></div>"
|
||||
)
|
||||
|
||||
if important:
|
||||
suspect_table = table(important)
|
||||
elif forms:
|
||||
suspect_table = "<p class=muted>Keine sinnverändernden Änderungen gefunden.</p>"
|
||||
else:
|
||||
suspect_table = "<p class=muted>Keine auffälligen Änderungen gefunden.</p>"
|
||||
if forms:
|
||||
suspect_table += (
|
||||
f"\n<h3>Wortform geändert ({len(forms)})</h3>\n"
|
||||
"<p class=muted>Nur Endung oder Umlaut anders („Punkten“ → „Punkte“, "
|
||||
"„das“ → „dass“), meist Grammatik; ein kurzer Blick genügt.</p>\n"
|
||||
+ table(forms)
|
||||
)
|
||||
|
||||
# Suspect words are framed; a suspect deletion shows ⚠ after the word before it
|
||||
warn_words, warn_after = {}, {}
|
||||
for s in suspects:
|
||||
note = esc(f"{s.reason}: „{s.old or EMPTY}“ → „{s.new or EMPTY}“", quote=True)
|
||||
css = "warn form" if s.reason == WORD_FORM else "warn"
|
||||
if s.j1 == s.j2:
|
||||
warn_after.setdefault(max(s.j1 - 1, 0), []).append(note)
|
||||
for j in range(s.j1, s.j2):
|
||||
warn_words[j] = note
|
||||
warn_words[j] = (css, note)
|
||||
|
||||
# Corrected text with its paragraphs; each word knows its paragraph
|
||||
parts = []
|
||||
|
|
@ -272,7 +316,8 @@ def render_html(comparison, title):
|
|||
current = comparison.paragraph[j]
|
||||
parts.append("</p>\n<p>")
|
||||
if j in warn_words:
|
||||
piece = f"<span class=warn title='{warn_words[j]}'>{piece}</span>"
|
||||
css, note = warn_words[j]
|
||||
piece = f"<span class='{css}' title='{note}'>{piece}</span>"
|
||||
parts.append(piece)
|
||||
for mark in comparison.marks.get(j, []):
|
||||
parts.append(f"<span class=check>{esc(mark)}</span>")
|
||||
|
|
@ -325,6 +370,12 @@ gültige Wörter ersetzt, stark veränderte, eingefügte oder gestrichene Wörte
|
|||
"""
|
||||
|
||||
|
||||
def split_suspects(suspects):
|
||||
"""(possible meaning changes, mere word forms), each in text order."""
|
||||
forms = [s for s in suspects if s.reason == WORD_FORM]
|
||||
return [s for s in suspects if s.reason != WORD_FORM], forms
|
||||
|
||||
|
||||
def suspect_context(comparison, suspect):
|
||||
"""HTML: the change in bold with a few corrected words around it."""
|
||||
esc = html.escape
|
||||
|
|
@ -346,8 +397,11 @@ def print_summary(comparison, limit=15, suspect_limit=10):
|
|||
if len(summary) > limit:
|
||||
print(f" … und {len(summary) - limit} weitere, siehe Bericht")
|
||||
|
||||
suspects = comparison.suspects
|
||||
print(f"Zu prüfen: {len(suspects)} Stellen")
|
||||
important, forms = split_suspects(comparison.suspects)
|
||||
# Meaning changes first, word forms after them
|
||||
suspects = important + forms
|
||||
of_which = f" (davon {len(forms)} Wortformen)" if forms else ""
|
||||
print(f"Zu prüfen: {len(suspects)} Stellen{of_which}")
|
||||
for s in suspects[:suspect_limit]:
|
||||
print(f" {s.reason}: „{s.old or EMPTY}“ → „{s.new or EMPTY}“")
|
||||
if len(suspects) > suspect_limit:
|
||||
|
|
@ -362,9 +416,7 @@ def write_report(raw, corrected, target, title, model_input=None, glossary=(), l
|
|||
"""
|
||||
comparison = compare(raw, corrected)
|
||||
by_model = compare(raw if model_input is None else model_input, corrected)
|
||||
comparison.suspects = review_candidates(
|
||||
by_model, load_wordlist(lang), glossary, FILLERS.get(lang, set())
|
||||
)
|
||||
comparison.suspects = review_candidates(by_model, load_wordlist(lang), glossary, lang)
|
||||
target.write_text(render_html(comparison, title), encoding="utf-8")
|
||||
return comparison
|
||||
|
||||
|
|
@ -375,6 +427,11 @@ def report_path(corrected_path):
|
|||
return corrected_path.with_name(f"{stem}.aenderungen.html")
|
||||
|
||||
|
||||
def without_headers(text):
|
||||
"""Text without time headers ("[00:04:57 - 00:06:04]") of excerpts."""
|
||||
return "\n\n".join(body for _, body in split_segments(text))
|
||||
|
||||
|
||||
def read_raw(path):
|
||||
text = path.read_text(encoding="utf-8-sig")
|
||||
if path.suffix.lower() == ".vtt":
|
||||
|
|
@ -430,6 +487,9 @@ def main():
|
|||
model_input = apply_replacements(" ".join(raw.split()), replacements)
|
||||
|
||||
corrected = args.korrigiert.read_text(encoding="utf-8")
|
||||
# A raw .vtt has no time headers: those of the excerpts are no insertions
|
||||
if without_headers(raw) == raw.strip():
|
||||
corrected = without_headers(corrected)
|
||||
comparison = write_report(
|
||||
raw, corrected, args.output, args.korrigiert.name,
|
||||
model_input=model_input, glossary=glossary_entries(terms, replacements),
|
||||
|
|
|
|||
Loading…
Reference in a new issue