fix: „Zu prüfen“ übersah Streichungen von „er“/„um“ und Glossar-Allerweltswörter

- Füllwörter je Sprache: „er“ und „um“ sind nur im Englischen Füllwörter;
  im Deutschen wurde ihre Streichung bisher nie gemeldet.
- Eine Änderung gilt nur noch als vom Glossar erklärt, wenn die neuen
  Wörter zusammenhängend in einem Eintrag stehen und dem alten Text
  ähneln. Bisher machte jedes Wort eines Eintrags („Sie“ aus „finden Sie“)
  Änderungen unsichtbar, auch eingefügte Vornamen.
- aenderungen.py -l en-US findet die englische Wortliste.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Dieter Schlüter 2026-10-11 21:53:17 +02:00
commit 45bf2da07b

View file

@ -30,7 +30,11 @@ WORDLISTS = {
}
WORDLIST_PACKAGES = {"de": "wngerman", "en": "wamerican"}
MIN_SIMILARITY = 0.6 # below this a replacement counts as heavily changed
FILLERS = {"äh", "ähm", "öh", "öhm", "hm", "hmm", "uh", "uhm", "um", "er", "erm"}
# Per language: "er" and "um" are fillers in English, real words in German
FILLERS = {
"de": {"äh", "ähm", "öh", "öhm", "hm", "hmm"},
"en": {"uh", "uhm", "um", "er", "erm", "hm", "hmm"},
}
CONTEXT_WORDS = 6
VALID_WORD = "gültiges Wort ersetzt"
@ -132,10 +136,10 @@ def load_wordlist(lang):
return {line.strip().lower() for line in words if line.strip()}
def glossary_words(terms, replacements):
"""Normalized words of glossary terms and replacement targets."""
def glossary_entries(terms, replacements):
"""Glossary terms and replacement targets as tuples of normalized words."""
texts = list(terms) + [right for _, right in replacements]
return {normalize(word) for text in texts for word in text.split()}
return {tuple(normalize(word) for word in text.split()) for text in texts} - {()}
def spelling_key(words):
@ -143,33 +147,47 @@ def spelling_key(words):
return re.sub(r"[\W_]+", "", "".join(words).lower())
def review_candidates(comparison, wordlist=None, known=()):
def similarity(old, new):
return SequenceMatcher(None, spelling_key(old), spelling_key(new)).ratio()
def glossary_spelling(old, new, glossary):
"""True if `new` is a glossary spelling of `old`: the new words stand
together in one glossary entry and resemble the old ones. Common words of
longer entries ("finden Sie") explain nothing on their own."""
words = tuple(normalize(word) for word in new)
size = len(words)
in_entry = any(
entry[start:start + size] == words
for entry in glossary for start in range(len(entry) - size + 1)
)
return in_entry and similarity(old, new) >= MIN_SIMILARITY
def review_candidates(comparison, wordlist=None, glossary=(), fillers=FILLERS["de"]):
"""Changes the model made on its own that may alter the meaning.
Not reported: changes of spacing, hyphens and punctuation only, new words
from the glossary, deleted filler words, and replacements of words not in
the word list (recognition errors) that stay similar.
Not reported: changes of spacing, hyphens and punctuation only, glossary
spellings of the old words, deleted filler words, and replacements of
words not in the word list (recognition errors) that stay similar.
"""
suspects = []
for tag, i1, i2, j1, j2 in comparison.opcodes:
if tag == "equal":
continue
old, new = comparison.raw[i1:i2], comparison.words[j1:j2]
if new and all(normalize(word) in known for word in new):
continue
if tag == "delete":
if all(normalize(word) in FILLERS for word in old):
if all(normalize(word) in fillers for word in old):
continue
reason = DELETED
elif tag == "insert":
reason = INSERTED
else:
old_key, new_key = spelling_key(old), spelling_key(new)
if old_key == new_key:
if spelling_key(old) == spelling_key(new) or glossary_spelling(old, new, glossary):
continue
if wordlist is not None and all(normalize(w) in wordlist for w in old):
reason = VALID_WORD
elif SequenceMatcher(None, old_key, new_key).ratio() < MIN_SIMILARITY:
elif similarity(old, new) < MIN_SIMILARITY:
reason = CHANGED_A_LOT
else:
continue
@ -336,7 +354,7 @@ def print_summary(comparison, limit=15, suspect_limit=10):
print(f" … und {len(suspects) - suspect_limit} weitere, siehe Bericht")
def write_report(raw, corrected, target, title, model_input=None, known=(), lang="de"):
def write_report(raw, corrected, target, title, model_input=None, glossary=(), lang="de"):
"""Write the HTML report; return the comparison with the suspect changes.
model_input is the text the model got (raw text after glossary
@ -344,7 +362,9 @@ def write_report(raw, corrected, target, title, model_input=None, known=(), lang
"""
comparison = compare(raw, corrected)
by_model = compare(raw if model_input is None else model_input, corrected)
comparison.suspects = review_candidates(by_model, load_wordlist(lang), set(known))
comparison.suspects = review_candidates(
by_model, load_wordlist(lang), glossary, FILLERS.get(lang, set())
)
target.write_text(render_html(comparison, title), encoding="utf-8")
return comparison
@ -397,6 +417,7 @@ def parse_args():
def main():
# abschrift_korrigieren imports this module: import it only when needed
from abschrift_korrigieren import apply_replacements, read_glossary
from yt_untertitel import base_language
args = parse_args()
raw = read_raw(args.roh)
@ -411,8 +432,8 @@ def main():
corrected = args.korrigiert.read_text(encoding="utf-8")
comparison = write_report(
raw, corrected, args.output, args.korrigiert.name,
model_input=model_input, known=glossary_words(terms, replacements),
lang=args.lang,
model_input=model_input, glossary=glossary_entries(terms, replacements),
lang=base_language(args.lang),
)
print_summary(comparison)
print(f"Bericht: {args.output}")