fix: „Zu prüfen“ übersah Streichungen von „er“/„um“ und Glossar-Allerweltswörter
- Füllwörter je Sprache: „er“ und „um“ sind nur im Englischen Füllwörter; im Deutschen wurde ihre Streichung bisher nie gemeldet. - Eine Änderung gilt nur noch als vom Glossar erklärt, wenn die neuen Wörter zusammenhängend in einem Eintrag stehen und dem alten Text ähneln. Bisher machte jedes Wort eines Eintrags („Sie“ aus „finden Sie“) Änderungen unsichtbar, auch eingefügte Vornamen. - aenderungen.py -l en-US findet die englische Wortliste. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
1e3c813137
commit
45bf2da07b
1 changed files with 85 additions and 31 deletions
|
|
@ -30,7 +30,11 @@ WORDLISTS = {
|
|||
}
|
||||
WORDLIST_PACKAGES = {"de": "wngerman", "en": "wamerican"}
|
||||
MIN_SIMILARITY = 0.6 # below this a replacement counts as heavily changed
|
||||
FILLERS = {"äh", "ähm", "öh", "öhm", "hm", "hmm", "uh", "uhm", "um", "er", "erm"}
|
||||
# Per language: "er" and "um" are fillers in English, real words in German
|
||||
FILLERS = {
|
||||
"de": {"äh", "ähm", "öh", "öhm", "hm", "hmm"},
|
||||
"en": {"uh", "uhm", "um", "er", "erm", "hm", "hmm"},
|
||||
}
|
||||
CONTEXT_WORDS = 6
|
||||
|
||||
VALID_WORD = "gültiges Wort ersetzt"
|
||||
|
|
@ -132,10 +136,10 @@ def load_wordlist(lang):
|
|||
return {line.strip().lower() for line in words if line.strip()}
|
||||
|
||||
|
||||
def glossary_words(terms, replacements):
|
||||
"""Normalized words of glossary terms and replacement targets."""
|
||||
def glossary_entries(terms, replacements):
|
||||
"""Glossary terms and replacement targets as tuples of normalized words."""
|
||||
texts = list(terms) + [right for _, right in replacements]
|
||||
return {normalize(word) for text in texts for word in text.split()}
|
||||
return {tuple(normalize(word) for word in text.split()) for text in texts} - {()}
|
||||
|
||||
|
||||
def spelling_key(words):
|
||||
|
|
@ -143,33 +147,47 @@ def spelling_key(words):
|
|||
return re.sub(r"[\W_]+", "", "".join(words).lower())
|
||||
|
||||
|
||||
def review_candidates(comparison, wordlist=None, known=()):
|
||||
def similarity(old, new):
|
||||
return SequenceMatcher(None, spelling_key(old), spelling_key(new)).ratio()
|
||||
|
||||
|
||||
def glossary_spelling(old, new, glossary):
|
||||
"""True if `new` is a glossary spelling of `old`: the new words stand
|
||||
together in one glossary entry and resemble the old ones. Common words of
|
||||
longer entries ("finden Sie") explain nothing on their own."""
|
||||
words = tuple(normalize(word) for word in new)
|
||||
size = len(words)
|
||||
in_entry = any(
|
||||
entry[start:start + size] == words
|
||||
for entry in glossary for start in range(len(entry) - size + 1)
|
||||
)
|
||||
return in_entry and similarity(old, new) >= MIN_SIMILARITY
|
||||
|
||||
|
||||
def review_candidates(comparison, wordlist=None, glossary=(), fillers=FILLERS["de"]):
|
||||
"""Changes the model made on its own that may alter the meaning.
|
||||
|
||||
Not reported: changes of spacing, hyphens and punctuation only, new words
|
||||
from the glossary, deleted filler words, and replacements of words not in
|
||||
the word list (recognition errors) that stay similar.
|
||||
Not reported: changes of spacing, hyphens and punctuation only, glossary
|
||||
spellings of the old words, deleted filler words, and replacements of
|
||||
words not in the word list (recognition errors) that stay similar.
|
||||
"""
|
||||
suspects = []
|
||||
for tag, i1, i2, j1, j2 in comparison.opcodes:
|
||||
if tag == "equal":
|
||||
continue
|
||||
old, new = comparison.raw[i1:i2], comparison.words[j1:j2]
|
||||
if new and all(normalize(word) in known for word in new):
|
||||
continue
|
||||
if tag == "delete":
|
||||
if all(normalize(word) in FILLERS for word in old):
|
||||
if all(normalize(word) in fillers for word in old):
|
||||
continue
|
||||
reason = DELETED
|
||||
elif tag == "insert":
|
||||
reason = INSERTED
|
||||
else:
|
||||
old_key, new_key = spelling_key(old), spelling_key(new)
|
||||
if old_key == new_key:
|
||||
if spelling_key(old) == spelling_key(new) or glossary_spelling(old, new, glossary):
|
||||
continue
|
||||
if wordlist is not None and all(normalize(w) in wordlist for w in old):
|
||||
reason = VALID_WORD
|
||||
elif SequenceMatcher(None, old_key, new_key).ratio() < MIN_SIMILARITY:
|
||||
elif similarity(old, new) < MIN_SIMILARITY:
|
||||
reason = CHANGED_A_LOT
|
||||
else:
|
||||
continue
|
||||
|
|
@ -336,7 +354,7 @@ def print_summary(comparison, limit=15, suspect_limit=10):
|
|||
print(f" … und {len(suspects) - suspect_limit} weitere, siehe Bericht")
|
||||
|
||||
|
||||
def write_report(raw, corrected, target, title, model_input=None, known=(), lang="de"):
|
||||
def write_report(raw, corrected, target, title, model_input=None, glossary=(), lang="de"):
|
||||
"""Write the HTML report; return the comparison with the suspect changes.
|
||||
|
||||
model_input is the text the model got (raw text after glossary
|
||||
|
|
@ -344,7 +362,9 @@ def write_report(raw, corrected, target, title, model_input=None, known=(), lang
|
|||
"""
|
||||
comparison = compare(raw, corrected)
|
||||
by_model = compare(raw if model_input is None else model_input, corrected)
|
||||
comparison.suspects = review_candidates(by_model, load_wordlist(lang), set(known))
|
||||
comparison.suspects = review_candidates(
|
||||
by_model, load_wordlist(lang), glossary, FILLERS.get(lang, set())
|
||||
)
|
||||
target.write_text(render_html(comparison, title), encoding="utf-8")
|
||||
return comparison
|
||||
|
||||
|
|
@ -397,6 +417,7 @@ def parse_args():
|
|||
def main():
|
||||
# abschrift_korrigieren imports this module: import it only when needed
|
||||
from abschrift_korrigieren import apply_replacements, read_glossary
|
||||
from yt_untertitel import base_language
|
||||
|
||||
args = parse_args()
|
||||
raw = read_raw(args.roh)
|
||||
|
|
@ -411,8 +432,8 @@ def main():
|
|||
corrected = args.korrigiert.read_text(encoding="utf-8")
|
||||
comparison = write_report(
|
||||
raw, corrected, args.output, args.korrigiert.name,
|
||||
model_input=model_input, known=glossary_words(terms, replacements),
|
||||
lang=args.lang,
|
||||
model_input=model_input, glossary=glossary_entries(terms, replacements),
|
||||
lang=base_language(args.lang),
|
||||
)
|
||||
print_summary(comparison)
|
||||
print(f"Bericht: {args.output}")
|
||||
|
|
|
|||
Loading…
Reference in a new issue