fix: Änderungsbericht erkennt Apostrophe im Wort; output/ ignorieren
- „Bahrs“ → „Bahr’s“ ist jetzt eine Änderung; ’ ' ` gelten als gleich, Anführungszeichen am Wortrand zählen weiterhin nicht - Prüfmarken auch englisch: [check: …] - output/ (eigene Ergebnisse und Testläufe) in .gitignore Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
547dcb358d
commit
95bdc660ca
1 changed files with 35 additions and 4 deletions
|
|
@ -11,9 +11,12 @@ import re
|
|||
|
||||
from vtt2txt import vtt_to_lines
|
||||
|
||||
REVIEW_MARK = re.compile(r"\[prüfen:[^\]]*\]")
|
||||
TOKEN = re.compile(r"\[prüfen:[^\]]*\]|\S+")
|
||||
# "[prüfen: …]" in German speeches, "[check: …]" in English ones
|
||||
REVIEW_MARK = re.compile(r"\[(?:prüfen|check):[^\]]*\]")
|
||||
TOKEN = re.compile(rf"{REVIEW_MARK.pattern}|\S+")
|
||||
NON_WORD = re.compile(r"[^\w]+")
|
||||
NON_WORD_KEEP_APOSTROPHE = re.compile(r"[^\w']+")
|
||||
APOSTROPHES = str.maketrans("’‘`´", "''''")
|
||||
ABBREVIATION = re.compile(r"\w\.") # one part of "z. B.", "d. h."
|
||||
ABBREVIATION_END = re.compile(r"(?:^|\s)\w\.$")
|
||||
EMPTY = "∅"
|
||||
|
|
@ -32,8 +35,13 @@ def add_token(words, token):
|
|||
|
||||
|
||||
def normalize(word):
|
||||
"""Compare words without case and punctuation; those changes are expected."""
|
||||
return NON_WORD.sub("", word.lower()) or word
|
||||
"""Compare words without case and punctuation; those changes are expected.
|
||||
|
||||
Apostrophes inside a word count ("Bahr’s" is not "Bahrs"), quotes at its
|
||||
edges do not.
|
||||
"""
|
||||
word = word.lower().translate(APOSTROPHES)
|
||||
return NON_WORD_KEEP_APOSTROPHE.sub("", word).strip("'") or word
|
||||
|
||||
|
||||
@dataclass
|
||||
|
|
|
|||
Loading…
Reference in a new issue