feat: Abkürzungen und Ordnungszahlen beenden keinen Satz
„z. B.“, „d. h.“, „Dr.“, „bzw.“, „ca.“, „e.g.“ und Ordnungszahlen wie „2. Weltkrieg“ oder „am 24.“ gelten bei Satz-, Abschnitts- und Absatzgrenzen nicht mehr als Satzende; „usw.“, „etc.“ und Jahreszahlen weiterhin. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
82ce30ca72
commit
f1d2e201df
1 changed files with 46 additions and 6 deletions
|
|
@ -35,6 +35,11 @@ MIN_PARAGRAPH_SENTENCES = 3 # kürzere Absätze werden mit dem Nachbarn verbun
|
|||
|
||||
SENTENCE_END = re.compile(r"[.!?…][\"'»«“”)\]]*$")
|
||||
SENTENCE_SPLIT = re.compile(r"(?:(?<=[.!?…])|(?<=[.!?…][\"'»«“”)\]]))\s+")
|
||||
# Abbreviations and ordinals ("z. B.", "Dr.", "2. Weltkrieg", "e.g.") end no
|
||||
# sentence; "usw." and "etc." usually do, four-digit years too
|
||||
NO_SENTENCE_END = re.compile(
|
||||
r"(?:^|[\s.])(?:\w|\d{1,2}|Dr|Prof|bzw|ca|Nr|St|vgl|sog|evtl|ggf|inkl|bspw|Mr|Mrs|Ms)\.$"
|
||||
)
|
||||
|
||||
SYSTEM = """Du korrigierst die automatische Abschrift eines deutschsprachigen
|
||||
gesprochenen Beitrags (z. B. Rede, Vortrag, Interview, Video).
|
||||
|
|
@ -237,7 +242,7 @@ def split_chunks(text):
|
|||
while start < len(units):
|
||||
end = min(start + CHUNK_SIZE, len(units))
|
||||
limit = min(end + MAX_EXTRA_WORDS, len(units))
|
||||
while end < limit and not SENTENCE_END.search(units[end - 1].rstrip()):
|
||||
while end < limit and not ends_sentence(units[end - 1].rstrip()):
|
||||
end += 1
|
||||
chunks.append("".join(units[start:end]))
|
||||
start = end
|
||||
|
|
@ -349,9 +354,21 @@ def correct_chunk(chunk, index, system):
|
|||
return corrected
|
||||
|
||||
|
||||
def ends_sentence(text):
|
||||
return bool(SENTENCE_END.search(text)) and not NO_SENTENCE_END.search(text)
|
||||
|
||||
|
||||
def split_sentences(text):
|
||||
"""Split flat text into sentences; joining them with spaces restores it."""
|
||||
return [s for s in SENTENCE_SPLIT.split(" ".join(text.split())) if s]
|
||||
sentences = []
|
||||
for piece in SENTENCE_SPLIT.split(" ".join(text.split())):
|
||||
if not piece:
|
||||
continue
|
||||
if sentences and NO_SENTENCE_END.search(sentences[-1]):
|
||||
sentences[-1] += " " + piece
|
||||
else:
|
||||
sentences.append(piece)
|
||||
return sentences
|
||||
|
||||
|
||||
def find_paragraph_starts(sentences, first, system):
|
||||
|
|
|
|||
Loading…
Reference in a new issue