feat(tts): Aussprache-Normalisierung vor Piper (Ordinalia/Einheiten/Abk./Lexikon)
In-Process-Layer ausgebaut statt neuer Lib/CLI. Leitprinzip: nicht duplizieren,
was espeak-ng schon kann (Kardinal-/Dezimalzahlen bleiben unangetastet) -- nur die
belegten Luecken fuellen.
- german_numbers.py: deutsche Ordinalzahlen 1.-31. (attributiv/adverbial)
- tts_normalizer.py: Ordinalia (Datum '1. Mai'->'erster Mai', Folgen '1. 2. 3.'->
'erstens, zweitens, ...'), Einheiten nach Zahl (kg/km/km-h/...), Abkuerzungen
(Dr./z.B./usw.), optionales YAML-Lexikon (config/pronunciation.<lang>.yaml).
Provider-abhaengige Stufen auto|full|light|off (TTS_NORMALIZE_LEVEL): piper=full,
Cloud=light (laesst Zahlen/Abk. fuer das Cloud-Modell in Ruhe).
- spoken_response_adapter.py: nummerierte Listen -> Ordinalwoerter statt Loeschen.
- sentence_chunker.py: trennt nicht mehr nach Ziffer+Punkt, Einzelbuchstabe+Punkt
('z. B.', Initialen) oder bekannten Abkuerzungen -> behebt das Streaming-Symptom
('1.' wurde als eigener 'Satz' zu 'eins').
- orchestrator/dependencies: normalize_level durchgereicht (auto: piper->full).
- Tests: tests/test_tts_normalizer.py + Chunker-Faelle (85 gruen).
- Doku: BEDIENUNGSANLEITUNG (Aussprache verbessern), .env.example.
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Dieter Schlüter
parent
cca423dac3
commit
c9d8552ec7
13 changed files with 411 additions and 56 deletions
|
|
@ -11,6 +11,12 @@ OPENROUTER_API_KEY=
|
|||
# false = Antwort erst komplett synthetisieren, dann abspielen.
|
||||
AUDIO_STREAM_DEFAULT=true
|
||||
|
||||
# Text-Normalisierung vor dem TTS (Aussprache): auto|full|light|off.
|
||||
# auto = piper bekommt 'full' (Ordinalia/Einheiten/Abk./Lexikon), Cloud-TTS 'light'
|
||||
# (nur Glaettung; Cloud spricht Zahlen/Abkuerzungen selbst gut). Lexikon:
|
||||
# config/pronunciation.de.yaml (erweitert die eingebauten Defaults).
|
||||
TTS_NORMALIZE_LEVEL=auto
|
||||
|
||||
# --- Authentifizierung -----------------------------------------------------
|
||||
# AUTH_ENABLED=true (Standard) schuetzt chat/speak/transcribe/sessions per Bearer-Token.
|
||||
# Fuer lokale Entwicklung/Tests auf false setzen (dann gilt ein anonymer Nutzer).
|
||||
|
|
|
|||
|
|
@ -261,6 +261,16 @@ piper-Stimme/Verzeichnis steuern `PIPER_VOICE`/`PIPER_VOICES_DIR` in `.env` (Def
|
|||
aber **nichts** und verlässt den Rechner nie. Liefert eine Stimme nicht 24000 Hz (z. B.
|
||||
`de_DE-thorsten-high` = 22050 Hz), resampelt das Gateway automatisch per `ffmpeg`.
|
||||
|
||||
**Aussprache verbessern (nur lokales TTS):** Vor Piper läuft ein Normalizer, der typische
|
||||
Stolpersteine glättet — Ordinalzahlen („1. Mai" → „erster Mai", „1. 2. 3." → „erstens,
|
||||
zweitens, drittens"), Einheiten („10 kg" → „… Kilogramm", „km/h" → „Kilometer pro Stunde")
|
||||
und Abkürzungen („Dr." → „Doktor", „z. B." → „zum Beispiel"). Reine Zahlen („123 Euro",
|
||||
„3,5") bleiben unangetastet — die spricht espeak-ng in Piper schon korrekt.
|
||||
- Stärke per `TTS_NORMALIZE_LEVEL` (`auto|full|light|off`): `auto` = Piper bekommt `full`,
|
||||
Cloud-TTS `light` (Cloud spricht Zahlen/Abkürzungen selbst gut, daher schonend).
|
||||
- Eigene Begriffe/Fachwörter pflegst du in **`config/pronunciation.de.yaml`** (Abkürzungen,
|
||||
Einheiten, Terms) — erweitert die eingebauten Defaults, ohne Code zu ändern.
|
||||
|
||||
## B2. Soundquelle & Ausgabe-Gerät wechseln (Mikrofon, Lautsprecher, Bluetooth, Handy)
|
||||
|
||||
> **Wichtig — aktueller Stand:** Die Geräte-Endpunkte **im Gateway**
|
||||
|
|
|
|||
|
|
@ -133,6 +133,8 @@ class Settings(BaseSettings):
|
|||
auth_enabled: bool = True
|
||||
history_max_messages: int = 10
|
||||
audio_stream_default: bool = True # satzweises TTS als Default (Admin kann abschalten)
|
||||
# TTS-Text-Normalisierung: auto|full|light|off. "auto" = piper -> full, Cloud -> light.
|
||||
tts_normalize_level: str = "auto"
|
||||
stt_fallback: str = "" # kommaseparierte Provider-Namen (Fallback-Kette)
|
||||
llm_fallback: str = ""
|
||||
tts_fallback: str = ""
|
||||
|
|
|
|||
|
|
@ -13,13 +13,15 @@ TTS_CHANNELS = 1
|
|||
|
||||
|
||||
class Orchestrator:
|
||||
def __init__(self, stt, llm, tts, input_cleaner, spoken_adapter, tts_normalizer):
|
||||
def __init__(self, stt, llm, tts, input_cleaner, spoken_adapter, tts_normalizer,
|
||||
normalize_level: str = "full"):
|
||||
self.stt = stt
|
||||
self.llm = llm
|
||||
self.tts = tts
|
||||
self.input_cleaner = input_cleaner
|
||||
self.spoken_adapter = spoken_adapter
|
||||
self.tts_normalizer = tts_normalizer
|
||||
self.normalize_level = normalize_level
|
||||
|
||||
async def _emit_to_output(self, audio: bytes, output) -> None:
|
||||
"""Schreibt das synthetisierte Audio durch den gewaehlten Output-Endpunkt.
|
||||
|
|
@ -72,7 +74,9 @@ class Orchestrator:
|
|||
output=None,
|
||||
):
|
||||
spoken = await self.spoken_adapter.run(text, language=language)
|
||||
normalized = await self.tts_normalizer.run(spoken, language=language)
|
||||
normalized = await self.tts_normalizer.run(
|
||||
spoken, language=language, level=self.normalize_level
|
||||
)
|
||||
with _stage("tts"):
|
||||
audio = await self.tts.synthesize(normalized, voice=voice)
|
||||
await self._emit_to_output(audio, output)
|
||||
|
|
@ -106,6 +110,7 @@ class Orchestrator:
|
|||
trace.tts_ready_text = await self.tts_normalizer.run(
|
||||
trace.spoken_response,
|
||||
language=language,
|
||||
level=self.normalize_level,
|
||||
)
|
||||
|
||||
with _stage("tts"):
|
||||
|
|
@ -141,7 +146,9 @@ class Orchestrator:
|
|||
|
||||
async def _emit_sentence(sentence: str) -> None:
|
||||
spoken = await self.spoken_adapter.run(sentence, language=language)
|
||||
ready = await self.tts_normalizer.run(spoken, language=language)
|
||||
ready = await self.tts_normalizer.run(
|
||||
spoken, language=language, level=self.normalize_level
|
||||
)
|
||||
if not ready.strip():
|
||||
return
|
||||
chunk = await self.tts.synthesize(ready, voice=voice)
|
||||
|
|
@ -179,6 +186,7 @@ class Orchestrator:
|
|||
trace.tts_ready_text = await self.tts_normalizer.run(
|
||||
trace.spoken_response,
|
||||
language=language,
|
||||
level=self.normalize_level,
|
||||
)
|
||||
|
||||
if chunker:
|
||||
|
|
|
|||
|
|
@ -226,6 +226,14 @@ def _provider_chain(registry, primary: str, fallback_csv: str, module: str, cfg:
|
|||
return _FALLBACK_CLASS[module](module, entries)
|
||||
|
||||
|
||||
def _resolve_normalize_level(tts_provider: str, cfg: Settings) -> str:
|
||||
"""auto -> piper bekommt 'full', Cloud-TTS 'light' (macht Zahlen/Abk. selbst gut)."""
|
||||
level = (cfg.tts_normalize_level or "auto").lower()
|
||||
if level == "auto":
|
||||
return "full" if tts_provider == "piper" else "light"
|
||||
return level
|
||||
|
||||
|
||||
def build_orchestrator(route: ResolvedRoute, cfg: Settings = settings) -> Orchestrator:
|
||||
return Orchestrator(
|
||||
stt=_provider_chain(STT_REGISTRY, route.stt_provider, cfg.stt_fallback, "stt", cfg),
|
||||
|
|
@ -234,6 +242,7 @@ def build_orchestrator(route: ResolvedRoute, cfg: Settings = settings) -> Orches
|
|||
input_cleaner=InputCleaner(),
|
||||
spoken_adapter=SpokenResponseAdapter(),
|
||||
tts_normalizer=TTSNormalizer(),
|
||||
normalize_level=_resolve_normalize_level(route.tts_provider, cfg),
|
||||
)
|
||||
|
||||
|
||||
|
|
|
|||
37
app/pipeline/german_numbers.py
Normal file
37
app/pipeline/german_numbers.py
Normal file
|
|
@ -0,0 +1,37 @@
|
|||
"""Deutsche Ordinalzahlen 1.–31. (für Datums- und Aufzählungs-Aussprache).
|
||||
|
||||
Bewusst eine kleine handgepflegte Tabelle statt `num2words`: die deutsche
|
||||
Ordinal-Flexion ist mit num2words nicht sauber abbildbar, und der Bereich 1–31
|
||||
deckt Datumsangaben und Listenpositionen vollständig ab.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
# Stamm der Ordinalzahl (ohne Endung). attributiv = Stamm + "er" ("erster"),
|
||||
# adverbial = Stamm + "ens" ("erstens").
|
||||
_STEMS: dict[int, str] = {
|
||||
1: "erst", 2: "zweit", 3: "dritt", 4: "viert", 5: "fünft",
|
||||
6: "sechst", 7: "siebt", 8: "acht", 9: "neunt", 10: "zehnt",
|
||||
11: "elft", 12: "zwölft", 13: "dreizehnt", 14: "vierzehnt", 15: "fünfzehnt",
|
||||
16: "sechzehnt", 17: "siebzehnt", 18: "achtzehnt", 19: "neunzehnt", 20: "zwanzigst",
|
||||
21: "einundzwanzigst", 22: "zweiundzwanzigst", 23: "dreiundzwanzigst",
|
||||
24: "vierundzwanzigst", 25: "fünfundzwanzigst", 26: "sechsundzwanzigst",
|
||||
27: "siebenundzwanzigst", 28: "achtundzwanzigst", 29: "neunundzwanzigst",
|
||||
30: "dreißigst", 31: "einunddreißigst",
|
||||
}
|
||||
|
||||
MIN, MAX = 1, 31
|
||||
|
||||
|
||||
def has_ordinal(n: int) -> bool:
|
||||
return n in _STEMS
|
||||
|
||||
|
||||
def ordinal_attributive(n: int) -> str:
|
||||
"""'1' -> 'erster' (z. B. 'erster Mai')."""
|
||||
return _STEMS[n] + "er"
|
||||
|
||||
|
||||
def ordinal_adverbial(n: int) -> str:
|
||||
"""'1' -> 'erstens' (Aufzählungen)."""
|
||||
return _STEMS[n] + "ens"
|
||||
|
|
@ -3,6 +3,18 @@ import re
|
|||
# Satzende: . ! ? … gefolgt von Whitespace (oder Stringende beim flush).
|
||||
_SENTENCE_END = re.compile(r"[.!?…]+(?=\s)")
|
||||
|
||||
# Abkürzungen, nach denen NICHT getrennt werden darf (sonst zerschneidet der
|
||||
# Streaming-Chunker mitten in "z. | B." und die Normalisierung greift nicht mehr).
|
||||
_ABBREVS = (
|
||||
"z.b.", "z. b.", "d.h.", "d. h.", "u.a.", "u. a.", "bzw.", "ca.", "usw.",
|
||||
"etc.", "dr.", "prof.", "nr.", "str.", "evtl.", "inkl.", "ggf.", "max.",
|
||||
"min.", "vgl.", "sog.", "u.ä.", "o.ä.", "bspw.",
|
||||
)
|
||||
_ABBR_END = re.compile(
|
||||
r"(?:^|[\s(„\"'])(" + "|".join(re.escape(a) for a in _ABBREVS) + r")$",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
|
||||
class SentenceChunker:
|
||||
"""Inkrementelle Satzsegmentierung fuer gestreamte LLM-Token.
|
||||
|
|
@ -11,21 +23,48 @@ class SentenceChunker:
|
|||
`flush()` den verbleibenden Rest (z. B. der letzte Satz ohne abschliessendes
|
||||
Leerzeichen). Damit kann pro Satz schon TTS erzeugt werden, waehrend das LLM
|
||||
noch weiterschreibt.
|
||||
|
||||
Es wird NICHT getrennt, wenn der Punkt zu einer Ordinal-/Datumszahl
|
||||
("1. Mai") oder einer bekannten Abkuerzung ("z. B.") gehoert.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
self._buffer = ""
|
||||
|
||||
def _is_real_end(self, match) -> bool:
|
||||
start, end = match.start(), match.end()
|
||||
punct = match.group()
|
||||
dot_only = set(punct) <= {".", "…"}
|
||||
if dot_only and start > 0:
|
||||
prev = self._buffer[start - 1]
|
||||
# Ziffer + Punkt ("1.") = Ordinal-/Listenmarker, kein Satzende.
|
||||
if prev.isdigit():
|
||||
return False
|
||||
# Einzelner Buchstabe + Punkt ("z. B.", Initialen "A.") -> kein Satzende.
|
||||
if prev.isalpha():
|
||||
before = self._buffer[start - 2] if start >= 2 else ""
|
||||
if before == "" or not before.isalpha():
|
||||
return False
|
||||
# Bekannte (mehrbuchstabige) Abkuerzung vor dem Punkt -> kein Satzende.
|
||||
if _ABBR_END.search(self._buffer[:end]):
|
||||
return False
|
||||
return True
|
||||
|
||||
def feed(self, text: str) -> list[str]:
|
||||
self._buffer += text
|
||||
sentences: list[str] = []
|
||||
search_start = 0
|
||||
while True:
|
||||
match = _SENTENCE_END.search(self._buffer)
|
||||
match = _SENTENCE_END.search(self._buffer, search_start)
|
||||
if not match:
|
||||
break
|
||||
end = match.end()
|
||||
if not self._is_real_end(match):
|
||||
search_start = end # diese Stelle nicht trennen, weitersuchen
|
||||
continue
|
||||
sentence = self._buffer[:end].strip()
|
||||
self._buffer = self._buffer[end:]
|
||||
search_start = 0
|
||||
if sentence:
|
||||
sentences.append(sentence)
|
||||
return sentences
|
||||
|
|
|
|||
|
|
@ -1,5 +1,7 @@
|
|||
import re
|
||||
|
||||
from app.pipeline import german_numbers as gn
|
||||
|
||||
|
||||
class SpokenResponseAdapter:
|
||||
async def run(self, text: str, language: str = "de") -> str:
|
||||
|
|
@ -14,9 +16,15 @@ class SpokenResponseAdapter:
|
|||
text = re.sub(r"\[([^\]]+)\]\([^)]+\)", r"\1", text) # markdown links
|
||||
text = re.sub(r"[*_~#>]+", " ", text) # markdown symbols
|
||||
|
||||
# Listen entschärfen
|
||||
# Listen entschärfen: Aufzählungspunkte weg, nummerierte Listen zu
|
||||
# Ordinalwörtern ("1. " -> "erstens, "), damit Piper nicht "eins" sagt.
|
||||
text = re.sub(r"(?m)^\s*[-•]\s+", "", text)
|
||||
text = re.sub(r"(?m)^\s*\d+\.\s+", "", text)
|
||||
|
||||
def _numbered(m):
|
||||
n = int(m.group(1))
|
||||
return f"{gn.ordinal_adverbial(n)}, " if gn.has_ordinal(n) else ""
|
||||
|
||||
text = re.sub(r"(?m)^\s*(\d{1,2})\.\s+", _numbered, text)
|
||||
|
||||
# Mehrfache Leerzeichen / Zeilenumbrüche glätten
|
||||
text = re.sub(r"\s+", " ", text).strip()
|
||||
|
|
|
|||
|
|
@ -1,59 +1,162 @@
|
|||
"""Text-Normalisierung vor dem TTS.
|
||||
|
||||
Leitprinzip: NICHT duplizieren, was espeak-ng (in piper) bereits gut kann
|
||||
(Kardinal-/Dezimalzahlen). Nur die belegten Lücken füllen: Ordinalzahlen,
|
||||
Einheiten-Abkürzungen, gängige Abkürzungen und ein optionales YAML-Lexikon.
|
||||
|
||||
Stufen (`level`):
|
||||
- "off": keine Änderung (Text unverändert durchreichen).
|
||||
- "light": nur harmlose Glättung (URLs/E-Mails, Whitespace, Abschlusspunkt) –
|
||||
für Cloud-TTS, das Zahlen/Abkürzungen selbst gut spricht.
|
||||
- "full": zusätzlich Ordinalia, Einheiten, Abkürzungen, Lexikon, Symbole –
|
||||
für lokales TTS (piper).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from functools import lru_cache
|
||||
from pathlib import Path
|
||||
|
||||
from app.pipeline import german_numbers as gn
|
||||
|
||||
try:
|
||||
import yaml
|
||||
except ModuleNotFoundError: # pragma: no cover - YAML optional
|
||||
yaml = None
|
||||
|
||||
_CONFIG_DIR = Path(__file__).resolve().parents[2] / "config"
|
||||
|
||||
_MONTHS = (
|
||||
"Januar|Februar|März|April|Mai|Juni|Juli|August|"
|
||||
"September|Oktober|November|Dezember"
|
||||
)
|
||||
_MONTH_ORDINAL = re.compile(rf"\b(\d{{1,2}})\.\s+(?=(?:{_MONTHS})\b)")
|
||||
# Aufzählungsmarker wie "1)" oder "1.)" -> adverbiale Ordinalzahl.
|
||||
_ENUM_MARKER = re.compile(r"\b(\d{1,2})\.?\)")
|
||||
# Folge von >=2 Zahl-Punkt-Markern ("1. 2. 3.") -> jeweils adverbial.
|
||||
_ENUM_SEQ = re.compile(r"(?:(?<![\d,])\d{1,2}\.\s*){2,}")
|
||||
_NUM_DOT = re.compile(r"(\d{1,2})\.")
|
||||
|
||||
# Eingebaute Defaults (auch ohne YAML aktiv). YAML erweitert/überschreibt sie.
|
||||
_DEFAULT_ABBREVS_DE = {
|
||||
"z.B.": "zum Beispiel", "z. B.": "zum Beispiel",
|
||||
"d.h.": "das heißt", "d. h.": "das heißt",
|
||||
"u.a.": "unter anderem", "bzw.": "beziehungsweise",
|
||||
"ca.": "circa", "usw.": "und so weiter", "etc.": "et cetera",
|
||||
"Dr.": "Doktor", "Prof.": "Professor", "Nr.": "Nummer", "Str.": "Straße",
|
||||
}
|
||||
_DEFAULT_UNITS_DE = {
|
||||
"kg": "Kilogramm", "g": "Gramm", "mg": "Milligramm",
|
||||
"km": "Kilometer", "m": "Meter", "cm": "Zentimeter", "mm": "Millimeter",
|
||||
"l": "Liter", "ml": "Milliliter", "h": "Stunden", "min": "Minuten",
|
||||
"km/h": "Kilometer pro Stunde",
|
||||
}
|
||||
_DEFAULT_SYMBOLS_DE = {
|
||||
"24/7": "vierundzwanzig sieben", "&": " und ", "%": " Prozent",
|
||||
"€": " Euro", "$": " Dollar",
|
||||
}
|
||||
_DEFAULT_SYMBOLS_EN = {
|
||||
"24/7": "twenty four seven", "&": " and ", "%": " percent",
|
||||
"€": " euros", "$": " dollars", "e.g.": "for example", "i.e.": "that is",
|
||||
}
|
||||
|
||||
|
||||
@lru_cache(maxsize=8)
|
||||
def _load_lexicon(language: str) -> tuple:
|
||||
"""Lädt config/pronunciation.<lang>.yaml; gibt (abbrevs, units, terms) zurück."""
|
||||
abbrevs, units, terms = {}, {}, {}
|
||||
path = _CONFIG_DIR / f"pronunciation.{language}.yaml"
|
||||
if yaml is not None and path.exists():
|
||||
try:
|
||||
data = yaml.safe_load(path.read_text(encoding="utf-8")) or {}
|
||||
abbrevs = {str(k): str(v) for k, v in (data.get("abbreviations") or {}).items()}
|
||||
units = {str(k): str(v) for k, v in (data.get("units") or {}).items()}
|
||||
terms = {str(k): str(v) for k, v in (data.get("terms") or {}).items()}
|
||||
except (OSError, ValueError, AttributeError):
|
||||
pass # defektes YAML -> nur Defaults
|
||||
return abbrevs, units, terms
|
||||
|
||||
|
||||
def _apply_ordinals(text: str) -> str:
|
||||
# Datum: "1. Mai" -> "erster Mai"
|
||||
def _date(m):
|
||||
n = int(m.group(1))
|
||||
return f"{gn.ordinal_attributive(n)} " if gn.has_ordinal(n) else m.group(0)
|
||||
|
||||
text = _MONTH_ORDINAL.sub(_date, text)
|
||||
|
||||
# Aufzählungs-Folge "1. 2. 3." -> "erstens zweitens drittens" (alle in der Folge).
|
||||
def _num(m):
|
||||
n = int(m.group(1))
|
||||
return f"{gn.ordinal_adverbial(n)} " if gn.has_ordinal(n) else m.group(0)
|
||||
|
||||
text = _ENUM_SEQ.sub(lambda m: _NUM_DOT.sub(_num, m.group(0)), text)
|
||||
|
||||
# Marker "1)" / "1.)" -> "erstens"
|
||||
def _marker(m):
|
||||
n = int(m.group(1))
|
||||
return gn.ordinal_adverbial(n) if gn.has_ordinal(n) else m.group(0)
|
||||
|
||||
text = _ENUM_MARKER.sub(_marker, text)
|
||||
return text
|
||||
|
||||
|
||||
def _apply_units(text: str, units: dict) -> str:
|
||||
# Nur DIREKT nach einer Zahl ersetzen, damit "Meter" nicht jedes "m" trifft.
|
||||
# Längere Einheitenkürzel zuerst (km/h vor km, mm vor m).
|
||||
for unit in sorted(units, key=len, reverse=True):
|
||||
spoken = units[unit]
|
||||
pattern = rf"(?<=\d)\s*{re.escape(unit)}\b"
|
||||
text = re.sub(pattern, f" {spoken}", text)
|
||||
return text
|
||||
|
||||
|
||||
def _apply_dict(text: str, mapping: dict) -> str:
|
||||
# Längste Schlüssel zuerst, wortgrenzen-bewusst (Abkürzungen enden oft auf ".").
|
||||
for key in sorted(mapping, key=len, reverse=True):
|
||||
repl = mapping[key]
|
||||
if key.isalnum(): # reines Wort/Akronym -> mit Wortgrenzen
|
||||
text = re.sub(rf"\b{re.escape(key)}\b", repl, text)
|
||||
else: # enthält Punkte/Sonderzeichen -> direkte Ersetzung
|
||||
text = text.replace(key, repl)
|
||||
return text
|
||||
|
||||
|
||||
class TTSNormalizer:
|
||||
async def run(self, text: str, language: str = "de") -> str:
|
||||
async def run(self, text: str, language: str = "de", level: str = "full") -> str:
|
||||
if not text:
|
||||
return ""
|
||||
if level == "off":
|
||||
return text
|
||||
|
||||
normalized = text
|
||||
t = text
|
||||
|
||||
if language == "de":
|
||||
replacements = {
|
||||
"24/7": "vierundzwanzig sieben",
|
||||
"&": " und ",
|
||||
"%": " Prozent",
|
||||
"€": " Euro",
|
||||
"$": " Dollar",
|
||||
"km/h": " Kilometer pro Stunde",
|
||||
"z.B.": "zum Beispiel",
|
||||
"bzw.": "beziehungsweise",
|
||||
"u.a.": "unter anderem",
|
||||
"ca.": "circa",
|
||||
}
|
||||
else:
|
||||
replacements = {
|
||||
"24/7": "twenty four seven",
|
||||
"&": " and ",
|
||||
"%": " percent",
|
||||
"€": " euros",
|
||||
"$": " dollars",
|
||||
"km/h": " kilometers per hour",
|
||||
"e.g.": "for example",
|
||||
"i.e.": "that is",
|
||||
}
|
||||
# --- immer (light + full): harmlose Glättung ---
|
||||
t = re.sub(r"https?://\S+", "Link", t)
|
||||
t = re.sub(r"\b[\w\.-]+@[\w\.-]+\.\w+\b", "E-Mail-Adresse", t)
|
||||
|
||||
for old, new in replacements.items():
|
||||
normalized = normalized.replace(old, new)
|
||||
if level == "full":
|
||||
abbrevs, units, terms = _load_lexicon(language)
|
||||
|
||||
# Slashes zwischen Wörtern/Zahlen sprachfreundlicher machen
|
||||
normalized = re.sub(r"(\w)/(\w)", r"\1 oder \2", normalized)
|
||||
|
||||
# Datums-/Versions-/Bereichsstriche etwas entschärfen
|
||||
normalized = normalized.replace("–", " bis ")
|
||||
normalized = normalized.replace("—", ", ")
|
||||
normalized = normalized.replace(" - ", ", ")
|
||||
|
||||
# URLs und E-Mails nicht roh vorlesen
|
||||
normalized = re.sub(r"https?://\S+", "Link", normalized)
|
||||
normalized = re.sub(r"\b[\w\.-]+@[\w\.-]+\.\w+\b", "E-Mail-Adresse", normalized)
|
||||
|
||||
# Mehrfache Leerzeichen glätten
|
||||
normalized = re.sub(r"\s+", " ", normalized).strip()
|
||||
|
||||
if normalized and not normalized.endswith((".", "!", "?")):
|
||||
normalized += "."
|
||||
|
||||
return normalized
|
||||
if language == "de":
|
||||
t = _apply_ordinals(t)
|
||||
t = _apply_units(t, {**_DEFAULT_UNITS_DE, **units})
|
||||
t = _apply_dict(t, {**_DEFAULT_ABBREVS_DE, **abbrevs})
|
||||
t = _apply_dict(t, terms)
|
||||
t = _apply_dict(t, _DEFAULT_SYMBOLS_DE)
|
||||
else:
|
||||
t = _apply_units(t, units)
|
||||
t = _apply_dict(t, abbrevs)
|
||||
t = _apply_dict(t, terms)
|
||||
t = _apply_dict(t, _DEFAULT_SYMBOLS_EN)
|
||||
|
||||
# Slash zwischen Wörtern/Zahlen sprachfreundlich, Striche entschärfen.
|
||||
t = re.sub(r"(\w)/(\w)", r"\1 oder \2", t)
|
||||
t = t.replace("–", " bis ").replace("—", ", ").replace(" - ", ", ")
|
||||
|
||||
# Mehrfache Leerzeichen glätten, Abschlusspunkt sicherstellen.
|
||||
t = re.sub(r"\s+", " ", t).strip()
|
||||
if t and not t.endswith((".", "!", "?")):
|
||||
t += "."
|
||||
return t
|
||||
|
|
|
|||
23
config/pronunciation.de.yaml
Normal file
23
config/pronunciation.de.yaml
Normal file
|
|
@ -0,0 +1,23 @@
|
|||
# Aussprache-Lexikon (Deutsch) für die TTS-Normalisierung vor Piper.
|
||||
#
|
||||
# Dies ist das GRUNDGERÜST – Inhalte werden später schrittweise ergänzt.
|
||||
# Einträge hier ERWEITERN/ÜBERSCHREIBEN die eingebauten Defaults des Normalizers.
|
||||
# Wirkt nur bei lokalem TTS (piper, Stufe "full"); Cloud-TTS bleibt unangetastet.
|
||||
#
|
||||
# Format:
|
||||
# abbreviations: ganze Token werden wortgrenzen-sicher ersetzt (Punkt mitschreiben!)
|
||||
# units: werden nur DIREKT NACH EINER ZAHL ersetzt (z. B. "10 kg")
|
||||
# terms: Fachwörter / Eigennamen / englische Begriffe
|
||||
#
|
||||
# Beispiele (auskommentiert – bei Bedarf aktivieren/ergänzen):
|
||||
|
||||
abbreviations:
|
||||
# "ggf.": "gegebenenfalls"
|
||||
# "inkl.": "inklusive"
|
||||
|
||||
units:
|
||||
# "kWh": "Kilowattstunden"
|
||||
|
||||
terms:
|
||||
# "Linux": "Linuks"
|
||||
# "CPU": "C P U"
|
||||
12
config/pronunciation.en.yaml
Normal file
12
config/pronunciation.en.yaml
Normal file
|
|
@ -0,0 +1,12 @@
|
|||
# Pronunciation lexicon (English) for TTS normalization before piper.
|
||||
# Grundgerüst – entries here extend/override the normalizer's built-in defaults.
|
||||
# Only applied for local TTS (piper, level "full").
|
||||
|
||||
abbreviations:
|
||||
# "approx.": "approximately"
|
||||
|
||||
units:
|
||||
# "kWh": "kilowatt hours"
|
||||
|
||||
terms:
|
||||
# "SQL": "sequel"
|
||||
|
|
@ -20,6 +20,15 @@ def test_sentence_chunker_incremental():
|
|||
assert SentenceChunker().feed("Eins. Zwei! Drei? Vier") == ["Eins.", "Zwei!", "Drei?"]
|
||||
|
||||
|
||||
def test_sentence_chunker_keeps_ordinals_and_abbreviations():
|
||||
# "1." (Ziffer+Punkt) darf KEINE Satzgrenze sein.
|
||||
ch = SentenceChunker()
|
||||
assert ch.feed("Am 1. Mai ist frei. ") == ["Am 1. Mai ist frei."]
|
||||
# Abkuerzung "z. B." darf den Satz nicht zerschneiden.
|
||||
ch2 = SentenceChunker()
|
||||
assert ch2.feed("Obst, z. B. Äpfel und Birnen. ") == ["Obst, z. B. Äpfel und Birnen."]
|
||||
|
||||
|
||||
def test_sse_delta_parsing():
|
||||
assert sse_delta('data: {"choices":[{"delta":{"content":"Hal"}}]}') == "Hal"
|
||||
assert sse_delta("data: [DONE]") is None
|
||||
|
|
|
|||
89
tests/test_tts_normalizer.py
Normal file
89
tests/test_tts_normalizer.py
Normal file
|
|
@ -0,0 +1,89 @@
|
|||
"""Tests fuer die TTS-Text-Normalisierung (Aussprache vor Piper)."""
|
||||
|
||||
import asyncio
|
||||
|
||||
from app.pipeline.tts_normalizer import TTSNormalizer
|
||||
from app.pipeline.spoken_response_adapter import SpokenResponseAdapter
|
||||
from app.pipeline import german_numbers as gn
|
||||
|
||||
|
||||
def _run(coro):
|
||||
return asyncio.run(coro)
|
||||
|
||||
|
||||
def _norm(text, level="full", language="de"):
|
||||
return _run(TTSNormalizer().run(text, language=language, level=level))
|
||||
|
||||
|
||||
# --- Ordinalzahlen ---------------------------------------------------------
|
||||
|
||||
def test_date_ordinal_to_attributive():
|
||||
assert "erster Mai" in _norm("Am 1. Mai feiern wir.")
|
||||
assert "dritter Oktober" in _norm("Der 3. Oktober ist ein Feiertag.")
|
||||
|
||||
|
||||
def test_enumeration_run_to_adverbial():
|
||||
out = _norm("1. 2. 3. fertig")
|
||||
assert "erstens" in out and "zweitens" in out and "drittens" in out
|
||||
|
||||
|
||||
def test_enum_marker_paren():
|
||||
assert "erstens" in _norm("1) Milch")
|
||||
assert "zweitens" in _norm("2.) Eier")
|
||||
|
||||
|
||||
def test_plain_trailing_number_not_touched():
|
||||
# Eine einzelne Zahl am Satzende ist KEINE Ordinalzahl -> bleibt Zahl.
|
||||
out = _norm("Ich nehme 5.")
|
||||
assert "fünftens" not in out and "fünfter" not in out
|
||||
assert "5" in out
|
||||
|
||||
|
||||
# --- Einheiten -------------------------------------------------------------
|
||||
|
||||
def test_units_after_number():
|
||||
assert "Kilogramm" in _norm("Das wiegt 10 kg.")
|
||||
assert "Kilometer pro Stunde" in _norm("Tempo 50 km/h.")
|
||||
|
||||
|
||||
def test_unit_letter_not_triggered_without_number():
|
||||
# "m" ohne vorangehende Zahl darf nicht zu "Meter" werden.
|
||||
assert "Meter" not in _norm("Am Morgen.")
|
||||
|
||||
|
||||
# --- Abkuerzungen / Lexikon ------------------------------------------------
|
||||
|
||||
def test_builtin_abbreviations():
|
||||
assert "Doktor" in _norm("Dr. Müller kommt.")
|
||||
assert "und so weiter" in _norm("Brot, Milch usw.")
|
||||
|
||||
|
||||
# --- Stufen ----------------------------------------------------------------
|
||||
|
||||
def test_light_leaves_numbers_and_abbrev_for_cloud():
|
||||
out = _norm("Am 1. Mai wiegt es 10 kg, Dr. Müller.", level="light")
|
||||
assert "erster" not in out and "Kilogramm" not in out and "Doktor" not in out
|
||||
assert "1." in out and "kg" in out # Cloud-TTS macht das selbst
|
||||
|
||||
def test_off_returns_unchanged():
|
||||
src = "Am 1. Mai, 10 kg."
|
||||
assert _norm(src, level="off") == src
|
||||
|
||||
|
||||
def test_url_and_email_squashed_in_light():
|
||||
out = _norm("Schau auf https://example.org oder mail@example.org.", level="light")
|
||||
assert "Link" in out and "E-Mail-Adresse" in out
|
||||
|
||||
|
||||
# --- Zusammenspiel mit dem SpokenResponseAdapter ---------------------------
|
||||
|
||||
def test_adapter_numbered_list_to_ordinal():
|
||||
md = "1. Milch\n2. Eier\n3. Brot"
|
||||
out = _run(SpokenResponseAdapter().run(md))
|
||||
assert "erstens, Milch" in out and "zweitens, Eier" in out
|
||||
|
||||
|
||||
def test_german_numbers_table():
|
||||
assert gn.ordinal_attributive(1) == "erster"
|
||||
assert gn.ordinal_adverbial(3) == "drittens"
|
||||
assert gn.has_ordinal(31) and not gn.has_ordinal(32)
|
||||
Loading…
Add table
Add a link
Reference in a new issue