feat: Barge-in/Turn-Manager und VAD-Aeusserungserkennung (#4 Ausbau)
- Barge-in: Antwort-Turn als abbrechbarer asyncio.Task; {"type":"interrupt"} oder
neue Eingabe bricht laufende Antwort ab -> interrupted-Event (/ws/chat + /ws/voice)
- VAD (app/audio/vad.py): energie-basierte Stille-Erkennung (reines Python, int16-PCM)
- /ws/voice opt-in {"type":"start","vad":true}: automatisches Aeusserungsende ohne end
- Tests: 52 gruen (+5: VAD-Unit, Barge-in, VAD-Auto-Segmentierung)
- Doku aktualisiert (README, BEDIENUNGSANLEITUNG, Architektur)
Offen (schwere Deps/Dienste): echte partielle Live-Transkripte (Streaming-STT),
WebRTC (aiortc). STT laeuft heute pro Aeusserung.
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
parent
9340d3f998
commit
093da817d8
6 changed files with 312 additions and 48 deletions
129
tests/test_realtime.py
Normal file
129
tests/test_realtime.py
Normal file
|
|
@ -0,0 +1,129 @@
|
|||
import array
|
||||
import asyncio
|
||||
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
import app.dependencies as deps
|
||||
from app.main import app
|
||||
from app.audio.vad import rms, EnergyVAD
|
||||
|
||||
client = TestClient(app)
|
||||
|
||||
|
||||
def _pcm(amplitude: int, n_samples: int) -> bytes:
|
||||
return array.array("h", [amplitude] * n_samples).tobytes()
|
||||
|
||||
|
||||
# --- VAD-Unit-Tests --------------------------------------------------------
|
||||
|
||||
def test_rms_silence_vs_loud():
|
||||
assert rms(b"") == 0.0
|
||||
assert rms(_pcm(0, 100)) == 0.0
|
||||
assert rms(_pcm(3000, 100)) > 2000
|
||||
|
||||
|
||||
def test_energy_vad_ends_after_speech_then_silence():
|
||||
vad = EnergyVAD(sample_rate=16000, threshold=500, silence_ms=300)
|
||||
loud = _pcm(3000, 1600) # 100 ms Sprache
|
||||
silent = _pcm(0, 1600) # 100 ms Stille
|
||||
assert vad.feed(loud) is False
|
||||
assert vad.feed(silent) is False # 100 ms
|
||||
assert vad.feed(silent) is False # 200 ms
|
||||
assert vad.feed(silent) is True # 300 ms -> Ende
|
||||
|
||||
|
||||
def test_energy_vad_ignores_silence_without_speech():
|
||||
vad = EnergyVAD(sample_rate=16000, threshold=500, silence_ms=100)
|
||||
for _ in range(10):
|
||||
assert vad.feed(_pcm(0, 1600)) is False
|
||||
|
||||
|
||||
# --- Barge-in --------------------------------------------------------------
|
||||
|
||||
def _install_slow_stream(monkeypatch):
|
||||
class SlowLLM:
|
||||
async def complete(self, text, history=None, session_id=None):
|
||||
return "fertig"
|
||||
|
||||
async def stream(self, text, history=None, session_id=None):
|
||||
for i in range(200):
|
||||
await asyncio.sleep(0.005)
|
||||
yield f"t{i} "
|
||||
|
||||
class StubTTS:
|
||||
async def synthesize(self, text, voice=None, audio_format="pcm"):
|
||||
return b"A"
|
||||
|
||||
monkeypatch.setitem(deps.LLM_REGISTRY, "slow", lambda s: SlowLLM())
|
||||
monkeypatch.setitem(deps.TTS_REGISTRY, "stub", lambda s: StubTTS())
|
||||
return {
|
||||
"llm_provider": "slow",
|
||||
"tts_provider": "stub",
|
||||
"output_endpoint": "loopback",
|
||||
"stream": True,
|
||||
}
|
||||
|
||||
|
||||
def test_ws_chat_interrupt_cancels_response(monkeypatch):
|
||||
base = _install_slow_stream(monkeypatch)
|
||||
with client.websocket_connect("/ws/chat") as ws:
|
||||
ws.send_json({"text": "Hallo", **base})
|
||||
assert ws.receive_json()["type"] == "ack"
|
||||
assert ws.receive_json()["type"] == "token" # Antwort laeuft
|
||||
|
||||
ws.send_json({"type": "interrupt"})
|
||||
|
||||
event = None
|
||||
for _ in range(500):
|
||||
event = ws.receive_json()
|
||||
if event["type"] in ("interrupted", "done"):
|
||||
break
|
||||
assert event["type"] == "interrupted" # abgebrochen, nicht fertig
|
||||
|
||||
|
||||
# --- VAD im WebSocket ------------------------------------------------------
|
||||
|
||||
def _install_voice_stubs(monkeypatch):
|
||||
class STT:
|
||||
async def transcribe(self, audio_bytes, fmt, language=None):
|
||||
return "ok"
|
||||
|
||||
class LLM:
|
||||
async def complete(self, text, history=None, session_id=None):
|
||||
return "antwort"
|
||||
|
||||
class TTS:
|
||||
async def synthesize(self, text, voice=None, audio_format="pcm"):
|
||||
return b"A"
|
||||
|
||||
monkeypatch.setitem(deps.STT_REGISTRY, "s", lambda x: STT())
|
||||
monkeypatch.setitem(deps.LLM_REGISTRY, "l", lambda x: LLM())
|
||||
monkeypatch.setitem(deps.TTS_REGISTRY, "t", lambda x: TTS())
|
||||
return {"stt_provider": "s", "llm_provider": "l", "tts_provider": "t", "output_endpoint": "loopback"}
|
||||
|
||||
|
||||
def test_ws_voice_vad_auto_segments_utterance(monkeypatch):
|
||||
opts = _install_voice_stubs(monkeypatch)
|
||||
start = {
|
||||
"type": "start",
|
||||
"vad": True,
|
||||
"sample_rate": 16000,
|
||||
"vad_silence_ms": 200,
|
||||
"format": "pcm",
|
||||
**opts,
|
||||
}
|
||||
loud = _pcm(3000, 1600) # 100 ms Sprache
|
||||
silent = _pcm(0, 1600) # je 100 ms Stille
|
||||
|
||||
with client.websocket_connect("/ws/voice") as ws:
|
||||
ws.send_json(start)
|
||||
ws.send_bytes(loud)
|
||||
ws.send_bytes(silent) # 100 ms
|
||||
ws.send_bytes(silent) # 200 ms -> VAD-Ende, Turn startet automatisch
|
||||
|
||||
transcript = ws.receive_json()
|
||||
assert transcript["type"] == "transcript" and transcript["text"] == "ok"
|
||||
assert ws.receive_json()["type"] == "ack"
|
||||
assert ws.receive_json()["type"] == "semantic"
|
||||
assert ws.receive_bytes() == b"A"
|
||||
assert ws.receive_json()["type"] == "done"
|
||||
Loading…
Add table
Add a link
Reference in a new issue