883 lines
39 KiB
Python
Executable file
883 lines
39 KiB
Python
Executable file
#!/usr/bin/env python3
|
|
"""
|
|
Sprachassistent mit 5-Zustands-Maschine
|
|
|
|
Zustände:
|
|
LAUSCH → wartet auf "Hey Jarvis"
|
|
AUFNAHME → nimmt User-Sprache auf
|
|
GENERATOR → sagt "Moment …", schickt Transkript an LLM
|
|
VORLESE → liest Antwort vor (Steuer-Wörter aktiv)
|
|
DIALOG → kurzes Zeitfenster für Rückfrage ohne Wake-Word
|
|
|
|
Steuer-Wörter (nur in VORLESE, "bitte" wird immer ignoriert):
|
|
"Stopp" → Abbruch → DIALOG
|
|
"Pause" → Wiedergabe anhalten (SIGSTOP)
|
|
"Weiter" → Weiterspielen ab Satzanfang
|
|
"Noch einmal" → letzten Satz wiederholen
|
|
"Absatz noch einmal" → zum Absatzanfang
|
|
"Alles von vorn" → von Anfang an
|
|
|
|
Verwendung:
|
|
python3 assistant.py # llama.cpp, ReSpeaker
|
|
python3 assistant.py --list-mics
|
|
python3 assistant.py --no-wakeword --silence-sec 3.0
|
|
python3 assistant.py --backend ollama --model llama3.2
|
|
python3 assistant.py --backend openai --model gpt-4o-mini
|
|
python3 assistant.py --mic motu --out motu
|
|
|
|
GPU-Belegung:
|
|
GPU 1 (phys.) RTX 3090 — llama.cpp / Port 8001
|
|
GPU 2 (phys.) RTX 3090 — Whisper (via --gpu 1 in CUDA_VISIBLE_DEVICES=1,2)
|
|
"""
|
|
|
|
import os
|
|
import re
|
|
import sys
|
|
import signal
|
|
import queue
|
|
import threading
|
|
import warnings
|
|
import argparse
|
|
import subprocess
|
|
import time
|
|
from enum import Enum, auto
|
|
|
|
import numpy as np
|
|
import sounddevice as sd
|
|
from scipy.signal import resample_poly
|
|
from faster_whisper import WhisperModel
|
|
|
|
warnings.filterwarnings("ignore", category=UserWarning, module="onnxruntime")
|
|
from openwakeword.model import Model as WakeWordModel # noqa: E402
|
|
|
|
import openai
|
|
|
|
sys.path.insert(0, os.path.dirname(__file__))
|
|
from speak import (resolve_sink, get_voice, synth_bytes, # noqa: E402
|
|
detect_lang, voice_for_lang, LANG_VOICES,
|
|
set_en_variant, EN_VARIANT_VOICES)
|
|
from piper.voice import PiperVoice # noqa: E402
|
|
from mic import resolve_mic, print_mic_list, DEFAULT_MIC # noqa: E402
|
|
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
# Zustände
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
|
|
class State(Enum):
|
|
LAUSCH = auto()
|
|
AUFNAHME = auto()
|
|
GENERATOR = auto()
|
|
VORLESE = auto()
|
|
DIALOG = auto()
|
|
|
|
STATE_LABEL = {
|
|
State.LAUSCH: "LAUSCH",
|
|
State.AUFNAHME: "AUFNAHME",
|
|
State.GENERATOR: "GENERATOR",
|
|
State.VORLESE: "VORLESE",
|
|
State.DIALOG: "DIALOG",
|
|
}
|
|
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
# Steuer-Befehle
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
|
|
CMD_STOP = "STOP"
|
|
CMD_PAUSE = "PAUSE"
|
|
CMD_RESUME = "RESUME"
|
|
CMD_REPEAT = "REPEAT"
|
|
CMD_PARAGRAPH = "PARAGRAPH"
|
|
CMD_RESTART = "RESTART"
|
|
|
|
# Längste Muster zuerst (Präfix-Priorität)
|
|
CMD_PATTERNS: list[tuple[str, str]] = [
|
|
("absatz noch einmal", CMD_PARAGRAPH),
|
|
("alles von vorn", CMD_RESTART),
|
|
("noch einmal", CMD_REPEAT),
|
|
("stopp", CMD_STOP),
|
|
("stop", CMD_STOP),
|
|
("pause", CMD_PAUSE),
|
|
("weiter", CMD_RESUME),
|
|
]
|
|
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
# Konstanten
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
|
|
WHISPER_RATE = 16000
|
|
WW_CHUNK_16K = 1280 # openwakeword: 80 ms @ 16 kHz
|
|
WW_COOLDOWN_SEC = 2.0 # Sekunden nach LAUSCH-Eintritt ohne Trigger
|
|
SILENCE_RMS = 0.008
|
|
MAX_RECORD_SEC = 60
|
|
STOP_CHECK_SEC = 2.0 # Intervall Stop-Wort-Check in AUFNAHME
|
|
STOP_WINDOW_SEC = 4.0 # Rollierendes Fenster Stop-Wort
|
|
CTRL_CHECK_SEC = 0.5 # Intervall Steuer-Wort-Check in VORLESE
|
|
CTRL_WINDOW_SEC = 3.0 # Rollierendes Fenster Steuer-Wort
|
|
|
|
BACKENDS: dict[str, dict] = {
|
|
"llama": {"base_url": "http://localhost:8001/v1", "api_key": "none",
|
|
"default_model": None},
|
|
"ollama": {"base_url": "http://localhost:11434/v1", "api_key": "ollama",
|
|
"default_model": "llama3"},
|
|
"openai": {"base_url": None, "api_key": os.getenv("OPENAI_API_KEY", ""),
|
|
"default_model": "gpt-4o-mini"},
|
|
}
|
|
|
|
DEFAULT_SYSTEM = (
|
|
"Du bist ein hilfreicher Sprachassistent. Antworte präzise und natürlich. "
|
|
"Deine Antworten werden direkt als Sprache ausgegeben, also verwende "
|
|
"keine Markdown-Formatierung, keine Aufzählungszeichen und keine Code-Blöcke."
|
|
)
|
|
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
# Hilfsfunktionen
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
|
|
def normalize_cmd(text: str) -> str:
|
|
"""Entfernt 'bitte' und normalisiert für Befehlsvergleich."""
|
|
text = re.sub(r'\bbitte\b', '', text, flags=re.IGNORECASE)
|
|
return re.sub(r'\s+', ' ', text).strip().lower()
|
|
|
|
|
|
def detect_command(text: str) -> str | None:
|
|
norm = normalize_cmd(text)
|
|
for pattern, cmd in CMD_PATTERNS:
|
|
if pattern in norm:
|
|
return cmd
|
|
return None
|
|
|
|
|
|
def show_status(state: State, info: str = "") -> None:
|
|
suffix = f" {info}" if info else ""
|
|
print(f"\r\033[K[{STATE_LABEL[state]}]{suffix}", end="", flush=True)
|
|
|
|
|
|
def status_newline() -> None:
|
|
print(flush=True)
|
|
|
|
|
|
def quick_play(text: str, voice: PiperVoice, out: str) -> None:
|
|
"""Spielt kurzen Text synchron (blockierend, kein Hintergrund-Thread)."""
|
|
sink = resolve_sink(out)
|
|
cmd = ["paplay", "--raw",
|
|
f"--rate={voice.config.sample_rate}",
|
|
"--channels=1", "--format=s16le"]
|
|
if sink:
|
|
cmd += [f"--device={sink}"]
|
|
audio = synth_bytes(text, voice)
|
|
proc = subprocess.Popen(cmd, stdin=subprocess.PIPE)
|
|
proc.stdin.write(audio)
|
|
proc.stdin.close()
|
|
proc.wait()
|
|
|
|
|
|
def play_chime(out: str) -> None:
|
|
rate = 22050
|
|
t = np.linspace(0, 0.2, int(rate * 0.2), endpoint=False)
|
|
beep = (np.sin(2 * np.pi * 880 * t) * 0.35 * 32767).astype(np.int16)
|
|
sink = resolve_sink(out)
|
|
cmd = ["paplay", "--raw", f"--rate={rate}", "--channels=1", "--format=s16le"]
|
|
if sink:
|
|
cmd += [f"--device={sink}"]
|
|
proc = subprocess.Popen(cmd, stdin=subprocess.PIPE)
|
|
proc.stdin.write(beep.tobytes())
|
|
proc.stdin.close()
|
|
proc.wait()
|
|
|
|
|
|
def _drain_audio(audio_q: queue.Queue, ctrl_audio_q: queue.Queue) -> None:
|
|
"""Leert audio_q und leitet Chunks an ctrl_audio_q weiter."""
|
|
while not audio_q.empty():
|
|
try:
|
|
ctrl_audio_q.put(audio_q.get_nowait())
|
|
except queue.Empty:
|
|
break
|
|
|
|
|
|
def get_model_name(args: argparse.Namespace, client: openai.OpenAI) -> str:
|
|
if args.model:
|
|
return args.model
|
|
if args.backend == "llama":
|
|
try:
|
|
models = client.models.list()
|
|
if models.data:
|
|
return models.data[0].id
|
|
except Exception:
|
|
pass
|
|
return "default"
|
|
return BACKENDS[args.backend]["default_model"] or "default"
|
|
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
# Steuer-Wort-Monitor (Hintergrund-Thread während VORLESE)
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
|
|
def ctrl_monitor_fn(
|
|
ctrl_audio_q: queue.Queue,
|
|
ctrl_cmd_q: queue.Queue,
|
|
stop_now: threading.Event,
|
|
pause_now: threading.Event,
|
|
stop_monitor: threading.Event,
|
|
whisper: WhisperModel,
|
|
capture_rate: int,
|
|
) -> None:
|
|
buffer = np.empty(0, dtype=np.float32)
|
|
window_samples = int(CTRL_WINDOW_SEC * capture_rate)
|
|
check_samples = int(CTRL_CHECK_SEC * capture_rate)
|
|
accumulated = 0
|
|
|
|
while not stop_monitor.is_set():
|
|
try:
|
|
chunk = ctrl_audio_q.get(timeout=0.1)
|
|
except queue.Empty:
|
|
continue
|
|
|
|
buffer = np.concatenate([buffer, chunk])
|
|
if len(buffer) > window_samples:
|
|
buffer = buffer[-window_samples:]
|
|
accumulated += len(chunk)
|
|
|
|
if accumulated < check_samples:
|
|
continue
|
|
accumulated = 0
|
|
|
|
window_16k = resample_poly(buffer, WHISPER_RATE, capture_rate).astype(np.float32)
|
|
segs, _ = whisper.transcribe(window_16k, language="de",
|
|
vad_filter=False, beam_size=1)
|
|
text = " ".join(s.text for s in segs).strip()
|
|
if not text:
|
|
continue
|
|
|
|
cmd = detect_command(text)
|
|
if cmd:
|
|
print(f"\n[Steuerbefehl: {cmd} ← \"{normalize_cmd(text)}\"]", flush=True)
|
|
ctrl_cmd_q.put(cmd)
|
|
if cmd == CMD_STOP:
|
|
stop_now.set()
|
|
elif cmd == CMD_PAUSE and not pause_now.is_set():
|
|
pause_now.set()
|
|
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
# Hauptschleife
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
|
|
def run(args: argparse.Namespace) -> None:
|
|
# ── Geräte & Modelle laden ────────────────────────────────────────────
|
|
device_idx, capture_rate, device_label = resolve_mic(args.mic)
|
|
capture_chunk = round(WW_CHUNK_16K * capture_rate / WHISPER_RATE)
|
|
|
|
print("Lade Whisper …", flush=True)
|
|
whisper = WhisperModel(args.whisper_model, device="cuda",
|
|
device_index=args.gpu, compute_type="float16")
|
|
|
|
print("Lade Piper-Stimme …", flush=True)
|
|
tts_voice = get_voice(args.voice)
|
|
|
|
if not args.no_wakeword:
|
|
print("Lade Wake-Word-Modell …", flush=True)
|
|
ww_model = WakeWordModel()
|
|
else:
|
|
ww_model = None
|
|
|
|
backend_cfg = dict(BACKENDS[args.backend]) # Kopie — kein Mutation globaler State
|
|
if args.api_url:
|
|
backend_cfg["base_url"] = args.api_url
|
|
if args.api_key:
|
|
backend_cfg["api_key"] = args.api_key
|
|
elif args.backend == "openai" and not backend_cfg["api_key"]:
|
|
backend_cfg["api_key"] = os.getenv("OPENAI_API_KEY", "")
|
|
if not backend_cfg["api_key"] and "openrouter" in (args.api_url or ""):
|
|
backend_cfg["api_key"] = os.getenv("OPENROUTER_API_KEY", "")
|
|
client = openai.OpenAI(
|
|
base_url=backend_cfg["base_url"],
|
|
api_key =backend_cfg["api_key"] or "none",
|
|
)
|
|
model_name = get_model_name(args, client)
|
|
|
|
print(f"Mikrofon: {device_label} ({capture_rate} Hz)")
|
|
print(f"Backend : {args.backend} | Modell: {model_name}")
|
|
print(f"Stimme : {args.voice} | Ausgabe: {args.out} | Englisch: {EN_VARIANT_VOICES[args.en_variant]}")
|
|
print(f"Sprache : {args.lang} | Verlauf: {args.history} Turns")
|
|
print(f"Stille : {args.silence_sec}s | Dialog: {args.dialog_timeout}s | Stop-Wort: {args.stop_word!r}")
|
|
|
|
set_en_variant(args.en_variant)
|
|
|
|
# ── Shared State ──────────────────────────────────────────────────────
|
|
history: list[dict] = []
|
|
audio_q: queue.Queue = queue.Queue()
|
|
stop_word_detected: threading.Event = threading.Event()
|
|
stop_check_busy: threading.Event = threading.Event()
|
|
abort_to_lausch: threading.Event = threading.Event() # Stopp in AUFNAHME/GENERATOR/DIALOG
|
|
abort_dialog_busy: threading.Event = threading.Event() # verhindert parallele DIALOG-Checks
|
|
dialog_buf: list[np.ndarray] = [] # rollierende Aufnahme für DIALOG-Check
|
|
dialog_check_n: list[int] = [0] # Chunk-Zähler für DIALOG-Check
|
|
|
|
# ── Audio-Callback (läuft in PortAudio-Thread) ────────────────────────
|
|
def audio_callback(indata: np.ndarray, frames: int, t, status) -> None:
|
|
if status and "input overflow" not in str(status):
|
|
print(f"[audio] {status}", file=sys.stderr)
|
|
audio_q.put(indata[:, 0].copy())
|
|
|
|
# ── Stop-Wort- / Stopp-Check für AUFNAHME (Hintergrund-Thread) ──────────
|
|
def _check_stop_word(buf: np.ndarray) -> None:
|
|
try:
|
|
w16k = resample_poly(buf, WHISPER_RATE, capture_rate).astype(np.float32)
|
|
segs, _ = whisper.transcribe(w16k, language=None, vad_filter=False, beam_size=1)
|
|
text = " ".join(s.text for s in segs).strip()
|
|
if detect_command(text) == CMD_STOP:
|
|
print("\n[Stopp erkannt — Aufnahme abbrechen]", flush=True)
|
|
abort_to_lausch.set()
|
|
elif args.stop_word and args.stop_word.lower() in text.lower().split()[-5:]:
|
|
print(f"\n[Stop-Wort '{args.stop_word}' erkannt]", flush=True)
|
|
stop_word_detected.set()
|
|
finally:
|
|
stop_check_busy.clear()
|
|
|
|
# ── Stopp-Check für DIALOG (Hintergrund-Thread) ───────────────────────
|
|
def _check_abort_dialog(buf: np.ndarray) -> None:
|
|
try:
|
|
w16k = resample_poly(buf, WHISPER_RATE, capture_rate).astype(np.float32)
|
|
segs, _ = whisper.transcribe(w16k, language=None, vad_filter=False, beam_size=1)
|
|
text = " ".join(s.text for s in segs).strip()
|
|
if detect_command(text) == CMD_STOP:
|
|
print("\n[Stopp erkannt in DIALOG]", flush=True)
|
|
abort_to_lausch.set()
|
|
finally:
|
|
abort_dialog_busy.clear()
|
|
|
|
# ── GENERATOR + VORLESE (blockierend, läuft im Haupt-Thread) ─────────
|
|
def _handle_turn(audio_data: np.ndarray) -> bool:
|
|
"""Transkription → LLM → TTS. Blockiert bis Vorlesen beendet.
|
|
Gibt True zurück wenn etwas verstanden wurde, sonst False."""
|
|
nonlocal history
|
|
|
|
# Sofort-Abbruch: wurde "Stopp" schon in AUFNAHME erkannt?
|
|
if abort_to_lausch.is_set():
|
|
abort_to_lausch.clear()
|
|
return False
|
|
|
|
# ── GENERATOR: Transkription ──────────────────────────────────────
|
|
show_status(State.GENERATOR, "Transkription …")
|
|
status_newline()
|
|
|
|
audio_16k = resample_poly(audio_data, WHISPER_RATE, capture_rate).astype(np.float32)
|
|
segments, _ = whisper.transcribe(
|
|
audio_16k, language=args.lang, vad_filter=True,
|
|
vad_parameters={"min_silence_duration_ms": 500}, beam_size=5,
|
|
)
|
|
user_text = " ".join(s.text.strip() for s in segments).strip()
|
|
|
|
if args.stop_word and user_text:
|
|
user_text = re.sub(
|
|
r'[,.]?\s*\b' + re.escape(args.stop_word) + r'[.,!?]?\s*$',
|
|
'', user_text, flags=re.IGNORECASE
|
|
).strip()
|
|
|
|
# Nach Transkription: Stopp nochmals prüfen
|
|
if abort_to_lausch.is_set():
|
|
abort_to_lausch.clear()
|
|
return False
|
|
|
|
if not user_text:
|
|
print("(nichts verstanden)", flush=True)
|
|
return False
|
|
|
|
print(f"Du: {user_text}", flush=True)
|
|
|
|
history.append({"role": "user", "content": user_text})
|
|
if len(history) > args.history * 2:
|
|
del history[: len(history) - args.history * 2]
|
|
|
|
messages = [{"role": "system", "content": args.system}] + history
|
|
|
|
# Quittungssatz sprechen
|
|
show_status(State.GENERATOR, args.ack_text)
|
|
quick_play(args.ack_text, tts_voice, args.out)
|
|
status_newline()
|
|
|
|
# Abort-Monitor für LLM-Phase (liest audio_q, kein Whisper-Konflikt jetzt)
|
|
_abort_gen_stop = threading.Event()
|
|
def _abort_gen_monitor() -> None:
|
|
buf_g = np.empty(0, dtype=np.float32)
|
|
window = int(CTRL_WINDOW_SEC * capture_rate)
|
|
check = int(CTRL_CHECK_SEC * capture_rate)
|
|
acc = 0
|
|
while not _abort_gen_stop.is_set():
|
|
try:
|
|
chunk = audio_q.get(timeout=0.1)
|
|
except queue.Empty:
|
|
continue
|
|
buf_g = np.concatenate([buf_g, chunk])[-window:]
|
|
acc += len(chunk)
|
|
if acc < check:
|
|
continue
|
|
acc = 0
|
|
w16k = resample_poly(buf_g, WHISPER_RATE, capture_rate).astype(np.float32)
|
|
segs, _ = whisper.transcribe(w16k, language=None, vad_filter=False, beam_size=1)
|
|
text = " ".join(s.text for s in segs).strip()
|
|
if detect_command(text) == CMD_STOP:
|
|
print("\n[Stopp erkannt in GENERATOR]", flush=True)
|
|
abort_to_lausch.set()
|
|
break
|
|
abort_gen_th = threading.Thread(target=_abort_gen_monitor, daemon=True)
|
|
abort_gen_th.start()
|
|
|
|
# ── LLM-Stream + Synthese (Hintergrund-Threads) ───────────────────
|
|
sentences: list[str] = []
|
|
sentence_bytes: list[bytes | None] = []
|
|
paragraph_starts: list[int] = [0]
|
|
full_parts: list[str] = []
|
|
llm_done = threading.Event()
|
|
|
|
# Spracherkennung: sobald ≥60 Zeichen LLM-Text vorliegen
|
|
response_voice = [tts_voice] # mutable holder
|
|
voice_load_ok = [True] # False → Voice-Download fehlgeschlagen
|
|
lang_detected_ev = threading.Event()
|
|
|
|
def _llm_thread() -> None:
|
|
token_buffer = ""
|
|
try:
|
|
stream = client.chat.completions.create(
|
|
model=model_name, messages=messages, stream=True
|
|
)
|
|
for llm_chunk in stream:
|
|
delta = (llm_chunk.choices[0].delta.content or "") \
|
|
if llm_chunk.choices else ""
|
|
token_buffer += delta
|
|
full_parts.append(delta)
|
|
|
|
# Spracherkennung auslösen sobald genug Text da ist
|
|
if not lang_detected_ev.is_set():
|
|
accumulated = "".join(full_parts)
|
|
if len(accumulated) >= 60:
|
|
lang = detect_lang(accumulated)
|
|
try:
|
|
v = voice_for_lang(lang, args.voice)
|
|
if v is not tts_voice:
|
|
voice_name = LANG_VOICES.get(lang or "", args.voice)
|
|
print(f"[Sprache erkannt: {lang} → {voice_name}]",
|
|
flush=True)
|
|
response_voice[0] = v
|
|
except Exception as exc:
|
|
print(f"[Stimme nicht geladen: {exc}]",
|
|
file=sys.stderr, flush=True)
|
|
voice_load_ok[0] = False
|
|
finally:
|
|
lang_detected_ev.set()
|
|
|
|
# Absatz-Grenze → neuen Absatz markieren
|
|
while "\n\n" in token_buffer:
|
|
idx = token_buffer.index("\n\n")
|
|
fragment = token_buffer[:idx].strip()
|
|
token_buffer = token_buffer[idx + 2:]
|
|
if len(fragment) > 3:
|
|
print(f" → {fragment}")
|
|
sentences.append(fragment)
|
|
sentence_bytes.append(None)
|
|
if sentences:
|
|
paragraph_starts.append(len(sentences))
|
|
|
|
# Satz-Grenzen extrahieren
|
|
while True:
|
|
m = re.search(r'(?<=[.!?])\s+(?=[^\s])', token_buffer)
|
|
if not m:
|
|
break
|
|
s = token_buffer[:m.start() + 1].strip()
|
|
token_buffer = token_buffer[m.end():]
|
|
if len(s) > 3:
|
|
print(f" → {s}")
|
|
sentences.append(s)
|
|
sentence_bytes.append(None)
|
|
|
|
except Exception as exc:
|
|
print(f"[LLM-Fehler] {exc}", file=sys.stderr)
|
|
|
|
remainder = token_buffer.strip()
|
|
if len(remainder) > 3:
|
|
print(f" → {remainder}")
|
|
sentences.append(remainder)
|
|
sentence_bytes.append(None)
|
|
|
|
# Fallback: LLM fertig aber <60 Zeichen → Default-Stimme
|
|
if not lang_detected_ev.is_set():
|
|
lang = detect_lang("".join(full_parts))
|
|
try:
|
|
response_voice[0] = voice_for_lang(lang, args.voice)
|
|
except Exception as exc:
|
|
print(f"[Stimme nicht geladen: {exc}]",
|
|
file=sys.stderr, flush=True)
|
|
voice_load_ok[0] = False
|
|
finally:
|
|
lang_detected_ev.set()
|
|
|
|
llm_done.set()
|
|
|
|
def _synth_thread() -> None:
|
|
lang_detected_ev.wait() # wartet auf Spracherkennung
|
|
if not voice_load_ok[0]:
|
|
return # keine Synthese mit falscher Stimme
|
|
voice_to_use = response_voice[0]
|
|
idx = 0
|
|
while True:
|
|
if idx < len(sentences) and sentence_bytes[idx] is None:
|
|
sentence_bytes[idx] = synth_bytes(sentences[idx], voice_to_use)
|
|
idx += 1
|
|
elif llm_done.is_set() and idx >= len(sentences):
|
|
break
|
|
else:
|
|
time.sleep(0.01)
|
|
|
|
threading.Thread(target=_llm_thread, daemon=True).start()
|
|
threading.Thread(target=_synth_thread, daemon=True).start()
|
|
|
|
# ── VORLESE ───────────────────────────────────────────────────────
|
|
show_status(State.VORLESE)
|
|
|
|
ctrl_audio_q: queue.Queue = queue.Queue()
|
|
ctrl_cmd_q: queue.Queue = queue.Queue()
|
|
stop_now = threading.Event()
|
|
pause_now = threading.Event()
|
|
stop_mon = threading.Event()
|
|
|
|
threading.Thread(
|
|
target=ctrl_monitor_fn,
|
|
args=(ctrl_audio_q, ctrl_cmd_q, stop_now, pause_now,
|
|
stop_mon, whisper, capture_rate),
|
|
daemon=True,
|
|
).start()
|
|
|
|
# paplay-Befehlsbasis: nach Spracherkennung (richtige Sample-Rate)
|
|
lang_detected_ev.wait()
|
|
|
|
# Abort-Monitor beenden bevor ctrl_monitor Whisper nutzt (kein GPU-Konflikt)
|
|
_abort_gen_stop.set()
|
|
abort_gen_th.join(timeout=0.6)
|
|
|
|
if abort_to_lausch.is_set():
|
|
abort_to_lausch.clear()
|
|
stop_mon.set()
|
|
status_newline()
|
|
return False
|
|
|
|
# ── Voice-Load fehlgeschlagen → Vorlesen überspringen ────────────
|
|
if not voice_load_ok[0]:
|
|
llm_done.wait() # warten bis LLM fertig (vollständiger Text)
|
|
stop_mon.set()
|
|
status_newline()
|
|
print("[Stimme nicht verfügbar — Text:]", flush=True)
|
|
print("".join(full_parts), flush=True)
|
|
quick_play(
|
|
"Entschuldigung, die Stimme für diese Sprache konnte nicht geladen werden. "
|
|
"Der Text ist auf dem Bildschirm zu sehen.",
|
|
tts_voice, args.out,
|
|
)
|
|
history.append({"role": "assistant", "content": "".join(full_parts)})
|
|
return True
|
|
|
|
sink = resolve_sink(args.out)
|
|
paplay_base = [
|
|
"paplay", "--raw",
|
|
f"--rate={response_voice[0].config.sample_rate}",
|
|
"--channels=1", "--format=s16le",
|
|
]
|
|
if sink:
|
|
paplay_base += [f"--device={sink}"]
|
|
|
|
def _wait_for_resume_or_stop() -> str:
|
|
"""Blockiert im Pause-Zustand bis RESUME oder STOP eintrifft."""
|
|
while True:
|
|
_drain_audio(audio_q, ctrl_audio_q)
|
|
try:
|
|
cmd2 = ctrl_cmd_q.get(timeout=0.1)
|
|
except queue.Empty:
|
|
continue
|
|
if cmd2 in (CMD_RESUME, CMD_STOP):
|
|
return cmd2
|
|
|
|
play_idx = 0
|
|
|
|
while True:
|
|
# Fertig?
|
|
if play_idx >= len(sentences) and llm_done.is_set():
|
|
break
|
|
|
|
# Warte auf synthetisierte Bytes (draint dabei audio_q)
|
|
if play_idx >= len(sentence_bytes) or sentence_bytes[play_idx] is None:
|
|
_drain_audio(audio_q, ctrl_audio_q)
|
|
time.sleep(0.02)
|
|
continue
|
|
|
|
# Befehl zwischen Sätzen prüfen (non-blocking)
|
|
try:
|
|
cmd = ctrl_cmd_q.get_nowait()
|
|
except queue.Empty:
|
|
cmd = None
|
|
|
|
if cmd == CMD_STOP:
|
|
break
|
|
elif cmd == CMD_REPEAT:
|
|
play_idx = max(0, play_idx - 1)
|
|
continue
|
|
elif cmd == CMD_PARAGRAPH:
|
|
para = 0
|
|
for ps in paragraph_starts:
|
|
if ps <= play_idx:
|
|
para = ps
|
|
play_idx = para
|
|
continue
|
|
elif cmd == CMD_RESTART:
|
|
play_idx = 0
|
|
continue
|
|
|
|
# Satz abspielen
|
|
label = sentences[play_idx][:50].replace('\n', ' ')
|
|
show_status(State.VORLESE, f"[{play_idx + 1}/{len(sentences)}] {label}")
|
|
|
|
proc = subprocess.Popen(paplay_base, stdin=subprocess.PIPE)
|
|
proc.stdin.write(sentence_bytes[play_idx])
|
|
proc.stdin.close()
|
|
|
|
finished_normally = True
|
|
while proc.poll() is None:
|
|
_drain_audio(audio_q, ctrl_audio_q)
|
|
if stop_now.is_set():
|
|
proc.kill()
|
|
finished_normally = False
|
|
break
|
|
if pause_now.is_set():
|
|
proc.send_signal(signal.SIGSTOP)
|
|
show_status(State.VORLESE, "[PAUSE — 'Weiter' zum Fortfahren]")
|
|
cmd2 = _wait_for_resume_or_stop()
|
|
proc.kill()
|
|
pause_now.clear()
|
|
if cmd2 == CMD_STOP:
|
|
stop_now.set()
|
|
# CMD_RESUME: finished_normally bleibt False → Satz von vorne
|
|
finished_normally = False
|
|
break
|
|
time.sleep(0.03)
|
|
|
|
if finished_normally:
|
|
play_idx += 1
|
|
if stop_now.is_set():
|
|
break
|
|
|
|
stop_mon.set()
|
|
status_newline()
|
|
history.append({"role": "assistant", "content": "".join(full_parts)})
|
|
return True
|
|
|
|
# ── Zustands-Maschine ─────────────────────────────────────────────────
|
|
state = State.AUFNAHME if args.no_wakeword else State.LAUSCH
|
|
record_buffer: list[np.ndarray] = []
|
|
silence_chunks = 0
|
|
check_chunks = 0
|
|
dialog_deadline = 0.0
|
|
ww_cooldown_chunks = 0 # Chunks verbleibend ohne Wake-Word-Trigger
|
|
|
|
print("\nBereit." if args.no_wakeword
|
|
else '\nWarte auf "Hey Jarvis" …', flush=True)
|
|
|
|
with sd.InputStream(
|
|
device=device_idx, channels=1, samplerate=capture_rate,
|
|
dtype="float32", blocksize=capture_chunk, callback=audio_callback,
|
|
):
|
|
while True:
|
|
chunk_raw = audio_q.get()
|
|
chunk_16k = resample_poly(chunk_raw, WHISPER_RATE, capture_rate).astype(np.float32)
|
|
rms = float(np.sqrt(np.mean(chunk_raw ** 2)))
|
|
|
|
# ── LAUSCH ────────────────────────────────────────────────────
|
|
if state == State.LAUSCH:
|
|
show_status(state, 'Warte auf "Hey Jarvis" …')
|
|
chunk_int16 = (chunk_16k * 32767).astype(np.int16)
|
|
score = ww_model.predict(chunk_int16).get("hey_jarvis", 0)
|
|
if ww_cooldown_chunks > 0:
|
|
ww_cooldown_chunks -= 1
|
|
elif score >= args.wakeword_threshold:
|
|
status_newline()
|
|
play_chime(args.out)
|
|
state = State.AUFNAHME
|
|
record_buffer = []
|
|
silence_chunks = check_chunks = 0
|
|
show_status(state, "Sprechen …")
|
|
status_newline()
|
|
|
|
# ── AUFNAHME ──────────────────────────────────────────────────
|
|
elif state == State.AUFNAHME:
|
|
if abort_to_lausch.is_set():
|
|
abort_to_lausch.clear()
|
|
stop_check_busy.clear()
|
|
record_buffer = []
|
|
silence_chunks = check_chunks = 0
|
|
status_newline()
|
|
state = State.AUFNAHME if args.no_wakeword else State.LAUSCH
|
|
if not args.no_wakeword:
|
|
ww_cooldown_chunks = round(WW_COOLDOWN_SEC * capture_rate / capture_chunk)
|
|
continue
|
|
|
|
record_buffer.append(chunk_raw)
|
|
silence_chunks = (silence_chunks + 1) if rms < SILENCE_RMS else 0
|
|
|
|
silence_elapsed = silence_chunks * capture_chunk / capture_rate
|
|
total_elapsed = len(record_buffer) * capture_chunk / capture_rate
|
|
show_status(state, f"{total_elapsed:.1f}s (Stille {silence_elapsed:.1f}s)")
|
|
|
|
# Stop-Wort-Check im Hintergrund
|
|
if args.stop_word:
|
|
check_chunks += 1
|
|
if check_chunks * capture_chunk / capture_rate >= STOP_CHECK_SEC:
|
|
check_chunks = 0
|
|
if not stop_check_busy.is_set():
|
|
stop_check_busy.set()
|
|
win = int(STOP_WINDOW_SEC * capture_rate / capture_chunk)
|
|
snap = np.concatenate(record_buffer[-win:]).copy()
|
|
threading.Thread(
|
|
target=_check_stop_word, args=(snap,), daemon=True
|
|
).start()
|
|
|
|
if silence_elapsed >= args.silence_sec \
|
|
or total_elapsed >= MAX_RECORD_SEC \
|
|
or stop_word_detected.is_set():
|
|
stop_word_detected.clear()
|
|
stop_check_busy.clear()
|
|
status_newline()
|
|
buf = np.concatenate(record_buffer)
|
|
record_buffer = []
|
|
silence_chunks = check_chunks = 0
|
|
|
|
understood = _handle_turn(buf) # blockiert während GENERATOR + VORLESE
|
|
|
|
# Audio aus der VORLESE-Phase verwerfen
|
|
while not audio_q.empty():
|
|
audio_q.get_nowait()
|
|
|
|
if understood:
|
|
state = State.DIALOG
|
|
dialog_deadline = time.time() + args.dialog_timeout
|
|
show_status(state, f"Sprechen? ({args.dialog_timeout:.0f}s)")
|
|
else:
|
|
state = State.AUFNAHME if args.no_wakeword else State.LAUSCH
|
|
if not args.no_wakeword:
|
|
ww_cooldown_chunks = round(WW_COOLDOWN_SEC * capture_rate / capture_chunk)
|
|
record_buffer = []
|
|
silence_chunks = check_chunks = 0
|
|
|
|
# ── DIALOG ────────────────────────────────────────────────────
|
|
elif state == State.DIALOG:
|
|
# Sofort-Abbruch durch "Stopp"
|
|
if abort_to_lausch.is_set():
|
|
abort_to_lausch.clear()
|
|
abort_dialog_busy.clear()
|
|
dialog_buf.clear()
|
|
dialog_check_n[0] = 0
|
|
status_newline()
|
|
state = State.AUFNAHME if args.no_wakeword else State.LAUSCH
|
|
if not args.no_wakeword:
|
|
ww_cooldown_chunks = round(WW_COOLDOWN_SEC * capture_rate / capture_chunk)
|
|
continue
|
|
|
|
remaining = dialog_deadline - time.time()
|
|
show_status(state, f"Sprechen? ({remaining:.1f}s)")
|
|
|
|
# Wakeword-Modell warm halten (kein Trigger im DIALOG)
|
|
if not args.no_wakeword:
|
|
chunk_int16 = (chunk_16k * 32767).astype(np.int16)
|
|
ww_model.predict(chunk_int16)
|
|
|
|
# Periodischer Stopp-Check im Hintergrund
|
|
dialog_buf.append(chunk_raw)
|
|
max_dbuf = round(CTRL_WINDOW_SEC * capture_rate / capture_chunk)
|
|
if len(dialog_buf) > max_dbuf:
|
|
dialog_buf.pop(0)
|
|
dialog_check_n[0] += 1
|
|
if dialog_check_n[0] * capture_chunk / capture_rate >= CTRL_CHECK_SEC:
|
|
dialog_check_n[0] = 0
|
|
if not abort_dialog_busy.is_set():
|
|
abort_dialog_busy.set()
|
|
snap = np.concatenate(dialog_buf).copy()
|
|
threading.Thread(
|
|
target=_check_abort_dialog, args=(snap,), daemon=True
|
|
).start()
|
|
|
|
if remaining <= 0:
|
|
dialog_buf.clear()
|
|
dialog_check_n[0] = 0
|
|
abort_dialog_busy.clear()
|
|
status_newline()
|
|
# --no-wakeword: sofort wieder aufnahmebereit
|
|
state = State.AUFNAHME if args.no_wakeword else State.LAUSCH
|
|
if args.no_wakeword:
|
|
record_buffer = []
|
|
silence_chunks = check_chunks = 0
|
|
show_status(state, "Sprechen …")
|
|
status_newline()
|
|
else:
|
|
ww_cooldown_chunks = round(WW_COOLDOWN_SEC * capture_rate / capture_chunk)
|
|
show_status(state, 'Warte auf "Hey Jarvis" …')
|
|
|
|
elif rms > SILENCE_RMS:
|
|
dialog_buf.clear()
|
|
dialog_check_n[0] = 0
|
|
status_newline()
|
|
state = State.AUFNAHME
|
|
record_buffer = [chunk_raw]
|
|
silence_chunks = check_chunks = 0
|
|
show_status(state, "Sprechen …")
|
|
status_newline()
|
|
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
# Argumente
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
|
|
def parse_args() -> argparse.Namespace:
|
|
p = argparse.ArgumentParser(
|
|
description="Sprachassistent: 5-Zustands-Maschine mit Steuer-Wörtern",
|
|
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
|
|
)
|
|
p.add_argument("--mic", default=DEFAULT_MIC,
|
|
help="Eingabegerät: respeaker | motu | camera | default | bluetooth | <idx>")
|
|
p.add_argument("--list-mics", action="store_true",
|
|
help="Verfügbare Mikrofone anzeigen und beenden")
|
|
p.add_argument("--backend", default="llama",
|
|
choices=["llama", "ollama", "openai"], help="LLM-Backend")
|
|
p.add_argument("--model", default=None,
|
|
help="LLM-Modellname (default: auto-detect)")
|
|
p.add_argument("--api-url", default=None,
|
|
help="Überschreibt die Backend-URL (z. B. https://openrouter.ai/api/v1)")
|
|
p.add_argument("--api-key", default=None,
|
|
help="API-Key (überschreibt OPENAI_API_KEY / Backend-Default)")
|
|
p.add_argument("--whisper-model", default="large-v3")
|
|
p.add_argument("--gpu", type=int, default=1,
|
|
help="CUDA-Index für Whisper (in CUDA_VISIBLE_DEVICES)")
|
|
p.add_argument("--lang", default="de")
|
|
p.add_argument("--voice", default="de_DE-thorsten-high")
|
|
p.add_argument("--en-variant", default="us", choices=["us", "gb"],
|
|
help="Englische TTS-Variante: us (en_US-ryan-high) | gb (en_GB-alan-medium)")
|
|
p.add_argument("--out", default="default",
|
|
help="Ausgabe-Sink: default | respeaker | hdmi | motu | bluetooth | <name>")
|
|
p.add_argument("--system", default=DEFAULT_SYSTEM)
|
|
p.add_argument("--history", type=int, default=10,
|
|
help="Max. Konversations-Turns")
|
|
p.add_argument("--wakeword-threshold", type=float, default=0.5)
|
|
p.add_argument("--no-wakeword", action="store_true",
|
|
help="Wake-Word deaktivieren — direkt in AUFNAHME starten")
|
|
p.add_argument("--stop-word", default="over",
|
|
help="Codewort zum Beenden der Aufnahme")
|
|
p.add_argument("--silence-sec", type=float, default=5.0,
|
|
help="Stille-Dauer bis Aufnahme endet")
|
|
p.add_argument("--dialog-timeout", type=float, default=5.0,
|
|
help="DIALOG-Fenster nach Vorlesen (Sekunden)")
|
|
p.add_argument("--ack-text", default="Moment, ich antworte gleich.",
|
|
help="Quittungssatz im GENERATOR-Zustand")
|
|
return p.parse_args()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
args = parse_args()
|
|
if args.list_mics:
|
|
print_mic_list()
|
|
sys.exit(0)
|
|
try:
|
|
run(args)
|
|
except KeyboardInterrupt:
|
|
print("\nBeendet.")
|