Jamulix: Optimierungen übernehmen, lokale Voice-Modelle ignorieren
This commit is contained in:
parent
e99508a727
commit
91b9ef3374
12 changed files with 555 additions and 94 deletions
|
|
@ -1,3 +1,5 @@
|
|||
import asyncio
|
||||
import time
|
||||
from collections.abc import AsyncIterator
|
||||
|
||||
import httpx
|
||||
|
|
@ -9,6 +11,37 @@ from app.providers.llm.base import (
|
|||
sse_delta,
|
||||
)
|
||||
|
||||
_RETRY_STATUS = {404, 429, 500, 502, 503}
|
||||
|
||||
|
||||
def _retry_wait(status: int, headers: httpx.Headers, attempt: int, body: str = "") -> float:
|
||||
"""Wartezeit: JSON retry_after_seconds > Retry-After > X-RateLimit-Reset > exponentiell."""
|
||||
if status == 429:
|
||||
if body:
|
||||
try:
|
||||
import json as _json
|
||||
d = _json.loads(body)
|
||||
secs = d.get("error", {}).get("metadata", {}).get("retry_after_seconds")
|
||||
if secs:
|
||||
return max(1.0, min(float(secs) + 0.5, 120.0))
|
||||
except Exception:
|
||||
pass
|
||||
retry_after = headers.get("Retry-After")
|
||||
if retry_after:
|
||||
try:
|
||||
return max(1.0, min(float(retry_after) + 0.5, 120.0))
|
||||
except ValueError:
|
||||
pass
|
||||
reset_ms = headers.get("X-RateLimit-Reset")
|
||||
if reset_ms:
|
||||
try:
|
||||
wait = (int(reset_ms) / 1000.0) - time.time() + 0.5
|
||||
return max(1.0, min(wait, 120.0))
|
||||
except ValueError:
|
||||
pass
|
||||
return 30.0
|
||||
return 2.0 * attempt
|
||||
|
||||
|
||||
SYSTEM_PROMPT = """
|
||||
You are a voice assistant for spoken conversations with older adults.
|
||||
|
|
@ -52,9 +85,10 @@ Always optimize your answer for listening, not for reading.
|
|||
|
||||
|
||||
class OpenRouterLLMProvider(LLMProvider):
|
||||
def __init__(self, api_key: str, model: str):
|
||||
def __init__(self, api_key: str, model: str, max_retries: int = 5):
|
||||
self.api_key = (api_key or "").strip()
|
||||
self.model = (model or "").strip()
|
||||
self.max_retries = max(1, max_retries)
|
||||
|
||||
def _build_messages(
|
||||
self, text: str, history: list[dict] | None, language: str | None = None
|
||||
|
|
@ -74,7 +108,6 @@ class OpenRouterLLMProvider(LLMProvider):
|
|||
messages = [{"role": "system", "content": system_content}]
|
||||
if history:
|
||||
messages.extend(history)
|
||||
# Sprach-Erinnerung direkt an der letzten Nutzer-Nachricht (schlägt History-Trägheit).
|
||||
messages.append({"role": "user", "content": with_lang_reminder(text.strip(), language)})
|
||||
return messages
|
||||
|
||||
|
|
@ -89,40 +122,52 @@ class OpenRouterLLMProvider(LLMProvider):
|
|||
"model": self.model,
|
||||
"messages": self._build_messages(text, history, language=language),
|
||||
}
|
||||
|
||||
timeout = httpx.Timeout(connect=10.0, read=120.0, write=30.0, pool=10.0)
|
||||
last_error: Exception | None = None
|
||||
|
||||
async with httpx.AsyncClient(timeout=timeout) as client:
|
||||
for attempt in range(1, self.max_retries + 1):
|
||||
async with httpx.AsyncClient(timeout=timeout) as client:
|
||||
try:
|
||||
response = await client.post(
|
||||
"https://openrouter.ai/api/v1/chat/completions",
|
||||
headers={
|
||||
"Authorization": f"Bearer {self.api_key}",
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
json=payload,
|
||||
)
|
||||
if response.status_code in _RETRY_STATUS and attempt < self.max_retries:
|
||||
last_error = RuntimeError(
|
||||
f"OpenRouter LLM error {response.status_code}: {response.text}"
|
||||
)
|
||||
await asyncio.sleep(_retry_wait(response.status_code, response.headers, attempt, response.text))
|
||||
continue
|
||||
response.raise_for_status()
|
||||
except httpx.HTTPStatusError as exc:
|
||||
last_error = RuntimeError(
|
||||
f"OpenRouter LLM error {exc.response.status_code}: {exc.response.text}"
|
||||
)
|
||||
if exc.response.status_code in _RETRY_STATUS and attempt < self.max_retries:
|
||||
await asyncio.sleep(_retry_wait(exc.response.status_code, exc.response.headers, attempt, exc.response.text))
|
||||
continue
|
||||
raise last_error from exc
|
||||
except httpx.TimeoutException as exc:
|
||||
raise RuntimeError("OpenRouter LLM timeout") from exc
|
||||
except httpx.HTTPError as exc:
|
||||
raise RuntimeError(f"OpenRouter LLM transport error: {exc}") from exc
|
||||
|
||||
data = response.json()
|
||||
try:
|
||||
response = await client.post(
|
||||
"https://openrouter.ai/api/v1/chat/completions",
|
||||
headers={
|
||||
"Authorization": f"Bearer {self.api_key}",
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
json=payload,
|
||||
)
|
||||
response.raise_for_status()
|
||||
except httpx.HTTPStatusError as exc:
|
||||
raise RuntimeError(
|
||||
f"OpenRouter LLM error {exc.response.status_code}: {exc.response.text}"
|
||||
) from exc
|
||||
except httpx.TimeoutException as exc:
|
||||
raise RuntimeError("OpenRouter LLM timeout") from exc
|
||||
except httpx.HTTPError as exc:
|
||||
raise RuntimeError(f"OpenRouter LLM transport error: {exc}") from exc
|
||||
content = data["choices"][0]["message"]["content"]
|
||||
except (KeyError, IndexError, TypeError) as exc:
|
||||
raise RuntimeError(f"Unexpected OpenRouter LLM response: {data}") from exc
|
||||
|
||||
data = response.json()
|
||||
if not content or not str(content).strip():
|
||||
raise RuntimeError("OpenRouter LLM returned empty content")
|
||||
|
||||
try:
|
||||
content = data["choices"][0]["message"]["content"]
|
||||
except (KeyError, IndexError, TypeError) as exc:
|
||||
raise RuntimeError(f"Unexpected OpenRouter LLM response: {data}") from exc
|
||||
return str(content).strip()
|
||||
|
||||
if not content or not str(content).strip():
|
||||
raise RuntimeError("OpenRouter LLM returned empty content")
|
||||
|
||||
return str(content).strip()
|
||||
raise last_error or RuntimeError("OpenRouter LLM: all attempts failed")
|
||||
|
||||
async def stream(
|
||||
self,
|
||||
|
|
@ -137,30 +182,42 @@ class OpenRouterLLMProvider(LLMProvider):
|
|||
"stream": True,
|
||||
}
|
||||
timeout = httpx.Timeout(connect=10.0, read=120.0, write=30.0, pool=10.0)
|
||||
last_error: Exception | None = None
|
||||
|
||||
async with httpx.AsyncClient(timeout=timeout) as client:
|
||||
try:
|
||||
async with client.stream(
|
||||
"POST",
|
||||
"https://openrouter.ai/api/v1/chat/completions",
|
||||
headers={
|
||||
"Authorization": f"Bearer {self.api_key}",
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
json=payload,
|
||||
) as response:
|
||||
if response.status_code >= 400:
|
||||
body = await response.aread()
|
||||
raise RuntimeError(
|
||||
f"OpenRouter LLM error {response.status_code}: "
|
||||
f"{body.decode(errors='replace')}"
|
||||
)
|
||||
async for line in response.aiter_lines():
|
||||
delta = sse_delta(line)
|
||||
if delta:
|
||||
yield delta
|
||||
except httpx.TimeoutException as exc:
|
||||
raise RuntimeError("OpenRouter LLM timeout") from exc
|
||||
except httpx.HTTPError as exc:
|
||||
raise RuntimeError(f"OpenRouter LLM transport error: {exc}") from exc
|
||||
for attempt in range(1, self.max_retries + 1):
|
||||
async with httpx.AsyncClient(timeout=timeout) as client:
|
||||
try:
|
||||
async with client.stream(
|
||||
"POST",
|
||||
"https://openrouter.ai/api/v1/chat/completions",
|
||||
headers={
|
||||
"Authorization": f"Bearer {self.api_key}",
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
json=payload,
|
||||
) as response:
|
||||
if response.status_code in _RETRY_STATUS and attempt < self.max_retries:
|
||||
body = await response.aread()
|
||||
last_error = RuntimeError(
|
||||
f"OpenRouter LLM error {response.status_code}: "
|
||||
f"{body.decode(errors='replace')}"
|
||||
)
|
||||
await asyncio.sleep(_retry_wait(response.status_code, response.headers, attempt, body.decode(errors='replace')))
|
||||
continue
|
||||
if response.status_code >= 400:
|
||||
body = await response.aread()
|
||||
raise RuntimeError(
|
||||
f"OpenRouter LLM error {response.status_code}: "
|
||||
f"{body.decode(errors='replace')}"
|
||||
)
|
||||
async for line in response.aiter_lines():
|
||||
delta = sse_delta(line)
|
||||
if delta:
|
||||
yield delta
|
||||
return
|
||||
except httpx.TimeoutException as exc:
|
||||
raise RuntimeError("OpenRouter LLM timeout") from exc
|
||||
except httpx.HTTPError as exc:
|
||||
raise RuntimeError(f"OpenRouter LLM transport error: {exc}") from exc
|
||||
|
||||
raise last_error or RuntimeError("OpenRouter LLM: all attempts failed")
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue