feat(llm): lokales llama.cpp-Modell (va_llm) als zentrale, sprachoptimierte KI

- scripts/llm-server/: start/stop/status fuer llama.cpp-Server (Port 8001, GPU 1,
  Modell Qwen3.6-35B-A3B-Uncensored, Alias va_llm) - alles per ENV ueberschreibbar
- Defaults auf den lokalen Server umgestellt (config.py, .example-Configs, .env.example)
- Provider local-openai-compatible sprachoptimiert: Reasoning aus
  (chat_template_kwargs.enable_thinking=false) + knapper Sprach-System-Prompt,
  optional max_tokens/temperature - Antwort ~9x schneller, kurze vorlesbare Texte
- Makefile-Targets llm-up/llm-down/llm-status
- Doku (README, BEDIENUNGSANLEITUNG) auf llama.cpp statt Ollama aktualisiert

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Dieter Schlüter 2026-06-18 02:57:57 +02:00
commit 28c729f1d4
11 changed files with 323 additions and 31 deletions

View file

@ -6,32 +6,61 @@ from app.providers.llm.base import LLMProvider, sse_delta
class LocalOpenAICompatibleLLM(LLMProvider):
def __init__(self, base_url: str, api_key: str, model: str):
def __init__(
self,
base_url: str,
api_key: str,
model: str,
system_prompt: str = "",
disable_reasoning: bool = True,
max_tokens: int = 0,
temperature: float = 0.3,
):
self.base_url = base_url.rstrip("/")
self.api_key = api_key
self.model = model
self.system_prompt = (system_prompt or "").strip()
self.disable_reasoning = disable_reasoning
self.max_tokens = max_tokens
self.temperature = temperature
def _build_messages(self, text: str, history: list[dict] | None) -> list[dict]:
messages = list(history) if history else []
messages: list[dict] = []
# Sprach-System-Prompt zuerst (knappe, vorlesbare Antworten). Etwaige
# System-Nachrichten aus der History (z. B. Nutzer-Erinnerungen) bleiben erhalten.
if self.system_prompt:
messages.append({"role": "system", "content": self.system_prompt})
if history:
messages.extend(history)
messages.append({"role": "user", "content": text})
return messages
def _payload(self, text: str, history: list[dict] | None, stream: bool) -> dict:
payload: dict = {
"model": self.model,
"messages": self._build_messages(text, history),
"temperature": self.temperature,
}
if stream:
payload["stream"] = True
if self.max_tokens > 0:
payload["max_tokens"] = self.max_tokens
if self.disable_reasoning:
# Qwen3/llama.cpp: Denkphase abschalten -> schnellere erste Antwort.
payload["chat_template_kwargs"] = {"enable_thinking": False}
return payload
async def complete(
self,
text: str,
history: list[dict] | None = None,
session_id: str | None = None,
) -> str:
payload = {
"model": self.model,
"messages": self._build_messages(text, history),
"temperature": 0.3,
}
async with httpx.AsyncClient(timeout=120) as client:
response = await client.post(
f"{self.base_url}/chat/completions",
headers={"Authorization": f"Bearer {self.api_key}"},
json=payload,
json=self._payload(text, history, stream=False),
)
response.raise_for_status()
data = response.json()
@ -43,18 +72,12 @@ class LocalOpenAICompatibleLLM(LLMProvider):
history: list[dict] | None = None,
session_id: str | None = None,
) -> AsyncIterator[str]:
payload = {
"model": self.model,
"messages": self._build_messages(text, history),
"temperature": 0.3,
"stream": True,
}
async with httpx.AsyncClient(timeout=120) as client:
async with client.stream(
"POST",
f"{self.base_url}/chat/completions",
headers={"Authorization": f"Bearer {self.api_key}"},
json=payload,
json=self._payload(text, history, stream=True),
) as response:
if response.status_code >= 400:
body = await response.aread()