feat(llm): lokales llama.cpp-Modell (va_llm) als zentrale, sprachoptimierte KI
- scripts/llm-server/: start/stop/status fuer llama.cpp-Server (Port 8001, GPU 1, Modell Qwen3.6-35B-A3B-Uncensored, Alias va_llm) - alles per ENV ueberschreibbar - Defaults auf den lokalen Server umgestellt (config.py, .example-Configs, .env.example) - Provider local-openai-compatible sprachoptimiert: Reasoning aus (chat_template_kwargs.enable_thinking=false) + knapper Sprach-System-Prompt, optional max_tokens/temperature - Antwort ~9x schneller, kurze vorlesbare Texte - Makefile-Targets llm-up/llm-down/llm-status - Doku (README, BEDIENUNGSANLEITUNG) auf llama.cpp statt Ollama aktualisiert Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
parent
cd7aef852d
commit
28c729f1d4
11 changed files with 323 additions and 31 deletions
|
|
@ -6,32 +6,61 @@ from app.providers.llm.base import LLMProvider, sse_delta
|
|||
|
||||
|
||||
class LocalOpenAICompatibleLLM(LLMProvider):
|
||||
def __init__(self, base_url: str, api_key: str, model: str):
|
||||
def __init__(
|
||||
self,
|
||||
base_url: str,
|
||||
api_key: str,
|
||||
model: str,
|
||||
system_prompt: str = "",
|
||||
disable_reasoning: bool = True,
|
||||
max_tokens: int = 0,
|
||||
temperature: float = 0.3,
|
||||
):
|
||||
self.base_url = base_url.rstrip("/")
|
||||
self.api_key = api_key
|
||||
self.model = model
|
||||
self.system_prompt = (system_prompt or "").strip()
|
||||
self.disable_reasoning = disable_reasoning
|
||||
self.max_tokens = max_tokens
|
||||
self.temperature = temperature
|
||||
|
||||
def _build_messages(self, text: str, history: list[dict] | None) -> list[dict]:
|
||||
messages = list(history) if history else []
|
||||
messages: list[dict] = []
|
||||
# Sprach-System-Prompt zuerst (knappe, vorlesbare Antworten). Etwaige
|
||||
# System-Nachrichten aus der History (z. B. Nutzer-Erinnerungen) bleiben erhalten.
|
||||
if self.system_prompt:
|
||||
messages.append({"role": "system", "content": self.system_prompt})
|
||||
if history:
|
||||
messages.extend(history)
|
||||
messages.append({"role": "user", "content": text})
|
||||
return messages
|
||||
|
||||
def _payload(self, text: str, history: list[dict] | None, stream: bool) -> dict:
|
||||
payload: dict = {
|
||||
"model": self.model,
|
||||
"messages": self._build_messages(text, history),
|
||||
"temperature": self.temperature,
|
||||
}
|
||||
if stream:
|
||||
payload["stream"] = True
|
||||
if self.max_tokens > 0:
|
||||
payload["max_tokens"] = self.max_tokens
|
||||
if self.disable_reasoning:
|
||||
# Qwen3/llama.cpp: Denkphase abschalten -> schnellere erste Antwort.
|
||||
payload["chat_template_kwargs"] = {"enable_thinking": False}
|
||||
return payload
|
||||
|
||||
async def complete(
|
||||
self,
|
||||
text: str,
|
||||
history: list[dict] | None = None,
|
||||
session_id: str | None = None,
|
||||
) -> str:
|
||||
payload = {
|
||||
"model": self.model,
|
||||
"messages": self._build_messages(text, history),
|
||||
"temperature": 0.3,
|
||||
}
|
||||
async with httpx.AsyncClient(timeout=120) as client:
|
||||
response = await client.post(
|
||||
f"{self.base_url}/chat/completions",
|
||||
headers={"Authorization": f"Bearer {self.api_key}"},
|
||||
json=payload,
|
||||
json=self._payload(text, history, stream=False),
|
||||
)
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
|
|
@ -43,18 +72,12 @@ class LocalOpenAICompatibleLLM(LLMProvider):
|
|||
history: list[dict] | None = None,
|
||||
session_id: str | None = None,
|
||||
) -> AsyncIterator[str]:
|
||||
payload = {
|
||||
"model": self.model,
|
||||
"messages": self._build_messages(text, history),
|
||||
"temperature": 0.3,
|
||||
"stream": True,
|
||||
}
|
||||
async with httpx.AsyncClient(timeout=120) as client:
|
||||
async with client.stream(
|
||||
"POST",
|
||||
f"{self.base_url}/chat/completions",
|
||||
headers={"Authorization": f"Bearer {self.api_key}"},
|
||||
json=payload,
|
||||
json=self._payload(text, history, stream=True),
|
||||
) as response:
|
||||
if response.status_code >= 400:
|
||||
body = await response.aread()
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue