92 lines
2.6 KiB
YAML
92 lines
2.6 KiB
YAML
|
services:
|
|||
|
|
qwen35b:
|
|||
|
|
image: ghcr.io/ggml-org/llama.cpp:server-cuda
|
|||
|
|
container_name: qwen35b-moe-uncensored-rag # eindeutiger Name, kein Konflikt mit RAG_faehig
|
|||
|
|
restart: unless-stopped
|
|||
|
|
|
|||
|
|
ports:
|
|||
|
|
- "8000:8000"
|
|||
|
|
|
|||
|
|
environment:
|
|||
|
|
HF_HOME: /hf_home
|
|||
|
|
NVIDIA_VISIBLE_DEVICES: "1,2" # Im Host‑System: 3090 = 1,2; T600 = 0
|
|||
|
|
|
|||
|
|
volumes:
|
|||
|
|
- /home/dschlueter/nvme2n1p7_home/huggingface:/hf_home:ro
|
|||
|
|
|
|||
|
|
command:
|
|||
|
|
- -m
|
|||
|
|
- /hf_home/models/qwen3/Qwen3.6-35B-A3B-Uncensored-HauhauCS-Aggressive-Q4_K_M.gguf
|
|||
|
|
|
|||
|
|
# Kontext & Ausgabe
|
|||
|
|
|
|||
|
|
- -c
|
|||
|
|
- "262144" # 256k: ideal für RAG mit langen Retrieval-Kontexten
|
|||
|
|
- -n
|
|||
|
|
- "16384" # 16k: Begrenzung verhindert Text-Generierungs-Loops
|
|||
|
|
|
|||
|
|
# Sampler
|
|||
|
|
|
|||
|
|
- --temp
|
|||
|
|
- "0.2" # niedrig: RAG braucht faktentreue, präzise Antworten
|
|||
|
|
- --top-p
|
|||
|
|
- "0.95" # Qwen-Empfehlung
|
|||
|
|
- --top-k
|
|||
|
|
- "40" # Qwen-Empfehlung (0 = deaktiviert wäre zu unscharf)
|
|||
|
|
- --min-p
|
|||
|
|
- "0.01" # stabilisiert Sampling-Verteilung
|
|||
|
|
- --repeat-penalty
|
|||
|
|
- "1.05" # minimal: verhindert Text-Wiederholungsschleifen
|
|||
|
|
|
|||
|
|
# GPU-/Multi-GPU-Setup
|
|||
|
|
|
|||
|
|
- --main-gpu
|
|||
|
|
- "0" # erste 3090 als Haupt-GPU im Container
|
|||
|
|
- --tensor-split
|
|||
|
|
- "0.5,0.5" # symmetrisch: beide 3090 haben je 24 GB VRAM
|
|||
|
|
- -ngl
|
|||
|
|
- "999" # alle Layer auf GPU auslagern
|
|||
|
|
- -fa
|
|||
|
|
- "on" # Flash Attention: optimierte Speicherzugriffe und Matmul
|
|||
|
|
|
|||
|
|
# KV-Cache
|
|||
|
|
|
|||
|
|
- --kv-unified
|
|||
|
|
- --cache-type-k
|
|||
|
|
- q8_0 # guter Speed/Qualitäts-Kompromiss
|
|||
|
|
- --cache-type-v
|
|||
|
|
- q8_0
|
|||
|
|
|
|||
|
|
# Batching & Parallelität
|
|||
|
|
|
|||
|
|
- --batch-size
|
|||
|
|
- "2048" # großer Prompt-Batch: schnellere Verarbeitung langer RAG-Kontexte
|
|||
|
|
- --ubatch-size
|
|||
|
|
- "512" # passend zu batch-size
|
|||
|
|
- --parallel
|
|||
|
|
- "2" # 2 parallele Slots für Single-User: spart ~10 GB KV-Cache
|
|||
|
|
- --cont-batching # kontinuierliches Batching aktivieren
|
|||
|
|
|
|||
|
|
# Server
|
|||
|
|
|
|||
|
|
- --jinja
|
|||
|
|
- --no-context-shift
|
|||
|
|
- --host
|
|||
|
|
- 0.0.0.0
|
|||
|
|
- --port
|
|||
|
|
- "8000"
|
|||
|
|
|
|||
|
|
healthcheck:
|
|||
|
|
test: ["CMD-SHELL", "curl -fs http://localhost:8000/ || exit 1"]
|
|||
|
|
interval: 30s
|
|||
|
|
timeout: 5s
|
|||
|
|
retries: 3
|
|||
|
|
start_period: 120s
|
|||
|
|
|
|||
|
|
deploy:
|
|||
|
|
resources:
|
|||
|
|
reservations:
|
|||
|
|
devices:
|
|||
|
|
- driver: nvidia
|
|||
|
|
device_ids: ["1", "2"] # beide 3090 (T600 = 0, nicht verwendet)
|
|||
|
|
capabilities: [gpu]
|