pi_coder/start-judge.sh

#!/usr/bin/env bash
set -euo pipefail

HF_HOME="${HF_HOME:-/home/dschlueter/nvme2n1p7_home/huggingface}"
MODEL_REL_PATH="models/qwen3/Qwen3.6-27B-Uncensored-HauhauCS-Aggressive-IQ4_XS.gguf"
IMAGE="ghcr.io/ggml-org/llama.cpp:server-cuda"
CONTAINER_NAME="qwen36-27b-judge"
HOST_PORT=8002
CONTAINER_PORT=8000
MODEL_ALIAS="qwen3.5-judge"

echo "[*] Verwende HF_HOME = $HF_HOME"
if [ ! -f "$HF_HOME/$MODEL_REL_PATH" ]; then
  echo "[!] Modell-Datei nicht gefunden: $HF_HOME/$MODEL_REL_PATH" >&2
  exit 1
fi

if docker ps -a --format '{{.Names}}' | grep -q "^${CONTAINER_NAME}\$"; then
  echo "[*] Stoppe existierenden Container $CONTAINER_NAME ..."
  docker rm -f "$CONTAINER_NAME" >/dev/null 2>&1 || true
fi

echo "[*] Starte llama.cpp-Server für Judge ..."
docker run -d \
  --gpus '"device=1,2"' \
  --name "$CONTAINER_NAME" \
  --restart unless-stopped \
  -e HF_HOME="/hf_home" \
  -v "$HF_HOME:/hf_home:ro" \
  -p "${HOST_PORT}:${CONTAINER_PORT}" \
  "$IMAGE" \
    -m "/hf_home/${MODEL_REL_PATH}" \
    --alias "${MODEL_ALIAS}" \
    -c 262144 \
    -n 8192 \
    --jinja \
    --no-context-shift \
    --temp 0.1 \
    --top-p 0.9 \
    --top-k 40 \
    --min-p 0.01 \
    --repeat-penalty 1.05 \
    --main-gpu 0 \
    --tensor-split 0.5,0.5 \
    -ngl 999 \
    -fa on \
    --kv-unified \
    --cache-type-k q4_0 \
    --cache-type-v q4_0 \
    --batch-size 512 \
    --ubatch-size 256 \
    --parallel 1 \
    --cont-batching \
    --host 0.0.0.0 \
    --port "$CONTAINER_PORT"

echo "[*] Warte auf HTTP ..."
HTTP_READY=0
for i in {1..90}; do
  if curl -s "http://localhost:${HOST_PORT}/health" >/dev/null 2>&1 || \
     curl -s "http://localhost:${HOST_PORT}/v1/models" >/dev/null 2>&1; then
    HTTP_READY=1
    break
  fi
  sleep 2
done

if [ "$HTTP_READY" -ne 1 ]; then
  echo "[!] HTTP-Server wurde nicht rechtzeitig erreichbar." >&2
  docker logs --tail 200 "$CONTAINER_NAME" || true
  exit 1
fi

echo "[*] Teste Judge-Endpoint ..."
curl -s -X POST "http://localhost:${HOST_PORT}/v1/chat/completions" \
  -H "Content-Type: application/json" \
  -d "{
    \"model\": \"${MODEL_ALIAS}\",
    \"messages\": [
      { \"role\": \"system\", \"content\": \"Du bist ein strenger Code-Reviewer.\" },
      { \"role\": \"user\", \"content\": \"Antworte nur mit dem Wort: bereit\" }
    ],
    \"max_tokens\": 8,
    \"temperature\": 0.0,
    \"stream\": false
  }"

echo
echo "[*] Server bereit auf http://0.0.0.0:${HOST_PORT}"
echo "[*] Stoppen mit: docker rm -f ${CONTAINER_NAME}"