dschlueter
b0c215abbb
classify_refusal() separates refusal (head), preamble, appended disclaimer
(tail) and moralising insert (inline). Only the first three are meaningful, and
only where a prompt forbids them: a preamble on a coding answer is normal, so it
no longer counts as evasion.
Two sources of false positives had to be removed first, both found by checking
the detector's hits against the archive rather than trusting them:
* llamacppctl's own truncation warning on stderr had leaked into an archived
output and was read as a model disclaimer. It also inflated that run's word
count, so count_words() strips it too.
* An AI character saying "Bitte beachten Sie:" inside a dystopian story is
plot, not distancing. Quoted speech is removed before markers are matched.
run_prompt_suite.sh gains a CASES filter so a single prompt can be re-run.
Measured result, recorded in KI_TOOLS_PROFILES.md: across seven prose prompts
neither the abliterated model nor the aligned base model evaded once. On
literary prose the abliteration buys nothing measurable.
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
102 lines
3.3 KiB
Bash
Executable file
102 lines
3.3 KiB
Bash
Executable file
#!/usr/bin/env bash
|
|
#
|
|
# Faehrt eine Domaene der Beispiel-Prompts gegen ein laufendes Modell-Profil und
|
|
# legt die Ausgaben unter der etablierten Namenskonvention im Test-Archiv ab:
|
|
#
|
|
# <label>_<domaene>_<nr>.out.txt
|
|
#
|
|
# Der Server fuer <profil> muss bereits laufen (llamacppctl --start --profile ...).
|
|
# Ausgewertet wird anschliessend mit scripts/eval_prompt_tests.py.
|
|
#
|
|
# Verwendung:
|
|
# scripts/run_prompt_suite.sh <profil> <label> <domaene> [<domaene>...]
|
|
#
|
|
# Beispiel (Kontrolllauf des nicht-abliterierten Basismodells auf Prosa):
|
|
# scripts/run_prompt_suite.sh qwen35base qwen35base prosa
|
|
#
|
|
# Environment:
|
|
# OUT_DIR Zielverzeichnis (Default: ~/llamacppctl_prompt_tests)
|
|
# CONFIG Config-Datei (Default: llama.cpp.config)
|
|
# MAX_TOKENS Antwortbudget (Default: 20000 -- Reasoning frisst viel; unter
|
|
# 16000 brechen Prosa/Reden mitten im Denken ab)
|
|
# CLI Pfad zum CLI (Default: ./.venv/bin/llamacppctl)
|
|
# OVERWRITE 1 = vorhandene Ausgaben ueberschreiben (Default: 0 = ueberspringen)
|
|
# CASES Leerzeichenliste von Fallnummern, z. B. "06 07 08" (Default: alle)
|
|
#
|
|
set -uo pipefail
|
|
|
|
cd "$(git rev-parse --show-toplevel)" || exit 1
|
|
|
|
if [ "$#" -lt 3 ]; then
|
|
sed -n '2,25p' "$0" | sed 's/^# \{0,1\}//'
|
|
exit 2
|
|
fi
|
|
|
|
PROFILE=$1; LABEL=$2; shift 2
|
|
OUT_DIR=${OUT_DIR:-"$HOME/llamacppctl_prompt_tests"}
|
|
CONFIG=${CONFIG:-llama.cpp.config}
|
|
MAX_TOKENS=${MAX_TOKENS:-20000}
|
|
CLI=${CLI:-./.venv/bin/llamacppctl}
|
|
OVERWRITE=${OVERWRITE:-0}
|
|
|
|
if [ ! -x "$CLI" ]; then
|
|
echo "CLI nicht ausfuehrbar: $CLI" >&2
|
|
exit 1
|
|
fi
|
|
|
|
# Ohne laufenden Server produziert jeder Prompt nur eine Fehlermeldung als
|
|
# "Ergebnis" -- lieber sofort abbrechen als 4 Muelldateien schreiben.
|
|
if ! "$CLI" --check --config "$CONFIG" --profile "$PROFILE" >/dev/null 2>&1; then
|
|
echo "Server fuer Profil '$PROFILE' ist nicht erreichbar (--check schlug fehl)." >&2
|
|
echo "Zuerst starten: $CLI --start --config $CONFIG --profile $PROFILE" >&2
|
|
exit 5
|
|
fi
|
|
|
|
mkdir -p "$OUT_DIR"
|
|
fail=0
|
|
|
|
for domain in "$@"; do
|
|
system_prompt="example_system_prompts/system_prompt_${domain}.md"
|
|
if [ ! -f "$system_prompt" ]; then
|
|
echo "System-Prompt fehlt: $system_prompt" >&2
|
|
fail=1
|
|
continue
|
|
fi
|
|
|
|
for prompt in example_user_prompts/"${domain}"_[0-9][0-9]_*.md; do
|
|
[ -e "$prompt" ] || continue
|
|
base=$(basename "$prompt" .md) # z. B. prosa_01_werkstatt
|
|
number=$(echo "$base" | cut -d_ -f2) # z. B. 01
|
|
out="$OUT_DIR/${LABEL}_${domain}_${number}.out.txt"
|
|
|
|
if [ -n "${CASES:-}" ] && ! echo " $CASES " | grep -q " $number "; then
|
|
continue
|
|
fi
|
|
|
|
if [ -e "$out" ] && [ "$OVERWRITE" != "1" ]; then
|
|
echo "== $base -> vorhanden, uebersprungen (OVERWRITE=1 erzwingt)"
|
|
continue
|
|
fi
|
|
|
|
echo "== $base -> $out"
|
|
if "$CLI" --chat --config "$CONFIG" --profile "$PROFILE" \
|
|
--system-file "$system_prompt" \
|
|
--prompt-file "$prompt" \
|
|
--stream --max-tokens "$MAX_TOKENS" > "$out.part" 2>"$out.err"; then
|
|
mv "$out.part" "$out"
|
|
rm -f "$out.err"
|
|
printf ' %s Woerter\n' "$(wc -w < "$out")"
|
|
else
|
|
echo " FEHLGESCHLAGEN -- siehe $out.err" >&2
|
|
rm -f "$out.part"
|
|
fail=1
|
|
fi
|
|
done
|
|
done
|
|
|
|
echo
|
|
if [ "$fail" -ne 0 ]; then
|
|
echo "Mindestens ein Lauf schlug fehl."
|
|
exit 1
|
|
fi
|
|
echo "Fertig. Auswerten mit: scripts/eval_prompt_tests.py"
|