dschlueter
5146b7fa30
Vollständiges Multi-Agenten-System für Fact-Checking, Artikelschreiben und Argumentationsanalyse. Zwei Backends: llama.cpp (★ bevorzugt) und Ollama. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
271 lines
8.9 KiB
Bash
Executable file
271 lines
8.9 KiB
Bash
Executable file
#!/usr/bin/env bash
|
|
# tests/run_corpus.sh
|
|
# Führt alle Testkorpus-Fälle durch verify-article und berechnet Precision/Recall.
|
|
#
|
|
# Verwendung:
|
|
# cd ~/Pi_Agent_Projekts/text_agent
|
|
# bash tests/run_corpus.sh # Alle Fälle
|
|
# bash tests/run_corpus.sh case_001 case_002 # Nur bestimmte Fälle
|
|
# bash tests/run_corpus.sh --mode deep # Perplexity-Modus
|
|
# bash tests/run_corpus.sh --no-cache # Cache umgehen
|
|
#
|
|
# Ausgabe:
|
|
# tests/results/<timestamp>/ ← JSON-Reports pro Fall
|
|
# tests/results/<timestamp>/summary.txt ← Precision/Recall-Zusammenfassung
|
|
|
|
set -euo pipefail
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
CORPUS_DIR="${SCRIPT_DIR}/corpus"
|
|
AGENT="${SCRIPT_DIR}/../agenten/llama-verify-article.ts"
|
|
TIMESTAMP="$(date +%Y-%m-%d_%H-%M-%S)"
|
|
RESULTS_DIR="${SCRIPT_DIR}/results/${TIMESTAMP}"
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Argument-Parsing
|
|
# ---------------------------------------------------------------------------
|
|
|
|
MODE="fast"
|
|
EXTRA_FLAGS=""
|
|
SELECTED_CASES=()
|
|
|
|
for arg in "$@"; do
|
|
case "$arg" in
|
|
--mode) shift; MODE="$1" ;;
|
|
--mode=*) MODE="${arg#--mode=}" ;;
|
|
--no-cache) EXTRA_FLAGS="${EXTRA_FLAGS} --no-cache" ;;
|
|
case_*) SELECTED_CASES+=("$arg") ;;
|
|
*) ;;
|
|
esac
|
|
done
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Setup
|
|
# ---------------------------------------------------------------------------
|
|
|
|
mkdir -p "${RESULTS_DIR}"
|
|
|
|
# Hilfsfunktionen
|
|
green() { echo -e "\033[0;32m$*\033[0m"; }
|
|
red() { echo -e "\033[0;31m$*\033[0m"; }
|
|
yellow() { echo -e "\033[0;33m$*\033[0m"; }
|
|
bold() { echo -e "\033[1m$*\033[0m"; }
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Fälle ermitteln
|
|
# ---------------------------------------------------------------------------
|
|
|
|
if [ ${#SELECTED_CASES[@]} -eq 0 ]; then
|
|
mapfile -t CASES < <(ls -d "${CORPUS_DIR}"/case_* 2>/dev/null | xargs -I{} basename {})
|
|
else
|
|
CASES=("${SELECTED_CASES[@]}")
|
|
fi
|
|
|
|
if [ ${#CASES[@]} -eq 0 ]; then
|
|
echo "Keine Fälle in ${CORPUS_DIR} gefunden."
|
|
exit 1
|
|
fi
|
|
|
|
bold "Pi Text-Agent — Testkorpus-Auswertung"
|
|
echo "Modus: ${MODE} | Fälle: ${#CASES[@]} | Ergebnisse: ${RESULTS_DIR}"
|
|
echo ""
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Metriken (Globale Zähler)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
TOTAL_CLAIMS=0
|
|
TRUE_POS=0 # Erwartet X → tatsächlich X
|
|
FALSE_POS=0 # Erwartet NOT contradicted → tatsächlich contradicted
|
|
FALSE_NEG=0 # Erwartet contradicted → tatsächlich NOT contradicted
|
|
TRUE_NEG=0 # Erwartet NOT contradicted → tatsächlich NOT contradicted
|
|
|
|
CASE_PASS=0
|
|
CASE_FAIL=0
|
|
CASE_ERROR=0
|
|
|
|
TOTAL_COST=0
|
|
TOTAL_TIME=0
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Pro-Fall-Verarbeitung
|
|
# ---------------------------------------------------------------------------
|
|
|
|
for case_name in "${CASES[@]}"; do
|
|
case_dir="${CORPUS_DIR}/${case_name}"
|
|
input_file="${case_dir}/input.txt"
|
|
expected_file="${case_dir}/expected.json"
|
|
|
|
if [ ! -f "${input_file}" ]; then
|
|
yellow " ${case_name}: input.txt nicht gefunden — übersprungen"
|
|
continue
|
|
fi
|
|
if [ ! -f "${expected_file}" ]; then
|
|
yellow " ${case_name}: expected.json nicht gefunden — übersprungen"
|
|
continue
|
|
fi
|
|
|
|
echo -n " ${case_name}: "
|
|
result_file="${RESULTS_DIR}/${case_name}.json"
|
|
t_start=$(date +%s%3N)
|
|
|
|
# verify-article aufrufen
|
|
if npx tsx "${AGENT}" \
|
|
--mode "${MODE}" \
|
|
--json \
|
|
${EXTRA_FLAGS} \
|
|
"$(cat "${input_file}")" \
|
|
> "${result_file}" 2>/dev/null; then
|
|
t_end=$(date +%s%3N)
|
|
elapsed_ms=$((t_end - t_start))
|
|
else
|
|
t_end=$(date +%s%3N)
|
|
elapsed_ms=$((t_end - t_start))
|
|
red "FEHLER (${elapsed_ms}ms)"
|
|
CASE_ERROR=$((CASE_ERROR + 1))
|
|
echo " Fehlerhafter Exit-Code von verify-article" >> "${RESULTS_DIR}/errors.log"
|
|
continue
|
|
fi
|
|
|
|
# Kosten aus Report
|
|
cost=$(python3 -c "
|
|
import json, sys
|
|
try:
|
|
r = json.load(open('${result_file}'))
|
|
print(r.get('totalCostUSD', 0))
|
|
except: print(0)
|
|
" 2>/dev/null || echo "0")
|
|
TOTAL_COST=$(python3 -c "print(${TOTAL_COST} + ${cost})" 2>/dev/null || echo "${TOTAL_COST}")
|
|
TOTAL_TIME=$((TOTAL_TIME + elapsed_ms))
|
|
|
|
# Erwartungen prüfen
|
|
case_pass=true
|
|
claim_results=""
|
|
|
|
while IFS= read -r expected_claim; do
|
|
text_contains=$(echo "${expected_claim}" | python3 -c "import json,sys; d=json.loads(sys.stdin.read()); print(d.get('text_contains',''))" 2>/dev/null)
|
|
expected_status=$(echo "${expected_claim}" | python3 -c "import json,sys; d=json.loads(sys.stdin.read()); print(d.get('expected_status',''))" 2>/dev/null)
|
|
note=$(echo "${expected_claim}" | python3 -c "import json,sys; d=json.loads(sys.stdin.read()); print(d.get('note',''))" 2>/dev/null)
|
|
|
|
if [ -z "${text_contains}" ] || [ -z "${expected_status}" ]; then
|
|
continue
|
|
fi
|
|
|
|
TOTAL_CLAIMS=$((TOTAL_CLAIMS + 1))
|
|
|
|
# Tatsächlichen Status aus Report ermitteln
|
|
actual_status=$(python3 -c "
|
|
import json, sys
|
|
try:
|
|
report = json.load(open('${result_file}'))
|
|
needle = '${text_contains}'.lower()
|
|
for r in report.get('results', []):
|
|
if needle in r.get('claim_text', '').lower():
|
|
print(r.get('status', 'not_found'))
|
|
sys.exit(0)
|
|
print('not_found')
|
|
except Exception as e:
|
|
print('error')
|
|
" 2>/dev/null)
|
|
|
|
# Metriken aktualisieren
|
|
if [ "${expected_status}" = "${actual_status}" ]; then
|
|
# Exakter Match
|
|
if [ "${expected_status}" = "contradicted" ]; then
|
|
TRUE_POS=$((TRUE_POS + 1))
|
|
else
|
|
TRUE_NEG=$((TRUE_NEG + 1))
|
|
fi
|
|
claim_results="${claim_results}\n ✓ [${actual_status}] ${text_contains:0:50}"
|
|
else
|
|
# Mismatch
|
|
case_pass=false
|
|
if [ "${expected_status}" = "contradicted" ] && [ "${actual_status}" != "contradicted" ]; then
|
|
FALSE_NEG=$((FALSE_NEG + 1))
|
|
claim_results="${claim_results}\n ✗ Erwartet contradicted, bekam ${actual_status}: ${text_contains:0:50}"
|
|
elif [ "${expected_status}" != "contradicted" ] && [ "${actual_status}" = "contradicted" ]; then
|
|
FALSE_POS=$((FALSE_POS + 1))
|
|
claim_results="${claim_results}\n ✗ Falsch widersprüchlich: ${text_contains:0:50}"
|
|
else
|
|
# z.B. supported vs mixed
|
|
claim_results="${claim_results}\n ~ Erwartet ${expected_status}, bekam ${actual_status}: ${text_contains:0:50}"
|
|
fi
|
|
fi
|
|
done < <(python3 -c "
|
|
import json
|
|
data = json.load(open('${expected_file}'))
|
|
for c in data.get('claims', []):
|
|
print(json.dumps(c))
|
|
" 2>/dev/null)
|
|
|
|
if [ "${case_pass}" = true ]; then
|
|
green "OK (${elapsed_ms}ms, \$${cost})"
|
|
CASE_PASS=$((CASE_PASS + 1))
|
|
else
|
|
red "FEHLGESCHLAGEN (${elapsed_ms}ms, \$${cost})"
|
|
CASE_FAIL=$((CASE_FAIL + 1))
|
|
fi
|
|
|
|
if [ -n "${claim_results}" ]; then
|
|
echo -e "${claim_results}"
|
|
fi
|
|
done
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Zusammenfassung
|
|
# ---------------------------------------------------------------------------
|
|
|
|
echo ""
|
|
bold "=============================="
|
|
bold "Ergebnisse"
|
|
bold "=============================="
|
|
echo ""
|
|
echo "Fälle: ${CASE_PASS} OK | ${CASE_FAIL} fehlgeschlagen | ${CASE_ERROR} Fehler"
|
|
echo "Claims: ${TOTAL_CLAIMS} geprüft"
|
|
echo ""
|
|
|
|
# Precision (wie viele der als contradicted markierten sind wirklich falsch)
|
|
if [ $((TRUE_POS + FALSE_POS)) -gt 0 ]; then
|
|
precision=$(python3 -c "print(f'{${TRUE_POS} / (${TRUE_POS} + ${FALSE_POS}) * 100:.1f}%')" 2>/dev/null || echo "n/a")
|
|
else
|
|
precision="n/a (keine contradicted-Urteile)"
|
|
fi
|
|
|
|
# Recall (wie viele der wirklich falschen Claims wurden erkannt)
|
|
if [ $((TRUE_POS + FALSE_NEG)) -gt 0 ]; then
|
|
recall=$(python3 -c "print(f'{${TRUE_POS} / (${TRUE_POS} + ${FALSE_NEG}) * 100:.1f}%')" 2>/dev/null || echo "n/a")
|
|
else
|
|
recall="n/a (keine erwarteten contradicted-Claims)"
|
|
fi
|
|
|
|
echo "Precision: ${precision} (Anteil korrekt widerlegter unter allen Widerlegungen)"
|
|
echo "Recall: ${recall} (Anteil erkannter Fehler unter allen bekannten Fehlern)"
|
|
echo ""
|
|
echo "True Positives: ${TRUE_POS} (korrekter contradicted-Fund)"
|
|
echo "False Positives: ${FALSE_POS} (fälschlich widerlegter korrekter Fakt)"
|
|
echo "False Negatives: ${FALSE_NEG} (nicht erkannter Fehler)"
|
|
echo "True Negatives: ${TRUE_NEG} (korrekt als nicht-widerlegbar bewertet)"
|
|
echo ""
|
|
echo "Kosten: \$${TOTAL_COST}"
|
|
echo "Zeit: $((TOTAL_TIME / 1000))s total"
|
|
echo ""
|
|
echo "Reports: ${RESULTS_DIR}/"
|
|
|
|
# Zusammenfassung speichern
|
|
{
|
|
echo "Testlauf: ${TIMESTAMP}"
|
|
echo "Modus: ${MODE}"
|
|
echo "Fälle: ${CASE_PASS} OK | ${CASE_FAIL} fehlgeschlagen | ${CASE_ERROR} Fehler"
|
|
echo "Claims: ${TOTAL_CLAIMS} geprüft"
|
|
echo "Precision: ${precision}"
|
|
echo "Recall: ${recall}"
|
|
echo "TP=${TRUE_POS} FP=${FALSE_POS} FN=${FALSE_NEG} TN=${TRUE_NEG}"
|
|
echo "Kosten: \$${TOTAL_COST}"
|
|
echo "Zeit: $((TOTAL_TIME / 1000))s"
|
|
} > "${RESULTS_DIR}/summary.txt"
|
|
|
|
# Exit-Code: 0 wenn alle Fälle bestanden
|
|
if [ "${CASE_FAIL}" -gt 0 ] || [ "${CASE_ERROR}" -gt 0 ]; then
|
|
exit 1
|
|
fi
|
|
exit 0
|