test: guard BEDIENUNGSANLEITUNG.md against drift #5
2 changed files with 194 additions and 1 deletions
test: guard BEDIENUNGSANLEITUNG.md against drift
bash tests/test_install_doc.sh # structure + live checks
bash tests/test_install_doc.sh --offline # structure only, no network
make test-doc
Two layers:
FAIL — local, deterministic: code-fence balance, internal section
references resolve, file-listing paths exist in repo, key tokens
(npm pkg, NodeSource URL, repo URL) present, model slugs
consistent between manual and profiles.example.yml.
WARN — online, flaky-tolerant: npm package resolves, NodeSource/nvm
URLs reachable, OpenRouter slugs still in the public model list.
A transient outage is a warning, not a failure.
Catches the failure classes that silently crept in before: a referenced
section number that no longer exists, a repo path renamed without updating
the manual, a stale npm package name, a renamed model slug.
Co-Authored-By: Claude <noreply@anthropic.com>
commit
d8bfe5fc3e
5
Makefile
5
Makefile
|
|
@ -1,3 +1,6 @@
|
|||
.PHONY: test
|
||||
.PHONY: test test-doc
|
||||
test:
|
||||
@bash tests/test_profiles.sh
|
||||
|
||||
test-doc:
|
||||
@bash tests/test_install_doc.sh
|
||||
|
|
|
|||
190
tests/test_install_doc.sh
Executable file
190
tests/test_install_doc.sh
Executable file
|
|
@ -0,0 +1,190 @@
|
|||
#!/usr/bin/env bash
|
||||
#
|
||||
# tests/test_install_doc.sh — verify BEDIENUNGSANLEITUNG.md stays accurate.
|
||||
#
|
||||
# Two layers:
|
||||
# FAIL — local, deterministic: markdown structure, internal section refs,
|
||||
# file-listing paths real, package/url tokens present. A failure
|
||||
# here means the manual is broken and MUST be fixed.
|
||||
# WARN — online, flaky-tolerant: npm package resolves, NodeSource/nvm
|
||||
# URLs return 2xx, OpenRouter model slugs still exist. A transient
|
||||
# outage of one endpoint is a warning, not a failure.
|
||||
#
|
||||
# Run: bash tests/test_install_doc.sh # offline + online
|
||||
# or: bash tests/test_install_doc.sh --offline # structure only, no network
|
||||
# or: make test-doc
|
||||
|
||||
set -u
|
||||
|
||||
REPO_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
DOC="$REPO_DIR/BEDIENUNGSANLEITUNG.md"
|
||||
|
||||
ONLINE=1
|
||||
[[ "${1:-}" == "--offline" ]] && ONLINE=0
|
||||
|
||||
FAILS=0
|
||||
WARNS=0
|
||||
CHECKS=0
|
||||
|
||||
red() { printf '\033[31m%s\033[0m\n' "$*"; }
|
||||
yellow() { printf '\033[33m%s\033[0m\n' "$*"; }
|
||||
green() { printf '\033[32m%s\033[0m\n' "$*"; }
|
||||
|
||||
# fail <message> — hard failure (structure)
|
||||
fail() { red " FAIL: $*"; FAILS=$((FAILS + 1)); }
|
||||
# warn <message> — soft warning (live endpoint)
|
||||
warn() { yellow " WARN: $*"; WARNS=$((WARNS + 1)); }
|
||||
# pass <message>
|
||||
pass() { green " ok: $*"; CHECKS=$((CHECKS + 1)); }
|
||||
|
||||
http_ok() {
|
||||
# http_ok <url> — prints body on success, empty on failure; follows redirects
|
||||
curl -fsSL --max-time 20 -o /dev/null -w '%{http_code}' "$1" 2>/dev/null
|
||||
}
|
||||
|
||||
echo "## BEDIENUNGSANLEITUNG.md — structure"
|
||||
|
||||
# --- markdown fences balanced ------------------------------------------------
|
||||
echo "-- code fences balanced"
|
||||
fence_count=$(grep -c '```' "$DOC")
|
||||
CHECKS=$((CHECKS + 1))
|
||||
if (( fence_count % 2 == 0 )); then
|
||||
pass "code fences balanced ($fence_count)"
|
||||
else
|
||||
fail "code fences unbalanced ($fence_count) — an open fence has no close"
|
||||
fi
|
||||
|
||||
# --- internal section references resolve --------------------------------------
|
||||
echo "-- internal section references resolve"
|
||||
# Collect every "Abschnitt N" / "Abschnitt N.M" reference and check a heading exists.
|
||||
CHECKS=$((CHECKS + 1))
|
||||
ref_broken=0
|
||||
refs=$(grep -oE 'Abschnitt [0-9]+(\.[0-9]+)?' "$DOC" | sort -u)
|
||||
while read -r ref; do
|
||||
[[ -z "$ref" ]] && continue
|
||||
num=${ref#Abschnitt }
|
||||
if [[ "$num" == *.* ]]; then
|
||||
# sub-section like 3.1 -> look for ### 3.1 (followed by space or .)
|
||||
grep -qE "^### ${num}( |\.|$)" "$DOC" || { fail "ref 'Abschnitt $num' has no matching heading"; ref_broken=1; }
|
||||
else
|
||||
grep -qE "^## ${num}( |\.|$)" "$DOC" || { fail "ref 'Abschnitt $num' has no matching heading"; ref_broken=1; }
|
||||
fi
|
||||
done <<< "$refs"
|
||||
[[ $ref_broken -eq 0 ]] && pass "all section references resolve"
|
||||
|
||||
# --- file-listing paths actually exist in repo --------------------------------
|
||||
echo "-- file-listing paths exist in repo"
|
||||
# Only check paths from the "Wo was liegt" file-listing block, not from
|
||||
# bash/dockerfile code fences (which contain /bin/bash or /var/lib/apt/lists/
|
||||
# that aren't repo files).
|
||||
CHECKS=$((CHECKS + 1))
|
||||
listing_missing=0
|
||||
# Extract the fenced block right after the "Wo was liegt" heading.
|
||||
listing_block=$(awk '/^## [0-9]+\. Wo was liegt/{f=1; next} /^```/{if(f){f=0}} f' "$DOC")
|
||||
mapfile -t paths < <(printf '%s\n' "$listing_block" | grep -oE '(bin|lib|shell|config|tests)/[A-Za-z0-9._-]+|^Makefile' | sort -u)
|
||||
for p in "${paths[@]}"; do
|
||||
if [[ ! -e "$REPO_DIR/$p" ]]; then
|
||||
fail "file-listing references '$p' but it does not exist in repo"
|
||||
listing_missing=1
|
||||
fi
|
||||
done
|
||||
[[ $listing_missing -eq 0 ]] && pass "all file-listing paths exist"
|
||||
|
||||
# --- package + URL tokens present --------------------------------------------
|
||||
echo "-- key tokens present in manual"
|
||||
CHECKS=$((CHECKS + 1))
|
||||
tok_missing=0
|
||||
for tok in "@anthropic-ai/claude-code" \
|
||||
"deb.nodesource.com/setup_lts.x" \
|
||||
"openrouter.ai/api" \
|
||||
"kitux.de/forgejo/dschlueter/claude-launcher-profiles"; do
|
||||
grep -qF "$tok" "$DOC" || { fail "token '$tok' missing from manual"; tok_missing=1; }
|
||||
done
|
||||
[[ $tok_missing -eq 0 ]] && pass "all key tokens present"
|
||||
|
||||
# --- model slugs mentioned in manual are in profiles.example.yml --------------
|
||||
echo "-- model slugs consistent with profiles.example.yml"
|
||||
# A slug is "consistent" if it appears at least once in BOTH the manual and
|
||||
# the example config. Counts need not match — the manual legitimately repeats
|
||||
# a slug across tables, examples and prose.
|
||||
CHECKS=$((CHECKS + 1))
|
||||
slug_drift=0
|
||||
for slug in "z-ai/glm-5.2" "moonshotai/kimi-k2.7-code" "openrouter/free"; do
|
||||
in_doc=$(grep -cF "$slug" "$DOC")
|
||||
in_cfg=$(grep -cF "$slug" "$REPO_DIR/config/profiles.example.yml")
|
||||
if [[ "$in_doc" -eq 0 ]]; then
|
||||
fail "slug '$slug' missing from manual (referenced in config)"
|
||||
slug_drift=1
|
||||
elif [[ "$in_cfg" -eq 0 ]]; then
|
||||
fail "slug '$slug' missing from profiles.example.yml (documented in manual)"
|
||||
slug_drift=1
|
||||
fi
|
||||
done
|
||||
[[ $slug_drift -eq 0 ]] && pass "slugs present in both manual and example config"
|
||||
|
||||
# ============================================================================
|
||||
if [[ $ONLINE -eq 1 ]]; then
|
||||
echo
|
||||
echo "## BEDIENUNGSANLEITUNG.md — live endpoints (warnings, not failures)"
|
||||
|
||||
echo "-- npm package @anthropic-ai/claude-code resolves"
|
||||
CHECKS=$((CHECKS + 1))
|
||||
if command -v npm >/dev/null 2>&1; then
|
||||
if npm view "@anthropic-ai/claude-code" version >/dev/null 2>&1; then
|
||||
ver=$(npm view "@anthropic-ai/claude-code" version 2>/dev/null | tail -1)
|
||||
pass "npm package resolves (latest: $ver)"
|
||||
else
|
||||
warn "npm view @anthropic-ai/claude-code failed — package missing or npm down"
|
||||
fi
|
||||
else
|
||||
warn "npm not installed — cannot verify package online"
|
||||
fi
|
||||
|
||||
echo "-- NodeSource setup URL reachable"
|
||||
CHECKS=$((CHECKS + 1))
|
||||
code=$(http_ok "https://deb.nodesource.com/setup_lts.x")
|
||||
if [[ "$code" =~ ^2 ]]; then
|
||||
pass "NodeSource setup URL returns $code"
|
||||
else
|
||||
warn "NodeSource setup URL returned '$code' — may be down or moved"
|
||||
fi
|
||||
|
||||
echo "-- nvm install URL reachable"
|
||||
CHECKS=$((CHECKS + 1))
|
||||
code=$(http_ok "https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.1/install.sh")
|
||||
if [[ "$code" =~ ^2 ]]; then
|
||||
pass "nvm install URL returns $code"
|
||||
else
|
||||
warn "nvm install URL returned '$code' — version may have moved (check latest nvm release)"
|
||||
fi
|
||||
|
||||
echo "-- OpenRouter model slugs still exist (public model-list endpoint)"
|
||||
# The per-model endpoint /models/{slug} returns 404 even for valid slugs,
|
||||
# so query the full list and check membership instead. No API key required.
|
||||
CHECKS=$((CHECKS + 1))
|
||||
or_list=$(curl -fsSL --max-time 25 "https://openrouter.ai/api/v1/models" 2>/dev/null)
|
||||
or_ok=1
|
||||
for slug in "z-ai/glm-5.2" "moonshotai/kimi-k2.7-code" "openrouter/free"; do
|
||||
if printf '%s' "$or_list" | grep -q "\"$slug\""; then
|
||||
:
|
||||
else
|
||||
warn "OpenRouter slug '$slug' not in model list — may have been renamed"
|
||||
or_ok=0
|
||||
fi
|
||||
done
|
||||
[[ $or_ok -eq 1 ]] && pass "all OpenRouter slugs present in model list"
|
||||
fi
|
||||
|
||||
# ============================================================================
|
||||
echo
|
||||
echo "------------------------------------------------------------"
|
||||
printf 'checks: %d ok, %d warnings, %d failures\n' "$CHECKS" "$WARNS" "$FAILS"
|
||||
if (( FAILS > 0 )); then
|
||||
echo "RESULT: FAIL (fix the structural failures above)"
|
||||
exit 1
|
||||
fi
|
||||
if (( WARNS > 0 )); then
|
||||
echo "RESULT: OK with warnings (live endpoints — recheck manually if persistent)"
|
||||
exit 0
|
||||
fi
|
||||
echo "RESULT: all good."
|
||||
Loading…
Add table
Add a link
Reference in a new issue