From d8bfe5fc3e238270cf8304956c1a50c649cdc75c Mon Sep 17 00:00:00 2001 From: dschlueter Date: Sun, 28 Jun 2026 04:37:12 +0200 Subject: [PATCH] test: guard BEDIENUNGSANLEITUNG.md against drift MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit bash tests/test_install_doc.sh # structure + live checks bash tests/test_install_doc.sh --offline # structure only, no network make test-doc Two layers: FAIL — local, deterministic: code-fence balance, internal section references resolve, file-listing paths exist in repo, key tokens (npm pkg, NodeSource URL, repo URL) present, model slugs consistent between manual and profiles.example.yml. WARN — online, flaky-tolerant: npm package resolves, NodeSource/nvm URLs reachable, OpenRouter slugs still in the public model list. A transient outage is a warning, not a failure. Catches the failure classes that silently crept in before: a referenced section number that no longer exists, a repo path renamed without updating the manual, a stale npm package name, a renamed model slug. Co-Authored-By: Claude --- Makefile | 5 +- tests/test_install_doc.sh | 190 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 194 insertions(+), 1 deletion(-) create mode 100755 tests/test_install_doc.sh diff --git a/Makefile b/Makefile index bad6abd..e076dae 100644 --- a/Makefile +++ b/Makefile @@ -1,3 +1,6 @@ -.PHONY: test +.PHONY: test test-doc test: @bash tests/test_profiles.sh + +test-doc: + @bash tests/test_install_doc.sh diff --git a/tests/test_install_doc.sh b/tests/test_install_doc.sh new file mode 100755 index 0000000..d68cc6e --- /dev/null +++ b/tests/test_install_doc.sh @@ -0,0 +1,190 @@ +#!/usr/bin/env bash +# +# tests/test_install_doc.sh — verify BEDIENUNGSANLEITUNG.md stays accurate. +# +# Two layers: +# FAIL — local, deterministic: markdown structure, internal section refs, +# file-listing paths real, package/url tokens present. A failure +# here means the manual is broken and MUST be fixed. +# WARN — online, flaky-tolerant: npm package resolves, NodeSource/nvm +# URLs return 2xx, OpenRouter model slugs still exist. A transient +# outage of one endpoint is a warning, not a failure. +# +# Run: bash tests/test_install_doc.sh # offline + online +# or: bash tests/test_install_doc.sh --offline # structure only, no network +# or: make test-doc + +set -u + +REPO_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +DOC="$REPO_DIR/BEDIENUNGSANLEITUNG.md" + +ONLINE=1 +[[ "${1:-}" == "--offline" ]] && ONLINE=0 + +FAILS=0 +WARNS=0 +CHECKS=0 + +red() { printf '\033[31m%s\033[0m\n' "$*"; } +yellow() { printf '\033[33m%s\033[0m\n' "$*"; } +green() { printf '\033[32m%s\033[0m\n' "$*"; } + +# fail — hard failure (structure) +fail() { red " FAIL: $*"; FAILS=$((FAILS + 1)); } +# warn — soft warning (live endpoint) +warn() { yellow " WARN: $*"; WARNS=$((WARNS + 1)); } +# pass +pass() { green " ok: $*"; CHECKS=$((CHECKS + 1)); } + +http_ok() { + # http_ok — prints body on success, empty on failure; follows redirects + curl -fsSL --max-time 20 -o /dev/null -w '%{http_code}' "$1" 2>/dev/null +} + +echo "## BEDIENUNGSANLEITUNG.md — structure" + +# --- markdown fences balanced ------------------------------------------------ +echo "-- code fences balanced" +fence_count=$(grep -c '```' "$DOC") +CHECKS=$((CHECKS + 1)) +if (( fence_count % 2 == 0 )); then + pass "code fences balanced ($fence_count)" +else + fail "code fences unbalanced ($fence_count) — an open fence has no close" +fi + +# --- internal section references resolve -------------------------------------- +echo "-- internal section references resolve" +# Collect every "Abschnitt N" / "Abschnitt N.M" reference and check a heading exists. +CHECKS=$((CHECKS + 1)) +ref_broken=0 +refs=$(grep -oE 'Abschnitt [0-9]+(\.[0-9]+)?' "$DOC" | sort -u) +while read -r ref; do + [[ -z "$ref" ]] && continue + num=${ref#Abschnitt } + if [[ "$num" == *.* ]]; then + # sub-section like 3.1 -> look for ### 3.1 (followed by space or .) + grep -qE "^### ${num}( |\.|$)" "$DOC" || { fail "ref 'Abschnitt $num' has no matching heading"; ref_broken=1; } + else + grep -qE "^## ${num}( |\.|$)" "$DOC" || { fail "ref 'Abschnitt $num' has no matching heading"; ref_broken=1; } + fi +done <<< "$refs" +[[ $ref_broken -eq 0 ]] && pass "all section references resolve" + +# --- file-listing paths actually exist in repo -------------------------------- +echo "-- file-listing paths exist in repo" +# Only check paths from the "Wo was liegt" file-listing block, not from +# bash/dockerfile code fences (which contain /bin/bash or /var/lib/apt/lists/ +# that aren't repo files). +CHECKS=$((CHECKS + 1)) +listing_missing=0 +# Extract the fenced block right after the "Wo was liegt" heading. +listing_block=$(awk '/^## [0-9]+\. Wo was liegt/{f=1; next} /^```/{if(f){f=0}} f' "$DOC") +mapfile -t paths < <(printf '%s\n' "$listing_block" | grep -oE '(bin|lib|shell|config|tests)/[A-Za-z0-9._-]+|^Makefile' | sort -u) +for p in "${paths[@]}"; do + if [[ ! -e "$REPO_DIR/$p" ]]; then + fail "file-listing references '$p' but it does not exist in repo" + listing_missing=1 + fi +done +[[ $listing_missing -eq 0 ]] && pass "all file-listing paths exist" + +# --- package + URL tokens present -------------------------------------------- +echo "-- key tokens present in manual" +CHECKS=$((CHECKS + 1)) +tok_missing=0 +for tok in "@anthropic-ai/claude-code" \ + "deb.nodesource.com/setup_lts.x" \ + "openrouter.ai/api" \ + "kitux.de/forgejo/dschlueter/claude-launcher-profiles"; do + grep -qF "$tok" "$DOC" || { fail "token '$tok' missing from manual"; tok_missing=1; } +done +[[ $tok_missing -eq 0 ]] && pass "all key tokens present" + +# --- model slugs mentioned in manual are in profiles.example.yml -------------- +echo "-- model slugs consistent with profiles.example.yml" +# A slug is "consistent" if it appears at least once in BOTH the manual and +# the example config. Counts need not match — the manual legitimately repeats +# a slug across tables, examples and prose. +CHECKS=$((CHECKS + 1)) +slug_drift=0 +for slug in "z-ai/glm-5.2" "moonshotai/kimi-k2.7-code" "openrouter/free"; do + in_doc=$(grep -cF "$slug" "$DOC") + in_cfg=$(grep -cF "$slug" "$REPO_DIR/config/profiles.example.yml") + if [[ "$in_doc" -eq 0 ]]; then + fail "slug '$slug' missing from manual (referenced in config)" + slug_drift=1 + elif [[ "$in_cfg" -eq 0 ]]; then + fail "slug '$slug' missing from profiles.example.yml (documented in manual)" + slug_drift=1 + fi +done +[[ $slug_drift -eq 0 ]] && pass "slugs present in both manual and example config" + +# ============================================================================ +if [[ $ONLINE -eq 1 ]]; then + echo + echo "## BEDIENUNGSANLEITUNG.md — live endpoints (warnings, not failures)" + + echo "-- npm package @anthropic-ai/claude-code resolves" + CHECKS=$((CHECKS + 1)) + if command -v npm >/dev/null 2>&1; then + if npm view "@anthropic-ai/claude-code" version >/dev/null 2>&1; then + ver=$(npm view "@anthropic-ai/claude-code" version 2>/dev/null | tail -1) + pass "npm package resolves (latest: $ver)" + else + warn "npm view @anthropic-ai/claude-code failed — package missing or npm down" + fi + else + warn "npm not installed — cannot verify package online" + fi + + echo "-- NodeSource setup URL reachable" + CHECKS=$((CHECKS + 1)) + code=$(http_ok "https://deb.nodesource.com/setup_lts.x") + if [[ "$code" =~ ^2 ]]; then + pass "NodeSource setup URL returns $code" + else + warn "NodeSource setup URL returned '$code' — may be down or moved" + fi + + echo "-- nvm install URL reachable" + CHECKS=$((CHECKS + 1)) + code=$(http_ok "https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.1/install.sh") + if [[ "$code" =~ ^2 ]]; then + pass "nvm install URL returns $code" + else + warn "nvm install URL returned '$code' — version may have moved (check latest nvm release)" + fi + + echo "-- OpenRouter model slugs still exist (public model-list endpoint)" + # The per-model endpoint /models/{slug} returns 404 even for valid slugs, + # so query the full list and check membership instead. No API key required. + CHECKS=$((CHECKS + 1)) + or_list=$(curl -fsSL --max-time 25 "https://openrouter.ai/api/v1/models" 2>/dev/null) + or_ok=1 + for slug in "z-ai/glm-5.2" "moonshotai/kimi-k2.7-code" "openrouter/free"; do + if printf '%s' "$or_list" | grep -q "\"$slug\""; then + : + else + warn "OpenRouter slug '$slug' not in model list — may have been renamed" + or_ok=0 + fi + done + [[ $or_ok -eq 1 ]] && pass "all OpenRouter slugs present in model list" +fi + +# ============================================================================ +echo +echo "------------------------------------------------------------" +printf 'checks: %d ok, %d warnings, %d failures\n' "$CHECKS" "$WARNS" "$FAILS" +if (( FAILS > 0 )); then + echo "RESULT: FAIL (fix the structural failures above)" + exit 1 +fi +if (( WARNS > 0 )); then + echo "RESULT: OK with warnings (live endpoints — recheck manually if persistent)" + exit 0 +fi +echo "RESULT: all good."