From 218c3fc791197a26cfc02b5e0784a18c2241107d Mon Sep 17 00:00:00 2001 From: dschlueter Date: Fri, 10 Jul 2026 16:24:47 +0200 Subject: [PATCH 1/4] feat(docker): support a multimodal projector via --mmproj Vision-capable GGUFs need a separate projector (mmproj) that maps image embeddings into the text model's space. Add `mmproj` and `mmproj_offload` as config keys and CLI overrides, and pass them through to llama-server. The projector path resolves under hf_home exactly like model_path, so it is covered by the existing read-only mount. validate_model_path() now also checks the projector, which means --change rejects a missing one *before* it removes the running container. --no-mmproj-offload is suppressed when no projector is configured, since llama.cpp rejects the flag on its own. Co-Authored-By: Claude Opus 4.8 --- llama.cpp.config.example | 18 ++++++++++++ src/llamacppctl/actions.py | 18 ++++++++---- src/llamacppctl/cli.py | 15 ++++++++++ src/llamacppctl/config.py | 6 ++++ src/llamacppctl/docker_ops.py | 22 +++++++++++--- src/llamacppctl/schema.py | 8 +++++ tests/test_actions.py | 31 ++++++++++++++++++++ tests/test_config.py | 55 +++++++++++++++++++++++++++++++++++ tests/test_docker_ops.py | 35 ++++++++++++++++++++++ 9 files changed, 199 insertions(+), 9 deletions(-) diff --git a/llama.cpp.config.example b/llama.cpp.config.example index b411535..4656f45 100644 --- a/llama.cpp.config.example +++ b/llama.cpp.config.example @@ -88,6 +88,24 @@ host_port = 8003 gpu_device = 1 ctx_size = 131072 +# Vision profile: `mmproj` points at the multimodal projector (vision encoder) +# GGUF. Like `model_path` it is resolved relative to `hf_home` unless absolute. +# The projector MUST match the base model's vision tower -- a projector built for +# a different base will not load. A missing projector aborts with exit code 3 +# (for --change: before the running container is removed). +# +# llama.cpp offloads the projector to the GPU by default; set mmproj_offload to +# false to keep it on the CPU when VRAM is tight. Images are then sent to +# /v1/chat/completions as an image_url content part -- `llamacppctl --chat` +# itself only sends text. +[model.vision] +model_path = qwen3/Qwen3-35B-A3B-Q4_K_M.gguf +mmproj = qwen3/mmproj.gguf +mmproj_offload = true +container_name = llama_cpp_vision +host_port = 8004 +gpu_device = 1 + [prompt.concise] system_prompt = Du antwortest kurz, präzise und technisch. diff --git a/src/llamacppctl/actions.py b/src/llamacppctl/actions.py index 9011338..f1bb902 100644 --- a/src/llamacppctl/actions.py +++ b/src/llamacppctl/actions.py @@ -77,15 +77,23 @@ def check_exit_code(result: CheckResult) -> int: return EXIT_HTTP_NOT_READY +def _host_path(cfg: ServerConfig, path_str: str) -> Path: + path = Path(path_str) + return path if path.is_absolute() else cfg.hf_home / path_str + + def validate_model_path(cfg: ServerConfig) -> None: - model_path = Path(cfg.model_path) - if model_path.is_absolute(): - target = model_path - else: - target = cfg.hf_home / cfg.model_path + """Check that the model file — and the vision projector, when one is + configured — exist on the host before the container is (re)created.""" + target = _host_path(cfg, cfg.model_path) if not target.exists(): raise FileNotFoundError(f"model file not found: {target}") + if cfg.mmproj: + projector = _host_path(cfg, cfg.mmproj) + if not projector.exists(): + raise FileNotFoundError(f"mmproj file not found: {projector}") + def print_effective_config(cfg: ServerConfig, prompt_cfg: PromptConfig) -> None: payload = { diff --git a/src/llamacppctl/cli.py b/src/llamacppctl/cli.py index 11ed47f..7230f42 100644 --- a/src/llamacppctl/cli.py +++ b/src/llamacppctl/cli.py @@ -46,6 +46,18 @@ def build_parser() -> argparse.ArgumentParser: p.add_argument("--image", help="Docker image (e.g. ghcr.io/ggml-org/llama.cpp:server-cuda)") p.add_argument("--hf-home", help="Host path mounted read-only as model storage") p.add_argument("--model-path", help="Model path, relative to --hf-home unless absolute") + p.add_argument( + "--mmproj", + help="Vision projector (mmproj) GGUF, relative to --hf-home unless absolute. " + "Must match the base model's vision tower; enables image input", + ) + p.add_argument( + "--no-mmproj-offload", + dest="mmproj_offload", + action="store_false", + default=None, + help="Keep the vision projector on the CPU instead of offloading it to the GPU", + ) p.add_argument("--container-name", help="Docker container name (must be unique per config)") p.add_argument("--host-port", type=int, help="Host port to publish") p.add_argument("--container-port", type=int, help="Container-internal port") @@ -185,3 +197,6 @@ def validate_args(args: argparse.Namespace, parser: argparse.ArgumentParser) -> if args.container_name is not None and not args.container_name.strip(): parser.error("--container-name must not be empty") + + if args.mmproj is not None and not args.mmproj.strip(): + parser.error("--mmproj must not be empty") diff --git a/src/llamacppctl/config.py b/src/llamacppctl/config.py index 684a6e9..331d899 100644 --- a/src/llamacppctl/config.py +++ b/src/llamacppctl/config.py @@ -71,6 +71,8 @@ def builtin_defaults() -> dict: "host": "0.0.0.0", "expose": "false", # publish only on loopback unless true "api_key": "", + "mmproj": "", # vision projector GGUF; empty => text-only server + "mmproj_offload": "true", "health_endpoint": "/health", "models_endpoint": "/v1/models", "chat_endpoint": "/v1/chat/completions", @@ -137,6 +139,8 @@ def _apply_cli_overrides(merged: dict, args) -> None: "lock_file": getattr(args, "lock_file", None), "expose": getattr(args, "expose", None), "api_key": getattr(args, "api_key", None), + "mmproj": getattr(args, "mmproj", None), + "mmproj_offload": getattr(args, "mmproj_offload", None), } for key, value in overrides.items(): if value is not None: @@ -193,6 +197,8 @@ def build_server_config(merged: dict) -> ServerConfig: host=str(merged["host"]), expose=_to_bool(merged.get("expose", "false"), "expose"), api_key=str(merged.get("api_key", "")), + mmproj=str(merged.get("mmproj") or "").strip(), + mmproj_offload=_to_bool(merged.get("mmproj_offload", "true"), "mmproj_offload"), health_endpoint=str(merged["health_endpoint"]), models_endpoint=str(merged["models_endpoint"]), chat_endpoint=str(merged["chat_endpoint"]), diff --git a/src/llamacppctl/docker_ops.py b/src/llamacppctl/docker_ops.py index 40c0669..d4184a3 100644 --- a/src/llamacppctl/docker_ops.py +++ b/src/llamacppctl/docker_ops.py @@ -105,11 +105,21 @@ def container_logs(name: str, tail: int = 100) -> str: tail_logs = container_logs +def _resolve_hf_path_in_container(path_str: str) -> str: + """Map a host-side model/projector path to its path inside the container. + Absolute paths are passed through; relative ones resolve under the + read-only /hf_home mount.""" + if Path(path_str).is_absolute(): + return path_str + return f"/hf_home/{path_str}" + + def _resolve_model_path_in_container(cfg: ServerConfig) -> str: - model_path = Path(cfg.model_path) - if model_path.is_absolute(): - return str(model_path) - return f"/hf_home/{cfg.model_path}" + return _resolve_hf_path_in_container(cfg.model_path) + + +def _resolve_mmproj_path_in_container(cfg: ServerConfig) -> str: + return _resolve_hf_path_in_container(cfg.mmproj) def _port_publish(cfg: ServerConfig) -> str: @@ -181,6 +191,10 @@ def build_run_command(cfg: ServerConfig) -> list: cmd.append("--kv-unified") if cfg.cont_batching: cmd.append("--cont-batching") + if cfg.mmproj: + cmd += ["--mmproj", _resolve_mmproj_path_in_container(cfg)] + if not cfg.mmproj_offload: + cmd.append("--no-mmproj-offload") if cfg.api_key: cmd += ["--api-key", cfg.api_key] cmd.extend(cfg.extra_args) diff --git a/src/llamacppctl/schema.py b/src/llamacppctl/schema.py index 13018c0..fab9b75 100644 --- a/src/llamacppctl/schema.py +++ b/src/llamacppctl/schema.py @@ -60,6 +60,14 @@ class ServerConfig: # as a Bearer token on every request. api_key: str = "" + # Optional multimodal projector (vision encoder) GGUF. Resolved relative to + # hf_home unless absolute, like model_path. Must match the base model's + # vision tower. Empty => text-only server. + mmproj: str = "" + # llama.cpp offloads the projector to the GPU by default; set False to keep + # it on the CPU when VRAM is tight (emits --no-mmproj-offload). + mmproj_offload: bool = True + extra_args: list = field(default_factory=list) diff --git a/tests/test_actions.py b/tests/test_actions.py index 3683444..8a881cc 100644 --- a/tests/test_actions.py +++ b/tests/test_actions.py @@ -213,3 +213,34 @@ def test_container_lock_force_bypasses_busy(tmp_path, capsys): entered = True assert entered is True assert "busy lock" in capsys.readouterr().err + + +# --- model / mmproj path validation --------------------------------------- + + +def test_validate_model_path_accepts_existing_model(tmp_path): + (tmp_path / "m.gguf").write_bytes(b"x") + cfg = make_cfg(hf_home=tmp_path, model_path="m.gguf") + actions.validate_model_path(cfg) # must not raise + + +def test_validate_model_path_rejects_missing_model(tmp_path): + cfg = make_cfg(hf_home=tmp_path, model_path="absent.gguf") + with pytest.raises(FileNotFoundError, match="model file not found"): + actions.validate_model_path(cfg) + + +def test_validate_model_path_rejects_missing_mmproj(tmp_path): + # The model exists but the configured projector does not: --change must fail + # here, before the running container is removed. + (tmp_path / "m.gguf").write_bytes(b"x") + cfg = make_cfg(hf_home=tmp_path, model_path="m.gguf", mmproj="absent-proj.gguf") + with pytest.raises(FileNotFoundError, match="mmproj file not found"): + actions.validate_model_path(cfg) + + +def test_validate_model_path_accepts_existing_mmproj(tmp_path): + (tmp_path / "m.gguf").write_bytes(b"x") + (tmp_path / "proj.gguf").write_bytes(b"x") + cfg = make_cfg(hf_home=tmp_path, model_path="m.gguf", mmproj="proj.gguf") + actions.validate_model_path(cfg) # must not raise diff --git a/tests/test_config.py b/tests/test_config.py index 38ddb2b..65f2785 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -55,6 +55,10 @@ model_alias = alt_llm [model.noname] host_port = 9002 +[model.vision] +container_name = test_vision +mmproj = qwen3/mmproj.gguf + [prompt.concise] system_prompt = Be brief. """ @@ -280,6 +284,57 @@ def test_stream_and_read_timeout_from_config(tmp_path): assert prompt_cfg.connect_timeout == 5.0 +def test_mmproj_empty_by_default(tmp_path): + cfg_path = _write_config(tmp_path) + server_cfg, _ = resolve_effective_config(_parse(["--start", "--config", str(cfg_path)])) + assert server_cfg.mmproj == "" + assert server_cfg.mmproj_offload is True # llama.cpp offloads by default + + +def test_mmproj_from_profile(tmp_path): + cfg_path = _write_config(tmp_path) + args = _parse(["--start", "--config", str(cfg_path), "--profile", "vision"]) + server_cfg, _ = resolve_effective_config(args) + assert server_cfg.mmproj == "qwen3/mmproj.gguf" + assert server_cfg.container_name == "test_vision" + + +def test_mmproj_cli_overrides_config(tmp_path): + cfg_path = _write_config(tmp_path) + args = _parse( + [ + "--start", + "--config", + str(cfg_path), + "--profile", + "vision", + "--mmproj", + "other/proj.gguf", + "--no-mmproj-offload", + ] + ) + server_cfg, _ = resolve_effective_config(args) + assert server_cfg.mmproj == "other/proj.gguf" + assert server_cfg.mmproj_offload is False + + +def test_mmproj_offload_false_from_config(tmp_path): + cfg = CONFIG_BASIC.replace( + "[model.vision]\ncontainer_name = test_vision", + "[model.vision]\ncontainer_name = test_vision\nmmproj_offload = false", + ) + cfg_path = _write_config(tmp_path, cfg) + args = _parse(["--start", "--config", str(cfg_path), "--profile", "vision"]) + server_cfg, _ = resolve_effective_config(args) + assert server_cfg.mmproj_offload is False + + +def test_blank_mmproj_via_cli_rejected(tmp_path): + cfg_path = _write_config(tmp_path) + with pytest.raises(SystemExit): + _parse(["--start", "--config", str(cfg_path), "--mmproj", " "]) + + def test_stream_and_read_timeout_cli_overrides_config(tmp_path): cfg = CONFIG_BASIC.replace("poll_interval = 2", "poll_interval = 2\nread_timeout = 900") cfg_path = _write_config(tmp_path, cfg) diff --git a/tests/test_docker_ops.py b/tests/test_docker_ops.py index 75a6c33..6c1a7b2 100644 --- a/tests/test_docker_ops.py +++ b/tests/test_docker_ops.py @@ -178,6 +178,41 @@ def test_build_run_command_relative_model_path(): assert cmd[idx + 1] == "/hf_home/qwen3/default.gguf" +def test_build_run_command_no_mmproj_by_default(): + cmd = build_run_command(make_cfg()) + assert "--mmproj" not in cmd + assert "--no-mmproj-offload" not in cmd + + +def test_build_run_command_mmproj_relative_path_resolves_under_hf_home(): + cfg = make_cfg(mmproj="qwen3/mmproj.gguf") + cmd = build_run_command(cfg) + idx = cmd.index("--mmproj") + assert cmd[idx + 1] == "/hf_home/qwen3/mmproj.gguf" + # offload is llama.cpp's default, so the opt-out flag must stay absent + assert "--no-mmproj-offload" not in cmd + + +def test_build_run_command_mmproj_absolute_path_passed_through(): + cfg = make_cfg(mmproj="/abs/mmproj.gguf") + cmd = build_run_command(cfg) + idx = cmd.index("--mmproj") + assert cmd[idx + 1] == "/abs/mmproj.gguf" + + +def test_build_run_command_mmproj_offload_disabled(): + cfg = make_cfg(mmproj="qwen3/mmproj.gguf", mmproj_offload=False) + cmd = build_run_command(cfg) + assert "--no-mmproj-offload" in cmd + + +def test_build_run_command_no_mmproj_offload_needs_mmproj(): + # Without a projector the offload opt-out is meaningless and must not leak + # into the command line (llama.cpp would reject it). + cfg = make_cfg(mmproj="", mmproj_offload=False) + assert "--no-mmproj-offload" not in build_run_command(cfg) + + def test_format_command_for_display_quotes_properly(): cmd = ["docker", "run", "--name", "has space"] out = format_command_for_display(cmd) From 3e3dd1c15045e99fffd91ce0a46254df76e98c28 Mon Sep 17 00:00:00 2001 From: dschlueter Date: Fri, 10 Jul 2026 16:25:07 +0200 Subject: [PATCH 2/4] fix(cli): stop --print-effective-config from starting a container MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit argparse requires exactly one action, so the documented diagnostic form `--print-effective-config --config … --start` never took the early-return branch in main.run(): it printed the resolved config and then executed a real do_start(), silently replacing a running container with the [default] model. README, installation guide and manual all recommended that form. Make it an action in the mutually exclusive group. It can no longer be combined with --start/--check/--stop/--change/--chat (argparse error, exit 2), and it skips the docker_available() check, so it now really is the offline config check the docs promise. --dry-run remains a modifier and still requires a reachable daemon. Co-Authored-By: Claude Opus 4.8 --- README.md | 11 ++++--- docs/BEDIENUNGSANLEITUNG.md | 2 +- docs/INSTALL_FROM_ARCHIVE.md | 2 +- docs/SECURITY_AND_OPERATIONS.md | 27 ++++++++++------ src/llamacppctl/cli.py | 13 +++++--- src/llamacppctl/main.py | 11 +++---- tests/test_cli.py | 11 +++++-- tests/test_main.py | 57 +++++++++++++++++++++++++++++++++ 8 files changed, 104 insertions(+), 30 deletions(-) create mode 100644 tests/test_main.py diff --git a/README.md b/README.md index a780db4..197c931 100644 --- a/README.md +++ b/README.md @@ -44,9 +44,10 @@ Voraussetzungen auf dem Zielsystem: - Docker (CLI + laufender Daemon), für `--start`/--check`/--stop`/--change` - Netzwerkzugriff auf den Container-Host-Port für `--chat`/--check` -`--print-effective-config` und `--dry-run` benötigen keinen laufenden Docker-Daemon -für die reine Konfigurationsprüfung; `--start`/--check`/--stop`/--change` erfordern -einen erreichbaren Docker-Daemon. +`--print-effective-config` ist eine eigenständige, nebenwirkungsfreie Aktion und +benötigt keinen laufenden Docker-Daemon. `--start`/--check`/--stop`/--change` +erfordern einen erreichbaren Docker-Daemon — auch zusammen mit `--dry-run`, weil +die Verfügbarkeit geprüft wird, bevor der Trockenlauf greift. ## Konfiguration @@ -93,8 +94,8 @@ llamacppctl --start --config llama.cpp.config # Nur den geplanten docker-run-Befehl anzeigen, nichts ausführen llamacppctl --start --config llama.cpp.config --dry-run -# Effektive Konfiguration als JSON ausgeben -llamacppctl --print-effective-config --config llama.cpp.config --start +# Effektive Konfiguration als JSON ausgeben (startet nichts, braucht kein Docker) +llamacppctl --print-effective-config --config llama.cpp.config # Status prüfen llamacppctl --check --config llama.cpp.config diff --git a/docs/BEDIENUNGSANLEITUNG.md b/docs/BEDIENUNGSANLEITUNG.md index 87b2605..7b8a91c 100644 --- a/docs/BEDIENUNGSANLEITUNG.md +++ b/docs/BEDIENUNGSANLEITUNG.md @@ -49,7 +49,7 @@ Siehe [`INSTALL_FROM_ARCHIVE.md`](INSTALL_FROM_ARCHIVE.md). ```bash llamacppctl --help -llamacppctl --print-effective-config --config llama.cpp.config --start --dry-run +llamacppctl --print-effective-config --config llama.cpp.config ``` --- diff --git a/docs/INSTALL_FROM_ARCHIVE.md b/docs/INSTALL_FROM_ARCHIVE.md index 2c7c2ef..ab0e641 100644 --- a/docs/INSTALL_FROM_ARCHIVE.md +++ b/docs/INSTALL_FROM_ARCHIVE.md @@ -64,7 +64,7 @@ cp llama.cpp.config.example llama.cpp.config Auflösung prüfen, ohne etwas zu starten: ```bash -llamacppctl --print-effective-config --config llama.cpp.config --start --dry-run +llamacppctl --print-effective-config --config llama.cpp.config ``` Das zeigt die vollständig aufgelöste Konfiguration als JSON **und** das diff --git a/docs/SECURITY_AND_OPERATIONS.md b/docs/SECURITY_AND_OPERATIONS.md index 4eaa144..254d7d6 100644 --- a/docs/SECURITY_AND_OPERATIONS.md +++ b/docs/SECURITY_AND_OPERATIONS.md @@ -258,22 +258,29 @@ keinen vermeidbaren Ausfall verursacht. ## 6. `--dry-run` und `--print-effective-config` als Sicherheitswerkzeuge -- `--print-effective-config` gibt die vollständig aufgelöste Konfiguration - (Server- und Prompt-Konfiguration) als JSON aus, **bevor** irgendeine - Docker- oder HTTP-Aktion ausgeführt wird. Damit lässt sich prüfen, welche - Werte aus welcher Quelle (Defaults/[default]/[model.*]/CLI) tatsächlich - gewonnen haben, ohne einen Container anzufassen. +- `--print-effective-config` ist eine **eigene Aktion** in der + Mutually-Exclusive-Gruppe: sie gibt die vollständig aufgelöste Konfiguration + (Server- und Prompt-Konfiguration) als JSON aus und beendet sich. Damit lässt + sich prüfen, welche Werte aus welcher Quelle (Defaults/[default]/[model.*]/CLI) + tatsächlich gewonnen haben, ohne einen Container anzufassen. + + Bis einschließlich 0.1.0 war dies ein *Flag*. Da argparse immer eine Aktion + verlangt, führte `--print-effective-config --start` die Konfigurationsausgabe + **und anschließend einen echten `do_start()`** aus — der einen laufenden + Container stillschweigend ersetzte. Die Kombination ist jetzt ein + argparse-Fehler (Exit-Code 2). - `--dry-run` (in Kombination mit `--start`/--change`) zeigt den vollständig zusammengesetzten `docker run`-Befehl (Shell-quotiert zur Anzeige) an, **ohne** ihn auszuführen. Empfohlen vor jeder Änderung an einer Produktionskonfiguration, insbesondere nach Anpassungen an `llama.cpp.config`. -Beide Flags erfordern **keinen** erreichbaren Docker-Daemon für ihre reine -Ausgabe — `main.run()` prüft `docker_available()` derzeit vor der -Konfigurationsauflösung; auf einem Host ganz ohne Docker (z. B. zur reinen -Konfigurationsvalidierung) schlägt der Aufruf entsprechend mit einer -expliziten Fehlermeldung fehl statt still falsche Annahmen zu treffen. +`--print-effective-config` erfordert **keinen** erreichbaren Docker-Daemon; +`main.run()` überspringt die `docker_available()`-Prüfung für diese Aktion. Für +`--dry-run` gilt das **nicht**: es ist ein Modifikator von `--start`/`--change`, +und die Docker-Prüfung läuft vor der Konfigurationsauflösung. Auf einem Host ganz +ohne Docker schlägt `--start --dry-run` daher mit einer expliziten Fehlermeldung +fehl, statt still falsche Annahmen zu treffen. ## 7. Betriebsmodell auf einem dedizierten GPU-Host diff --git a/src/llamacppctl/cli.py b/src/llamacppctl/cli.py index 7230f42..d32da57 100644 --- a/src/llamacppctl/cli.py +++ b/src/llamacppctl/cli.py @@ -20,6 +20,14 @@ def build_parser() -> argparse.ArgumentParser: action.add_argument("--stop", action="store_true", help="Stop/remove the container") action.add_argument("--change", action="store_true", help="Stop, reconfigure, and restart") action.add_argument("--chat", action="store_true", help="Send a prompt to a running server") + # A diagnostic action, not a modifier: combining it with --start used to print + # the config and then really start the container. It is mutually exclusive with + # the other actions so it can never have a side effect. + action.add_argument( + "--print-effective-config", + action="store_true", + help="Print the fully merged configuration as JSON and exit. Touches nothing", + ) p.add_argument("--config", default="llama.cpp.config", help="Path to llama.cpp.config") p.add_argument("--profile", help="Model profile name: [model.] in config") @@ -146,11 +154,6 @@ def build_parser() -> argparse.ArgumentParser: p.add_argument("--log-lines", type=int, default=100) p.add_argument("--dry-run", action="store_true", help="Show the docker run command, do not execute it") p.add_argument("--force", action="store_true", help="Force stop/change despite inconsistencies") - p.add_argument( - "--print-effective-config", - action="store_true", - help="Print the fully merged configuration as JSON and exit (unless combined with an action)", - ) return p diff --git a/src/llamacppctl/main.py b/src/llamacppctl/main.py index b633b96..0c82d56 100644 --- a/src/llamacppctl/main.py +++ b/src/llamacppctl/main.py @@ -3,7 +3,7 @@ Orchestration order: 1. Parse CLI arguments (cli.build_parser) 2. Run semantic validation (cli.validate_args) - 3. Check that docker is available + 3. Check that docker is available (skipped for --print-effective-config) 4. Resolve prompt sources (prompt_io) under an InputPolicy built from CLI flags 5. Resolve effective server/prompt config (config.resolve_effective_config) 6. Dispatch to the requested action (actions.do_*) @@ -55,7 +55,9 @@ def run(argv: Optional[Sequence[str]] = None) -> int: args = parser.parse_args(argv) validate_args(args, parser) - if not docker_available(): + # --print-effective-config is purely diagnostic: it must work on a host + # without a Docker daemon, and it must never touch a container. + if not args.print_effective_config and not docker_available(): print("docker is not available on PATH (or the daemon is not reachable)", file=sys.stderr) return 1 @@ -74,10 +76,7 @@ def run(argv: Optional[Sequence[str]] = None) -> int: if args.print_effective_config: actions.print_effective_config(server_cfg, prompt_cfg) - # An action is always required by argparse; only exit early if the run - # is purely diagnostic (no action would actually do anything). - if not (args.start or args.check or args.stop or args.change or args.chat): - return 0 + return 0 if args.start: return actions.do_start(server_cfg, prompt_cfg, args) diff --git a/tests/test_cli.py b/tests/test_cli.py index 051035c..b792e41 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -147,9 +147,16 @@ def test_dry_run_flag(): assert args.dry_run is True -def test_print_effective_config_flag(): - args = parse(["--start", "--print-effective-config"]) +def test_print_effective_config_is_a_standalone_action(): + args = parse(["--print-effective-config"]) assert args.print_effective_config is True + assert args.start is False + + +def test_print_effective_config_cannot_be_combined_with_start(): + # Regression: as a plain flag this printed the config and then really started + # the container, silently replacing a running one. + parse_error(["--print-effective-config", "--start"]) def test_reasoning_choice_validated(): diff --git a/tests/test_main.py b/tests/test_main.py new file mode 100644 index 0000000..6b1e554 --- /dev/null +++ b/tests/test_main.py @@ -0,0 +1,57 @@ +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src")) + +from llamacppctl import actions, main as main_mod # noqa: E402 + +CONFIG = """ +[default] +hf_home = /srv/models +model_path = qwen3/default.gguf +container_name = test_main +""" + + +def _config(tmp_path: Path) -> Path: + path = tmp_path / "llama.cpp.config" + path.write_text(CONFIG, encoding="utf-8") + return path + + +def test_print_effective_config_has_no_side_effects(tmp_path, monkeypatch, capsys): + """Regression: --print-effective-config used to fall through into do_start() + because argparse always required an action, silently replacing a running + container.""" + started = [] + monkeypatch.setattr(actions, "do_start", lambda *a, **k: started.append(1)) + monkeypatch.setattr(actions, "do_change", lambda *a, **k: started.append(1)) + monkeypatch.setattr(actions, "do_stop", lambda *a, **k: started.append(1)) + + rc = main_mod.run(["--print-effective-config", "--config", str(_config(tmp_path))]) + + assert rc == 0 + assert started == [] + assert '"container_name": "test_main"' in capsys.readouterr().out + + +def test_print_effective_config_works_without_docker(tmp_path, monkeypatch, capsys): + """It is a pure config check, so it must not require a reachable daemon.""" + def boom() -> bool: + raise AssertionError("docker_available() must not be consulted") + + monkeypatch.setattr(main_mod, "docker_available", boom) + + rc = main_mod.run(["--print-effective-config", "--config", str(_config(tmp_path))]) + + assert rc == 0 + assert '"hf_home": "/srv/models"' in capsys.readouterr().out + + +def test_start_still_requires_docker(tmp_path, monkeypatch, capsys): + monkeypatch.setattr(main_mod, "docker_available", lambda: False) + + rc = main_mod.run(["--start", "--config", str(_config(tmp_path))]) + + assert rc == 1 + assert "docker is not available" in capsys.readouterr().err From 6f2f8aff6ca27a763d9320666f676fc7bf59288c Mon Sep 17 00:00:00 2001 From: dschlueter Date: Fri, 10 Jul 2026 16:25:26 +0200 Subject: [PATCH 3/4] feat(scripts): add prompt-test evaluator and suite runner MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit eval_prompt_tests.py measures the objective half of docs/EVAL_RUBRIC.md over the manual test archive: word count against the target stated in each prompt, truncation suspicion, and — for the coding domain — it writes the generated module and tests to a temp dir and actually runs pytest against them. Deriving the module's filename is the delicate part: a name taken from a test's `import sqlite3` would shadow the stdlib and fail the run for a reason the model is not responsible for. Names now come from the last *.py mention before the block, then from `from X import`, and anything in sys.stdlib_module_names is rejected. A module that no test imports is reported as such, since that is a finding about test quality rather than a guess the runner got wrong. run_prompt_suite.sh drives one prompt domain against a running profile and stores the outputs under the archive's naming convention. Both scripts join the ruff gate. Co-Authored-By: Claude Opus 4.8 --- .forgejo/workflows/ci.yml | 2 +- scripts/check.sh | 2 +- scripts/eval_prompt_tests.py | 457 +++++++++++++++++++++++++++++++++++ scripts/run_prompt_suite.sh | 97 ++++++++ 4 files changed, 556 insertions(+), 2 deletions(-) create mode 100755 scripts/eval_prompt_tests.py create mode 100755 scripts/run_prompt_suite.sh diff --git a/.forgejo/workflows/ci.yml b/.forgejo/workflows/ci.yml index 8041541..c931ba7 100644 --- a/.forgejo/workflows/ci.yml +++ b/.forgejo/workflows/ci.yml @@ -19,7 +19,7 @@ jobs: run: pip install -e ".[dev]" - name: Ruff (lint) - run: ruff check src/ tests/ build_archive.py + run: ruff check src/ tests/ build_archive.py scripts/eval_prompt_tests.py - name: Mypy (type check) run: mypy diff --git a/scripts/check.sh b/scripts/check.sh index a7e59c7..9773ca7 100755 --- a/scripts/check.sh +++ b/scripts/check.sh @@ -27,7 +27,7 @@ run() { fi } -run "ruff (lint)" "${BIN}ruff" check src/ tests/ build_archive.py +run "ruff (lint)" "${BIN}ruff" check src/ tests/ build_archive.py scripts/eval_prompt_tests.py run "mypy (types)" "${BIN}mypy" # Use `python -m pytest` (not the pytest console script) so the repo root is on # sys.path — the test modules import `from tests.test_docker_ops import ...`. diff --git a/scripts/eval_prompt_tests.py b/scripts/eval_prompt_tests.py new file mode 100755 index 0000000..290db3d --- /dev/null +++ b/scripts/eval_prompt_tests.py @@ -0,0 +1,457 @@ +#!/usr/bin/env python3 +"""eval_prompt_tests.py + +Auswertung der manuellen Modell-Prompt-Testlaeufe unter +``~/llamacppctl_prompt_tests/`` gegen die Prompts in ``example_user_prompts/``. + +Misst die objektive Ebene der Rubrik (docs/EVAL_RUBRIC.md): + + * prosa/reden: Wortzahl und Abweichung vom im Prompt genannten Zielwert, + Trunkierungsverdacht. + * coding: extrahiert die Python-Codebloecke, schreibt Modul und Tests in + ein temporaeres Verzeichnis und **fuehrt pytest tatsaechlich aus**. + +WARNUNG: ``--exec`` fuehrt modellgenerierten Code aus. Der Runner nutzt ein +temporaeres Verzeichnis, ein Zeitlimit und verweigert den Start als root, bietet +aber keine Netz- oder Dateisystem-Isolation. Siehe docs/EVAL_RUBRIC.md. + +Beispiele +--------- + scripts/eval_prompt_tests.py # messen + Tests ausfuehren + scripts/eval_prompt_tests.py --no-exec # nur statisch messen + scripts/eval_prompt_tests.py --json report.json # maschinenlesbar + scripts/eval_prompt_tests.py --anonymize blind/ # blinde Bewertung vorbereiten + +Exit-Codes +---------- + 0 Auswertung durchgelaufen (auch wenn einzelne Testlaeufe rot sind) + 1 Eingabeverzeichnis fehlt oder unsichere Ausfuehrungsumgebung +""" + +from __future__ import annotations + +import argparse +import json +import os +import random +import re +import shutil +import subprocess +import sys +import tempfile +from dataclasses import asdict, dataclass, field +from pathlib import Path +from typing import Optional + +DEFAULT_RESULTS_DIR = Path.home() / "llamacppctl_prompt_tests" +DEFAULT_PROMPTS_DIR = Path(__file__).resolve().parents[1] / "example_user_prompts" + +DOMAINS = ("prosa", "reden", "coding") +# Dateien ohne Modellpraefix stammen vom Default-Modell aus llama.cpp.config. +KNOWN_MODELS = ("carnice", "qwen27b", "qwopus", "ornith", "qwen35base") +DEFAULT_MODEL_LABEL = "default" +VARIANT_MARKERS = frozenset({"r2", "highbudget", "explizit"}) + +# "etwa 1200 Wörtern", "ca. 700 Wörter", "rund 900 Worten" +LENGTH_TARGET_RE = re.compile( + r"(?:etwa|ca\.|circa|rund)\s*(\d{2,5})\s*(?:Wörter|Wörtern|Worte|Worten)", + re.IGNORECASE, +) +FENCE_RE = re.compile(r"^```([A-Za-z+#]*)\s*$(.*?)^```\s*$", re.MULTILINE | re.DOTALL) +PY_FILENAME_RE = re.compile(r"`([A-Za-z_][A-Za-z0-9_]*)\.py`") +FROM_IMPORT_RE = re.compile(r"^\s*from\s+([A-Za-z_][A-Za-z0-9_]*)\s+import", re.MULTILINE) +PLAIN_IMPORT_RE = re.compile(r"^\s*import\s+([A-Za-z_][A-Za-z0-9_]*)", re.MULTILINE) +PYTEST_COUNT_RE = re.compile(r"(\d+)\s+(passed|failed|error|errors)") + +# Endet der Text sauber? Fehlendes Satzzeichen deutet auf finish_reason=length. +SENTENCE_END = tuple('.!?"»“’—-)') + +# Ein Modul darf niemals nach einem Stdlib-Modul benannt werden: eine Datei +# sqlite3.py im Arbeitsverzeichnis ueberschattet die echte Stdlib und laesst den +# Testlauf mit einem Importfehler scheitern, der faelschlich dem Modell +# angelastet wuerde. sys.stdlib_module_names ist exakt (Python >= 3.10). +RESERVED_MODULE_NAMES = frozenset(sys.stdlib_module_names) | {"pytest", "conftest"} + + +@dataclass +class Run: + """Ein einzelner Testlauf, abgeleitet aus dem Dateinamen.""" + + path: Path + model: str + domain: str + case: str + variant: str = "" + + @property + def label(self) -> str: + base = f"{self.model}/{self.domain}_{self.case}" + return f"{base}[{self.variant}]" if self.variant else base + + +@dataclass +class Metrics: + label: str + model: str + domain: str + case: str + variant: str + words: int = 0 + target_words: Optional[int] = None + deviation_pct: Optional[float] = None + looks_truncated: bool = False + # coding + executed: bool = False + exec_status: str = "" # ok | failed | timeout | no-tests | skipped- + tests_passed: int = 0 + tests_failed: int = 0 + detail: str = "" + code_langs: list = field(default_factory=list) + + +def parse_run(path: Path) -> Optional[Run]: + """Zerlegt ``[_]_[_].out.txt``. + + Die Namenskonvention ist historisch uneinheitlich (mal steht der Fallname + hinter der Nummer, mal eine Variante wie ``r2``), deshalb wird tolerant + geparst statt streng validiert. + """ + stem = path.name + for suffix in (".out.txt", ".txt", ".md"): + if stem.endswith(suffix): + stem = stem[: -len(suffix)] + break + + tokens = stem.split("_") + if not tokens: + return None + + model = DEFAULT_MODEL_LABEL + if tokens[0] in KNOWN_MODELS: + model = tokens[0] + tokens = tokens[1:] + + domain = next((t for t in tokens if t in DOMAINS), None) + if domain is None: + return None + rest = tokens[tokens.index(domain) + 1 :] + + case = next((t for t in rest if t.isdigit()), None) + if case is None: + return None + + # Der Fallname (z. B. "jsonl_validator") ist keine Variante. Nur bekannte + # Marker zaehlen -- sie muessen erhalten bleiben, sonst kollidieren zwei + # verschiedene Laeufe (prosa_04_dialog vs. prosa_04_dialog_highbudget) unter + # demselben Label. + variant = "_".join(t for t in rest[rest.index(case) + 1 :] if t in VARIANT_MARKERS) + + return Run(path=path, model=model, domain=domain, case=case, variant=variant) + + +def load_length_targets(prompts_dir: Path) -> dict: + """Mappt (domaene, nr) auf die im Prompt genannte Ziel-Wortzahl.""" + targets: dict = {} + if not prompts_dir.is_dir(): + return targets + for prompt in sorted(prompts_dir.glob("*.md")): + tokens = prompt.stem.split("_") + if len(tokens) < 2 or tokens[0] not in DOMAINS or not tokens[1].isdigit(): + continue + match = LENGTH_TARGET_RE.search(prompt.read_text(encoding="utf-8")) + if match: + targets[(tokens[0], tokens[1])] = int(match.group(1)) + return targets + + +def count_words(text: str) -> int: + """Woerter ohne Markdown-Auszeichnung und ohne Codebloecke.""" + without_code = FENCE_RE.sub("", text) + cleaned = re.sub(r"[#*_>`]", " ", without_code) + return len(cleaned.split()) + + +def looks_truncated(text: str) -> bool: + stripped = text.strip() + if not stripped: + return True + return not stripped.endswith(SENTENCE_END) + + +def extract_code_blocks(text: str) -> list: + """Liefert (sprache, code, start_offset) je Fence-Block. + + Der Offset wird gebraucht, um den im Fliesstext *vor* einem Block genannten + Dateinamen zu finden; ein Textvergleich waere mehrdeutig, wenn derselbe Code + mehrfach vorkommt. + """ + return [ + (match.group(1).lower(), match.group(2), match.start()) + for match in FENCE_RE.finditer(text) + ] + + +def is_test_block(code: str) -> bool: + return "def test_" in code or "import pytest" in code + + +def _usable(name: str) -> bool: + return bool(name) and name not in RESERVED_MODULE_NAMES and not name.startswith("test_") + + +def infer_module_name(markdown: str, block_start: int, tests: list) -> str: + """Ermittelt den Dateinamen des Moduls. + + Reihenfolge: (1) letzte im Fliesstext *vor* dem Block genannte ``*.py``-Datei, + (2) ``from X import`` in den Tests -- die Tests importieren das Modul unter + genau diesem Namen, (3) ``import X``. Stdlib-Namen und ``test_*`` werden in + jeder Stufe verworfen. + """ + for name in reversed(PY_FILENAME_RE.findall(markdown[:block_start])): + if _usable(name): + return name + + for pattern in (FROM_IMPORT_RE, PLAIN_IMPORT_RE): + for test_code in tests: + for candidate in pattern.findall(test_code): + if _usable(candidate): + return candidate + + return "generated_module" + + +def _parse_pytest_counts(output: str) -> tuple: + passed = failed = 0 + for count, kind in PYTEST_COUNT_RE.findall(output): + if kind == "passed": + passed = int(count) + else: + failed += int(count) + return passed, failed + + +def run_python_case(markdown: str, timeout: int, keep_dir: Optional[Path]) -> dict: + """Schreibt Modul + Tests in ein Temp-Verzeichnis und ruft pytest auf.""" + blocks = extract_code_blocks(markdown) + py_blocks = [(body, start) for lang, body, start in blocks if lang in ("python", "py")] + if not py_blocks: + return {"exec_status": "no-code", "detail": "kein Python-Block gefunden"} + + tests = [body for body, _ in py_blocks if is_test_block(body)] + modules = [(body, start) for body, start in py_blocks if not is_test_block(body)] + if not tests: + return {"exec_status": "no-tests", "detail": f"{len(modules)} Modul-Block(loecke), keine Tests"} + + workdir = Path(tempfile.mkdtemp(prefix="llamaeval_")) + warnings = [] + try: + used: set = set() + for module_body, block_start in modules: + name = infer_module_name(markdown, block_start, tests) + if name == "generated_module": + # Weder der Fliesstext nennt einen Dateinamen noch importieren die + # Tests das Modul: ein echter Mangel der Testqualitaet, kein + # Ratefehler des Runners. + warnings.append("Tests importieren das Modul nicht") + while name in used: # zwei Modul-Bloecke, gleicher geratener Name + name += "_x" + used.add(name) + (workdir / f"{name}.py").write_text(module_body, encoding="utf-8") + for index, test_body in enumerate(tests): + suffix = "" if index == 0 else f"_{index}" + (workdir / f"test_generated{suffix}.py").write_text(test_body, encoding="utf-8") + + try: + proc = subprocess.run( + [sys.executable, "-m", "pytest", "-q", "--tb=no", "-p", "no:cacheprovider"], + cwd=workdir, + capture_output=True, + text=True, + timeout=timeout, + ) + except subprocess.TimeoutExpired: + return {"exec_status": "timeout", "detail": f"pytest > {timeout}s"} + except FileNotFoundError: + return {"exec_status": "skipped-no-pytest", "detail": "pytest nicht gefunden"} + + output = proc.stdout + proc.stderr + passed, failed = _parse_pytest_counts(output) + status = "ok" if proc.returncode == 0 else "failed" + last = [ln for ln in output.strip().splitlines() if ln.strip()] + detail = "; ".join(warnings + ([last[-1]] if last else [])) + return { + "exec_status": status, + "tests_passed": passed, + "tests_failed": failed, + "detail": detail[:120], + } + finally: + if keep_dir is not None: + shutil.move(str(workdir), str(keep_dir / workdir.name)) + else: + shutil.rmtree(workdir, ignore_errors=True) + + +def evaluate(run: Run, targets: dict, do_exec: bool, timeout: int, keep_dir) -> Metrics: + text = run.path.read_text(encoding="utf-8", errors="replace") + metrics = Metrics( + label=run.label, + model=run.model, + domain=run.domain, + case=run.case, + variant=run.variant, + words=count_words(text), + looks_truncated=looks_truncated(text), + code_langs=sorted({lang for lang, _, _ in extract_code_blocks(text) if lang}), + ) + + target = targets.get((run.domain, run.case)) + if target: + metrics.target_words = target + metrics.deviation_pct = round((metrics.words - target) / target * 100, 1) + + if run.domain != "coding": + return metrics + + if not do_exec: + metrics.exec_status = "skipped-no-exec" + return metrics + if "typescript" in metrics.code_langs and "python" not in metrics.code_langs: + metrics.exec_status = "skipped-typescript" + metrics.detail = "kein tsc im PATH; TypeScript-Lauf nicht implementiert" + return metrics + + result = run_python_case(text, timeout, keep_dir) + metrics.executed = result["exec_status"] in ("ok", "failed") + metrics.exec_status = result["exec_status"] + metrics.tests_passed = result.get("tests_passed", 0) + metrics.tests_failed = result.get("tests_failed", 0) + metrics.detail = result.get("detail", "") + return metrics + + +def anonymize(runs: list, out_dir: Path) -> Path: + """Kopiert die Laeufe unter Zufalls-IDs und schreibt die Aufloesungstabelle. + + Fuer blinde Bewertung: der Bewertende darf das Modell nicht kennen. + """ + out_dir.mkdir(parents=True, exist_ok=True) + ids = [f"text_{i:03d}" for i in range(len(runs))] + random.shuffle(ids) + mapping = {} + for run, blind_id in zip(runs, ids): + shutil.copyfile(run.path, out_dir / f"{blind_id}.txt") + mapping[blind_id] = {"model": run.model, "domain": run.domain, "case": run.case, + "variant": run.variant, "source": run.path.name} + key_path = out_dir / "AUFLOESUNG.json" + key_path.write_text(json.dumps(mapping, indent=2, ensure_ascii=False), encoding="utf-8") + return key_path + + +def _fmt_dev(metrics: Metrics) -> str: + if metrics.deviation_pct is None: + return "—" + return f"{metrics.deviation_pct:+.1f}%" + + +def print_report(all_metrics: list) -> None: + for domain in DOMAINS: + rows = [m for m in all_metrics if m.domain == domain] + if not rows: + continue + print(f"\n=== {domain} ===") + if domain == "coding": + print(f"{'Lauf':<28} {'Status':<18} {'passed':>6} {'failed':>6} Detail") + for m in sorted(rows, key=lambda r: (r.model, r.case)): + print(f"{m.label:<28} {m.exec_status:<18} {m.tests_passed:>6} " + f"{m.tests_failed:>6} {m.detail[:60]}") + else: + print(f"{'Lauf':<28} {'Wörter':>7} {'Ziel':>6} {'Abw.':>8} {'trunkiert':<9}") + for m in sorted(rows, key=lambda r: (r.model, r.case)): + target = str(m.target_words) if m.target_words else "—" + trunc = "ja" if m.looks_truncated else "" + print(f"{m.label:<28} {m.words:>7} {target:>6} {_fmt_dev(m):>8} {trunc:<9}") + + coding = [m for m in all_metrics if m.domain == "coding" and m.executed] + if coding: + green = sum(1 for m in coding if m.exec_status == "ok") + total_p = sum(m.tests_passed for m in coding) + total_f = sum(m.tests_failed for m in coding) + print(f"\nCoding gesamt: {green}/{len(coding)} Läufe grün, " + f"{total_p} Tests bestanden, {total_f} durchgefallen.") + + lengths = [m for m in all_metrics if m.deviation_pct is not None] + if lengths: + within5 = sum(1 for m in lengths if abs(m.deviation_pct) <= 5) + worst = max(lengths, key=lambda m: abs(m.deviation_pct)) + print(f"Längentreue: {within5}/{len(lengths)} Läufe im ±5%-Fenster; " + f"größte Abweichung {_fmt_dev(worst)} ({worst.label}).") + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[1] if __doc__ else "") + parser.add_argument("--results-dir", type=Path, default=DEFAULT_RESULTS_DIR) + parser.add_argument("--prompts-dir", type=Path, default=DEFAULT_PROMPTS_DIR) + parser.add_argument("--no-exec", dest="exec_code", action="store_false", default=True, + help="Generierten Code NICHT ausführen, nur statisch messen") + parser.add_argument("--timeout", type=int, default=60, help="Sekunden pro pytest-Lauf") + parser.add_argument("--json", type=Path, help="Report zusätzlich als JSON schreiben") + parser.add_argument("--keep-workdirs", type=Path, + help="Temp-Verzeichnisse der Testläufe hierhin retten (Debugging)") + parser.add_argument("--anonymize", type=Path, + help="Anonymisierte Kopien + Auflösungstabelle für blinde Bewertung") + args = parser.parse_args() + + if not args.results_dir.is_dir(): + print(f"Ergebnisverzeichnis nicht gefunden: {args.results_dir}", file=sys.stderr) + return 1 + + if args.exec_code and hasattr(os, "geteuid") and os.geteuid() == 0: + print("Verweigert: modellgenerierten Code nicht als root ausführen " + "(--no-exec erzwingt die statische Auswertung).", file=sys.stderr) + return 1 + + runs = [] + skipped = [] + for path in sorted(args.results_dir.iterdir()): + if not path.is_file(): + continue + run = parse_run(path) + (runs.append(run) if run else skipped.append(path.name)) + + if not runs: + print(f"Keine auswertbaren Läufe in {args.results_dir}", file=sys.stderr) + return 1 + + if args.anonymize: + key = anonymize(runs, args.anonymize) + print(f"{len(runs)} Texte anonymisiert nach {args.anonymize}; Auflösung: {key}") + + if args.keep_workdirs: + args.keep_workdirs.mkdir(parents=True, exist_ok=True) + + targets = load_length_targets(args.prompts_dir) + if not targets: + print(f"Warnung: keine Ziel-Wortzahlen aus {args.prompts_dir} gelesen", file=sys.stderr) + + all_metrics = [ + evaluate(run, targets, args.exec_code, args.timeout, args.keep_workdirs) + for run in runs + ] + + print_report(all_metrics) + if skipped: + print(f"\nÜbersprungen ({len(skipped)}): {', '.join(skipped)}") + + if args.json: + args.json.write_text( + json.dumps([asdict(m) for m in all_metrics], indent=2, ensure_ascii=False), + encoding="utf-8", + ) + print(f"JSON-Report: {args.json}") + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/run_prompt_suite.sh b/scripts/run_prompt_suite.sh new file mode 100755 index 0000000..bc73570 --- /dev/null +++ b/scripts/run_prompt_suite.sh @@ -0,0 +1,97 @@ +#!/usr/bin/env bash +# +# Faehrt eine Domaene der Beispiel-Prompts gegen ein laufendes Modell-Profil und +# legt die Ausgaben unter der etablierten Namenskonvention im Test-Archiv ab: +# +#