diff --git a/llama.cpp.config.example b/llama.cpp.config.example index b411535..4656f45 100644 --- a/llama.cpp.config.example +++ b/llama.cpp.config.example @@ -88,6 +88,24 @@ host_port = 8003 gpu_device = 1 ctx_size = 131072 +# Vision profile: `mmproj` points at the multimodal projector (vision encoder) +# GGUF. Like `model_path` it is resolved relative to `hf_home` unless absolute. +# The projector MUST match the base model's vision tower -- a projector built for +# a different base will not load. A missing projector aborts with exit code 3 +# (for --change: before the running container is removed). +# +# llama.cpp offloads the projector to the GPU by default; set mmproj_offload to +# false to keep it on the CPU when VRAM is tight. Images are then sent to +# /v1/chat/completions as an image_url content part -- `llamacppctl --chat` +# itself only sends text. +[model.vision] +model_path = qwen3/Qwen3-35B-A3B-Q4_K_M.gguf +mmproj = qwen3/mmproj.gguf +mmproj_offload = true +container_name = llama_cpp_vision +host_port = 8004 +gpu_device = 1 + [prompt.concise] system_prompt = Du antwortest kurz, präzise und technisch. diff --git a/src/llamacppctl/actions.py b/src/llamacppctl/actions.py index 9011338..f1bb902 100644 --- a/src/llamacppctl/actions.py +++ b/src/llamacppctl/actions.py @@ -77,15 +77,23 @@ def check_exit_code(result: CheckResult) -> int: return EXIT_HTTP_NOT_READY +def _host_path(cfg: ServerConfig, path_str: str) -> Path: + path = Path(path_str) + return path if path.is_absolute() else cfg.hf_home / path_str + + def validate_model_path(cfg: ServerConfig) -> None: - model_path = Path(cfg.model_path) - if model_path.is_absolute(): - target = model_path - else: - target = cfg.hf_home / cfg.model_path + """Check that the model file — and the vision projector, when one is + configured — exist on the host before the container is (re)created.""" + target = _host_path(cfg, cfg.model_path) if not target.exists(): raise FileNotFoundError(f"model file not found: {target}") + if cfg.mmproj: + projector = _host_path(cfg, cfg.mmproj) + if not projector.exists(): + raise FileNotFoundError(f"mmproj file not found: {projector}") + def print_effective_config(cfg: ServerConfig, prompt_cfg: PromptConfig) -> None: payload = { diff --git a/src/llamacppctl/cli.py b/src/llamacppctl/cli.py index 11ed47f..7230f42 100644 --- a/src/llamacppctl/cli.py +++ b/src/llamacppctl/cli.py @@ -46,6 +46,18 @@ def build_parser() -> argparse.ArgumentParser: p.add_argument("--image", help="Docker image (e.g. ghcr.io/ggml-org/llama.cpp:server-cuda)") p.add_argument("--hf-home", help="Host path mounted read-only as model storage") p.add_argument("--model-path", help="Model path, relative to --hf-home unless absolute") + p.add_argument( + "--mmproj", + help="Vision projector (mmproj) GGUF, relative to --hf-home unless absolute. " + "Must match the base model's vision tower; enables image input", + ) + p.add_argument( + "--no-mmproj-offload", + dest="mmproj_offload", + action="store_false", + default=None, + help="Keep the vision projector on the CPU instead of offloading it to the GPU", + ) p.add_argument("--container-name", help="Docker container name (must be unique per config)") p.add_argument("--host-port", type=int, help="Host port to publish") p.add_argument("--container-port", type=int, help="Container-internal port") @@ -185,3 +197,6 @@ def validate_args(args: argparse.Namespace, parser: argparse.ArgumentParser) -> if args.container_name is not None and not args.container_name.strip(): parser.error("--container-name must not be empty") + + if args.mmproj is not None and not args.mmproj.strip(): + parser.error("--mmproj must not be empty") diff --git a/src/llamacppctl/config.py b/src/llamacppctl/config.py index 684a6e9..331d899 100644 --- a/src/llamacppctl/config.py +++ b/src/llamacppctl/config.py @@ -71,6 +71,8 @@ def builtin_defaults() -> dict: "host": "0.0.0.0", "expose": "false", # publish only on loopback unless true "api_key": "", + "mmproj": "", # vision projector GGUF; empty => text-only server + "mmproj_offload": "true", "health_endpoint": "/health", "models_endpoint": "/v1/models", "chat_endpoint": "/v1/chat/completions", @@ -137,6 +139,8 @@ def _apply_cli_overrides(merged: dict, args) -> None: "lock_file": getattr(args, "lock_file", None), "expose": getattr(args, "expose", None), "api_key": getattr(args, "api_key", None), + "mmproj": getattr(args, "mmproj", None), + "mmproj_offload": getattr(args, "mmproj_offload", None), } for key, value in overrides.items(): if value is not None: @@ -193,6 +197,8 @@ def build_server_config(merged: dict) -> ServerConfig: host=str(merged["host"]), expose=_to_bool(merged.get("expose", "false"), "expose"), api_key=str(merged.get("api_key", "")), + mmproj=str(merged.get("mmproj") or "").strip(), + mmproj_offload=_to_bool(merged.get("mmproj_offload", "true"), "mmproj_offload"), health_endpoint=str(merged["health_endpoint"]), models_endpoint=str(merged["models_endpoint"]), chat_endpoint=str(merged["chat_endpoint"]), diff --git a/src/llamacppctl/docker_ops.py b/src/llamacppctl/docker_ops.py index 40c0669..d4184a3 100644 --- a/src/llamacppctl/docker_ops.py +++ b/src/llamacppctl/docker_ops.py @@ -105,11 +105,21 @@ def container_logs(name: str, tail: int = 100) -> str: tail_logs = container_logs +def _resolve_hf_path_in_container(path_str: str) -> str: + """Map a host-side model/projector path to its path inside the container. + Absolute paths are passed through; relative ones resolve under the + read-only /hf_home mount.""" + if Path(path_str).is_absolute(): + return path_str + return f"/hf_home/{path_str}" + + def _resolve_model_path_in_container(cfg: ServerConfig) -> str: - model_path = Path(cfg.model_path) - if model_path.is_absolute(): - return str(model_path) - return f"/hf_home/{cfg.model_path}" + return _resolve_hf_path_in_container(cfg.model_path) + + +def _resolve_mmproj_path_in_container(cfg: ServerConfig) -> str: + return _resolve_hf_path_in_container(cfg.mmproj) def _port_publish(cfg: ServerConfig) -> str: @@ -181,6 +191,10 @@ def build_run_command(cfg: ServerConfig) -> list: cmd.append("--kv-unified") if cfg.cont_batching: cmd.append("--cont-batching") + if cfg.mmproj: + cmd += ["--mmproj", _resolve_mmproj_path_in_container(cfg)] + if not cfg.mmproj_offload: + cmd.append("--no-mmproj-offload") if cfg.api_key: cmd += ["--api-key", cfg.api_key] cmd.extend(cfg.extra_args) diff --git a/src/llamacppctl/schema.py b/src/llamacppctl/schema.py index 13018c0..fab9b75 100644 --- a/src/llamacppctl/schema.py +++ b/src/llamacppctl/schema.py @@ -60,6 +60,14 @@ class ServerConfig: # as a Bearer token on every request. api_key: str = "" + # Optional multimodal projector (vision encoder) GGUF. Resolved relative to + # hf_home unless absolute, like model_path. Must match the base model's + # vision tower. Empty => text-only server. + mmproj: str = "" + # llama.cpp offloads the projector to the GPU by default; set False to keep + # it on the CPU when VRAM is tight (emits --no-mmproj-offload). + mmproj_offload: bool = True + extra_args: list = field(default_factory=list) diff --git a/tests/test_actions.py b/tests/test_actions.py index 3683444..8a881cc 100644 --- a/tests/test_actions.py +++ b/tests/test_actions.py @@ -213,3 +213,34 @@ def test_container_lock_force_bypasses_busy(tmp_path, capsys): entered = True assert entered is True assert "busy lock" in capsys.readouterr().err + + +# --- model / mmproj path validation --------------------------------------- + + +def test_validate_model_path_accepts_existing_model(tmp_path): + (tmp_path / "m.gguf").write_bytes(b"x") + cfg = make_cfg(hf_home=tmp_path, model_path="m.gguf") + actions.validate_model_path(cfg) # must not raise + + +def test_validate_model_path_rejects_missing_model(tmp_path): + cfg = make_cfg(hf_home=tmp_path, model_path="absent.gguf") + with pytest.raises(FileNotFoundError, match="model file not found"): + actions.validate_model_path(cfg) + + +def test_validate_model_path_rejects_missing_mmproj(tmp_path): + # The model exists but the configured projector does not: --change must fail + # here, before the running container is removed. + (tmp_path / "m.gguf").write_bytes(b"x") + cfg = make_cfg(hf_home=tmp_path, model_path="m.gguf", mmproj="absent-proj.gguf") + with pytest.raises(FileNotFoundError, match="mmproj file not found"): + actions.validate_model_path(cfg) + + +def test_validate_model_path_accepts_existing_mmproj(tmp_path): + (tmp_path / "m.gguf").write_bytes(b"x") + (tmp_path / "proj.gguf").write_bytes(b"x") + cfg = make_cfg(hf_home=tmp_path, model_path="m.gguf", mmproj="proj.gguf") + actions.validate_model_path(cfg) # must not raise diff --git a/tests/test_config.py b/tests/test_config.py index 38ddb2b..65f2785 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -55,6 +55,10 @@ model_alias = alt_llm [model.noname] host_port = 9002 +[model.vision] +container_name = test_vision +mmproj = qwen3/mmproj.gguf + [prompt.concise] system_prompt = Be brief. """ @@ -280,6 +284,57 @@ def test_stream_and_read_timeout_from_config(tmp_path): assert prompt_cfg.connect_timeout == 5.0 +def test_mmproj_empty_by_default(tmp_path): + cfg_path = _write_config(tmp_path) + server_cfg, _ = resolve_effective_config(_parse(["--start", "--config", str(cfg_path)])) + assert server_cfg.mmproj == "" + assert server_cfg.mmproj_offload is True # llama.cpp offloads by default + + +def test_mmproj_from_profile(tmp_path): + cfg_path = _write_config(tmp_path) + args = _parse(["--start", "--config", str(cfg_path), "--profile", "vision"]) + server_cfg, _ = resolve_effective_config(args) + assert server_cfg.mmproj == "qwen3/mmproj.gguf" + assert server_cfg.container_name == "test_vision" + + +def test_mmproj_cli_overrides_config(tmp_path): + cfg_path = _write_config(tmp_path) + args = _parse( + [ + "--start", + "--config", + str(cfg_path), + "--profile", + "vision", + "--mmproj", + "other/proj.gguf", + "--no-mmproj-offload", + ] + ) + server_cfg, _ = resolve_effective_config(args) + assert server_cfg.mmproj == "other/proj.gguf" + assert server_cfg.mmproj_offload is False + + +def test_mmproj_offload_false_from_config(tmp_path): + cfg = CONFIG_BASIC.replace( + "[model.vision]\ncontainer_name = test_vision", + "[model.vision]\ncontainer_name = test_vision\nmmproj_offload = false", + ) + cfg_path = _write_config(tmp_path, cfg) + args = _parse(["--start", "--config", str(cfg_path), "--profile", "vision"]) + server_cfg, _ = resolve_effective_config(args) + assert server_cfg.mmproj_offload is False + + +def test_blank_mmproj_via_cli_rejected(tmp_path): + cfg_path = _write_config(tmp_path) + with pytest.raises(SystemExit): + _parse(["--start", "--config", str(cfg_path), "--mmproj", " "]) + + def test_stream_and_read_timeout_cli_overrides_config(tmp_path): cfg = CONFIG_BASIC.replace("poll_interval = 2", "poll_interval = 2\nread_timeout = 900") cfg_path = _write_config(tmp_path, cfg) diff --git a/tests/test_docker_ops.py b/tests/test_docker_ops.py index 75a6c33..6c1a7b2 100644 --- a/tests/test_docker_ops.py +++ b/tests/test_docker_ops.py @@ -178,6 +178,41 @@ def test_build_run_command_relative_model_path(): assert cmd[idx + 1] == "/hf_home/qwen3/default.gguf" +def test_build_run_command_no_mmproj_by_default(): + cmd = build_run_command(make_cfg()) + assert "--mmproj" not in cmd + assert "--no-mmproj-offload" not in cmd + + +def test_build_run_command_mmproj_relative_path_resolves_under_hf_home(): + cfg = make_cfg(mmproj="qwen3/mmproj.gguf") + cmd = build_run_command(cfg) + idx = cmd.index("--mmproj") + assert cmd[idx + 1] == "/hf_home/qwen3/mmproj.gguf" + # offload is llama.cpp's default, so the opt-out flag must stay absent + assert "--no-mmproj-offload" not in cmd + + +def test_build_run_command_mmproj_absolute_path_passed_through(): + cfg = make_cfg(mmproj="/abs/mmproj.gguf") + cmd = build_run_command(cfg) + idx = cmd.index("--mmproj") + assert cmd[idx + 1] == "/abs/mmproj.gguf" + + +def test_build_run_command_mmproj_offload_disabled(): + cfg = make_cfg(mmproj="qwen3/mmproj.gguf", mmproj_offload=False) + cmd = build_run_command(cfg) + assert "--no-mmproj-offload" in cmd + + +def test_build_run_command_no_mmproj_offload_needs_mmproj(): + # Without a projector the offload opt-out is meaningless and must not leak + # into the command line (llama.cpp would reject it). + cfg = make_cfg(mmproj="", mmproj_offload=False) + assert "--no-mmproj-offload" not in build_run_command(cfg) + + def test_format_command_for_display_quotes_properly(): cmd = ["docker", "run", "--name", "has space"] out = format_command_for_display(cmd)