feat(docker): support a multimodal projector via --mmproj

Vision-capable GGUFs need a separate projector (mmproj) that maps image
embeddings into the text model's space. Add `mmproj` and `mmproj_offload`
as config keys and CLI overrides, and pass them through to llama-server.

The projector path resolves under hf_home exactly like model_path, so it
is covered by the existing read-only mount. validate_model_path() now also
checks the projector, which means --change rejects a missing one *before*
it removes the running container.

--no-mmproj-offload is suppressed when no projector is configured, since
llama.cpp rejects the flag on its own.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

llama.cpp.config.example

@@ -88,6 +88,24 @@ host_port = 8003
 gpu_device = 1
 ctx_size = 131072
 
+# Vision profile: `mmproj` points at the multimodal projector (vision encoder)
+# GGUF. Like `model_path` it is resolved relative to `hf_home` unless absolute.
+# The projector MUST match the base model's vision tower -- a projector built for
+# a different base will not load. A missing projector aborts with exit code 3
+# (for --change: before the running container is removed).
+#
+# llama.cpp offloads the projector to the GPU by default; set mmproj_offload to
+# false to keep it on the CPU when VRAM is tight. Images are then sent to
+# /v1/chat/completions as an image_url content part -- `llamacppctl --chat`
+# itself only sends text.
+[model.vision]
+model_path = qwen3/Qwen3-35B-A3B-Q4_K_M.gguf
+mmproj = qwen3/mmproj.gguf
+mmproj_offload = true
+container_name = llama_cpp_vision
+host_port = 8004
+gpu_device = 1
+
 [prompt.concise]
 system_prompt = Du antwortest kurz, präzise und technisch.
 

src/llamacppctl/actions.py

@@ -77,15 +77,23 @@ def check_exit_code(result: CheckResult) -> int:
     return EXIT_HTTP_NOT_READY
 
 
+def _host_path(cfg: ServerConfig, path_str: str) -> Path:
+    path = Path(path_str)
+    return path if path.is_absolute() else cfg.hf_home / path_str
+
+
 def validate_model_path(cfg: ServerConfig) -> None:
-    model_path = Path(cfg.model_path)
+    """Check that the model file — and the vision projector, when one is
-    if model_path.is_absolute():
+    configured — exist on the host before the container is (re)created."""
-        target = model_path
+    target = _host_path(cfg, cfg.model_path)
-    else:
-        target = cfg.hf_home / cfg.model_path
     if not target.exists():
         raise FileNotFoundError(f"model file not found: {target}")
 
+    if cfg.mmproj:
+        projector = _host_path(cfg, cfg.mmproj)
+        if not projector.exists():
+            raise FileNotFoundError(f"mmproj file not found: {projector}")
+
 
 def print_effective_config(cfg: ServerConfig, prompt_cfg: PromptConfig) -> None:
     payload = {

src/llamacppctl/cli.py

@@ -46,6 +46,18 @@ def build_parser() -> argparse.ArgumentParser:
     p.add_argument("--image", help="Docker image (e.g. ghcr.io/ggml-org/llama.cpp:server-cuda)")
     p.add_argument("--hf-home", help="Host path mounted read-only as model storage")
     p.add_argument("--model-path", help="Model path, relative to --hf-home unless absolute")
+    p.add_argument(
+        "--mmproj",
+        help="Vision projector (mmproj) GGUF, relative to --hf-home unless absolute. "
+        "Must match the base model's vision tower; enables image input",
+    )
+    p.add_argument(
+        "--no-mmproj-offload",
+        dest="mmproj_offload",
+        action="store_false",
+        default=None,
+        help="Keep the vision projector on the CPU instead of offloading it to the GPU",
+    )
     p.add_argument("--container-name", help="Docker container name (must be unique per config)")
     p.add_argument("--host-port", type=int, help="Host port to publish")
     p.add_argument("--container-port", type=int, help="Container-internal port")
@@ -185,3 +197,6 @@ def validate_args(args: argparse.Namespace, parser: argparse.ArgumentParser) ->
 
     if args.container_name is not None and not args.container_name.strip():
         parser.error("--container-name must not be empty")
+
+    if args.mmproj is not None and not args.mmproj.strip():
+        parser.error("--mmproj must not be empty")

src/llamacppctl/config.py

@@ -71,6 +71,8 @@ def builtin_defaults() -> dict:
         "host": "0.0.0.0",
         "expose": "false",  # publish only on loopback unless true
         "api_key": "",
+        "mmproj": "",  # vision projector GGUF; empty => text-only server
+        "mmproj_offload": "true",
         "health_endpoint": "/health",
         "models_endpoint": "/v1/models",
         "chat_endpoint": "/v1/chat/completions",
@@ -137,6 +139,8 @@ def _apply_cli_overrides(merged: dict, args) -> None:
         "lock_file": getattr(args, "lock_file", None),
         "expose": getattr(args, "expose", None),
         "api_key": getattr(args, "api_key", None),
+        "mmproj": getattr(args, "mmproj", None),
+        "mmproj_offload": getattr(args, "mmproj_offload", None),
     }
     for key, value in overrides.items():
         if value is not None:
@@ -193,6 +197,8 @@ def build_server_config(merged: dict) -> ServerConfig:
         host=str(merged["host"]),
         expose=_to_bool(merged.get("expose", "false"), "expose"),
         api_key=str(merged.get("api_key", "")),
+        mmproj=str(merged.get("mmproj") or "").strip(),
+        mmproj_offload=_to_bool(merged.get("mmproj_offload", "true"), "mmproj_offload"),
         health_endpoint=str(merged["health_endpoint"]),
         models_endpoint=str(merged["models_endpoint"]),
         chat_endpoint=str(merged["chat_endpoint"]),

src/llamacppctl/docker_ops.py

@@ -105,11 +105,21 @@ def container_logs(name: str, tail: int = 100) -> str:
 tail_logs = container_logs
 
 
+def _resolve_hf_path_in_container(path_str: str) -> str:
+    """Map a host-side model/projector path to its path inside the container.
+    Absolute paths are passed through; relative ones resolve under the
+    read-only /hf_home mount."""
+    if Path(path_str).is_absolute():
+        return path_str
+    return f"/hf_home/{path_str}"
+
+
 def _resolve_model_path_in_container(cfg: ServerConfig) -> str:
-    model_path = Path(cfg.model_path)
+    return _resolve_hf_path_in_container(cfg.model_path)
-    if model_path.is_absolute():
+
-        return str(model_path)
+
-    return f"/hf_home/{cfg.model_path}"
+def _resolve_mmproj_path_in_container(cfg: ServerConfig) -> str:
+    return _resolve_hf_path_in_container(cfg.mmproj)
 
 
 def _port_publish(cfg: ServerConfig) -> str:
@@ -181,6 +191,10 @@ def build_run_command(cfg: ServerConfig) -> list:
         cmd.append("--kv-unified")
     if cfg.cont_batching:
         cmd.append("--cont-batching")
+    if cfg.mmproj:
+        cmd += ["--mmproj", _resolve_mmproj_path_in_container(cfg)]
+        if not cfg.mmproj_offload:
+            cmd.append("--no-mmproj-offload")
     if cfg.api_key:
         cmd += ["--api-key", cfg.api_key]
     cmd.extend(cfg.extra_args)

src/llamacppctl/schema.py

@@ -60,6 +60,14 @@ class ServerConfig:
     # as a Bearer token on every request.
     api_key: str = ""
 
+    # Optional multimodal projector (vision encoder) GGUF. Resolved relative to
+    # hf_home unless absolute, like model_path. Must match the base model's
+    # vision tower. Empty => text-only server.
+    mmproj: str = ""
+    # llama.cpp offloads the projector to the GPU by default; set False to keep
+    # it on the CPU when VRAM is tight (emits --no-mmproj-offload).
+    mmproj_offload: bool = True
+
     extra_args: list = field(default_factory=list)
 
 

tests/test_actions.py

@@ -213,3 +213,34 @@ def test_container_lock_force_bypasses_busy(tmp_path, capsys):
             entered = True
     assert entered is True
     assert "busy lock" in capsys.readouterr().err
+
+
+# --- model / mmproj path validation ---------------------------------------
+
+
+def test_validate_model_path_accepts_existing_model(tmp_path):
+    (tmp_path / "m.gguf").write_bytes(b"x")
+    cfg = make_cfg(hf_home=tmp_path, model_path="m.gguf")
+    actions.validate_model_path(cfg)  # must not raise
+
+
+def test_validate_model_path_rejects_missing_model(tmp_path):
+    cfg = make_cfg(hf_home=tmp_path, model_path="absent.gguf")
+    with pytest.raises(FileNotFoundError, match="model file not found"):
+        actions.validate_model_path(cfg)
+
+
+def test_validate_model_path_rejects_missing_mmproj(tmp_path):
+    # The model exists but the configured projector does not: --change must fail
+    # here, before the running container is removed.
+    (tmp_path / "m.gguf").write_bytes(b"x")
+    cfg = make_cfg(hf_home=tmp_path, model_path="m.gguf", mmproj="absent-proj.gguf")
+    with pytest.raises(FileNotFoundError, match="mmproj file not found"):
+        actions.validate_model_path(cfg)
+
+
+def test_validate_model_path_accepts_existing_mmproj(tmp_path):
+    (tmp_path / "m.gguf").write_bytes(b"x")
+    (tmp_path / "proj.gguf").write_bytes(b"x")
+    cfg = make_cfg(hf_home=tmp_path, model_path="m.gguf", mmproj="proj.gguf")
+    actions.validate_model_path(cfg)  # must not raise

tests/test_config.py

@@ -55,6 +55,10 @@ model_alias = alt_llm
 [model.noname]
 host_port = 9002
 
+[model.vision]
+container_name = test_vision
+mmproj = qwen3/mmproj.gguf
+
 [prompt.concise]
 system_prompt = Be brief.
 """
@@ -280,6 +284,57 @@ def test_stream_and_read_timeout_from_config(tmp_path):
     assert prompt_cfg.connect_timeout == 5.0
 
 
+def test_mmproj_empty_by_default(tmp_path):
+    cfg_path = _write_config(tmp_path)
+    server_cfg, _ = resolve_effective_config(_parse(["--start", "--config", str(cfg_path)]))
+    assert server_cfg.mmproj == ""
+    assert server_cfg.mmproj_offload is True  # llama.cpp offloads by default
+
+
+def test_mmproj_from_profile(tmp_path):
+    cfg_path = _write_config(tmp_path)
+    args = _parse(["--start", "--config", str(cfg_path), "--profile", "vision"])
+    server_cfg, _ = resolve_effective_config(args)
+    assert server_cfg.mmproj == "qwen3/mmproj.gguf"
+    assert server_cfg.container_name == "test_vision"
+
+
+def test_mmproj_cli_overrides_config(tmp_path):
+    cfg_path = _write_config(tmp_path)
+    args = _parse(
+        [
+            "--start",
+            "--config",
+            str(cfg_path),
+            "--profile",
+            "vision",
+            "--mmproj",
+            "other/proj.gguf",
+            "--no-mmproj-offload",
+        ]
+    )
+    server_cfg, _ = resolve_effective_config(args)
+    assert server_cfg.mmproj == "other/proj.gguf"
+    assert server_cfg.mmproj_offload is False
+
+
+def test_mmproj_offload_false_from_config(tmp_path):
+    cfg = CONFIG_BASIC.replace(
+        "[model.vision]\ncontainer_name = test_vision",
+        "[model.vision]\ncontainer_name = test_vision\nmmproj_offload = false",
+    )
+    cfg_path = _write_config(tmp_path, cfg)
+    args = _parse(["--start", "--config", str(cfg_path), "--profile", "vision"])
+    server_cfg, _ = resolve_effective_config(args)
+    assert server_cfg.mmproj_offload is False
+
+
+def test_blank_mmproj_via_cli_rejected(tmp_path):
+    cfg_path = _write_config(tmp_path)
+    with pytest.raises(SystemExit):
+        _parse(["--start", "--config", str(cfg_path), "--mmproj", "   "])
+
+
 def test_stream_and_read_timeout_cli_overrides_config(tmp_path):
     cfg = CONFIG_BASIC.replace("poll_interval = 2", "poll_interval = 2\nread_timeout = 900")
     cfg_path = _write_config(tmp_path, cfg)

tests/test_docker_ops.py

@@ -178,6 +178,41 @@ def test_build_run_command_relative_model_path():
     assert cmd[idx + 1] == "/hf_home/qwen3/default.gguf"
 
 
+def test_build_run_command_no_mmproj_by_default():
+    cmd = build_run_command(make_cfg())
+    assert "--mmproj" not in cmd
+    assert "--no-mmproj-offload" not in cmd
+
+
+def test_build_run_command_mmproj_relative_path_resolves_under_hf_home():
+    cfg = make_cfg(mmproj="qwen3/mmproj.gguf")
+    cmd = build_run_command(cfg)
+    idx = cmd.index("--mmproj")
+    assert cmd[idx + 1] == "/hf_home/qwen3/mmproj.gguf"
+    # offload is llama.cpp's default, so the opt-out flag must stay absent
+    assert "--no-mmproj-offload" not in cmd
+
+
+def test_build_run_command_mmproj_absolute_path_passed_through():
+    cfg = make_cfg(mmproj="/abs/mmproj.gguf")
+    cmd = build_run_command(cfg)
+    idx = cmd.index("--mmproj")
+    assert cmd[idx + 1] == "/abs/mmproj.gguf"
+
+
+def test_build_run_command_mmproj_offload_disabled():
+    cfg = make_cfg(mmproj="qwen3/mmproj.gguf", mmproj_offload=False)
+    cmd = build_run_command(cfg)
+    assert "--no-mmproj-offload" in cmd
+
+
+def test_build_run_command_no_mmproj_offload_needs_mmproj():
+    # Without a projector the offload opt-out is meaningless and must not leak
+    # into the command line (llama.cpp would reject it).
+    cfg = make_cfg(mmproj="", mmproj_offload=False)
+    assert "--no-mmproj-offload" not in build_run_command(cfg)
+
+
 def test_format_command_for_display_quotes_properly():
     cmd = ["docker", "run", "--name", "has space"]
     out = format_command_for_display(cmd)